Media-Server-Setup/bin/vanguard-alerts.sh

#!/bin/bash
# Deliver Kibana's alerts to Discord. Runs from a systemd timer every minute; curl and python only.
#
# WHY THIS EXISTS
# On a basic license Kibana's only connector types are .index and .server-log — every connector that leaves
# the stack is Gold or above. So Kibana rules write their alerts into ONE Elasticsearch index (the alert bus,
# created by observability/kibana-setup.py) and this script is the delivery side. Alerts are therefore
# ordinary documents: queryable, replayable, and visible in Discover alongside the logs that caused them.
#
# Heartbeat documents (kind=heartbeat) are deliberately NOT delivered. They exist for bin/vanguard-deadman.sh
# to notice their absence, which is the only way to catch the alerting pipeline itself dying.
#
# Repeat suppression is a safety net, not the primary throttle: a rule should use "On status change" or its
# own throttle in Kibana. This stops a misconfigured rule turning into a message every minute, and it is per
# rule name, so one noisy rule never silences another.
#
# Exit status: 0 when there was nothing to send or everything was sent; 1 when alerts existed and could not be
# delivered, so `systemctl status vanguard-alerts` and the journal both carry the failure.
set -uo pipefail

ES="${ES_URL:-http://127.0.0.1:9200}"
ES_NETRC="${ES_NETRC_FILE:-/etc/vanguard/es-netrc}"
INDEX="${ALERT_INDEX:-vanguard-alerts}"
STATE="${ALERT_STATE:-/var/lib/vanguard/alerts.state}"
WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}"
SUPPRESS_S="${ALERT_SUPPRESS_S:-900}"
BATCH="${ALERT_BATCH:-50}"

vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=alerts %s\n' "$(date -Is)" "$lvl" "$*"; }
es_curl() { if [[ -r $ES_NETRC ]]; then curl -s --netrc-file "$ES_NETRC" "$@"; else curl -s "$@"; fi; }

state_get() { [[ -r $STATE ]] || return 0; grep -m1 "^$1=" "$STATE" 2>/dev/null | cut -d= -f2-; }
state_set() {
  local key="$1" value="$2" tmp
  install -d "$(dirname "$STATE")" 2>/dev/null || return 0
  tmp=$(mktemp "${STATE}.XXXXXX" 2>/dev/null) || return 0
  { [[ -r $STATE ]] && grep -v "^${key}=" "$STATE"; printf '%s=%s\n' "$key" "$value"; } > "$tmp" 2>/dev/null
  mv -f "$tmp" "$STATE" 2>/dev/null || rm -f "$tmp"
}

webhook=""
[[ -r $WEBHOOK_FILE ]] && webhook=$(tr -d '[:space:]' < "$WEBHOOK_FILE")
# Same guard as the deadman: a typo that is not an https URL must not be posted to, and must not look like
# delivery succeeded. Treated as "no webhook", which means journal-only and a non-zero exit.
case "${webhook:-}" in
  "")       ;;
  https://*) ;;
  *)        vglog error 'event=bad_webhook msg="ALERT_WEBHOOK_URL is not an https URL; ignoring it"' >&2; webhook="" ;;
esac

now=$(date +%s)
cursor=$(state_get cursor); cursor="${cursor:-0}"

# Everything newer than the cursor that is not a heartbeat, oldest first so the cursor advances safely.
query=$(printf '{"size":%d,"sort":[{"@timestamp":"asc"}],"query":{"bool":{"filter":[{"range":{"@timestamp":{"gt":%s,"format":"epoch_millis"}}}],"must_not":[{"term":{"kind":"heartbeat"}}]}}}' "$BATCH" "$cursor")
response=$(es_curl -m 15 -H 'Content-Type: application/json' "$ES/$INDEX/_search" -d "$query")

if [[ -z $response ]]; then
  vglog error 'event=unreachable msg="Elasticsearch did not answer; cannot read the alert bus"' >&2
  exit 1
fi
# A missing index just means no rule has fired yet. Not a fault, and not worth a line every minute.
if grep -q 'index_not_found_exception' <<<"$response"; then
  exit 0
fi

# Retention. A heartbeat document arrives every minute, so without this the alert bus grows without bound.
# At most once every 6 h: a delete_by_query is cheap, but not worth doing sixty times an hour. Deliberately
# before the "nothing new" exit below, because most runs have nothing to deliver and would skip it.
retention_days="${ALERT_RETENTION_DAYS:-90}"
last_purge=$(state_get purged); last_purge="${last_purge:-0}"
if (( now - last_purge > 21600 )); then
  purge=$(es_curl -m 120 -H 'Content-Type: application/json' \
            -XPOST "$ES/$INDEX/_delete_by_query?conflicts=proceed" \
            -d "{\"query\":{\"range\":{\"@timestamp\":{\"lt\":\"now-${retention_days}d\"}}}}")
  gone=$(printf '%s' "${purge:-}" | python3 -c 'import json,sys
try: print(json.load(sys.stdin).get("deleted", 0))
except Exception: print("?")')
  state_set purged "$now"
  vglog info "event=purged older_than=${retention_days}d deleted=${gone:-?}"
fi

rows=$(printf '%s' "$response" | python3 -c '
import json, sys
from datetime import datetime
try:
    data = json.load(sys.stdin)
except Exception:
    sys.exit(0)
for hit in data.get("hits", {}).get("hits", []):
    source = hit.get("_source", {})
    stamp = str(source.get("@timestamp", ""))
    try:
        when = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
        millis = int(when.timestamp() * 1000)
    except Exception:
        continue
    def clean(value, limit):
        return " ".join(str(value).split())[:limit] or "-"
    print("\t".join((str(millis),
                     clean(source.get("rule", "unnamed rule"), 120),
                     clean(source.get("level", "warn"), 12),
                     clean(source.get("message", json.dumps(source)[:400]), 1200))))
')

[[ -z $rows ]] && exit 0

# One per line: "a=0 b=0 now" would make bash run `now` as a COMMAND with a and b as temporary
# assignments, leaving every counter unset for the rest of the script.
sent=0
suppressed=0
failed=0
while IFS=$'\t' read -r millis rule level message; do
  [[ -z ${millis:-} ]] && continue
  last=$(state_get "sent:$rule"); last="${last:-0}"
  if (( now - last < SUPPRESS_S )); then
    suppressed=$((suppressed + 1))
    vglog info "event=suppressed rule=\"$rule\" within=${SUPPRESS_S}s"
    state_set cursor "$millis"
    continue
  fi
  case "$level" in
    error|critical) mark="[ERROR]" ;;
    warn|warning)   mark="[WARN]" ;;
    *)              mark="[$level]" ;;
  esac
  text="$mark $rule — $message"
  if [[ -z $webhook ]]; then
    vglog warn "event=undeliverable rule=\"$rule\" msg=\"no Discord webhook configured; alert only in the journal\"" >&2
    vglog warn "event=alert rule=\"$rule\" level=$level msg=\"$message\"" >&2
    failed=$((failed + 1))
    state_set cursor "$millis"
    continue
  fi
  payload=$(VG_TEXT="$text" python3 -c 'import json, os; print(json.dumps({"content": os.environ["VG_TEXT"][:1900]}))')
  # curl already prints 000 on a connection failure; "|| echo 000" would concatenate and give 000000.
  code=$(curl -s -o /dev/null -w '%{http_code}' -m 15 -X POST -H 'Content-Type: application/json' \
           -d "$payload" "$webhook" || true)
  code="${code:-000}"
  if [[ $code == 2* ]]; then
    sent=$((sent + 1)); state_set "sent:$rule" "$now"
    vglog info "event=delivered rule=\"$rule\" level=$level"
  else
    failed=$((failed + 1))
    vglog error "event=delivery_failed rule=\"$rule\" http=$code" >&2
  fi
  state_set cursor "$millis"
done <<<"$rows"

vglog info "event=run sent=$sent suppressed=$suppressed failed=$failed"
(( failed > 0 )) && exit 1
exit 0