Media-Server-Setup/bin/vanguard-deadman.sh

#!/bin/bash
# The deadman: the one check that does not depend on the Elastic stack being alive.
#
# Kibana tells you when something on the box is wrong. Nothing inside Elasticsearch, Kibana or the Beats can
# tell you that they are the thing that broke, and silence looks exactly like health. So this runs on the HOST
# from a systemd timer, uses nothing but curl, and posts straight to a Discord webhook.
#
# It checks six things, in the order they go wrong in practice:
#   1. Elasticsearch answers and the cluster is not red
#   2. no index has been flipped to read-only by the disk watermark — the classic Elastic failure, because it
#      does NOT clear itself when space is freed, so ingestion stays silently dead until someone resets it
#   3. Kibana reports itself available (it is the only UI; if it is down you are blind)
#   4. Filebeat has actually written a document recently — "running" is not the same as "shipping"
#   5. Metricbeat likewise
#   6. the heartbeat a Kibana rule posts to dumpd is fresh, which proves rule evaluation AND delivery work
#   …and the disks, because that is the failure this box is most likely to die of, and it must be reported
#   even when the thing that would normally report it is what broke.
#
# State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes,
# and says so once when everything recovers.
#
# No webhook configured? The checks still run and still print to the journal, and the unit exits non-zero
# when it found something it could not report — so `systemctl is-failed vanguard-deadman.service` and
# `journalctl -u vanguard-deadman` remain a signal. It never fails because the webhook is absent alone.
set -uo pipefail

# systemd passes these in via EnvironmentFile. Load the same file when run by hand, or a manual run behaves
# differently from the timer — which cost a confusing round of "the mention is configured but never appears".
# Anything already in the environment wins, so `ALERT_SUPPRESS_S=0 vanguard-alerts.sh` still overrides.
if [[ -r /etc/vanguard/alerts.env ]]; then
  while IFS='=' read -r _k _v; do
    [[ -z ${_k:-} || ${_k:0:1} == '#' ]] && continue
    [[ -n ${!_k:-} ]] || export "$_k=$_v"
  done < /etc/vanguard/alerts.env
fi

ES="${ES_URL:-http://127.0.0.1:9200}"
# Elasticsearch requires credentials (security is on; observability/kibana.yml says why). configure.py writes
# this netrc, root-only: netrc rather than -u so the password is never visible in a process list. If the file
# is missing, the calls below get 401 and the deadman reports Elasticsearch as unreachable — which is correct,
# because from its point of view it is. Kibana needs nothing: its anonymous provider signs callers in.
ES_NETRC="${ES_NETRC_FILE:-/etc/vanguard/es-netrc}"
es_curl() {
  if [[ -r $ES_NETRC ]]; then curl -s --netrc-file "$ES_NETRC" "$@"; else curl -s "$@"; fi
}
KIBANA="${KIBANA_URL:-http://127.0.0.1:5601}"
WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}"
ALERT_INDEX="${ALERT_INDEX:-vanguard-alerts}"            # the alert bus Kibana's rules write into
ALERT_STATE="${ALERT_STATE:-/var/lib/vanguard/alerts.state}"  # touched by every vanguard-alerts.sh run
ALERT_STATE_MAX_S="${ALERT_STATE_MAX_S:-900}"
HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-900}"
STALE_MAX_S="${BEAT_STALE_MAX_S:-900}"
STATE="${DEADMAN_STATE:-/var/lib/vanguard/deadman.state}"
DISK_PCT="${DEADMAN_DISK_PCT:-90}"

problems=()
note() { problems+=("$1"); }
jq_get() { python3 -c 'import json,sys
try: d=json.load(sys.stdin)
except Exception: sys.exit(1)
for k in sys.argv[1].split("."):
    d = d.get(k) if isinstance(d, dict) else None
    if d is None: sys.exit(1)
print(d)' "$1" 2>/dev/null; }

# 1. Elasticsearch
health=$(es_curl -m 8 "$ES/_cluster/health" || true)
if [[ -z $health ]]; then
  note "Elasticsearch is not answering on ${ES#http://} — nothing is being stored"
else
  status=$(printf '%s' "$health" | jq_get status)
  case "$status" in
    green|yellow) ;;
    red) note "Elasticsearch cluster health is RED: some index has no usable shard" ;;
    *)   note "Elasticsearch health is unreadable (got: ${status:-nothing})" ;;
  esac

  # 2. read-only indices. Elastic sets this when the disk passes the flood watermark and never unsets it.
  blocked=$(es_curl -m 8 "$ES/_all/_settings/index.blocks.read_only_allow_delete" \
            | python3 -c 'import json,sys
try: d=json.load(sys.stdin)
except Exception: sys.exit(0)
bad=[n for n,v in d.items() if str(v.get("settings",{}).get("index",{}).get("blocks",{}).get("read_only_allow_delete","")).lower()=="true"]
print(len(bad))' 2>/dev/null || echo 0)
  (( ${blocked:-0} > 0 )) && note "$blocked Elasticsearch index/indices are READ-ONLY from the disk watermark; ingestion is dead until this is reset (see README 5h)"

  # 4 and 5. are the Beats actually writing?
  for beat in filebeat metricbeat; do
    count=$(es_curl -m 8 -H 'Content-Type: application/json' \
              "$ES/${beat}-*/_count" \
              -d "{\"query\":{\"range\":{\"@timestamp\":{\"gte\":\"now-${STALE_MAX_S}s\"}}}}" | jq_get count)
    if [[ -z ${count:-} ]]; then
      note "cannot tell whether $beat is shipping (no index yet, or the query failed)"
    elif (( count == 0 )); then
      note "$beat has written nothing in the last $((STALE_MAX_S/60)) minutes — it may be up but not shipping"
    fi
  done
fi

# 3. Kibana
level=$(curl -s -m 8 "$KIBANA/api/status" | jq_get status.overall.level)
[[ ${level:-} == available ]] || note "Kibana is not available (${level:-no answer}) — the only UI is down"

# 6. the heartbeat. A Kibana rule writes a kind=heartbeat document into the alert index every minute (see
# observability/kibana-setup.py). Its absence is the ONLY thing that catches the alerting pipeline itself
# dying: if Kibana, its rule engine or the index connector stops, no ordinary alert can report it, because
# the thing that would report it is what broke. Checked in Elasticsearch, not as a file: a basic licence has
# no webhook connector, so nothing posts to dumpd any more.
if [[ -n $health ]]; then
  beats=$(es_curl -m 8 -H 'Content-Type: application/json' "$ES/$ALERT_INDEX/_count" \
            -d "{\"query\":{\"bool\":{\"filter\":[{\"term\":{\"kind\":\"heartbeat\"}},{\"range\":{\"@timestamp\":{\"gte\":\"now-${HEARTBEAT_MAX_S}s\"}}}]}}}")
  if grep -q 'index_not_found_exception' <<<"${beats:-}"; then
    note "the alert index $ALERT_INDEX does not exist — the Kibana connector and heartbeat rule have not been created (observability/kibana-setup.py)"
  else
    hb=$(printf '%s' "${beats:-}" | jq_get count)
    if [[ -z ${hb:-} ]]; then
      note "cannot read the alert heartbeat (query against $ALERT_INDEX failed)"
    elif (( hb == 0 )); then
      note "no alert heartbeat in the last $((HEARTBEAT_MAX_S/60)) minutes — Kibana's rule engine or the index connector has stopped, so NO alert can reach Discord"
    fi
  fi
fi

# 7. is the delivery side running? The heartbeat proves Kibana still writes alerts; this proves something
# still takes them to Discord. Both halves have to be alive for an alarm to arrive.
if [[ -f $ALERT_STATE ]]; then
  age=$(( $(date +%s) - $(stat -c %Y "$ALERT_STATE") ))
  (( age <= ALERT_STATE_MAX_S )) || note "vanguard-alerts.timer has not run for ${age}s — alerts are being written but not delivered"
fi
# Freshness above proves the timer fires; this proves the run finished. A unit that starts every minute and
# dies immediately would otherwise look alive, because the file is written at the start of the run.
if systemctl is-failed --quiet vanguard-alerts.service 2>/dev/null; then
  note "vanguard-alerts.service is in a failed state — $(systemctl show -p Result --value vanguard-alerts.service 2>/dev/null); alerts are written but not delivered"
fi

# the disks, independently of any agent
while read -r pct mount; do
  (( pct >= DISK_PCT )) && note "$mount is ${pct}% full"
done < <(df --output=pcent,target /srv / 2>/dev/null | tail -n +2 | tr -d '%' | awk '{print $1, $2}')

# ---- report only on change ---------------------------------------------------------------------------
now="$(IFS='|'; echo "${problems[*]-}")"
mkdir -p "$(dirname "$STATE")"
was="$(cat "$STATE" 2>/dev/null || true)"
printf '%s' "$now" > "$STATE"
[[ $now == "$was" ]] && exit 0

# Delivery and mention handling are shared with vanguard-alerts.sh. A missing helper falls back to a plain
# post: losing the mention is acceptable, losing the alert is not.
if [[ -r "$(dirname "$0")/vgnotify.sh" ]]; then
  . "$(dirname "$0")/vgnotify.sh"
else
  vg_webhook() { [[ -r $WEBHOOK_FILE ]] && tr -d '[:space:]' < "$WEBHOOK_FILE" | grep '^https://'; }
  vg_post() {
    local url; url=$(vg_webhook) || return 1
    curl -s -o /dev/null -m 15 -X POST -H 'Content-Type: application/json' \
      --data "$(VG_T="$2" python3 -c 'import json,os; print(json.dumps({"content": os.environ["VG_T"][:1900]}))')" \
      "$url"
  }
fi
webhook=$(vg_webhook 2>/dev/null || true)

# A deadman alarm is the one that must wake somebody: it means no other alert can get through. Recovery is
# sent at info level so good news never pings.
post() { vg_post "${2:-error}" "$1"; }

if [[ -z $now ]]; then
  echo "deadman: recovered — Elastic, Kibana, both Beats and the disks are healthy"
  post "✅ **Vanguard monitoring recovered** — Elasticsearch, Kibana and both Beats are healthy again." info || true
  exit 0
fi

printf 'deadman: %s\n' "${problems[@]}" >&2
body="⚠️ **Vanguard monitoring is not healthy** (sent from the host, outside Elastic):"
for item in "${problems[@]}"; do body+=$'\n'"• $item"; done
body+=$'\n'"Until this clears, assume no other alert can reach you."
if post "$body"; then exit 0; fi
[[ -z $webhook ]] && echo "deadman: no Discord webhook configured (ALERT_WEBHOOK_URL in .env -> $WEBHOOK_FILE), so the above was not sent" >&2
exit 1