#!/bin/bash
# The deadman: the one check that does not depend on the observability stack being alive.
#
# vmalert and Alertmanager tell you when something on the box is wrong. Nothing inside that chain can tell you
# the chain itself is down, and silence looks exactly like health. So this runs on the HOST from a systemd
# timer, uses nothing but curl, and posts straight to a Discord webhook.
#
# It checks three things:
#   1. every piece of the stack answers its health endpoint
#   2. Alertmanager's Watchdog heartbeat is fresh (proves rule evaluation AND routing work end to end,
#      not merely that the processes are up)
#   3. the disks, because that is the failure this box is most likely to die of, and it must be reported
#      even when the thing that would normally report it is the thing that broke
#
# State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes,
# and says so once when everything recovers.
set -uo pipefail

WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}"
HEARTBEAT="${ALERT_HEARTBEAT_FILE:-/srv/media/.markers/alert-heartbeat}"
HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-600}"
STATE="${DEADMAN_STATE:-/var/lib/vanguard/deadman.state}"
DISK_PCT="${DEADMAN_DISK_PCT:-90}"

problems=()
note() { problems+=("$1"); }

alive() {   # alive <label> <url>
  local code
  code=$(curl -s -o /dev/null -m 5 -w '%{http_code}' "$2" || true)
  [[ $code == 2* || $code == 3* ]] || note "$1 is not answering (HTTP ${code:-no response})"
}

alive "VictoriaLogs"    "http://127.0.0.1:9428/logs/health"
alive "VictoriaMetrics" "http://127.0.0.1:8428/health"
alive "vmalert (metrics)" "http://127.0.0.1:8880/health"
alive "vmalert (logs)"    "http://127.0.0.1:8881/health"
alive "Alertmanager"    "http://127.0.0.1:9093/-/healthy"

# 2. the heartbeat. A missing file right after a restart is not yet a fault, so only age counts.
if [[ -f $HEARTBEAT ]]; then
  age=$(( $(date +%s) - $(stat -c %Y "$HEARTBEAT") ))
  (( age <= HEARTBEAT_MAX_S )) || note "no alert heartbeat for ${age}s (vmalert → Alertmanager → dumpd path is broken)"
else
  note "no alert heartbeat has ever arrived (is the Watchdog rule loaded and routed?)"
fi

# 3. disks, independently of any exporter
while read -r pct mount; do
  (( pct >= DISK_PCT )) && note "$mount is ${pct}% full"
done < <(df --output=pcent,target /srv / 2>/dev/null | tail -n +2 | tr -d '%' | awk '{print $1, $2}')

# ---- report only on change ---------------------------------------------------------------------------
now="$(IFS='|'; echo "${problems[*]-}")"
mkdir -p "$(dirname "$STATE")"
was="$(cat "$STATE" 2>/dev/null || true)"
printf '%s' "$now" > "$STATE"
[[ $now == "$was" ]] && exit 0                      # nothing changed since the last run: stay quiet

post() {                                            # post <text>
  [[ -s $WEBHOOK_FILE ]] || { echo "deadman: no webhook in $WEBHOOK_FILE; not sent: $1" >&2; return; }
  curl -s -m 15 -H 'Content-Type: application/json' \
    --data "$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1][:1900]}))' "$1")" \
    "$(cat "$WEBHOOK_FILE")" >/dev/null || echo "deadman: webhook post failed" >&2
}

if [[ -z $now ]]; then
  echo "deadman: recovered"
  post "✅ **Vanguard monitoring recovered** — the alerting stack and disks are healthy again."
else
  printf 'deadman: %s\n' "${problems[@]}"
  body="⚠️ **Vanguard monitoring is not healthy** (this message comes from the host, outside the stack):"
  for item in "${problems[@]}"; do body+=$'\n'"• $item"; done
  body+=$'\n'"Until this clears, assume no other alert can reach you."
  post "$body"
fi