Brandyn / Techy
da9a1f4fa6
Each service now runs behind the prefix it is published under (-http.pathPrefix, or route-prefix plus
external-url for Alertmanager). Without that, three of the four UIs would have loaded at their sub-path and
then fetched their assets from the domain root, so the pages would have come up broken. The prefixes cascade,
so the scrape paths, vmalert's notifier and remote-write URLs, and the deadman's health probes all follow.
The index is diag/index.html mounted into Caddy, not an inline respond: Caddy reads {...} as placeholders and
the page's CSS is full of braces, so inline would have been mangled. It also matches how the gateway, dock and
clips pages are already served.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
103 lines
4.9 KiB
Bash
Executable File
103 lines
4.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# The deadman: the one check that does not depend on the observability stack being alive.
|
|
#
|
|
# vmalert and Alertmanager tell you when something on the box is wrong. Nothing inside that chain can tell you
|
|
# the chain itself is down, and silence looks exactly like health. So this runs on the HOST from a systemd
|
|
# timer, uses nothing but curl, and posts straight to a Discord webhook.
|
|
#
|
|
# It checks three things:
|
|
# 1. every piece of the stack answers its health endpoint
|
|
# 2. Alertmanager's Watchdog heartbeat is fresh (proves rule evaluation AND routing work end to end,
|
|
# not merely that the processes are up)
|
|
# 3. the disks, because that is the failure this box is most likely to die of, and it must be reported
|
|
# even when the thing that would normally report it is the thing that broke
|
|
#
|
|
# State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes,
|
|
# and says so once when everything recovers.
|
|
#
|
|
# No webhook configured? The checks still run and still print to the journal, and the unit exits non-zero
|
|
# when it found something it could not report — so `systemctl is-failed vanguard-deadman.service` and
|
|
# `journalctl -u vanguard-deadman` remain a signal. It never fails because the webhook is absent alone.
|
|
set -uo pipefail
|
|
|
|
WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}"
|
|
HEARTBEAT="${ALERT_HEARTBEAT_FILE:-/srv/media/.markers/alert-heartbeat}"
|
|
HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-600}"
|
|
STATE="${DEADMAN_STATE:-/var/lib/vanguard/deadman.state}"
|
|
DISK_PCT="${DEADMAN_DISK_PCT:-90}"
|
|
|
|
problems=()
|
|
note() { problems+=("$1"); }
|
|
|
|
alive() { # alive <label> <url>
|
|
local code
|
|
code=$(curl -s -o /dev/null -m 5 -w '%{http_code}' "$2" || true)
|
|
[[ $code == 2* || $code == 3* ]] || note "$1 is not answering (HTTP ${code:-no response})"
|
|
}
|
|
|
|
# Each service runs behind the path prefix it is published under on diag.HOST, so its health endpoint
|
|
# lives under that prefix too.
|
|
alive "VictoriaLogs" "http://127.0.0.1:9428/logs/health"
|
|
alive "VictoriaMetrics" "http://127.0.0.1:8428/metrics-db/health"
|
|
alive "vmalert (metrics)" "http://127.0.0.1:8880/rules-metrics/health"
|
|
alive "vmalert (logs)" "http://127.0.0.1:8881/rules-logs/health"
|
|
alive "Alertmanager" "http://127.0.0.1:9093/alerts/-/healthy"
|
|
|
|
# 2. the heartbeat. A missing file right after a restart is not yet a fault, so only age counts.
|
|
if [[ -f $HEARTBEAT ]]; then
|
|
age=$(( $(date +%s) - $(stat -c %Y "$HEARTBEAT") ))
|
|
(( age <= HEARTBEAT_MAX_S )) || note "no alert heartbeat for ${age}s (vmalert → Alertmanager → dumpd path is broken)"
|
|
else
|
|
note "no alert heartbeat has ever arrived (is the Watchdog rule loaded and routed?)"
|
|
fi
|
|
|
|
# 3. disks, independently of any exporter
|
|
while read -r pct mount; do
|
|
(( pct >= DISK_PCT )) && note "$mount is ${pct}% full"
|
|
done < <(df --output=pcent,target /srv / 2>/dev/null | tail -n +2 | tr -d '%' | awk '{print $1, $2}')
|
|
|
|
# ---- report only on change ---------------------------------------------------------------------------
|
|
now="$(IFS='|'; echo "${problems[*]-}")"
|
|
mkdir -p "$(dirname "$STATE")"
|
|
was="$(cat "$STATE" 2>/dev/null || true)"
|
|
printf '%s' "$now" > "$STATE"
|
|
[[ $now == "$was" ]] && exit 0 # nothing changed since the last run: stay quiet
|
|
|
|
# The webhook is optional. With none configured the checks still run and still land in the journal
|
|
# (journalctl -u vanguard-deadman), so nothing silently stops working; only the Discord delivery is skipped.
|
|
webhook=""
|
|
if [[ -r $WEBHOOK_FILE ]]; then webhook="$(tr -d '[:space:]' < "$WEBHOOK_FILE" 2>/dev/null || true)"; fi
|
|
case "$webhook" in
|
|
https://*) ;;
|
|
"") webhook="" ;;
|
|
*) echo "deadman: ALERT_WEBHOOK_URL is not an https URL; ignoring it" >&2; webhook="" ;;
|
|
esac
|
|
|
|
post() { # post <text>; a no-op without a webhook
|
|
[[ -n $webhook ]] || return 1
|
|
curl -s -m 15 -H 'Content-Type: application/json' \
|
|
--data "$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1][:1900]}))' "$1")" \
|
|
"$webhook" >/dev/null && return 0
|
|
echo "deadman: the webhook post failed" >&2
|
|
return 1
|
|
}
|
|
|
|
if [[ -z $now ]]; then
|
|
echo "deadman: recovered — the alerting stack and disks are healthy again"
|
|
post "✅ **Vanguard monitoring recovered** — the alerting stack and disks are healthy again." || true
|
|
exit 0
|
|
fi
|
|
|
|
printf 'deadman: %s\n' "${problems[@]}" >&2
|
|
body="⚠️ **Vanguard monitoring is not healthy** (sent from the host, outside the stack):"
|
|
for item in "${problems[@]}"; do body+=$'\n'"• $item"; done
|
|
body+=$'\n'"Until this clears, assume no other alert can reach you."
|
|
if post "$body"; then exit 0; fi
|
|
|
|
# Problems found and nobody could be told. Exit non-zero so the unit shows as failed: with no webhook
|
|
# configured, `systemctl is-failed vanguard-deadman.service` is the only signal there is.
|
|
if [[ -z $webhook ]]; then
|
|
echo "deadman: no Discord webhook configured (ALERT_WEBHOOK_URL in .env -> $WEBHOOK_FILE), so the above was not sent" >&2
|
|
fi
|
|
exit 1
|