Brandyn / Techy
8ae43905dc
Six containers, all loopback-published, reachable only through the gated diag. host: victorialogs 30d retention, -http.pathPrefix=/logs so Caddy passes the prefix through unchanged vector docker_logs + journald -> VictoriaLogs (elasticsearch bulk, the documented path) victoriametrics 90d, scrapes the four taps that were already live plus the new services vmalert-metrics 11 metric rules: disk, load, memory, RAID, restart loops, a core container gone vmalert-logs 5 LogsQL rules: sweep failures, publish-deny bursts, failed bot deploys, cert trouble alertmanager groups and inhibits, posts to Discord, webhook read from a 0600 file not the config The deadman (bin/vanguard-deadman.sh, host timer, 5 min) is the part that does not trust any of the above: curl only, it checks each component answers, that the always-firing Watchdog alert is still arriving through vmalert -> Alertmanager -> dumpd /heartbeat, and the disks. It reports on change and announces recovery once. Verified here: failures listed, silence when nothing changes, disk threshold, recovery fired once. Collection uses the Docker socket (Phase 13 option A, chosen explicitly). That is root-equivalent on this host and is recorded as such in vector.yaml with the two upgrade paths. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
77 lines
3.6 KiB
Bash
Executable File
77 lines
3.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# The deadman: the one check that does not depend on the observability stack being alive.
|
|
#
|
|
# vmalert and Alertmanager tell you when something on the box is wrong. Nothing inside that chain can tell you
|
|
# the chain itself is down, and silence looks exactly like health. So this runs on the HOST from a systemd
|
|
# timer, uses nothing but curl, and posts straight to a Discord webhook.
|
|
#
|
|
# It checks three things:
|
|
# 1. every piece of the stack answers its health endpoint
|
|
# 2. Alertmanager's Watchdog heartbeat is fresh (proves rule evaluation AND routing work end to end,
|
|
# not merely that the processes are up)
|
|
# 3. the disks, because that is the failure this box is most likely to die of, and it must be reported
|
|
# even when the thing that would normally report it is the thing that broke
|
|
#
|
|
# State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes,
|
|
# and says so once when everything recovers.
|
|
set -uo pipefail
|
|
|
|
WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}"
|
|
HEARTBEAT="${ALERT_HEARTBEAT_FILE:-/srv/media/.markers/alert-heartbeat}"
|
|
HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-600}"
|
|
STATE="${DEADMAN_STATE:-/var/lib/vanguard/deadman.state}"
|
|
DISK_PCT="${DEADMAN_DISK_PCT:-90}"
|
|
|
|
problems=()
|
|
note() { problems+=("$1"); }
|
|
|
|
alive() { # alive <label> <url>
|
|
local code
|
|
code=$(curl -s -o /dev/null -m 5 -w '%{http_code}' "$2" || true)
|
|
[[ $code == 2* || $code == 3* ]] || note "$1 is not answering (HTTP ${code:-no response})"
|
|
}
|
|
|
|
alive "VictoriaLogs" "http://127.0.0.1:9428/logs/health"
|
|
alive "VictoriaMetrics" "http://127.0.0.1:8428/health"
|
|
alive "vmalert (metrics)" "http://127.0.0.1:8880/health"
|
|
alive "vmalert (logs)" "http://127.0.0.1:8881/health"
|
|
alive "Alertmanager" "http://127.0.0.1:9093/-/healthy"
|
|
|
|
# 2. the heartbeat. A missing file right after a restart is not yet a fault, so only age counts.
|
|
if [[ -f $HEARTBEAT ]]; then
|
|
age=$(( $(date +%s) - $(stat -c %Y "$HEARTBEAT") ))
|
|
(( age <= HEARTBEAT_MAX_S )) || note "no alert heartbeat for ${age}s (vmalert → Alertmanager → dumpd path is broken)"
|
|
else
|
|
note "no alert heartbeat has ever arrived (is the Watchdog rule loaded and routed?)"
|
|
fi
|
|
|
|
# 3. disks, independently of any exporter
|
|
while read -r pct mount; do
|
|
(( pct >= DISK_PCT )) && note "$mount is ${pct}% full"
|
|
done < <(df --output=pcent,target /srv / 2>/dev/null | tail -n +2 | tr -d '%' | awk '{print $1, $2}')
|
|
|
|
# ---- report only on change ---------------------------------------------------------------------------
|
|
now="$(IFS='|'; echo "${problems[*]-}")"
|
|
mkdir -p "$(dirname "$STATE")"
|
|
was="$(cat "$STATE" 2>/dev/null || true)"
|
|
printf '%s' "$now" > "$STATE"
|
|
[[ $now == "$was" ]] && exit 0 # nothing changed since the last run: stay quiet
|
|
|
|
post() { # post <text>
|
|
[[ -s $WEBHOOK_FILE ]] || { echo "deadman: no webhook in $WEBHOOK_FILE; not sent: $1" >&2; return; }
|
|
curl -s -m 15 -H 'Content-Type: application/json' \
|
|
--data "$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1][:1900]}))' "$1")" \
|
|
"$(cat "$WEBHOOK_FILE")" >/dev/null || echo "deadman: webhook post failed" >&2
|
|
}
|
|
|
|
if [[ -z $now ]]; then
|
|
echo "deadman: recovered"
|
|
post "✅ **Vanguard monitoring recovered** — the alerting stack and disks are healthy again."
|
|
else
|
|
printf 'deadman: %s\n' "${problems[@]}"
|
|
body="⚠️ **Vanguard monitoring is not healthy** (this message comes from the host, outside the stack):"
|
|
for item in "${problems[@]}"; do body+=$'\n'"• $item"; done
|
|
body+=$'\n'"Until this clears, assume no other alert can reach you."
|
|
post "$body"
|
|
fi
|