Phase 13 integration: install.sh, configure.py, and a deadman that degrades cleanly
install.sh: the three data dirs with correct owners (uid 1000 for the VictoriaMetrics pair, 65534 for alertmanager), /var/lib/vanguard, the deadman script and timer, and /etc/vanguard/alert-webhook as a FILE — compose bind-mounts it, so without it Docker creates a directory and alertmanager will not start. configure.py: --full now asks for the alert webhook and stores it; --show says plainly when none is set; the webhook lands at 0640 root:65534 rather than world-readable, since it is a credential for that channel. The deadman no longer assumes a webhook. It trims and validates the file (the previous -s test passed on the newline configure.py writes), skips delivery when there is none, still logs every finding, and exits non-zero only when it found something it could not report — so the unit state is the signal. Verified for a missing file, a newline-only file, a junk value, and the healthy case. check.sh validates all five observability configs; confirmed it catches a deliberate break. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,6 +14,10 @@
|
||||
#
|
||||
# State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes,
|
||||
# and says so once when everything recovers.
|
||||
#
|
||||
# No webhook configured? The checks still run and still print to the journal, and the unit exits non-zero
|
||||
# when it found something it could not report — so `systemctl is-failed vanguard-deadman.service` and
|
||||
# `journalctl -u vanguard-deadman` remain a signal. It never fails because the webhook is absent alone.
|
||||
set -uo pipefail
|
||||
|
||||
WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}"
|
||||
@@ -57,20 +61,40 @@ was="$(cat "$STATE" 2>/dev/null || true)"
|
||||
printf '%s' "$now" > "$STATE"
|
||||
[[ $now == "$was" ]] && exit 0 # nothing changed since the last run: stay quiet
|
||||
|
||||
post() { # post <text>
|
||||
[[ -s $WEBHOOK_FILE ]] || { echo "deadman: no webhook in $WEBHOOK_FILE; not sent: $1" >&2; return; }
|
||||
# The webhook is optional. With none configured the checks still run and still land in the journal
|
||||
# (journalctl -u vanguard-deadman), so nothing silently stops working; only the Discord delivery is skipped.
|
||||
webhook=""
|
||||
if [[ -r $WEBHOOK_FILE ]]; then webhook="$(tr -d '[:space:]' < "$WEBHOOK_FILE" 2>/dev/null || true)"; fi
|
||||
case "$webhook" in
|
||||
https://*) ;;
|
||||
"") webhook="" ;;
|
||||
*) echo "deadman: ALERT_WEBHOOK_URL is not an https URL; ignoring it" >&2; webhook="" ;;
|
||||
esac
|
||||
|
||||
post() { # post <text>; a no-op without a webhook
|
||||
[[ -n $webhook ]] || return 1
|
||||
curl -s -m 15 -H 'Content-Type: application/json' \
|
||||
--data "$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1][:1900]}))' "$1")" \
|
||||
"$(cat "$WEBHOOK_FILE")" >/dev/null || echo "deadman: webhook post failed" >&2
|
||||
"$webhook" >/dev/null && return 0
|
||||
echo "deadman: the webhook post failed" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ -z $now ]]; then
|
||||
echo "deadman: recovered"
|
||||
post "✅ **Vanguard monitoring recovered** — the alerting stack and disks are healthy again."
|
||||
else
|
||||
printf 'deadman: %s\n' "${problems[@]}"
|
||||
body="⚠️ **Vanguard monitoring is not healthy** (this message comes from the host, outside the stack):"
|
||||
for item in "${problems[@]}"; do body+=$'\n'"• $item"; done
|
||||
body+=$'\n'"Until this clears, assume no other alert can reach you."
|
||||
post "$body"
|
||||
echo "deadman: recovered — the alerting stack and disks are healthy again"
|
||||
post "✅ **Vanguard monitoring recovered** — the alerting stack and disks are healthy again." || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
printf 'deadman: %s\n' "${problems[@]}" >&2
|
||||
body="⚠️ **Vanguard monitoring is not healthy** (sent from the host, outside the stack):"
|
||||
for item in "${problems[@]}"; do body+=$'\n'"• $item"; done
|
||||
body+=$'\n'"Until this clears, assume no other alert can reach you."
|
||||
if post "$body"; then exit 0; fi
|
||||
|
||||
# Problems found and nobody could be told. Exit non-zero so the unit shows as failed: with no webhook
|
||||
# configured, `systemctl is-failed vanguard-deadman.service` is the only signal there is.
|
||||
if [[ -z $webhook ]]; then
|
||||
echo "deadman: no Discord webhook configured (ALERT_WEBHOOK_URL in .env -> $WEBHOOK_FILE), so the above was not sent" >&2
|
||||
fi
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user