diff --git a/.gitignore b/.gitignore index c1fa658..abf5d25 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ bot/app/__pycache__/ # generated by configure.py on the server — never commit or scp these mediamtx.yml copyparty.conf -observability/alertmanager.yml Caddyfile stream-token.secret deploy.env diff --git a/bin/vanguard-deadman.sh b/bin/vanguard-deadman.sh index 5a37a7a..f40a368 100755 --- a/bin/vanguard-deadman.sh +++ b/bin/vanguard-deadman.sh @@ -1,16 +1,20 @@ #!/bin/bash -# The deadman: the one check that does not depend on the observability stack being alive. +# The deadman: the one check that does not depend on the Elastic stack being alive. # -# vmalert and Alertmanager tell you when something on the box is wrong. Nothing inside that chain can tell you -# the chain itself is down, and silence looks exactly like health. So this runs on the HOST from a systemd -# timer, uses nothing but curl, and posts straight to a Discord webhook. +# Kibana tells you when something on the box is wrong. Nothing inside Elasticsearch, Kibana or the Beats can +# tell you that they are the thing that broke, and silence looks exactly like health. So this runs on the HOST +# from a systemd timer, uses nothing but curl, and posts straight to a Discord webhook. # -# It checks three things: -# 1. every piece of the stack answers its health endpoint -# 2. Alertmanager's Watchdog heartbeat is fresh (proves rule evaluation AND routing work end to end, -# not merely that the processes are up) -# 3. the disks, because that is the failure this box is most likely to die of, and it must be reported -# even when the thing that would normally report it is the thing that broke +# It checks six things, in the order they go wrong in practice: +# 1. Elasticsearch answers and the cluster is not red +# 2. no index has been flipped to read-only by the disk watermark — the classic Elastic failure, because it +# does NOT clear itself when space is freed, so ingestion stays silently dead until someone resets it +# 3. Kibana reports itself available (it is the only UI; if it is down you are blind) +# 4. Filebeat has actually written a document recently — "running" is not the same as "shipping" +# 5. Metricbeat likewise +# 6. the heartbeat a Kibana rule posts to dumpd is fresh, which proves rule evaluation AND delivery work +# …and the disks, because that is the failure this box is most likely to die of, and it must be reported +# even when the thing that would normally report it is what broke. # # State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes, # and says so once when everything recovers. @@ -20,38 +24,72 @@ # `journalctl -u vanguard-deadman` remain a signal. It never fails because the webhook is absent alone. set -uo pipefail +ES="${ES_URL:-http://127.0.0.1:9200}" +KIBANA="${KIBANA_URL:-http://127.0.0.1:5601}" WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}" HEARTBEAT="${ALERT_HEARTBEAT_FILE:-/srv/media/.markers/alert-heartbeat}" -HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-600}" +HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-900}" +STALE_MAX_S="${BEAT_STALE_MAX_S:-900}" STATE="${DEADMAN_STATE:-/var/lib/vanguard/deadman.state}" DISK_PCT="${DEADMAN_DISK_PCT:-90}" problems=() note() { problems+=("$1"); } +jq_get() { python3 -c 'import json,sys +try: d=json.load(sys.stdin) +except Exception: sys.exit(1) +for k in sys.argv[1].split("."): + d = d.get(k) if isinstance(d, dict) else None + if d is None: sys.exit(1) +print(d)' "$1" 2>/dev/null; } -alive() { # alive