diff --git a/.gitignore b/.gitignore index abf5d25..c1fa658 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ bot/app/__pycache__/ # generated by configure.py on the server — never commit or scp these mediamtx.yml copyparty.conf +observability/alertmanager.yml Caddyfile stream-token.secret deploy.env diff --git a/LOGGING.md b/LOGGING.md index a189ea9..2be1831 100644 --- a/LOGGING.md +++ b/LOGGING.md @@ -104,8 +104,9 @@ vmalert → Alertmanager → dumpd `/heartbeat`, and the disks. It speaks on cha with no webhook configured still logs and exits non-zero so `systemctl is-failed vanguard-deadman.service` is a signal. Nothing inside the stack can report its own death; this can. -`ALERT_WEBHOOK_URL` in `.env` → `/etc/vanguard/alert-webhook` (0640 root:65534). Empty means log-only, which -is a working configuration, not an error. +`ALERT_WEBHOOK_URL` in `.env` → `/etc/vanguard/alert-webhook` (0640 root:65534). Empty means log-only, which is +a working configuration, not an error: `configure.py` generates `observability/alertmanager.yml` with the +Discord notifier omitted, so nothing is half-configured and nothing fails to load. ## Wiring sketches (the options that were weighed) diff --git a/README.md b/README.md index 44c89f9..d3defdf 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,10 @@ alert by itself. Metric rules (`observability/alerts-metrics.yml`) cover disk, l loops and a missing core container; log rules (`observability/alerts-logs.yml`, LogsQL) cover sweep failures, publish-deny bursts, failed bot releases and certificate trouble. Both notify one Discord webhook, `ALERT_WEBHOOK_URL` in `.env`, written to `/etc/vanguard/alert-webhook` so it never sits in a config file. +**Running without a webhook is a supported state**, not a half-configured one: `configure.py` generates +`observability/alertmanager.yml` with the Discord notifier left out entirely, so alerts still evaluate, group +and appear at `/alerts`, they are simply not delivered. The deadman behaves the same way, logging to the +journal and exiting non-zero when it finds something it could not report. Set the variable, re-apply, done. **The deadman is the part that matters most.** `vanguard-deadman.timer` runs on the host every 5 minutes with nothing but curl: are all six answering, is the always-firing `Watchdog` alert still coming through the whole diff --git a/check.sh b/check.sh index 5a37aea..3830042 100755 --- a/check.sh +++ b/check.sh @@ -13,7 +13,7 @@ for f in configure.py bin/dumpd.py bin/vgstrip.py; do python3 -m py_compile "$f" && say "$f" ok || { say "$f" "PYTHON SYNTAX"; fail=1; } done rm -rf __pycache__ bin/__pycache__ -python3 -c "import yaml,sys; [yaml.safe_load(open(f)) for f in sys.argv[1:]]" docker-compose.yml mediamtx.example.yml observability/vector.yaml observability/scrape.yml observability/alerts-metrics.yml observability/alerts-logs.yml observability/alertmanager.yml && say "yaml" ok || { say "yaml" "INVALID"; fail=1; } +python3 -c "import yaml,sys; [yaml.safe_load(open(f)) for f in sys.argv[1:]]" docker-compose.yml mediamtx.example.yml observability/vector.yaml observability/scrape.yml observability/alerts-metrics.yml observability/alerts-logs.yml observability/alertmanager.example.yml && say "yaml" ok || { say "yaml" "INVALID"; fail=1; } node -e " const fs=require('fs'); for (const f of ['dock/index.html','dock/clock/index.html','clips/index.html','about/index.html']) { const html=fs.readFileSync(f,'utf8'); for (const m of html.matchAll(/]*)>([\s\S]*?)<\/script>/g)) if (m[1].trim()) new Function(m[1]); } diff --git a/configure.py b/configure.py index 9dca7cd..96538ca 100755 --- a/configure.py +++ b/configure.py @@ -17,7 +17,7 @@ Reads : Caddyfile (hostnames), mediamtx.yml (streamers), .env (editors/admins are Authentik groups now: editors, media-admins, admins — nothing local) (these are GENERATED on the server and gitignored; *.example.* in the repo are reference only — never scp them over the live ones: that is how streamers got wiped once) -Writes : Caddyfile, mediamtx.yml, copyparty.conf, stream-token.secret -> /etc/vanguard, .env (generated secrets + +Writes : Caddyfile, mediamtx.yml, copyparty.conf, observability/alertmanager.yml, stream-token.secret -> /etc/vanguard, .env (generated secrets + Discord bot token/guild), folders, restarts Set VG_DRY=1 to write files only (no sudo / docker / folders). """ @@ -500,6 +500,59 @@ paths: ~^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$: {{}} """) +def write_alertmanager(): + """Alertmanager config. Generated, because the Discord receiver must be ABSENT when no webhook is set: + a receiver with no notifiers is valid and simply drops notifications, whereas pointing discord_configs at + an empty webhook_url_file may stop Alertmanager from loading at all. The URL itself stays out of this file + and in /etc/vanguard/alert-webhook (0640 root:65534), which is what webhook_url_file reads.""" + webhook = read_env().get("ALERT_WEBHOOK_URL", "").strip() + discord = """ discord_configs: + - webhook_url_file: /etc/vanguard/alert-webhook + title: '{{ if eq .Status "firing" }}🔴{{ else }}✅{{ end }} {{ .CommonLabels.alertname }}' + message: | + {{ range .Alerts }}{{ .Annotations.summary }} + {{ if .Annotations.detail }}{{ .Annotations.detail }} + {{ end }}{{ end }} +""" if webhook.startswith("https://") else """ # No ALERT_WEBHOOK_URL in .env, so this receiver sends nowhere: alerts still group and still show at + # https://DIAG_HOST/alerts, they just are not delivered. Set the variable and re-apply to turn it on. +""" + Path("observability/alertmanager.yml").write_text(f"""# Alertmanager — GENERATED by configure.py from ALERT_WEBHOOK_URL. Reference copy: alertmanager.example.yml +# The heartbeat route is half of the deadman: the Watchdog alert always fires, so Alertmanager posts to dumpd +# every minute and bin/vanguard-deadman.sh treats a STALE heartbeat as the alarm. +global: + resolve_timeout: 5m + +route: + receiver: discord + group_by: [alertname, service] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + routes: + - matchers: [severity="none"] + receiver: heartbeat + group_wait: 0s + group_interval: 1m + repeat_interval: 1m + - matchers: [severity="critical"] + receiver: discord + group_wait: 10s + repeat_interval: 1h + +inhibit_rules: + - source_matchers: [alertname="MediaDiskCritical"] + target_matchers: [alertname="MediaDiskFilling"] + - source_matchers: [alertname="ContainerGone"] + target_matchers: [alertname="ContainerRestartLoop"] + +receivers: + - name: discord +{discord} - name: heartbeat + webhook_configs: + - url: http://host.docker.internal:8081/heartbeat + send_resolved: false +""") + def write_copyparty(): """copyparty behind Caddy + Authentik: identity and groups arrive as headers; permissions are per group.""" key = read_env().get("COPYPARTY_IDP_KEY", "key") @@ -560,7 +613,7 @@ def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", dis if not yes(" continue anyway?", False): print("aborted; add streamers first ([s] in the menu)"); sys.exit(1) print("\n== writing configs") write_env(watch_host, discord_token, discord_guild, git_host, bot_host, live_host, domain, live_webhook, files_host, sync_host, clips_host, diag_host) - write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); write_mediamtx(domain); write_copyparty() + write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); write_mediamtx(domain); write_copyparty(); write_alertmanager() Path("stream-token.secret").write_text(read_env()["STREAM_TOKEN_SECRET"] + "\n"); os.chmod("stream-token.secret", 0o600) run(["sudo", "install", "-d", "-o", os.environ.get("SUDO_USER") or os.environ.get("USER") or "debian", "/srv/live"]) print("== app data folders") diff --git a/observability/alertmanager.yml b/observability/alertmanager.example.yml similarity index 86% rename from observability/alertmanager.yml rename to observability/alertmanager.example.yml index 504ec34..a0f1e44 100644 --- a/observability/alertmanager.yml +++ b/observability/alertmanager.example.yml @@ -1,3 +1,7 @@ +# REFERENCE COPY ONLY — configure.py generates observability/alertmanager.yml from ALERT_WEBHOOK_URL and that +# generated file is what runs (it omits the Discord receiver's notifier entirely when no webhook is set, so +# the config stays valid and alerts simply are not delivered). Never scp this over the generated one. +# # Alertmanager: groups what vmalert fires and posts it to Discord. The webhook URL is NOT in here — it is a # secret, so configure.py writes it to /etc/vanguard/alert-webhook with mode 0600 and this reads it from there. #