diff --git a/Caddyfile.example b/Caddyfile.example index 45e9e5e..87f074d 100644 --- a/Caddyfile.example +++ b/Caddyfile.example @@ -153,6 +153,67 @@ clips.example.com { } } +# vanguard:diag +diag.example.com { + encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } + # /logout on every gated host = sign out of the Discord login (domain-wide session) + redir /logout /outpost.goauthentik.io/sign_out 302 + # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the + # whole domain (recommended) or single-application for this host — assigned to the embedded outpost. + # Until then: 4xx, never open. Per-host group requirements are enforced below from X-Authentik-Groups. + reverse_proxy /outpost.goauthentik.io/* 127.0.0.1:9000 + forward_auth 127.0.0.1:9000 { + uri /outpost.goauthentik.io/auth/caddy + copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Entitlements X-Authentik-Email X-Authentik-Name X-Authentik-Uid X-Authentik-Jwt X-Authentik-Meta-Jwks X-Authentik-Meta-Outpost X-Authentik-Meta-Provider X-Authentik-Meta-App X-Authentik-Meta-Version + trusted_proxies private_ranges + } + @lacks_group not header_regexp X-Authentik-Groups (^|\|)media\-admins(\||$) + handle @lacks_group { + respond "This page is for members of the 'media-admins' group." 403 + } + # VictoriaLogs runs with -http.pathPrefix=/logs and emits /logs/... URLs itself, so the prefix is + # passed through unchanged. Stripping it would break the UI's own asset paths. + redir /logs /logs/select/vmui/ 302 + redir /logs/ /logs/select/vmui/ 302 + handle /logs* { + reverse_proxy 127.0.0.1:9428 + } + handle /metrics-db* { + reverse_proxy 127.0.0.1:8428 # VictoriaMetrics UI + query API + } + handle /alerts* { + reverse_proxy 127.0.0.1:9093 # Alertmanager: what is firing, silences + } + handle /rules-metrics* { + reverse_proxy 127.0.0.1:8880 # vmalert over VictoriaMetrics + } + handle /rules-logs* { + reverse_proxy 127.0.0.1:8881 # vmalert over VictoriaLogs + } + handle { + respond "Vanguard diagnostics + +/logs logs (VictoriaLogs) +/metrics-db metrics (VictoriaMetrics) +/alerts what is firing (Alertmanager) +/rules-metrics metric rules (vmalert) +/rules-logs log rules (vmalert) + +Alerts go to Discord. The deadman runs on the host and watches this stack." 200 + } +} + # vanguard:live live.example.com { encode zstd gzip diff --git a/CommunityDiscordMedia.md b/CommunityDiscordMedia.md index 77462f0..ff1b074 100644 --- a/CommunityDiscordMedia.md +++ b/CommunityDiscordMedia.md @@ -320,7 +320,53 @@ permanent reference is needed for long sessions, not just a start burst. export at 2560×1440) - rejected: LTC audio (SRT multitrack + per-PC generator); clock inside the picture; cropping at remux (re-encode) -### Phase 13 — push-to-deploy (built; OFF — `DEPLOY_AUTO=0`, CI workflow in draft-workflows/) +### Phase 13 — `diag.` host: logs, metrics, alerting, deadman (NEXT) + +**Decided 2026-11-03: VictoriaLogs**, on its own hostname `diag.BASE`, with the log UI at `diag.BASE/logs`. +Rejected: ELK (a JVM heap and a growing index on a box that spends its nights running x264, to search one +host's logs), and Grafana + Loki (two services and a dashboard tool we would only use as a log viewer). +VictoriaLogs is one container with its own UI and the same `/metrics` endpoint style as the taps we already +expose. The framework everything feeds from is `LOGGING.md`; this phase only adds the store and the collector. + +- [ ] CNAME `diag` → the main host +- [ ] `configure.py`: new host role `diag`, Caddy block gated by `DIAG_REQUIRE_GROUP` (default `media-admins`) +- [ ] `victoria-logs` container, `127.0.0.1:9428`, data in `/srv/appdata/victorialogs`, run with + `-http.pathPrefix=/logs` so its own UI emits `/logs/...` URLs; Caddy uses `handle /logs*` and does **not** + strip the prefix (stripping breaks vmui's asset paths). `redir /logs /logs/select/vmui/ 302` for humans. +- [ ] collector: **Vector**, one container, two sources — `journald` for our units (dumpd, normalize-sweep, + trash-purge, deploy-pending, vgdeband) and Docker logs for the 15 containers. The + `vanguard.service` / `vanguard.category` labels already on every container become stream fields, so + filtering by category works without parsing. Parse stages per source as `LOGGING.md` lists them + (JSON for caddy/authentik, logfmt for syncthing and our own scripts, one regex for mediamtx, + multiline + regex for jellyfin). +- [ ] **Decide at build time how Vector reads container logs.** Its `docker_logs` source needs the Docker API, + and mounting the socket is root-equivalent on this host — the same exposure act_runner already has. The + alternative is a read-only socket proxy in front of it, one more container but no root. Reading the + json files directly avoids both and loses the container labels, which is most of the value. +- [ ] retention: one global `-retentionPeriod` (start at 30d and watch the disk). Keeping `auth` and `deploy` + longer than the rest would mean separate tenants — not worth it unless an audit needs it. +- [ ] switch on the metrics scrape at the same time if wanted: the four taps are already live + (node_exporter 9100, cAdvisor 9101, MediaMTX 9998, Caddy 2019) and VictoriaMetrics beside VictoriaLogs + would consume them. Not required for logs. +- [ ] **alerts — VictoriaLogs does NOT alert.** It stores and queries, and exposes `/select/logsql/stats_query` + (+ `_range`) returning Prometheus-compatible results. Alerting is **vmalert** evaluating rules against + those endpoints, and **Alertmanager** receiving what fires (it has a native Discord receiver). That chain + covers *log* conditions only — a sweep `event=failed`, a burst of `publish_deny`, a failed bot deploy. + Disk, load and restart loops are *metrics*, so alerting on them additionally needs VictoriaMetrics + scraping the four taps that are already live. Full picture: four containers. +- [ ] **Do this first, instead, and keep it either way**: `bin/vanguard-watch.sh` on a timer — disk on `/srv` + over 85 %, any `*.normalize-failed.log`, a container missing from `docker compose ps`, dumpd `level=error` + in the last interval → one Discord webhook post. About forty lines, no new services, and it covers the + failure modes that actually take the platform down. The vmalert/Alertmanager stack is the grown-up + version for thresholds, grouping and silencing, not a prerequisite for being warned. + +**Later on the same host — ops controls.** Restart buttons and similar, so `diag.` becomes the one place an +admin looks and acts. Design note so it is not built the easy wrong way: the page must not get Docker access. +Put each action behind a dumpd endpoint with a fixed allowlist (restart *this* named service, run the sweep now, +purge trash now), authorise from the Authentik headers as everything else does, and log every invocation as a +logfmt event so the action lands in the same place as the logs it is diagnosing. No generic "run a command". + +### Phase 14 — push-to-deploy (built; OFF — `DEPLOY_AUTO=0`, CI workflow in draft-workflows/) - server folder = git clone (`switchToClone.sh`); generated files gitignored and preserved - `vanguard-deploy pull|apply|rollback|pending|status` applies only the changed parts; `check.sh` gates it; MediaMTX restarts are deferred while anything is live (`deploy-pending.timer`) @@ -384,52 +430,6 @@ Checklist if adopted: Swap, not redesign: same compose, same `/srv` layout; restore `/srv/appdata` + `/srv/media`. - [ ] home archive puller (`home/rclone-archive.sh`) on the home box, nightly cron -### Phase 14 — `diag.` host: logs now, ops controls later - -**Decided 2026-11-03: VictoriaLogs**, on its own hostname `diag.BASE`, with the log UI at `diag.BASE/logs`. -Rejected: ELK (a JVM heap and a growing index on a box that spends its nights running x264, to search one -host's logs), and Grafana + Loki (two services and a dashboard tool we would only use as a log viewer). -VictoriaLogs is one container with its own UI and the same `/metrics` endpoint style as the taps we already -expose. The framework everything feeds from is `LOGGING.md`; this phase only adds the store and the collector. - -- [ ] CNAME `diag` → the main host -- [ ] `configure.py`: new host role `diag`, Caddy block gated by `DIAG_REQUIRE_GROUP` (default `media-admins`) -- [ ] `victoria-logs` container, `127.0.0.1:9428`, data in `/srv/appdata/victorialogs`, run with - `-http.pathPrefix=/logs` so its own UI emits `/logs/...` URLs; Caddy uses `handle /logs*` and does **not** - strip the prefix (stripping breaks vmui's asset paths). `redir /logs /logs/select/vmui/ 302` for humans. -- [ ] collector: **Vector**, one container, two sources — `journald` for our units (dumpd, normalize-sweep, - trash-purge, deploy-pending, vgdeband) and Docker logs for the 15 containers. The - `vanguard.service` / `vanguard.category` labels already on every container become stream fields, so - filtering by category works without parsing. Parse stages per source as `LOGGING.md` lists them - (JSON for caddy/authentik, logfmt for syncthing and our own scripts, one regex for mediamtx, - multiline + regex for jellyfin). -- [ ] **Decide at build time how Vector reads container logs.** Its `docker_logs` source needs the Docker API, - and mounting the socket is root-equivalent on this host — the same exposure act_runner already has. The - alternative is a read-only socket proxy in front of it, one more container but no root. Reading the - json files directly avoids both and loses the container labels, which is most of the value. -- [ ] retention: one global `-retentionPeriod` (start at 30d and watch the disk). Keeping `auth` and `deploy` - longer than the rest would mean separate tenants — not worth it unless an audit needs it. -- [ ] switch on the metrics scrape at the same time if wanted: the four taps are already live - (node_exporter 9100, cAdvisor 9101, MediaMTX 9998, Caddy 2019) and VictoriaMetrics beside VictoriaLogs - would consume them. Not required for logs. -- [ ] **alerts — VictoriaLogs does NOT alert.** It stores and queries, and exposes `/select/logsql/stats_query` - (+ `_range`) returning Prometheus-compatible results. Alerting is **vmalert** evaluating rules against - those endpoints, and **Alertmanager** receiving what fires (it has a native Discord receiver). That chain - covers *log* conditions only — a sweep `event=failed`, a burst of `publish_deny`, a failed bot deploy. - Disk, load and restart loops are *metrics*, so alerting on them additionally needs VictoriaMetrics - scraping the four taps that are already live. Full picture: four containers. -- [ ] **Do this first, instead, and keep it either way**: `bin/vanguard-watch.sh` on a timer — disk on `/srv` - over 85 %, any `*.normalize-failed.log`, a container missing from `docker compose ps`, dumpd `level=error` - in the last interval → one Discord webhook post. About forty lines, no new services, and it covers the - failure modes that actually take the platform down. The vmalert/Alertmanager stack is the grown-up - version for thresholds, grouping and silencing, not a prerequisite for being warned. - -**Later on the same host — ops controls.** Restart buttons and similar, so `diag.` becomes the one place an -admin looks and acts. Design note so it is not built the easy wrong way: the page must not get Docker access. -Put each action behind a dumpd endpoint with a fixed allowlist (restart *this* named service, run the sweep now, -purge trash now), authorise from the Authentik headers as everything else does, and log every invocation as a -logfmt event so the action lands in the same place as the logs it is diagnosing. No generic "run a command". - ## Risks / gotchas diff --git a/bin/dumpd.py b/bin/dumpd.py index 1ce3243..1c6b406 100755 --- a/bin/dumpd.py +++ b/bin/dumpd.py @@ -12,6 +12,8 @@ POST /me/clips -> {"device_id": "..."}: create clips/ (+ .stignore), add POST /deploy -> Gitea push webhook (bridge port only, HMAC-verified with DEPLOY_WEBHOOK_SECRET from /etc/vanguard/deploy.env). Runs /usr/local/bin/vanguard-deploy pull ONLY when DEPLOY_AUTO=1; otherwise logs the push and answers 202 "auto-deploy disabled". +POST /heartbeat -> Alertmanager's Watchdog route: touches a file the deadman timer checks, so a dead + alerting pipeline becomes visible (bin/vanguard-deadman.sh). POST /deploy-bot -> Gitea Actions (job container -> host gateway) hands over a Discord-bot release bundle, HMAC-SHA256 signed with BOT_DEPLOY_SECRET; installs it with vanguard-deploy-bot. May carry X-Vanguard-Env (base64 JSON): the bot's own settings, kept in Gitea, written @@ -326,6 +328,8 @@ def apply_bot_env(values): log("bot_env_applied", keys=",".join(applied), refused=",".join(refused) or None) return applied, refused +HEARTBEAT_FILE = Path(os.environ.get("ALERT_HEARTBEAT_FILE", "/srv/media/.markers/alert-heartbeat")) + _bot_deploy_lock = threading.Lock() BOT_DEPLOY_MAX_BYTES = 300 * 1024 * 1024 BOT_DEPLOY_INSTALLER = os.environ.get("BOT_DEPLOY_INSTALLER", "/usr/local/bin/vanguard-deploy-bot") @@ -493,6 +497,17 @@ class H(BaseHTTPRequestHandler): "files": [str(f.relative_to(ROOT)) for f in files], "gb": round(sum(f.stat().st_size for f in files) / 1e9, 2)}) def do_POST(self): + if self.server.server_port == AUTH_LISTEN[1] and self.path.startswith("/heartbeat"): + # Alertmanager posts here every minute because the Watchdog alert always fires. Nothing is read from + # the body and nothing is authenticated: the port only accepts the Docker bridge, and the only thing + # this grants is keeping a timestamp fresh. The deadman treats a STALE file as the alarm. + try: + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + HEARTBEAT_FILE.parent.mkdir(parents=True, exist_ok=True) + HEARTBEAT_FILE.write_text(f"{int(_time.time())}\n") + except OSError as error: + return self._json(500, {"error": str(error)}) + return self._json(200, {"ok": True}) if self.server.server_port == AUTH_LISTEN[1] and self.path.startswith("/deploy-bot"): # The bot's CI posts its release bundle here (bridge-only port; the firewall allows docker0/br-* alone). # The body is the artifact; the signature proves it came from a runner holding BOT_DEPLOY_SECRET. diff --git a/bin/vanguard-deadman.sh b/bin/vanguard-deadman.sh new file mode 100755 index 0000000..1e56b93 --- /dev/null +++ b/bin/vanguard-deadman.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# The deadman: the one check that does not depend on the observability stack being alive. +# +# vmalert and Alertmanager tell you when something on the box is wrong. Nothing inside that chain can tell you +# the chain itself is down, and silence looks exactly like health. So this runs on the HOST from a systemd +# timer, uses nothing but curl, and posts straight to a Discord webhook. +# +# It checks three things: +# 1. every piece of the stack answers its health endpoint +# 2. Alertmanager's Watchdog heartbeat is fresh (proves rule evaluation AND routing work end to end, +# not merely that the processes are up) +# 3. the disks, because that is the failure this box is most likely to die of, and it must be reported +# even when the thing that would normally report it is the thing that broke +# +# State in /var/lib/vanguard/deadman.state: it announces a change, not the same thing every five minutes, +# and says so once when everything recovers. +set -uo pipefail + +WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-/etc/vanguard/alert-webhook}" +HEARTBEAT="${ALERT_HEARTBEAT_FILE:-/srv/media/.markers/alert-heartbeat}" +HEARTBEAT_MAX_S="${HEARTBEAT_MAX_S:-600}" +STATE="${DEADMAN_STATE:-/var/lib/vanguard/deadman.state}" +DISK_PCT="${DEADMAN_DISK_PCT:-90}" + +problems=() +note() { problems+=("$1"); } + +alive() { # alive