Metrics taps: node_exporter + cAdvisor containers, MediaMTX and Caddy metrics on loopback (Prometheus format, store-agnostic)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-01 19:55:42 -06:00
parent 704bc8c4d3
commit 855651c5a4
5 changed files with 163 additions and 13 deletions
--- a/Caddyfile.example
+++ b/Caddyfile.example
@@ -1,6 +1,23 @@
+{
+	servers {
+		metrics                           # Prometheus metrics at http://127.0.0.1:2019/metrics (LOGGING.md)
+	}
+}
+
 # vanguard:main
 media.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	# /logout on every gated host = sign out of the Discord login (domain-wide session)
 	redir /logout /outpost.goauthentik.io/sign_out 302
 	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
@@ -32,14 +49,20 @@ name:   {http.request.header.X-Authentik-Name}"
 	}
 }

-# vanguard:apex
-example.com {
-	redir https://media.example.com/ 302
-}
-
 # vanguard:files
 files.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	# /logout on every gated host = sign out of the Discord login (domain-wide session)
 	redir /logout /outpost.goauthentik.io/sign_out 302
 	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
@@ -54,7 +77,7 @@ files.example.com {
 	# copyparty at the root. It honours X-Authentik-Username / -Groups only when the request also carries a header
 	# whose secret NAME only this proxy knows (copyparty idp-h-key). Clients cannot forge identity.
 	reverse_proxy 127.0.0.1:3923 {
-		header_up X-Vanguard-EXAMPLEKEY 1
+		header_up X-Vanguard-k 1
 		flush_interval -1
 	}
 }
@@ -62,6 +85,17 @@ files.example.com {
 # vanguard:sync
 sync.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	# /logout on every gated host = sign out of the Discord login (domain-wide session)
 	redir /logout /outpost.goauthentik.io/sign_out 302
 	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
@@ -86,6 +120,17 @@ sync.example.com {
 # vanguard:clips
 clips.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	# /logout on every gated host = sign out of the Discord login (domain-wide session)
 	redir /logout /outpost.goauthentik.io/sign_out 302
 	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
@@ -111,6 +156,17 @@ clips.example.com {
 # vanguard:live
 live.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	# /logout on every gated host = sign out of the Discord login (domain-wide session)
 	redir /logout /outpost.goauthentik.io/sign_out 302
 	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
@@ -167,6 +223,17 @@ groups: {http.request.header.X-Authentik-Groups}"
 # vanguard:bot
 bot.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	# /logout on every gated host = sign out of the Discord login (domain-wide session)
 	redir /logout /outpost.goauthentik.io/sign_out 302
 	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
@@ -188,17 +255,50 @@ bot.example.com {
 # vanguard:git
 git.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	reverse_proxy 127.0.0.1:3000
 }

 # vanguard:auth
 auth.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	reverse_proxy 127.0.0.1:9000
 }

 # vanguard:watch
 watch.example.com {
 	encode zstd gzip
+	log {
+		output stderr
+		format filter {
+			wrap json
+			fields {
+				request>headers>Cookie delete
+				request>headers>Authorization delete
+				request>headers>X-Stream-Auth delete
+			}
+		}
+	}
 	reverse_proxy 127.0.0.1:8096
 }
--- a/LOGGING.md
+++ b/LOGGING.md
@@ -10,9 +10,19 @@ upstream of it is fixed now, so the choice later is a collector config, not a co
 | **journald** | our systemd units: `dumpd`, `normalize-sweep`, `trash-purge`, `deploy-pending`, `vgdeband` (transient), `deploy.sh` output | `journalctl -u <unit>`; collectors: Alloy `loki.source.journal`, Vector `journald`, VictoriaLogs journald ingestion, Netdata's journal plugin |
 | **Docker json-file logs** | every container (`docker-compose.yml`), rotated 5 × 50 MB each | `docker logs <name>`; collectors read `/var/lib/docker/containers/*/*-json.log` and get the container **labels** below for free |

-Every collector on the shortlist reads both. Metrics are the same story: node_exporter / cAdvisor expose the
-Prometheus text format that Alloy, vmagent and Netdata all scrape; MediaMTX (`metrics: yes`) and Caddy have
-endpoints too, currently off.
+Every collector on the shortlist reads both. Metrics are the same story: see the taps table below — all four endpoints are on now, on loopback.
+
+## Metrics taps (all Prometheus text format, loopback only — scrape them with Alloy, vmagent or Netdata)
+
+| endpoint | what | from |
+|---|---|---|
+| `http://127.0.0.1:9100/metrics` | host: CPU, RAM, disk space + I/O, network, load, md RAID | node_exporter (host network, pid host) |
+| `http://127.0.0.1:9101/metrics` | per container: CPU, memory, network, block I/O — labelled with `vanguard.service` / `vanguard.category` | cAdvisor |
+| `http://127.0.0.1:9998/metrics` | MediaMTX: per path bytes/s, readers, publishers, WebRTC/HLS/RTMP sessions, dropped packets | MediaMTX `metrics: yes` |
+| `http://127.0.0.1:2019/metrics` | Caddy: requests by host/status, durations, TLS | Caddy admin API (`servers { metrics }`) |
+
+None of these are reachable from outside (loopback binds; nothing in the firewall). Alert thresholds to start with:
+disk on `/srv` > 85 %, load > cores for 15 min, a container restarting, MediaMTX dropped packets rising on a live path.

 ## Container labels (set in `docker-compose.yml`, `x-logging` anchor + `labels:`)

@@ -74,8 +84,7 @@ On for every site block (`CADDY_LOG` in `configure.py`): JSON to stderr; `Cookie

 - **Grafana + Loki**: `alloy` container with `loki.source.journal` + `loki.source.docker` (labels from the container
  labels above) → `loki` on loopback → `grafana` behind Authentik OIDC (third provider, like Jellyfin/Gitea).
-  Metrics: Alloy scrapes node_exporter, cAdvisor, MediaMTX `:9998/metrics`, Caddy `:2019/metrics` → Prometheus or
-  VictoriaMetrics, same Grafana.
+  Metrics: Alloy scrapes the four taps above → Prometheus or VictoriaMetrics, same Grafana.
 - **VictoriaLogs**: single container, journald + docker via its agent or Vector; built-in UI behind the Caddy
  forward-auth gate (`media-admins`). VictoriaMetrics beside it for metrics.
 - **Netdata**: single container, host + container metrics and alerts (Discord) out of the box, journald viewer;
--- a/STATUS.md
+++ b/STATUS.md
@@ -59,6 +59,7 @@ Source of truth: this repo, `techy/Media-Server-Setup` on the stack's own Gitea
 | normalize sweep | editing standard, one copy per clip | live |
 | nftables | host firewall (22, 80/443, 1935, 2222, 3306, 8890/udp, 22000) | live |
 | Caddy | TLS + routing for all hostnames; JSON access logs per site (identity headers kept, cookies/tokens dropped) | live (access logs: deploy pending) |
+| Metrics taps | node_exporter (host), cAdvisor (containers), MediaMTX and Caddy metrics — Prometheus format on loopback, no store yet | **committed 2026-11-01, deploy pending** |
 | Logging framework | rotated Docker logs + `vanguard.service/category` labels on every container; dumpd/sweep/vgdeband/deploy emit logfmt events (`LOGGING.md`); store not chosen (Loki vs VictoriaLogs vs Netdata) | **committed 2026-11-01, deploy pending** (recreates every container) |
 | Authentik (+Postgres, Redis) | Discord login, groups, guild-membership policy | live |
 | Jellyfin | watch page, SSO via Authentik, direct play only | live |
--- a/configure.py
+++ b/configure.py
@@ -303,7 +303,13 @@ groups: {{http.request.header.X-Authentik-Groups}}"
 {CADDY_LOG}	reverse_proxy 127.0.0.1:8096
 }}
 """
-    Path("Caddyfile").write_text(f"""# vanguard:main
+    Path("Caddyfile").write_text(f"""{{
+	servers {{
+		metrics                           # Prometheus metrics at http://127.0.0.1:2019/metrics (LOGGING.md)
+	}}
+}}
+
+# vanguard:main
 {domain} {{
 	encode zstd gzip
 {gate_any}	# who does Authentik say I am? (debug: shows exactly the identity headers the apps receive)
@@ -393,7 +399,8 @@ rtspAddress: 127.0.0.1:8554
 rtspTransports: [tcp]
 rtspEncryption: "no"
 moq: no
-metrics: no
+metrics: yes                     # Prometheus text format for whichever metrics store is chosen (LOGGING.md); loopback only
+metricsAddress: :9998
 pprof: no

 # live portal (served to the browser only via Caddy + Authentik on live.HOST; these ports are published on 127.0.0.1)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -23,6 +23,7 @@ services:
      - "127.0.0.1:8888:8888"    # HLS      -> Caddy live.HOST/hls/*
      - "127.0.0.1:8889:8889"    # WebRTC   -> Caddy live.HOST/webrtc/*
      - "127.0.0.1:9997:9997"    # API      -> Caddy live.HOST/api/*
+      - "127.0.0.1:9998:9998"    # Prometheus metrics (per-stream bitrate, readers, drops) -> scraper on the host
    volumes:
      - ./mediamtx.yml:/mediamtx.yml:ro
      - ./bin/mtx-on-ready.sh:/vanguard/on-ready.sh:ro   # is-live notice + WebRTC audio mirror (runOnReady)
@@ -299,6 +300,38 @@ services:
      - gitea
    restart: unless-stopped

+  # ---- metrics taps (LOGGING.md): Prometheus text format on loopback, scraped by whichever store is chosen later
+  node_exporter:
+    logging: *vglog
+    labels: { vanguard.service: node_exporter, vanguard.category: infra }
+    image: quay.io/prometheus/node-exporter:latest
+    container_name: node_exporter
+    network_mode: host               # listens on 127.0.0.1:9100 only
+    pid: host
+    command: ["--path.rootfs=/host", "--web.listen-address=127.0.0.1:9100", "--collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker/.+)($$|/)"]
+    volumes:
+      - /:/host:ro,rslave
+    restart: unless-stopped
+
+  cadvisor:
+    logging: *vglog
+    labels: { vanguard.service: cadvisor, vanguard.category: infra }
+    image: gcr.io/cadvisor/cadvisor:latest
+    container_name: cadvisor
+    ports:
+      - "127.0.0.1:9101:8080"        # per-container CPU / memory / network / disk
+    command: ["--housekeeping_interval=10s", "--docker_only=true", "--store_container_labels=false", "--whitelisted_container_labels=vanguard.service,vanguard.category"]
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk/:/dev/disk:ro
+    devices:
+      - /dev/kmsg
+    privileged: true
+    restart: unless-stopped
+
 volumes:
  copyparty_hist:
  caddy_data: