diff --git a/Caddyfile.example b/Caddyfile.example index 8098d57..45e9e5e 100644 --- a/Caddyfile.example +++ b/Caddyfile.example @@ -1,6 +1,23 @@ +{ + servers { + metrics # Prometheus metrics at http://127.0.0.1:2019/metrics (LOGGING.md) + } +} + # vanguard:main media.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } # /logout on every gated host = sign out of the Discord login (domain-wide session) redir /logout /outpost.goauthentik.io/sign_out 302 # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the @@ -32,14 +49,20 @@ name: {http.request.header.X-Authentik-Name}" } } -# vanguard:apex -example.com { - redir https://media.example.com/ 302 -} - # vanguard:files files.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } # /logout on every gated host = sign out of the Discord login (domain-wide session) redir /logout /outpost.goauthentik.io/sign_out 302 # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the @@ -54,7 +77,7 @@ files.example.com { # copyparty at the root. It honours X-Authentik-Username / -Groups only when the request also carries a header # whose secret NAME only this proxy knows (copyparty idp-h-key). Clients cannot forge identity. reverse_proxy 127.0.0.1:3923 { - header_up X-Vanguard-EXAMPLEKEY 1 + header_up X-Vanguard-k 1 flush_interval -1 } } @@ -62,6 +85,17 @@ files.example.com { # vanguard:sync sync.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } # /logout on every gated host = sign out of the Discord login (domain-wide session) redir /logout /outpost.goauthentik.io/sign_out 302 # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the @@ -86,6 +120,17 @@ sync.example.com { # vanguard:clips clips.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } # /logout on every gated host = sign out of the Discord login (domain-wide session) redir /logout /outpost.goauthentik.io/sign_out 302 # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the @@ -111,6 +156,17 @@ clips.example.com { # vanguard:live live.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } # /logout on every gated host = sign out of the Discord login (domain-wide session) redir /logout /outpost.goauthentik.io/sign_out 302 # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the @@ -167,6 +223,17 @@ groups: {http.request.header.X-Authentik-Groups}" # vanguard:bot bot.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } # /logout on every gated host = sign out of the Discord login (domain-wide session) redir /logout /outpost.goauthentik.io/sign_out 302 # Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the @@ -188,17 +255,50 @@ bot.example.com { # vanguard:git git.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } reverse_proxy 127.0.0.1:3000 } # vanguard:auth auth.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } reverse_proxy 127.0.0.1:9000 } # vanguard:watch watch.example.com { encode zstd gzip + log { + output stderr + format filter { + wrap json + fields { + request>headers>Cookie delete + request>headers>Authorization delete + request>headers>X-Stream-Auth delete + } + } + } reverse_proxy 127.0.0.1:8096 } diff --git a/LOGGING.md b/LOGGING.md index 7471cf8..44d86a6 100644 --- a/LOGGING.md +++ b/LOGGING.md @@ -10,9 +10,19 @@ upstream of it is fixed now, so the choice later is a collector config, not a co | **journald** | our systemd units: `dumpd`, `normalize-sweep`, `trash-purge`, `deploy-pending`, `vgdeband` (transient), `deploy.sh` output | `journalctl -u `; collectors: Alloy `loki.source.journal`, Vector `journald`, VictoriaLogs journald ingestion, Netdata's journal plugin | | **Docker json-file logs** | every container (`docker-compose.yml`), rotated 5 × 50 MB each | `docker logs `; collectors read `/var/lib/docker/containers/*/*-json.log` and get the container **labels** below for free | -Every collector on the shortlist reads both. Metrics are the same story: node_exporter / cAdvisor expose the -Prometheus text format that Alloy, vmagent and Netdata all scrape; MediaMTX (`metrics: yes`) and Caddy have -endpoints too, currently off. +Every collector on the shortlist reads both. Metrics are the same story: see the taps table below — all four endpoints are on now, on loopback. + +## Metrics taps (all Prometheus text format, loopback only — scrape them with Alloy, vmagent or Netdata) + +| endpoint | what | from | +|---|---|---| +| `http://127.0.0.1:9100/metrics` | host: CPU, RAM, disk space + I/O, network, load, md RAID | node_exporter (host network, pid host) | +| `http://127.0.0.1:9101/metrics` | per container: CPU, memory, network, block I/O — labelled with `vanguard.service` / `vanguard.category` | cAdvisor | +| `http://127.0.0.1:9998/metrics` | MediaMTX: per path bytes/s, readers, publishers, WebRTC/HLS/RTMP sessions, dropped packets | MediaMTX `metrics: yes` | +| `http://127.0.0.1:2019/metrics` | Caddy: requests by host/status, durations, TLS | Caddy admin API (`servers { metrics }`) | + +None of these are reachable from outside (loopback binds; nothing in the firewall). Alert thresholds to start with: +disk on `/srv` > 85 %, load > cores for 15 min, a container restarting, MediaMTX dropped packets rising on a live path. ## Container labels (set in `docker-compose.yml`, `x-logging` anchor + `labels:`) @@ -74,8 +84,7 @@ On for every site block (`CADDY_LOG` in `configure.py`): JSON to stderr; `Cookie - **Grafana + Loki**: `alloy` container with `loki.source.journal` + `loki.source.docker` (labels from the container labels above) → `loki` on loopback → `grafana` behind Authentik OIDC (third provider, like Jellyfin/Gitea). - Metrics: Alloy scrapes node_exporter, cAdvisor, MediaMTX `:9998/metrics`, Caddy `:2019/metrics` → Prometheus or - VictoriaMetrics, same Grafana. + Metrics: Alloy scrapes the four taps above → Prometheus or VictoriaMetrics, same Grafana. - **VictoriaLogs**: single container, journald + docker via its agent or Vector; built-in UI behind the Caddy forward-auth gate (`media-admins`). VictoriaMetrics beside it for metrics. - **Netdata**: single container, host + container metrics and alerts (Discord) out of the box, journald viewer; diff --git a/STATUS.md b/STATUS.md index 26113fc..66cd581 100644 --- a/STATUS.md +++ b/STATUS.md @@ -59,6 +59,7 @@ Source of truth: this repo, `techy/Media-Server-Setup` on the stack's own Gitea | normalize sweep | editing standard, one copy per clip | live | | nftables | host firewall (22, 80/443, 1935, 2222, 3306, 8890/udp, 22000) | live | | Caddy | TLS + routing for all hostnames; JSON access logs per site (identity headers kept, cookies/tokens dropped) | live (access logs: deploy pending) | +| Metrics taps | node_exporter (host), cAdvisor (containers), MediaMTX and Caddy metrics — Prometheus format on loopback, no store yet | **committed 2026-11-01, deploy pending** | | Logging framework | rotated Docker logs + `vanguard.service/category` labels on every container; dumpd/sweep/vgdeband/deploy emit logfmt events (`LOGGING.md`); store not chosen (Loki vs VictoriaLogs vs Netdata) | **committed 2026-11-01, deploy pending** (recreates every container) | | Authentik (+Postgres, Redis) | Discord login, groups, guild-membership policy | live | | Jellyfin | watch page, SSO via Authentik, direct play only | live | diff --git a/configure.py b/configure.py index 187f5fd..aacb627 100755 --- a/configure.py +++ b/configure.py @@ -303,7 +303,13 @@ groups: {{http.request.header.X-Authentik-Groups}}" {CADDY_LOG} reverse_proxy 127.0.0.1:8096 }} """ - Path("Caddyfile").write_text(f"""# vanguard:main + Path("Caddyfile").write_text(f"""{{ + servers {{ + metrics # Prometheus metrics at http://127.0.0.1:2019/metrics (LOGGING.md) + }} +}} + +# vanguard:main {domain} {{ encode zstd gzip {gate_any} # who does Authentik say I am? (debug: shows exactly the identity headers the apps receive) @@ -393,7 +399,8 @@ rtspAddress: 127.0.0.1:8554 rtspTransports: [tcp] rtspEncryption: "no" moq: no -metrics: no +metrics: yes # Prometheus text format for whichever metrics store is chosen (LOGGING.md); loopback only +metricsAddress: :9998 pprof: no # live portal (served to the browser only via Caddy + Authentik on live.HOST; these ports are published on 127.0.0.1) diff --git a/docker-compose.yml b/docker-compose.yml index 451191e..233b245 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ services: - "127.0.0.1:8888:8888" # HLS -> Caddy live.HOST/hls/* - "127.0.0.1:8889:8889" # WebRTC -> Caddy live.HOST/webrtc/* - "127.0.0.1:9997:9997" # API -> Caddy live.HOST/api/* + - "127.0.0.1:9998:9998" # Prometheus metrics (per-stream bitrate, readers, drops) -> scraper on the host volumes: - ./mediamtx.yml:/mediamtx.yml:ro - ./bin/mtx-on-ready.sh:/vanguard/on-ready.sh:ro # is-live notice + WebRTC audio mirror (runOnReady) @@ -299,6 +300,38 @@ services: - gitea restart: unless-stopped + # ---- metrics taps (LOGGING.md): Prometheus text format on loopback, scraped by whichever store is chosen later + node_exporter: + logging: *vglog + labels: { vanguard.service: node_exporter, vanguard.category: infra } + image: quay.io/prometheus/node-exporter:latest + container_name: node_exporter + network_mode: host # listens on 127.0.0.1:9100 only + pid: host + command: ["--path.rootfs=/host", "--web.listen-address=127.0.0.1:9100", "--collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker/.+)($$|/)"] + volumes: + - /:/host:ro,rslave + restart: unless-stopped + + cadvisor: + logging: *vglog + labels: { vanguard.service: cadvisor, vanguard.category: infra } + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + ports: + - "127.0.0.1:9101:8080" # per-container CPU / memory / network / disk + command: ["--housekeeping_interval=10s", "--docker_only=true", "--store_container_labels=false", "--whitelisted_container_labels=vanguard.service,vanguard.category"] + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + devices: + - /dev/kmsg + privileged: true + restart: unless-stopped + volumes: copyparty_hist: caddy_data: