Brandyn / Techy
88bb28989c
Eight observability containers become four. Elasticsearch stores logs and metrics, Kibana is the one UI for Discover, dashboards and alerting, Filebeat ships container logs plus our journal units, and Metricbeat covers host and Docker metrics — which retires node_exporter and cAdvisor too — while still scraping MediaMTX's and Caddy's Prometheus endpoints. Pinned rather than left to chance: a fixed 2 GB heap (a self-sizing JVM would claim a quarter of a 62 GB box whose spare capacity belongs to x264), single-node discovery, security off because it only listens on loopback behind the Authentik gate, ML off, and vm.max_map_count set and persisted since ES will not start without it. Kibana gets a fixed encryption key or it loses its saved objects on every restart. diag.HOST is now just Kibana at the root: no path prefixes to get wrong and no index page of links. The deadman is the only thing kept from the old stack, repointed at Elastic. It checks that ES answers and is not red, that no index has been flipped read-only by the disk watermark (which does not clear itself when space is freed, so ingestion stays silently dead), that Kibana is available, that each Beat has actually written a document recently rather than merely running, that the Kibana heartbeat still arrives, and the disks. Verified against a mock Elasticsearch for all five failure modes plus the healthy case staying silent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
421 lines
18 KiB
YAML
421 lines
18 KiB
YAML
# Log framework (LOGGING.md): every container keeps rotated json-file logs and carries labels a collector can use as
|
||
# low-cardinality stream labels without parsing: vanguard.service / vanguard.category.
|
||
x-logging: &vglog
|
||
driver: json-file
|
||
options: { max-size: "50m", max-file: "5" }
|
||
|
||
services:
|
||
mediamtx:
|
||
logging: *vglog
|
||
labels: { vanguard.service: mediamtx, vanguard.category: stream }
|
||
image: bluenviron/mediamtx:latest-ffmpeg # alpine variant: has sh + wget for the is-live Discord hook
|
||
container_name: mediamtx
|
||
user: "1000:1000" # recordings owned by the debian user, so the archive pull can delete them
|
||
environment:
|
||
TZ: America/New_York # recording filenames are stamped in the container's clock; must match the host
|
||
LIVE_WEBHOOK_URL: ${LIVE_WEBHOOK_URL:-}
|
||
LIVE_HOST: ${LIVE_HOST:-live.example.com}
|
||
ports:
|
||
- "1935:1935" # RTMP ingest
|
||
- "8890:8890/udp" # SRT ingest
|
||
- "8189:8189/udp" # WebRTC media (viewers + Go Live)
|
||
- "8189:8189/tcp" # WebRTC ICE-TCP fallback
|
||
- "127.0.0.1:8888:8888" # HLS -> Caddy live.HOST/hls/*
|
||
- "127.0.0.1:8889:8889" # WebRTC -> Caddy live.HOST/webrtc/*
|
||
- "127.0.0.1:9997:9997" # API -> Caddy live.HOST/api/*
|
||
- "127.0.0.1:9998:9998" # Prometheus metrics (per-stream bitrate, readers, drops) -> scraper on the host
|
||
volumes:
|
||
- ./mediamtx.yml:/mediamtx.yml:ro
|
||
- ./bin/mtx-on-ready.sh:/vanguard/on-ready.sh:ro # is-live notice + WebRTC audio mirror (runOnReady)
|
||
- /srv/media/streams:/recordings
|
||
restart: unless-stopped
|
||
|
||
copyparty:
|
||
logging: *vglog
|
||
labels: { vanguard.service: copyparty, vanguard.category: media }
|
||
image: copyparty/ac:latest
|
||
container_name: copyparty
|
||
ports:
|
||
- "127.0.0.1:3923:3923"
|
||
volumes:
|
||
- /srv/media:/w
|
||
- ./copyparty.conf:/cfg/copyparty.conf:ro
|
||
- copyparty_hist:/cfg/hist
|
||
restart: unless-stopped
|
||
|
||
syncthing:
|
||
logging: *vglog
|
||
labels: { vanguard.service: syncthing, vanguard.category: intake }
|
||
image: syncthing/syncthing:latest
|
||
container_name: syncthing
|
||
environment:
|
||
- PUID=1000
|
||
- PGID=1000
|
||
ports:
|
||
- "22000:22000/tcp"
|
||
- "22000:22000/udp"
|
||
- "127.0.0.1:8384:8384" # admin UI, reach via ssh tunnel
|
||
volumes:
|
||
- /srv/media/.syncthing:/var/syncthing/config # created by install.sh, owned by uid 1000
|
||
- /srv/media/clips:/var/syncthing/clips
|
||
restart: unless-stopped
|
||
|
||
caddy:
|
||
logging: *vglog
|
||
labels: { vanguard.service: caddy, vanguard.category: auth }
|
||
image: caddy:latest
|
||
container_name: caddy
|
||
depends_on: [copyparty]
|
||
network_mode: host # talks to copyparty (127.0.0.1:3923) and dumpd (127.0.0.1:8080) on loopback
|
||
volumes:
|
||
- ./Caddyfile:/etc/caddy/Caddyfile:ro
|
||
- ./dock:/srv/dock:ro
|
||
- ./clips:/srv/clips:ro # clip-intake self-service page
|
||
- ./about:/srv/about:ro # gateway page (/about)
|
||
- ./diag:/srv/diag:ro # diagnostics index (diag.HOST)
|
||
- /srv/live:/srv/live:ro # live portal static build (deployLive.sh)
|
||
- caddy_data:/data
|
||
- caddy_config:/config
|
||
restart: unless-stopped
|
||
|
||
# ---------------------------------------------------------------- identity (Authentik)
|
||
authentik-db:
|
||
logging: *vglog
|
||
labels: { vanguard.service: authentik-db, vanguard.category: auth }
|
||
image: docker.io/library/postgres:16-alpine
|
||
container_name: authentik-db
|
||
environment:
|
||
POSTGRES_DB: authentik
|
||
POSTGRES_USER: authentik
|
||
POSTGRES_PASSWORD: ${AUTHENTIK_PG_PASS}
|
||
volumes:
|
||
- /srv/appdata/authentik/db:/var/lib/postgresql/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "pg_isready -d authentik -U authentik"]
|
||
interval: 30s
|
||
timeout: 5s
|
||
retries: 5
|
||
start_period: 20s
|
||
restart: unless-stopped
|
||
|
||
authentik-redis:
|
||
logging: *vglog
|
||
labels: { vanguard.service: authentik-redis, vanguard.category: auth }
|
||
image: docker.io/library/redis:alpine
|
||
container_name: authentik-redis
|
||
command: --save 60 1 --loglevel warning
|
||
volumes:
|
||
- /srv/appdata/authentik/redis:/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
|
||
interval: 30s
|
||
timeout: 3s
|
||
retries: 5
|
||
restart: unless-stopped
|
||
|
||
authentik-server:
|
||
logging: *vglog
|
||
labels: { vanguard.service: authentik-server, vanguard.category: auth }
|
||
image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-2025.8}
|
||
container_name: authentik-server
|
||
command: server
|
||
environment: &authentik_env
|
||
AUTHENTIK_SECRET_KEY: ${AUTHENTIK_SECRET_KEY}
|
||
AUTHENTIK_BOOTSTRAP_PASSWORD: ${AUTHENTIK_BOOTSTRAP_PASSWORD:-} # akadmin password on FIRST start (no initial-setup page)
|
||
AUTHENTIK_BOOTSTRAP_TOKEN: ${AUTHENTIK_BOOTSTRAP_TOKEN:-} # API token for akadmin, created on first start (blueprints/automation)
|
||
AUTHENTIK_REDIS__HOST: authentik-redis
|
||
AUTHENTIK_POSTGRESQL__HOST: authentik-db
|
||
AUTHENTIK_POSTGRESQL__USER: authentik
|
||
AUTHENTIK_POSTGRESQL__NAME: authentik
|
||
AUTHENTIK_POSTGRESQL__PASSWORD: ${AUTHENTIK_PG_PASS}
|
||
AUTHENTIK_ERROR_REPORTING__ENABLED: "false"
|
||
ports:
|
||
- "127.0.0.1:9000:9000"
|
||
volumes:
|
||
- /srv/appdata/authentik/media:/media
|
||
- /srv/appdata/authentik/templates:/templates
|
||
depends_on:
|
||
authentik-db: { condition: service_healthy }
|
||
authentik-redis: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
authentik-worker:
|
||
logging: *vglog
|
||
labels: { vanguard.service: authentik-worker, vanguard.category: auth }
|
||
image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-2025.8}
|
||
container_name: authentik-worker
|
||
command: worker
|
||
environment: *authentik_env
|
||
volumes:
|
||
- /srv/appdata/authentik/media:/media
|
||
- /srv/appdata/authentik/templates:/templates
|
||
- /srv/appdata/authentik/certs:/certs
|
||
depends_on:
|
||
authentik-db: { condition: service_healthy }
|
||
authentik-redis: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
# ---------------------------------------------------------------- watching (Jellyfin, direct play only)
|
||
jellyfin:
|
||
logging: *vglog
|
||
labels: { vanguard.service: jellyfin, vanguard.category: media }
|
||
image: jellyfin/jellyfin:latest
|
||
container_name: jellyfin
|
||
user: "1000:1000"
|
||
environment:
|
||
JELLYFIN_PublishedServerUrl: https://${WATCH_HOST:-watch.example.com}
|
||
ports:
|
||
- "127.0.0.1:8096:8096"
|
||
volumes:
|
||
- /srv/appdata/jellyfin/config:/config
|
||
- /srv/appdata/jellyfin/cache:/cache
|
||
- /srv/media/streams:/media/streams:ro
|
||
- /srv/media/clips:/media/clips:ro
|
||
- /srv/media/finals:/media/finals:ro
|
||
restart: unless-stopped
|
||
|
||
# ---------------------------------------------------------------- database (for the Discord bot and friends)
|
||
mariadb:
|
||
logging: *vglog
|
||
labels: { vanguard.service: mariadb, vanguard.category: infra }
|
||
image: mariadb:11
|
||
container_name: mariadb
|
||
network_mode: host # PUBLIC on 3306: host networking so the nftables input rules (db_allow / rate
|
||
# limit) actually apply — Docker-published ports bypass the host firewall.
|
||
environment:
|
||
MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD}
|
||
MARIADB_ROOT_HOST: localhost # root only from the box itself; the bot user may connect from anywhere
|
||
MARIADB_DATABASE: ${BOT_DB_NAME:-vanguardbot}
|
||
MARIADB_USER: ${BOT_DB_USER:-bot}
|
||
MARIADB_PASSWORD: ${BOT_DB_PASSWORD}
|
||
GITEA_DB_PASSWORD: ${GITEA_DB_PASSWORD} # used by mariadb/init/10-gitea.sh on first init
|
||
volumes:
|
||
- /srv/appdata/mariadb:/var/lib/mysql
|
||
- ./mariadb/init:/docker-entrypoint-initdb.d:ro
|
||
healthcheck:
|
||
test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
|
||
interval: 20s
|
||
timeout: 5s
|
||
retries: 6
|
||
start_period: 30s
|
||
restart: unless-stopped
|
||
|
||
# ------------------------------------------- Discord bot (python; its release is installed in /srv/bot/app)
|
||
discord-bot:
|
||
logging: *vglog
|
||
labels: { vanguard.service: discord-bot, vanguard.category: bot }
|
||
build: ./bot
|
||
container_name: discord-bot
|
||
user: "1000:1000"
|
||
environment:
|
||
DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN:-}
|
||
DISCORD_GUILD_ID: ${DISCORD_GUILD_ID:-}
|
||
DB_HOST: host.docker.internal # mariadb runs on the host network
|
||
DB_PORT: "3306"
|
||
DB_NAME: ${BOT_DB_NAME:-vanguardbot}
|
||
DB_USER: ${BOT_DB_USER:-bot}
|
||
DB_PASSWORD: ${BOT_DB_PASSWORD}
|
||
DB_BACKEND: mariadb
|
||
DATA_DIR: /data
|
||
WEB_ALLOWED_GROUPS: ${BOT_WEB_ALLOWED_GROUPS:-bot-users,admins} # the bot's own gate; Caddy already requires BOT_REQUIRE_GROUP
|
||
WEB_SERVE_UI: "true" # serve the bot's own pages at / on this port (same origin as /api)
|
||
PYTHONPATH: /app/vendor # the bundle's dependencies; the image installs none
|
||
MEDIA_CLIPS: /media/clips
|
||
WEB_HOST: 0.0.0.0
|
||
WEB_PORT: "8000"
|
||
WEB_PUBLIC_URL: https://${BOT_HOST:-bot.example.com}
|
||
AUTH_HEADER_USER: X-Authentik-Username
|
||
AUTH_HEADER_GROUPS: X-Authentik-Groups
|
||
AUTH_HEADER_EMAIL: X-Authentik-Email
|
||
TZ: America/New_York
|
||
PYTHONUNBUFFERED: "1"
|
||
ports:
|
||
- "127.0.0.1:8000:8000" # web UI; public only via Caddy + Authentik forward-auth at https://BOT_HOST/
|
||
volumes:
|
||
- ${BOT_APP_DIR:-/srv/bot/app}:/app # CI bundle installed by ./deployBotBundle.sh (code + vendor/)
|
||
- /srv/media/clips:/media/clips # rw: for a clip-submissions feature (writes into clips/<user>/)
|
||
- /srv/appdata/discord-bot:/data # persistent scratch (sqlite, caches) if the bot wants files
|
||
extra_hosts:
|
||
- "host.docker.internal:host-gateway"
|
||
networks:
|
||
- botnet # its OWN network: the bot trusts the identity headers Caddy injects, so
|
||
# nothing else must be able to reach :8000 and forge them. Caddy is on the
|
||
# host network and uses the published 127.0.0.1:8000; MariaDB is reached
|
||
# through host.docker.internal. The bot needs no sibling container.
|
||
depends_on:
|
||
mariadb: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
# ---------------------------------------------------------------- Gitea + Actions runner
|
||
gitea:
|
||
logging: *vglog
|
||
labels: { vanguard.service: gitea, vanguard.category: code }
|
||
image: docker.gitea.com/gitea:1.24
|
||
container_name: gitea
|
||
environment:
|
||
USER_UID: "1000"
|
||
USER_GID: "1000"
|
||
GITEA__database__DB_TYPE: mysql
|
||
GITEA__database__HOST: host.docker.internal:3306 # mariadb runs on the host network
|
||
GITEA__database__NAME: gitea
|
||
GITEA__database__USER: gitea
|
||
GITEA__database__PASSWD: ${GITEA_DB_PASSWORD}
|
||
GITEA__server__DOMAIN: ${GIT_HOST:-git.example.com}
|
||
GITEA__server__ROOT_URL: https://${GIT_HOST:-git.example.com}/
|
||
GITEA__server__HTTP_PORT: "3000"
|
||
GITEA__server__SSH_DOMAIN: ${GIT_HOST:-git.example.com}
|
||
GITEA__server__SSH_PORT: "2222" # what users put in their remotes
|
||
GITEA__server__SSH_LISTEN_PORT: "22" # inside the container
|
||
# accounts come from Authentik (Discord) via OIDC: external auto-registration on, local self-registration off
|
||
GITEA__service__DISABLE_REGISTRATION: "false"
|
||
GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION: "true"
|
||
GITEA__service__SHOW_REGISTRATION_BUTTON: "false"
|
||
GITEA__service__REQUIRE_SIGNIN_VIEW: "false"
|
||
GITEA__oauth2_client__ENABLE_AUTO_REGISTRATION: "true"
|
||
GITEA__oauth2_client__ACCOUNT_LINKING: auto
|
||
GITEA__oauth2_client__USERNAME: nickname # Authentik preferred_username
|
||
GITEA__oauth2_client__UPDATE_AVATAR: "true"
|
||
GITEA__oauth2_client__OPENID_CONNECT_SCOPES: openid email profile
|
||
GITEA__actions__ENABLED: "true"
|
||
GITEA__actions__DEFAULT_ACTIONS_URL: github
|
||
GITEA__webhook__ALLOWED_HOST_LIST: private,loopback,external
|
||
GITEA__security__INSTALL_LOCK: "true" # no install page: db/settings come from env; admin user via configure.py
|
||
GITEA__time__DEFAULT_UI_LOCATION: America/New_York
|
||
ports:
|
||
- "127.0.0.1:3000:3000"
|
||
- "2222:22"
|
||
volumes:
|
||
- /srv/appdata/gitea:/data
|
||
- /etc/timezone:/etc/timezone:ro
|
||
- /etc/localtime:/etc/localtime:ro
|
||
extra_hosts:
|
||
- "host.docker.internal:host-gateway"
|
||
depends_on:
|
||
mariadb: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
act_runner:
|
||
logging: *vglog
|
||
labels: { vanguard.service: act_runner, vanguard.category: deploy }
|
||
image: docker.gitea.com/act_runner:latest
|
||
container_name: act_runner
|
||
environment:
|
||
GITEA_INSTANCE_URL: https://${GIT_HOST:-git.example.com}/
|
||
GITEA_RUNNER_REGISTRATION_TOKEN: ${GITEA_RUNNER_TOKEN:-} # set by ./registerGiteaRunner.sh
|
||
GITEA_RUNNER_NAME: vanguard-runner
|
||
# JavaScript actions (checkout, upload-artifact) need node INSIDE the job image, so a label used with
|
||
# them must point at an image that has it. "python" is bare CPython for plain run: steps; "python312"
|
||
# adds node and is what the Discord bot's CI uses, so its wheels match the python:3.12-slim runtime.
|
||
GITEA_RUNNER_LABELS: "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04,dotnet:docker://mcr.microsoft.com/dotnet/sdk:10.0,python:docker://python:3.12,python312:docker://nikolaik/python-nodejs:python3.12-nodejs22"
|
||
volumes:
|
||
- /srv/appdata/act_runner:/data
|
||
- /var/run/docker.sock:/var/run/docker.sock # jobs run as sibling containers; root-equivalent on the host
|
||
depends_on:
|
||
- gitea
|
||
restart: unless-stopped
|
||
|
||
|
||
|
||
# ================================================================ observability — Elastic (Phase 13)
|
||
# One store, one UI, one collector pair. Replaces VictoriaLogs+Vector+VictoriaMetrics+2×vmalert+Alertmanager
|
||
# +node_exporter+cAdvisor: eight containers for four, and Kibana is the single pane for logs, dashboards and
|
||
# alerting. Everything binds loopback and is published only through diag.HOST behind the Authentik gate.
|
||
elasticsearch:
|
||
logging: *vglog
|
||
labels: { vanguard.service: elasticsearch, vanguard.category: infra }
|
||
image: docker.elastic.co/elasticsearch/elasticsearch:9.2.0
|
||
container_name: elasticsearch
|
||
environment:
|
||
# Pinned deliberately. A JVM that sizes its own heap on a 62 GB box would take a quarter of it, on a
|
||
# machine whose spare CPU and RAM belong to x264. 2 GB is ample for one host's logs and metrics.
|
||
ES_JAVA_OPTS: -Xms2g -Xmx2g
|
||
discovery.type: single-node
|
||
xpack.security.enabled: "false" # loopback only, behind Caddy + Authentik; no second login
|
||
xpack.ml.enabled: "false" # not used, and it reserves memory
|
||
cluster.routing.allocation.disk.watermark.low: 85%
|
||
cluster.routing.allocation.disk.watermark.high: 90%
|
||
cluster.routing.allocation.disk.watermark.flood_stage: 95%
|
||
bootstrap.memory_lock: "true"
|
||
ulimits:
|
||
memlock: { soft: -1, hard: -1 }
|
||
nofile: { soft: 65536, hard: 65536 }
|
||
ports:
|
||
- "127.0.0.1:9200:9200"
|
||
volumes:
|
||
- /srv/appdata/elasticsearch:/usr/share/elasticsearch/data
|
||
healthcheck:
|
||
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s || exit 1"]
|
||
interval: 20s
|
||
timeout: 10s
|
||
retries: 12
|
||
start_period: 90s
|
||
restart: unless-stopped
|
||
|
||
kibana:
|
||
logging: *vglog
|
||
labels: { vanguard.service: kibana, vanguard.category: infra }
|
||
image: docker.elastic.co/kibana/kibana:9.2.0
|
||
container_name: kibana
|
||
environment:
|
||
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||
SERVER_PUBLICBASEURL: https://${DIAG_HOST:-diag.example.com}
|
||
SERVER_BASEPATH: "" # served at the root of diag.HOST, so no prefix rewriting to get wrong
|
||
TELEMETRY_OPTIN: "false"
|
||
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY}
|
||
XPACK_REPORTING_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY}
|
||
XPACK_SECURITY_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY}
|
||
ports:
|
||
- "127.0.0.1:5601:5601"
|
||
depends_on:
|
||
elasticsearch: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
filebeat:
|
||
logging: *vglog
|
||
labels: { vanguard.service: filebeat, vanguard.category: infra }
|
||
image: docker.elastic.co/beats/filebeat:9.2.0
|
||
container_name: filebeat
|
||
user: root
|
||
command: ["filebeat", "-e", "--strict.perms=false"]
|
||
volumes:
|
||
- ./observability/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||
- /var/run/docker.sock:/var/run/docker.sock # autodiscover reads container names and our vanguard.* labels
|
||
- /var/log/journal:/var/log/journal:ro # our own systemd units
|
||
- /etc/machine-id:/etc/machine-id:ro
|
||
- filebeat_data:/usr/share/filebeat/data
|
||
depends_on:
|
||
elasticsearch: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
metricbeat:
|
||
logging: *vglog
|
||
labels: { vanguard.service: metricbeat, vanguard.category: infra }
|
||
image: docker.elastic.co/beats/metricbeat:9.2.0
|
||
container_name: metricbeat
|
||
user: root
|
||
command: ["metricbeat", "-e", "--strict.perms=false"]
|
||
# system + docker modules replace node_exporter and cAdvisor, and ship their own Kibana dashboards.
|
||
network_mode: host # real interfaces, and loopback reach to MediaMTX/Caddy/ES/Kibana
|
||
pid: host
|
||
volumes:
|
||
- ./observability/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
|
||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||
- /proc:/hostfs/proc:ro
|
||
- /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
|
||
- /:/hostfs:ro
|
||
- metricbeat_data:/usr/share/metricbeat/data
|
||
depends_on:
|
||
elasticsearch: { condition: service_healthy }
|
||
restart: unless-stopped
|
||
|
||
networks:
|
||
botnet: # isolates discord-bot from the shared default network (see above)
|
||
|
||
volumes:
|
||
copyparty_hist:
|
||
filebeat_data:
|
||
metricbeat_data:
|
||
caddy_data:
|
||
caddy_config:
|