Media-Server-Setup/docker-compose.yml

# Log framework (LOGGING.md): every container keeps rotated json-file logs and carries labels a collector can use as
# low-cardinality stream labels without parsing: vanguard.service / vanguard.category.
x-logging: &vglog
  driver: json-file
  options: { max-size: "50m", max-file: "5" }

services:
  mediamtx:
    logging: *vglog
    labels: { vanguard.service: mediamtx, vanguard.category: stream }
    image: bluenviron/mediamtx:latest-ffmpeg     # alpine variant: has sh + wget for the is-live Discord hook
    container_name: mediamtx
    user: "1000:1000"             # recordings owned by the debian user, so the archive pull can delete them
    environment:
      TZ: America/New_York          # recording filenames are stamped in the container's clock; must match the host
      LIVE_WEBHOOK_URL: ${LIVE_WEBHOOK_URL:-}
      LIVE_HOST: ${LIVE_HOST:-live.example.com}
    # Host network, as MediaMTX recommends for WebRTC. On the bridge, every WebRTC media packet on 8189/udp
    # was relayed by Docker's userland proxy (docker-proxy), which lost ~0.4% of them at a thousand packets a
    # second; each loss corrupted an access unit, the recorder reopened its file, and one evening produced 471
    # fragments. Publishers also all appeared as the bridge gateway (172.18.0.1). Now MediaMTX binds the host
    # directly: ingest ports (1935, 8890/udp, 8189) are open in the nftables INPUT chain, and everything Caddy
    # or Metricbeat talks to is bound to 127.0.0.1 in mediamtx.yml. No ports: block, nothing to publish.
    network_mode: host
    volumes:
      - ./mediamtx.yml:/mediamtx.yml:ro
      - ./bin/mtx-on-ready.sh:/vanguard/on-ready.sh:ro   # is-live notice + WebRTC audio mirror (runOnReady)
      - /srv/media/streams:/recordings
    restart: unless-stopped

  copyparty:
    logging: *vglog
    labels: { vanguard.service: copyparty, vanguard.category: media }
    image: copyparty/ac:latest
    container_name: copyparty
    ports:
      - "127.0.0.1:3923:3923"
    volumes:
      - /srv/media:/w
      - ./copyparty.conf:/cfg/copyparty.conf:ro
      - copyparty_hist:/cfg/hist
    restart: unless-stopped

  syncthing:
    logging: *vglog
    labels: { vanguard.service: syncthing, vanguard.category: intake }
    image: syncthing/syncthing:latest
    container_name: syncthing
    environment:
      - PUID=1000
      - PGID=1000
    ports:
      - "22000:22000/tcp"
      - "22000:22000/udp"
      - "127.0.0.1:8384:8384"   # admin UI, reach via ssh tunnel
    volumes:
      - /srv/media/.syncthing:/var/syncthing/config   # created by install.sh, owned by uid 1000
      - /srv/media/clips:/var/syncthing/clips
    restart: unless-stopped

  caddy:
    logging: *vglog
    labels: { vanguard.service: caddy, vanguard.category: auth }
    image: caddy:latest
    container_name: caddy
    depends_on: [copyparty]
    network_mode: host            # talks to copyparty (127.0.0.1:3923) and dumpd (127.0.0.1:8080) on loopback
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile:ro
      - ./dock:/srv/dock:ro
      - ./clips:/srv/clips:ro               # clip-intake self-service page
      - ./about:/srv/about:ro               # gateway page (/about)
      - ./streaming:/srv/streaming:ro       # Live + Stuck tabs (diag host /streaming)
      - /srv/live:/srv/live:ro              # live portal static build (deployLive.sh)
      - caddy_data:/data
      - caddy_config:/config
    restart: unless-stopped

  # ---------------------------------------------------------------- identity (Authentik)
  authentik-db:
    logging: *vglog
    labels: { vanguard.service: authentik-db, vanguard.category: auth }
    image: docker.io/library/postgres:16-alpine
    container_name: authentik-db
    environment:
      POSTGRES_DB: authentik
      POSTGRES_USER: authentik
      POSTGRES_PASSWORD: ${AUTHENTIK_PG_PASS}
    volumes:
      - /srv/appdata/authentik/db:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -d authentik -U authentik"]
      interval: 30s
      timeout: 5s
      retries: 5
      start_period: 20s
    restart: unless-stopped

  authentik-redis:
    logging: *vglog
    labels: { vanguard.service: authentik-redis, vanguard.category: auth }
    image: docker.io/library/redis:alpine
    container_name: authentik-redis
    command: --save 60 1 --loglevel warning
    volumes:
      - /srv/appdata/authentik/redis:/data
    healthcheck:
      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
      interval: 30s
      timeout: 3s
      retries: 5
    restart: unless-stopped

  authentik-server:
    logging: *vglog
    labels: { vanguard.service: authentik-server, vanguard.category: auth }
    image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-2025.8}
    container_name: authentik-server
    command: server
    environment: &authentik_env
      AUTHENTIK_SECRET_KEY: ${AUTHENTIK_SECRET_KEY}
      AUTHENTIK_BOOTSTRAP_PASSWORD: ${AUTHENTIK_BOOTSTRAP_PASSWORD:-}   # akadmin password on FIRST start (no initial-setup page)
      AUTHENTIK_BOOTSTRAP_TOKEN: ${AUTHENTIK_BOOTSTRAP_TOKEN:-}         # API token for akadmin, created on first start (blueprints/automation)
      AUTHENTIK_REDIS__HOST: authentik-redis
      AUTHENTIK_POSTGRESQL__HOST: authentik-db
      AUTHENTIK_POSTGRESQL__USER: authentik
      AUTHENTIK_POSTGRESQL__NAME: authentik
      AUTHENTIK_POSTGRESQL__PASSWORD: ${AUTHENTIK_PG_PASS}
      AUTHENTIK_ERROR_REPORTING__ENABLED: "false"
    ports:
      - "127.0.0.1:9000:9000"
    volumes:
      - /srv/appdata/authentik/media:/media
      - /srv/appdata/authentik/templates:/templates
    depends_on:
      authentik-db: { condition: service_healthy }
      authentik-redis: { condition: service_healthy }
    restart: unless-stopped

  authentik-worker:
    logging: *vglog
    labels: { vanguard.service: authentik-worker, vanguard.category: auth }
    image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-2025.8}
    container_name: authentik-worker
    command: worker
    environment: *authentik_env
    volumes:
      - /srv/appdata/authentik/media:/media
      - /srv/appdata/authentik/templates:/templates
      - /srv/appdata/authentik/certs:/certs
    depends_on:
      authentik-db: { condition: service_healthy }
      authentik-redis: { condition: service_healthy }
    restart: unless-stopped

  # ---------------------------------------------------------------- watching (Jellyfin, direct play only)
  jellyfin:
    logging: *vglog
    labels: { vanguard.service: jellyfin, vanguard.category: media }
    image: jellyfin/jellyfin:latest
    container_name: jellyfin
    user: "1000:1000"
    environment:
      JELLYFIN_PublishedServerUrl: https://${WATCH_HOST:-watch.example.com}
    ports:
      - "127.0.0.1:8096:8096"
    volumes:
      - /srv/appdata/jellyfin/config:/config
      - /srv/appdata/jellyfin/cache:/cache
      - /srv/media/streams:/media/streams:ro
      - /srv/media/clips:/media/clips:ro
      - /srv/media/finals:/media/finals:ro
    restart: unless-stopped

  # ---------------------------------------------------------------- database (for the Discord bot and friends)
  mariadb:
    logging: *vglog
    labels: { vanguard.service: mariadb, vanguard.category: infra }
    image: mariadb:11
    container_name: mariadb
    network_mode: host              # PUBLIC on 3306: host networking so the nftables input rules (db_allow / rate
                                    # limit) actually apply — Docker-published ports bypass the host firewall.
    environment:
      MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD}
      MARIADB_ROOT_HOST: localhost  # root only from the box itself; the bot user may connect from anywhere
      MARIADB_DATABASE: ${BOT_DB_NAME:-vanguardbot}
      MARIADB_USER: ${BOT_DB_USER:-bot}
      MARIADB_PASSWORD: ${BOT_DB_PASSWORD}
      GITEA_DB_PASSWORD: ${GITEA_DB_PASSWORD}       # used by mariadb/init/10-gitea.sh on first init
    volumes:
      - /srv/appdata/mariadb:/var/lib/mysql
      - ./mariadb/init:/docker-entrypoint-initdb.d:ro
    healthcheck:
      test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"]
      interval: 20s
      timeout: 5s
      retries: 6
      start_period: 30s
    restart: unless-stopped

  # ------------------------------------------- Discord bot (python; its release is installed in /srv/bot/app)
  discord-bot:
    logging: *vglog
    labels: { vanguard.service: discord-bot, vanguard.category: bot }
    build: ./bot
    container_name: discord-bot
    user: "1000:1000"
    environment:
      DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN:-}
      DISCORD_GUILD_ID: ${DISCORD_GUILD_ID:-}
      DB_HOST: host.docker.internal   # mariadb runs on the host network
      DB_PORT: "3306"
      DB_NAME: ${BOT_DB_NAME:-vanguardbot}
      DB_USER: ${BOT_DB_USER:-bot}
      DB_PASSWORD: ${BOT_DB_PASSWORD}
      DB_BACKEND: mariadb
      DATA_DIR: /data
      WEB_ALLOWED_GROUPS: ${BOT_WEB_ALLOWED_GROUPS:-bot-users,admins}   # the bot's own gate; Caddy already requires BOT_REQUIRE_GROUP
      WEB_SERVE_UI: "true"                # serve the bot's own pages at / on this port (same origin as /api)
      PYTHONPATH: /app/vendor             # the bundle's dependencies; the image installs none
      MEDIA_CLIPS: /media/clips
      WEB_HOST: 0.0.0.0
      WEB_PORT: "8000"
      WEB_PUBLIC_URL: https://${BOT_HOST:-bot.example.com}
      AUTH_HEADER_USER: X-Authentik-Username
      AUTH_HEADER_GROUPS: X-Authentik-Groups
      AUTH_HEADER_EMAIL: X-Authentik-Email
      TZ: America/New_York
      PYTHONUNBUFFERED: "1"
    ports:
      - "127.0.0.1:8000:8000"             # web UI; public only via Caddy + Authentik forward-auth at https://BOT_HOST/
    volumes:
      - ${BOT_APP_DIR:-/srv/bot/app}:/app  # CI bundle installed by ./deployBotBundle.sh (code + vendor/)
      - /srv/media/clips:/media/clips     # rw: for a clip-submissions feature (writes into clips/<user>/)
      - /srv/appdata/discord-bot:/data    # persistent scratch (sqlite, caches) if the bot wants files
    # Asks the BOT about itself rather than just proving the port accepts a connection: /health answers 503
    # unless the Discord gateway is connected AND the database responds, and -f turns that into an unhealthy
    # container, which Metricbeat's docker healthcheck metricset turns into an alarm. Requires the bot release
    # that made /health unauthenticated and status-bearing — on older code every check 401s and -f fails,
    # so ship the bot first. start_period covers command sync and the first roster load.
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS -m 5 http://127.0.0.1:8000/health >/dev/null || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s
    extra_hosts:
      - "host.docker.internal:host-gateway"
    networks:
      - botnet                          # its OWN network: the bot trusts the identity headers Caddy injects, so
                                        # nothing else must be able to reach :8000 and forge them. Caddy is on the
                                        # host network and uses the published 127.0.0.1:8000; MariaDB is reached
                                        # through host.docker.internal. The bot needs no sibling container.
    depends_on:
      mariadb: { condition: service_healthy }
    restart: unless-stopped

  # ---------------------------------------------------------------- Gitea + Actions runner
  gitea:
    logging: *vglog
    labels: { vanguard.service: gitea, vanguard.category: code }
    image: docker.gitea.com/gitea:1.24
    container_name: gitea
    environment:
      USER_UID: "1000"
      USER_GID: "1000"
      GITEA__database__DB_TYPE: mysql
      GITEA__database__HOST: host.docker.internal:3306      # mariadb runs on the host network
      GITEA__database__NAME: gitea
      GITEA__database__USER: gitea
      GITEA__database__PASSWD: ${GITEA_DB_PASSWORD}
      GITEA__server__DOMAIN: ${GIT_HOST:-git.example.com}
      GITEA__server__ROOT_URL: https://${GIT_HOST:-git.example.com}/
      GITEA__server__HTTP_PORT: "3000"
      GITEA__server__SSH_DOMAIN: ${GIT_HOST:-git.example.com}
      GITEA__server__SSH_PORT: "2222"                       # what users put in their remotes
      GITEA__server__SSH_LISTEN_PORT: "22"                  # inside the container
      # accounts come from Authentik (Discord) via OIDC: external auto-registration on, local self-registration off
      GITEA__service__DISABLE_REGISTRATION: "false"
      GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION: "true"
      GITEA__service__SHOW_REGISTRATION_BUTTON: "false"
      GITEA__service__REQUIRE_SIGNIN_VIEW: "false"
      GITEA__oauth2_client__ENABLE_AUTO_REGISTRATION: "true"
      GITEA__oauth2_client__ACCOUNT_LINKING: auto
      GITEA__oauth2_client__USERNAME: nickname                # Authentik preferred_username
      GITEA__oauth2_client__UPDATE_AVATAR: "true"
      GITEA__oauth2_client__OPENID_CONNECT_SCOPES: openid email profile
      GITEA__actions__ENABLED: "true"
      GITEA__actions__DEFAULT_ACTIONS_URL: github
      GITEA__webhook__ALLOWED_HOST_LIST: private,loopback,external
      GITEA__security__INSTALL_LOCK: "true"                  # no install page: db/settings come from env; admin user via configure.py
      GITEA__time__DEFAULT_UI_LOCATION: America/New_York
    ports:
      - "127.0.0.1:3000:3000"
      - "2222:22"
    volumes:
      - /srv/appdata/gitea:/data
      - /etc/timezone:/etc/timezone:ro
      - /etc/localtime:/etc/localtime:ro
    extra_hosts:
      - "host.docker.internal:host-gateway"
    depends_on:
      mariadb: { condition: service_healthy }
    restart: unless-stopped

  act_runner:
    logging: *vglog
    labels: { vanguard.service: act_runner, vanguard.category: deploy }
    image: docker.gitea.com/act_runner:latest
    container_name: act_runner
    environment:
      GITEA_INSTANCE_URL: https://${GIT_HOST:-git.example.com}/
      GITEA_RUNNER_REGISTRATION_TOKEN: ${GITEA_RUNNER_TOKEN:-}   # set by ./registerGiteaRunner.sh
      GITEA_RUNNER_NAME: vanguard-runner
      # JavaScript actions (checkout, upload-artifact) need node INSIDE the job image, so a label used with
      # them must point at an image that has it. "python" is bare CPython for plain run: steps; "python312"
      # adds node and is what the Discord bot's CI uses, so its wheels match the python:3.12-slim runtime.
      GITEA_RUNNER_LABELS: "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04,dotnet:docker://mcr.microsoft.com/dotnet/sdk:10.0,python:docker://python:3.12,python312:docker://nikolaik/python-nodejs:python3.12-nodejs22"
    volumes:
      - /srv/appdata/act_runner:/data
      - /var/run/docker.sock:/var/run/docker.sock   # jobs run as sibling containers; root-equivalent on the host
    depends_on:
      - gitea
    restart: unless-stopped



  # ================================================================ observability — Elastic (Phase 13)
  # One store, one UI, one collector pair. Replaces VictoriaLogs+Vector+VictoriaMetrics+2×vmalert+Alertmanager
  # +node_exporter+cAdvisor: eight containers for four, and Kibana is the single pane for logs, dashboards and
  # alerting. Everything binds loopback and is published only through diag.HOST behind the Authentik gate.
  elasticsearch:
    logging: *vglog
    labels: { vanguard.service: elasticsearch, vanguard.category: infra }
    image: docker.elastic.co/elasticsearch/elasticsearch:9.2.0
    container_name: elasticsearch
    environment:
      # Pinned deliberately. A JVM that sizes its own heap on a 62 GB box would take a quarter of it, on a
      # machine whose spare CPU and RAM belong to x264. 2 GB is ample for one host's logs and metrics.
      ES_JAVA_OPTS: -Xms2g -Xmx2g
      discovery.type: single-node
      # ON. Kibana's Observability apps (Infrastructure/Metrics) ask Elasticsearch whether the current user
      # holds a privilege; with security off that API does not exist and they fail on every view. Humans still
      # never see a password: Kibana signs everyone in as one user (observability/kibana.yml). Caddy +
      # Authentik + DIAG_REQUIRE_GROUP remains the only real gate.
      xpack.security.enabled: "true"
      xpack.security.http.ssl.enabled: "false"      # plain HTTP, published on loopback only
      xpack.security.transport.ssl.enabled: "false" # single-node: no transport traffic, and the check is skipped
      xpack.security.enrollment.enabled: "false"
      xpack.security.autoconfiguration.enabled: "false"
      # Bootstraps the elastic superuser on first start with security on. configure.py generates it and then
      # sets up kibana_system and the service users; the error below means .env predates that.
      ELASTIC_PASSWORD: "${ES_ELASTIC_PASSWORD:?run ./configure.py --apply first — it generates the Elastic passwords}"
      xpack.ml.enabled: "false"             # not used, and it reserves memory
      cluster.routing.allocation.disk.watermark.low: 85%
      cluster.routing.allocation.disk.watermark.high: 90%
      cluster.routing.allocation.disk.watermark.flood_stage: 95%
      bootstrap.memory_lock: "true"
    ulimits:
      memlock: { soft: -1, hard: -1 }
      nofile: { soft: 65536, hard: 65536 }
    ports:
      - "127.0.0.1:9200:9200"
    volumes:
      - /srv/appdata/elasticsearch:/usr/share/elasticsearch/data
    healthcheck:
      test: ["CMD-SHELL", "curl -sf -u elastic:$${ELASTIC_PASSWORD} http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s || exit 1"]
      interval: 20s
      timeout: 10s
      retries: 12
      start_period: 90s
    restart: unless-stopped

  kibana:
    logging: *vglog
    labels: { vanguard.service: kibana, vanguard.category: infra }
    image: docker.elastic.co/kibana/kibana:9.2.0
    container_name: kibana
    # Everything non-secret lives in observability/kibana.yml (which explains the auth model); only the
    # generated secrets arrive as environment variables, which that file interpolates.
    environment:
      DIAG_HOST: ${DIAG_HOST:-diag.example.com}
      KIBANA_ENCRYPTION_KEY: ${KIBANA_ENCRYPTION_KEY}
      ES_KIBANA_PASSWORD: "${ES_KIBANA_PASSWORD:?run ./configure.py --apply first}"
      VG_DIAG_USER: ${VG_DIAG_USER:-vanguard_diag}
      VG_DIAG_PASSWORD: "${VG_DIAG_PASSWORD:?run ./configure.py --apply first}"
    volumes:
      - ./observability/kibana.yml:/usr/share/kibana/config/kibana.yml:ro
    ports:
      - "127.0.0.1:5601:5601"
    depends_on:
      elasticsearch: { condition: service_healthy }
    restart: unless-stopped

  filebeat:
    logging: *vglog
    labels: { vanguard.service: filebeat, vanguard.category: infra }
    image: docker.elastic.co/beats/filebeat:9.2.0
    container_name: filebeat
    environment:                            # Elasticsearch requires auth now; the config interpolates these
      ES_BEATS_USER: ${VG_BEATS_USER:-vanguard_beats}
      ES_BEATS_PASSWORD: "${VG_BEATS_PASSWORD:?run ./configure.py --apply first}"
    user: root
    command: ["filebeat", "-e", "--strict.perms=false"]
    volumes:
      - ./observability/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock     # autodiscover reads container names and our vanguard.* labels
      - /var/log/journal:/var/log/journal:ro          # our own systemd units
      - /run/log/journal:/run/log/journal:ro          # ...and the volatile journal, if this host has no persistent one
      - /etc/machine-id:/etc/machine-id:ro
      - filebeat_data:/usr/share/filebeat/data
    depends_on:
      elasticsearch: { condition: service_healthy }
    restart: unless-stopped

  metricbeat:
    logging: *vglog
    labels: { vanguard.service: metricbeat, vanguard.category: infra }
    image: docker.elastic.co/beats/metricbeat:9.2.0
    container_name: metricbeat
    environment:                            # Elasticsearch requires auth now; the config interpolates these
      ES_BEATS_USER: ${VG_BEATS_USER:-vanguard_beats}
      ES_BEATS_PASSWORD: "${VG_BEATS_PASSWORD:?run ./configure.py --apply first}"
    user: root
    command: ["metricbeat", "-e", "--strict.perms=false"]
    # system + docker modules replace node_exporter and cAdvisor, and ship their own Kibana dashboards.
    network_mode: host                      # real interfaces, and loopback reach to MediaMTX/Caddy/ES/Kibana
    pid: host
    volumes:
      - ./observability/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /proc:/hostfs/proc:ro
      - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
      - /:/hostfs:ro
      - metricbeat_data:/usr/share/metricbeat/data
    depends_on:
      elasticsearch: { condition: service_healthy }
    restart: unless-stopped

networks:
  botnet:                               # isolates discord-bot from the shared default network (see above)

volumes:
  copyparty_hist:
  filebeat_data:
  metricbeat_data:
  caddy_data:
  caddy_config: