Media-Server-Setup/configure.py

#!/usr/bin/env python3
"""vanguard-stream post-install configuration. Stateless: the config files ARE the state.

  ./configure.py --full     first-time setup: asks every question (current values as defaults)
  ./configure.py            manage: skips what is already set; add/remove/reset streamers & editors
  ./configure.py --show     print current streamers, editors, domain and the OBS settings
  ./configure.py --set K=V  change ONE setting and apply only what it needs (Discord tokens and webhooks,
                            retention, alert tuning). Run it with no arguments to list what is settable.
                            Everything else regenerates a config or a container, so it needs --apply.
  ./configure.py --apply    regenerate everything from the values already on disk and apply (non-interactive;
                            used by deploy.sh). Fails if the main hostname was never set.
  ./configure.py --hosts AUTH WATCH [GIT [BOT [LIVE [FILES [SYNC [CLIPS [DIAG]]]]]]]
                            set the other hostnames and apply (non-interactive; used by addCommunityMediaServer.sh).
                            Pass "" to leave one unset. NOTE: --apply reads the hostnames back out of the
                            Caddyfile, so it can only regenerate hosts that are already there. Introducing a
                            NEW host needs --full or this.

Hosts: main = the gateway page at /, files = copyparty, clips = clip intake, sync = Syncthing UI,
       live = portal + OBS dock (/dock/, /dock/clock/, /dump, /me, /time), diag = logs/metrics/alerts,
       auth, watch, git, bot.

Reads  : Caddyfile (hostnames), mediamtx.yml (streamers), .env
         (editors/admins are Authentik groups now: editors, media-admins, admins — nothing local)
         (these are GENERATED on the server and gitignored; *.example.* in the repo are reference only —
          never scp them over the live ones: that is how streamers got wiped once)
Writes : Caddyfile, mediamtx.yml, copyparty.conf, stream-token.secret -> /etc/vanguard, .env (generated secrets +
         Discord bot token/guild), folders, restarts
Set VG_DRY=1 to write files only (no sudo / docker / folders).
"""
import base64, getpass, hashlib, json, os, re, secrets, shutil, subprocess, sys, time
import urllib.error, urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent
os.chdir(HERE)
DRY = os.environ.get("VG_DRY") == "1"
NAME_RE = re.compile(r"^[a-z0-9_-]{2,32}$")
BAD_PW = set(':&?#/ "\'\\\t')          # would break the RTMP query string / SRT streamid / YAML

# ------------------------------------------------------------------ read current
def read_hosts():
    """{'main': host, 'auth': host, 'watch': host} from '# vanguard:<role>' markers; first block = main."""
    hosts, role = {}, None
    try:
        for line in Path("Caddyfile").read_text().splitlines():
            if line[:1].isspace(): continue          # site blocks start at column 0
            t = line.strip()
            m = re.match(r"#\s*vanguard:(\w+)", t)
            if m: role = m.group(1); continue
            if t and not t.startswith(("#", "}", "{")) and t.endswith("{") and " " in t:
                host = t.split()[0]
                if "example.com" in host: role = None; continue
                hosts[role or ("main" if "main" not in hosts else host)] = host
                role = None
    except FileNotFoundError:
        pass
    return hosts

def read_domain(): return read_hosts().get("main", "")

# Every Discord credential and webhook, in one block, in this order, with these comments written into .env.
# They used to be scattered through the file in the order they were added, so "where does the alarm webhook
# go" had no obvious answer. read_env ignores any line starting with #, so the comments are free.
DISCORD_KEYS = (
    ("DISCORD_BOT_TOKEN",         "the bot's own token (Discord dev portal -> Bot). The bot's CI also sets this from Gitea"),
    ("DISCORD_GUILD_ID",          "the server the bot and the Authentik membership policies work against"),
    ("ALERT_WEBHOOK_URL",         "webhook: alarms (vanguard-alerts.timer) and the deadman"),
    ("LIVE_WEBHOOK_URL",          "webhook: 'a stream went live' (MediaMTX runOnReady)"),
    ("DEPLOY_NOTIFY_WEBHOOK_URL", "webhook: deploy results (deploy.sh)"),
)
DEPLOY_ENV_KEYS = ("DEPLOY_AUTO", "DEPLOY_WEBHOOK_SECRET", "DEPLOY_BRANCH", "DEPLOY_NOTIFY_WEBHOOK_URL",
                   "BOT_DEPLOY_AUTO", "BOT_DEPLOY_SECRET", "BOT_ENV_KEYS")

def render_env(env):
    """.env text with the Discord block first and commented, everything else after it."""
    named = dict(DISCORD_KEYS)
    out = ["# ---- Discord: tokens and webhooks ----\n",
           "# change any of these with:  ./configure.py --set KEY=VALUE\n"]
    for key, why in DISCORD_KEYS:
        out.append(f"# {why}\n{key}={env.get(key, '')}\n")
    out.append("\n# ---- everything else ----\n")
    out += [f"{k}={v}\n" for k, v in env.items() if k not in named]
    return "".join(out)

def write_deploy_env(env=None):
    """/etc/vanguard/deploy.env — read by dumpd's /deploy webhook and by deploy.sh."""
    env = env or read_env()
    Path("deploy.env").write_text("".join(f"{k}={env.get(k, '')}\n" for k in DEPLOY_ENV_KEYS)
                                  + f"DEPLOY_REPO_DIR={Path.cwd()}\n")
    os.chmod("deploy.env", 0o600)
    run(["sudo", "install", "-m", "0600", "deploy.env", "/etc/vanguard/deploy.env"])
    Path("deploy.env").unlink(missing_ok=True)

# Settings that can be changed on their own, with the one side effect each needs. Everything else has to go
# through --apply or --full because it regenerates a config file or a container: DIAG_REQUIRE_GROUP needs the
# Caddyfile rewritten and reloaded, DIAG_KIBANA_ROLES needs the Elasticsearch user changed and Kibana
# restarted, hostnames need the whole pass. Keeping this list short is the point.
SETTABLE = {
    "DISCORD_BOT_TOKEN":   "the bot's token (restarts the bot container)",
    "DISCORD_GUILD_ID":    "the Discord server id (restarts the bot container)",
    "ALERT_WEBHOOK_URL":   "Discord webhook for alarms and the deadman (blank = journal only)",
    "LIVE_WEBHOOK_URL":    "Discord webhook for 'went live' (needs a MediaMTX recreate — NOT done for you)",
    "DEPLOY_NOTIFY_WEBHOOK_URL": "Discord webhook for deploy results",
    "ALERT_RETENTION_DAYS": "how long delivered alerts stay queryable in vanguard-alerts",
    "ALERT_SUPPRESS_S":    "per-rule repeat suppression in the delivery timer, seconds",
    "ALERT_MENTION":       "who an alarm pings, e.g. role:123,role:456 or user:789 (blank = nobody)",
    "ALERT_MENTION_LEVELS": "which levels ping, comma separated (default error; add warn to widen)",
    "HEARTBEAT_MAX_S":     "how stale the alerting heartbeat may get before the deadman alarms, seconds",
    "LOG_RETENTION_DAYS":  "ILM delete age for filebeat-*/metricbeat-* (re-applied immediately)",
}

def set_env(updates):
    """Change individual .env keys in place and apply only what each one needs.

    --full re-prompts for every hostname and secret, which is far too blunt for "point the alarms at a new
    Discord channel" — mistyping a hostname there has broken TLS before. This writes the keys, rewrites the
    files that derive from them, and touches nothing else.
    """
    env = read_env()
    if not env:
        print("no .env here — run ./configure.py --full first"); sys.exit(1)
    for key, value in updates.items():
        if key not in SETTABLE:
            print(f"{key} is not settable on its own. Settable keys:")
            for name, why in SETTABLE.items():
                print(f"   {name:22} {why}")
            print("Anything else regenerates a config or a container: use --apply (or --full for hostnames).")
            sys.exit(1)
        env[key] = value
    Path(".env").write_text(render_env(env))
    os.chmod(".env", 0o600)
    for key, value in updates.items():
        print(f"   {key}={value if key != 'ALERT_WEBHOOK_URL' else (value[:40] + '…' if value else '(cleared)')}")

    # The webhook is a credential for that channel, so it stays in its own root-only file.
    if "ALERT_WEBHOOK_URL" in updates:
        Path("alert-webhook").write_text((env.get("ALERT_WEBHOOK_URL") or "") + "\n"); os.chmod("alert-webhook", 0o600)
        run(["sudo", "install", "-m", "0600", "-o", "root", "-g", "root", "alert-webhook", "/etc/vanguard/alert-webhook"])
        Path("alert-webhook").unlink(missing_ok=True)
        print("   wrote /etc/vanguard/alert-webhook")
    # Always, not only when one of its keys changed: it is two file writes, it is idempotent, and making it
    # conditional means the file is missing on any system where the first --set happened to be the webhook.
    # A systemd timer never reads the repo's .env, so this file is the only way those knobs reach the scripts.
    write_alerts_env(env)
    if "DEPLOY_NOTIFY_WEBHOOK_URL" in updates:
        write_deploy_env(env); print("   wrote /etc/vanguard/deploy.env")
    if "LOG_RETENTION_DAYS" in updates:
        es_security(compose_cmd())          # re-injects the ILM delete phase at the new age
    # The bot reads these from its container environment, so they only take effect on recreate. Its own CI
    # does the same thing when it installs a bundle, so a restart here is nothing unusual for the bot.
    if any(k in updates for k in ("DISCORD_BOT_TOKEN", "DISCORD_GUILD_ID")):
        run(compose_cmd() + ["up", "-d", "discord-bot"]); print("   discord-bot recreated with the new settings")
    # MediaMTX reads LIVE_WEBHOOK_URL from ITS environment too, but recreating MediaMTX cuts whatever is
    # recording right now. That is never worth doing automatically, so it is left to a human and a quiet moment.
    if "LIVE_WEBHOOK_URL" in updates:
        print("   NOTE: MediaMTX still has the old value. It only reads this on start, and a restart cuts a")
        print("         live recording, so run this yourself between streams:  docker compose up -d mediamtx")

def write_alerts_env(env=None):
    """/etc/vanguard/alerts.env — read by vanguard-alerts.service and vanguard-deadman.service."""
    env = env or read_env()
    keys = ("ALERT_INDEX", "ALERT_RETENTION_DAYS", "ALERT_SUPPRESS_S", "HEARTBEAT_MAX_S",
            "ALERT_MENTION", "ALERT_MENTION_LEVELS")                                       # what the two scripts read
    Path("alerts.env").write_text("".join(f"{k}={env[k]}\n" for k in keys if env.get(k)))
    os.chmod("alerts.env", 0o644)
    run(["sudo", "install", "-m", "0644", "-o", "root", "-g", "root", "alerts.env", "/etc/vanguard/alerts.env"])
    Path("alerts.env").unlink(missing_ok=True)
    print("   wrote /etc/vanguard/alerts.env")

def read_env():
    env = {}
    try:
        for line in Path(".env").read_text().splitlines():
            if "=" in line and not line.startswith("#"):
                k, v = line.split("=", 1); env[k.strip()] = v.strip()
    except FileNotFoundError:
        pass
    return env

def default_sub(main, sub):
    labels = main.split(".")
    base = ".".join(labels[1:]) if len(labels) >= 3 else main
    return f"{sub}.{base}" if main else ""

def read_streamers():
    out, cur = {}, None
    try:
        in_users = False
        for line in Path("mediamtx.yml").read_text().splitlines():
            if line.startswith("authInternalUsers:"):
                in_users = True; continue
            if in_users and line and not line.startswith(" "):
                break
            if in_users:
                m = re.match(r"\s*-\s*user:\s*(\S+)", line)
                if m: cur = m.group(1); continue
                m = re.match(r"\s*pass:\s*(\S+)", line)
                if m and cur:
                    if not m.group(1).startswith("CHANGEME"): out[cur] = m.group(1)
                    cur = None
    except FileNotFoundError:
        pass
    return out

def read_accounts():
    out = {}
    try:
        in_acc = False
        for line in Path("copyparty.conf").read_text().splitlines():
            if line.strip() == "[accounts]": in_acc = True; continue
            if in_acc and line.startswith("["): break
            if in_acc:
                m = re.match(r"\s*([A-Za-z0-9_-]+):\s*(\S+)", line)
                if m and not m.group(2).startswith("CHANGEME"): out[m.group(1)] = m.group(2)
    except FileNotFoundError:
        pass
    return out

# ------------------------------------------------------------------ prompts
def ask(q, default=""):
    s = input(f"{q}{f' [{default}]' if default else ''}: ").strip()
    return s or default

def yes(q, default=True):
    d = "Y/n" if default else "y/N"
    s = input(f"{q} [{d}]: ").strip().lower()
    return default if not s else s.startswith("y")

def gen_pw(): return "".join(c for c in secrets.token_urlsafe(24) if c.isalnum())[:20]

def ask_pw(label, current=None):
    """keep / generate / type. Returns the password."""
    opts = "(enter = generate" + (", k = keep current" if current else "") + ", or type one)"
    while True:
        prompt = f"  password for {label} {opts}: "
        s = (getpass.getpass(prompt) if sys.stdin.isatty() else input(prompt)).strip()
        if not s: p = gen_pw(); print(f"  generated: {p}"); return p
        if s == "k" and current: return current
        if BAD_PW & set(s): print("  avoid : & ? # / \\ quotes and spaces (they break stream keys)"); continue
        if len(s) < 8: print("  at least 8 characters"); continue
        return s

def ask_name(kind, taken):
    while True:
        n = ask(f"  {kind} name (blank to stop)").lower()
        if not n: return None
        if not NAME_RE.match(n): print("  lowercase letters, digits, _ -  (2-32 chars)"); continue
        if n in taken: print("  already exists"); continue
        return n

# ------------------------------------------------------------------ writers
# Access log per site, JSON on stderr (docker logs). Identity headers Authentik injected are kept (who did what);
# cookies and tokens are dropped. See LOGGING.md.
CADDY_LOG = """	log {
		output stderr
		format filter {
			wrap json
			fields {
				request>headers>Cookie delete
				request>headers>Authorization delete
				request>headers>X-Stream-Auth delete
			}
		}
	}
"""
AUTHENTIK_FORWARD_AUTH = """	# /logout on every gated host = sign out of the Discord login (domain-wide session)
	redir /logout /outpost.goauthentik.io/sign_out 302
	# Authentik forward-auth (embedded outpost). Needs a Proxy Provider in Authentik — domain-level for the
	# whole domain (recommended) or single-application for this host — assigned to the embedded outpost.
	# Until then: 4xx, never open. Per-host group requirements are enforced below from X-Authentik-Groups.
	reverse_proxy /outpost.goauthentik.io/* 127.0.0.1:9000
	forward_auth 127.0.0.1:9000 {
		uri /outpost.goauthentik.io/auth/caddy
		copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Entitlements X-Authentik-Email X-Authentik-Name X-Authentik-Uid X-Authentik-Jwt X-Authentik-Meta-Jwks X-Authentik-Meta-Outpost X-Authentik-Meta-Provider X-Authentik-Meta-App X-Authentik-Meta-Version
		trusted_proxies private_ranges
	}
"""

def gate(required_group=""):
    """Caddy lines: Authentik forward-auth, then deny unless X-Authentik-Groups contains required_group."""
    block = CADDY_LOG + AUTHENTIK_FORWARD_AUTH
    if required_group:
        g = re.escape(required_group)
        block += f"""	@lacks_group not header_regexp X-Authentik-Groups (^|\\|){g}(\\||$)
	handle @lacks_group {{
		respond "This page is for members of the '{required_group}' group." 403
	}}
"""
    return block

def write_caddy(domain, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host="", diag_host=""):
    env = read_env()
    gate_any = gate("")
    extra = ""
    apex, target = env.get("APEX_HOST", ""), env.get("APEX_REDIRECT", "")
    if apex and target:
        extra += f"""
# vanguard:apex
{apex} {{
	redir {target} 302
}}
"""
    if files_host:
        extra += f"""
# vanguard:files
{files_host} {{
	encode zstd gzip
{gate_any}	# copyparty at the root. It honours X-Authentik-Username / -Groups only when the request also carries a header
	# whose secret NAME only this proxy knows (copyparty idp-h-key). Clients cannot forge identity.
	reverse_proxy 127.0.0.1:3923 {{
		header_up X-Vanguard-{env.get("COPYPARTY_IDP_KEY", "key")} 1
		flush_interval -1
	}}
}}
"""
    if sync_host:
        extra += f"""
# vanguard:sync
{sync_host} {{
	encode zstd gzip
{gate(env.get("SYNC_REQUIRE_GROUP", "media-admins"))}	# Syncthing admin UI (its own GUI login is disabled; the gate + group is the login)
	reverse_proxy 127.0.0.1:8384 {{
		header_up Host {{upstream_hostport}}
	}}
}}
"""
    if clips_host:
        extra += f"""
# vanguard:clips
{clips_host} {{
	encode zstd gzip
{gate_any}	@me path /me /me/*
	handle @me {{
		reverse_proxy 127.0.0.1:8080          # dumpd: identity + /me/clips self-service
	}}
	handle {{
		root * /srv/clips
		try_files {{path}} /index.html
		file_server
	}}
}}
"""
    if diag_host:
        extra += f"""
# vanguard:diag
{diag_host} {{
	encode zstd gzip
{gate(env.get("DIAG_REQUIRE_GROUP", "media-admins"))}	# Triage lives on this host, alongside the dashboards it belongs with. Both handles come BEFORE the
	# catch-all below, and the Kibana proxy is wrapped in its own handle so the three are evaluated in the
	# order written rather than by Caddy's directive ordering.
	handle /stuck/api {{
		reverse_proxy 127.0.0.1:8080          # dumpd: the list, and retry/bin for the admin groups
	}}
	handle_path /stuck* {{
		root * /srv/stuck
		try_files {{path}} /index.html
		file_server                           # stuck-recording triage; this host is already media-admins only
	}}
	# Kibana IS the rest of this host: logs (Discover), dashboards and alerting in one app, served at the
	# root so there is no path prefix to get wrong. Elasticsearch itself is never published.
	handle {{
		reverse_proxy 127.0.0.1:5601 {{
			flush_interval -1
		}}
	}}
}}
"""
    if live_host:
        extra += f"""
# vanguard:live
{live_host} {{
	encode zstd gzip
{gate(env.get("LIVE_REQUIRE_GROUP", ""))}	handle /whoami {{
		header Content-Type text/plain
		respond "user:   {{http.request.header.X-Authentik-Username}}
groups: {{http.request.header.X-Authentik-Groups}}"
	}}
	@me path /me /me/*
	handle @me {{
		reverse_proxy 127.0.0.1:8080          # dumpd: identity, stream tokens (Go Live + dock)
	}}
	handle /time {{
		reverse_proxy 127.0.0.1:8080          # server clock for the timecode strip (clock page + Go Live)
	}}
	handle /dump* {{
		reverse_proxy 127.0.0.1:8080          # dump tonight (dock)
	}}
	handle_path /dock* {{
		root * /srv/dock
		file_server                           # OBS browser dock + /dock/clock/
	}}
	handle_path /api/* {{
		reverse_proxy 127.0.0.1:9997
	}}
	handle_path /webrtc/* {{
		# Go Live sends MediaMTX credentials in X-Stream-Auth so Authentik's forward-auth never sees an
		# Authorization header; Caddy turns it into Authorization only on the hop to MediaMTX.
		@creds header X-Stream-Auth *
		reverse_proxy @creds 127.0.0.1:8889 {{
			header_up Authorization {{http.request.header.X-Stream-Auth}}
			header_up -X-Stream-Auth
		}}
		reverse_proxy 127.0.0.1:8889
	}}
	handle_path /hls/* {{
		# handle_path strips /hls before proxying, so MediaMTX's own redirects come back root-relative and
		# lose the prefix. It redirects every playlist request once, to ?cookieCheck=1, so without this the
		# browser follows Location: /<user>/<Title>/index.m3u8 into the SPA catch-all below, receives
		# index.html, and hls.js reports "manifestParsingError" — a web page where a manifest should be.
		reverse_proxy 127.0.0.1:8888 {{
			header_down Location ^/ /hls/
			flush_interval -1          # low-latency HLS sends partial segments; do not buffer them
		}}
	}}
	handle {{
		root * /srv/live
		try_files {{path}} /index.html
		file_server
	}}
}}
"""
    if bot_host:
        extra += f"""
# vanguard:bot
{bot_host} {{
	encode zstd gzip
{gate(env.get("BOT_REQUIRE_GROUP", "bot-users"))}	reverse_proxy 127.0.0.1:8000
}}
"""
    if git_host:
        extra += f"""
# vanguard:git
{git_host} {{
	encode zstd gzip
{CADDY_LOG}	reverse_proxy 127.0.0.1:3000
}}
"""
    if auth_host:
        extra += f"""
# vanguard:auth
{auth_host} {{
	encode zstd gzip
{CADDY_LOG}	reverse_proxy 127.0.0.1:9000
}}
"""
    if watch_host:
        extra += f"""
# vanguard:watch
{watch_host} {{
	encode zstd gzip
{CADDY_LOG}	reverse_proxy 127.0.0.1:8096
}}
"""
    Path("Caddyfile").write_text(f"""{{
	servers {{
		metrics                           # Prometheus metrics at http://127.0.0.1:2019/metrics (LOGGING.md)
	}}
}}

# vanguard:main
{domain} {{
	encode zstd gzip
{gate_any}	# who does Authentik say I am? (debug: shows exactly the identity headers the apps receive)
	handle /whoami {{
		header Content-Type text/plain
		respond "user:   {{http.request.header.X-Authentik-Username}}
groups: {{http.request.header.X-Authentik-Groups}}
email:  {{http.request.header.X-Authentik-Email}}
name:   {{http.request.header.X-Authentik-Name}}"
	}}
	@me path /me /me/*
	handle @me {{
		reverse_proxy 127.0.0.1:8080          # dumpd: identity for the gateway page
	}}
	redir /about* / 308                       # old bookmarks: the gateway page used to live at /about/
	handle {{
		root * /srv/about
		try_files {{path}} /index.html
		file_server                           # the gateway page is the root of the main host
	}}
}}
""" + extra)

def write_env(watch_host, discord_token=None, discord_guild=None, git_host="", bot_host="", live_host="", main_host="", live_webhook=None, files_host="", sync_host="", clips_host="", diag_host=""):
    env = read_env()
    if files_host: env["FILES_HOST"] = files_host
    if sync_host: env["SYNC_HOST"] = sync_host
    if clips_host: env["CLIPS_HOST"] = clips_host
    if diag_host: env["DIAG_HOST"] = diag_host
    env["LIVE_HOST"] = live_host or env.get("LIVE_HOST", "live.example.com")
    if main_host: env["MAIN_HOST"] = main_host
    if live_webhook is not None: env["LIVE_WEBHOOK_URL"] = live_webhook
    env.setdefault("LIVE_WEBHOOK_URL", "")
    env.setdefault("APEX_HOST", "")          # bare domain to serve (A record -> this box); empty = skip
    env.setdefault("APEX_REDIRECT", "")      # where the bare domain sends people, e.g. https://watch.HOST/
    env.setdefault("SYNC_REQUIRE_GROUP", "media-admins")  # Caddy gate for /sync on the main host
    env.setdefault("STREAM_TOKEN_SECRET", secrets.token_urlsafe(48))   # signs publish tokens (dumpd)
    env.setdefault("AUTHENTIK_BOOTSTRAP_PASSWORD", "".join(c for c in secrets.token_urlsafe(30) if c.isalnum())[:20])  # akadmin on first start
    env.setdefault("AUTHENTIK_BOOTSTRAP_TOKEN", secrets.token_urlsafe(40))
    env.setdefault("GITEA_ADMIN_USER", "vanguard")
    env.setdefault("GITEA_ADMIN_PASSWORD", "".join(c for c in secrets.token_urlsafe(30) if c.isalnum())[:20])
    env.setdefault("DEPLOY_AUTO", "0")                                  # 1 = Gitea push webhook runs deploy.sh (off until trusted)
    env.setdefault("DEPLOY_WEBHOOK_SECRET", secrets.token_urlsafe(32))  # Gitea webhook secret (HMAC)
    env.setdefault("DEPLOY_BRANCH", "main")
    env.setdefault("DEPLOY_NOTIFY_WEBHOOK_URL", "")                    # optional Discord webhook for deploy notices
    env.setdefault("COPYPARTY_IDP_KEY", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32])  # Caddy->copyparty idp-h-key
    env.setdefault("BOT_REQUIRE_GROUP", "bot-users")   # Caddy 403s anyone not in this Authentik group; empty = any logged-in member
    env.setdefault("LIVE_REQUIRE_GROUP", "")
    env["GIT_HOST"] = git_host or env.get("GIT_HOST", "git.example.com")
    env["BOT_HOST"] = bot_host or env.get("BOT_HOST", "bot.example.com")
    env.setdefault("GITEA_DB_PASSWORD", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32])
    env.setdefault("GITEA_RUNNER_TOKEN", "")          # filled by ./registerGiteaRunner.sh
    if discord_token is not None: env["DISCORD_BOT_TOKEN"] = discord_token
    if discord_guild is not None: env["DISCORD_GUILD_ID"] = discord_guild
    env.setdefault("AUTHENTIK_SECRET_KEY", secrets.token_urlsafe(48))
    env.setdefault("AUTHENTIK_PG_PASS", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32])
    env.setdefault("AUTHENTIK_TAG", "2025.8")
    env.setdefault("MARIADB_ROOT_PASSWORD", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32])
    env.setdefault("BOT_DB_NAME", "vanguardbot")
    env.setdefault("BOT_DB_USER", "bot")
    env.setdefault("BOT_DB_PASSWORD", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32])
    env.setdefault("BOT_DEPLOY_SECRET", secrets.token_hex(32))   # Gitea repo secret BOT_DEPLOY_SECRET: signs the bot's release bundle
    env.setdefault("BOT_DEPLOY_AUTO", "1")                       # 0 = refuse bundles posted by CI; install by hand instead
    env.setdefault("BOT_ENV_KEYS", "DISCORD_BOT_TOKEN,DISCORD_GUILD_ID")   # the only .env keys the bot's CI may set
    env.setdefault("DIAG_REQUIRE_GROUP", "media-admins")     # who may see logs, metrics and alerts
    env.setdefault("ALERT_WEBHOOK_URL", "")                  # Discord webhook: vanguard-alerts.sh delivery AND the deadman
    env.setdefault("KIBANA_ENCRYPTION_KEY", secrets.token_hex(32))   # without a fixed key Kibana loses saved objects on restart
    # Elasticsearch security is ON (observability/kibana.yml explains why, and why nobody sees a login).
    # These are service credentials, never typed by a person: Discord remains the only human login.
    def pw(): return "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32]
    env.setdefault("ES_ELASTIC_PASSWORD", pw())      # the elastic superuser; bootstrapped on first secured start
    env.setdefault("ES_KIBANA_PASSWORD", pw())       # kibana_system: Kibana's own internal account
    env.setdefault("VG_BEATS_USER", "vanguard_beats")
    env.setdefault("VG_BEATS_PASSWORD", pw())        # what Filebeat and Metricbeat ship as
    env.setdefault("VG_DIAG_USER", "vanguard_diag")  # who every Kibana visitor silently becomes
    env.setdefault("VG_DIAG_PASSWORD", pw())
    # What that visitor may do. superuser keeps today's behaviour exactly (everyone past the gate is an admin).
    # Narrowing this to "editor" is step one of the read-only tier in STATUS.md 9b.
    env.setdefault("DIAG_KIBANA_ROLES", "superuser")
    env.setdefault("ALERT_INDEX", "vanguard-alerts")         # the alert bus: Kibana writes, vanguard-alerts.sh delivers
    env.setdefault("LOG_RETENTION_DAYS", "30")               # ILM delete phase for filebeat-*/metricbeat-*
    env.setdefault("ALERT_RETENTION_DAYS", "90")             # how long delivered alerts stay queryable
    env.setdefault("ALERT_MENTION", "")                      # role:<id>/user:<id> list an alarm pings
    env.setdefault("ALERT_MENTION_LEVELS", "error")          # only errors ping; warnings arrive quietly
    env.setdefault("DISCORD_BOT_TOKEN", "")          # fill in: Discord dev portal -> the Authentik app -> Bot
    env.setdefault("DISCORD_GUILD_ID", "")
    env["WATCH_HOST"] = watch_host or "watch.example.com"
    Path(".env").write_text(render_env(env))
    os.chmod(".env", 0o600)

def docker_gateway():
    """Host IP as seen from the compose network: MediaMTX (bridge) calls dumpd's /mtx-auth there."""
    if DRY: return "172.18.0.1"
    for net in ("vanguard-stream_default", "vanguard_default"):
        try:
            out = subprocess.run(["docker", "network", "inspect", net, "-f", "{{(index .IPAM.Config 0).Gateway}}"], capture_output=True, text=True, timeout=10)
            if out.returncode == 0 and out.stdout.strip(): return out.stdout.strip()
        except Exception:
            pass
    return "172.18.0.1"

def write_mediamtx(public_host=""):
    auth_gw = docker_gateway()
    Path("mediamtx.yml").write_text(f"""# MediaMTX — ingest only. No playback endpoints exposed.  (generated by configure.py)
logLevel: info

# RTSP only on the container loopback: the audio-mirror hook (bin/mtx-on-ready.sh) reads the source and publishes
# "<path>.webrtc" through it. Not published by compose, unreachable from outside.
rtsp: yes
rtspAddress: 127.0.0.1:8554
rtspTransports: [tcp]
rtspEncryption: "no"
moq: no
metrics: yes                     # Prometheus text format, scraped by Metricbeat (LOGGING.md); loopback only
metricsAddress: :9998
pprof: no

# live portal (served to the browser only via Caddy + Authentik on live.HOST; these ports are published on 127.0.0.1)
api: yes
apiAddress: :9997
hls: yes
hlsAddress: :8888
hlsVariant: lowLatency
hlsAlwaysRemux: no
webrtc: yes
webrtcAddress: :8889
webrtcLocalUDPAddress: :8189
webrtcLocalTCPAddress: :8189
webrtcIPsFromInterfaces: no
webrtcAdditionalHosts: [{public_host or "127.0.0.1"}]     # public hostname browsers can reach for ICE (written by configure.py)

rtmp: yes
rtmpAddress: :1935

srt: yes
srtAddress: :8890

# ---- auth -------------------------------------------------------------
# Publishing is authorised by dumpd (/mtx-auth) with short-lived tokens minted for Discord-authenticated
# members of the streamers group. Reads are allowed; viewers are gated by Caddy + Authentik. No passwords.
# NOTE: on this setup (bind-mounted file) MediaMTX does not reload this file live: restart after any change here.
# Day-to-day operations (streamers, tokens, titles) never touch this file.
authMethod: http
authHTTPAddress: http://{auth_gw}:8081/mtx-auth
authHTTPExclude:
  - action: api
  - action: metrics
  - action: pprof

# ---- recording --------------------------------------------------------
pathDefaults:
  record: yes
  recordPath: /recordings/%path/.live/%Y-%m-%d_%H-%M-%S-%f   # hidden while recording; microseconds = server arrival time (timecode fallback)
  recordFormat: fmp4
  recordPartDuration: 1s
  recordSegmentDuration: 12h      # one file per stream unless it runs >12h
  recordDeleteAfter: 0s           # never auto-delete; the archive job handles it
  # on ready: optional Discord "is live" notice (LIVE_WEBHOOK_URL in .env) + the WebRTC audio mirror
  # (bin/mtx-on-ready.sh, bind-mounted by compose; AAC cannot cross WebRTC, so audio is transcoded to Opus into <path>.webrtc)
  runOnReady: /vanguard/on-ready.sh
  runOnReadyRestart: no

paths:
  # audio mirrors (first rule wins): never recorded, no hook; only publishable from inside the container (dumpd)
  ~^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\\.webrtc$:
    record: no
    runOnReady: ""
  # <username>/<Title>; dumpd decides who may publish where (token user == prefix)
  ~^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$: {{}}
""")

def write_copyparty():
    """copyparty behind Caddy + Authentik: identity and groups arrive as headers; permissions are per group."""
    key = read_env().get("COPYPARTY_IDP_KEY", "key")
    Path("copyparty.conf").write_text(f"""# copyparty — behind Caddy + Authentik forward-auth.  (generated by configure.py)
# Identity comes from the proxy: X-Authentik-Username / X-Authentik-Groups (groups separated by |).
# No local accounts. Groups: editors (read streams/clips, write finals), media-admins (everything incl. trash).
# No WebDAV: login is Discord only (browser session); download via the web UI (zip/tar for folders).
# Docs: https://github.com/9001/copyparty#identity-providers

[global]
  e2dsa                 # index files for search
  e2ts                  # read media tags
  th-ff-jpg             # video thumbnails via ffmpeg (cached under /cfg/hist)
  hist: /cfg/hist
  rproxy: 1             # trust first X-Forwarded-For hop (Caddy)
  xff-src: 172.16.0.0/12, 10.0.0.0/8, 192.168.0.0/16, 127.0.0.0/8
  idp-h-usr: X-Authentik-Username
  idp-h-grp: X-Authentik-Groups
  idp-h-key: X-Vanguard-{key}     # header Caddy inserts on every request; IdP headers are ignored without it
  idp-logout: /outpost.goauthentik.io/sign_out   # copyparty's logout link ends the Authentik session (not just its own cookie)
  no-robots
  name: vanguard

[/streams]
  /w/streams
  accs:
    r:    @editors
    rwmd: @media-admins

[/clips]
  /w/clips
  accs:
    r:    @editors
    rwmd: @media-admins

[/finals]
  /w/finals
  accs:
    rw:   @editors
    rwmd: @media-admins

[/trash]
  /w/trash
  accs:
    rwmd: @media-admins
""")

def run(cmd, **kw):
    print("  $", " ".join(cmd))
    if DRY: return 0
    return subprocess.call(cmd, **kw)

def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", discord_token=None, discord_guild=None, git_host="", bot_host="", live_host="", live_webhook=None, files_host="", sync_host="", clips_host="", diag_host=""):
    # admin_pw / editors are legacy no-ops: access is by Authentik group now
    if not streamers and Path("mediamtx.yml").exists() and "CHANGEME" in Path("mediamtx.yml").read_text():
        print("\n!! mediamtx.yml contains CHANGEME placeholders — it looks like the repo template was copied over the")
        print("   generated config. Applying now would leave NO streamers (OBS ingest and the dump button stop working).")
        if not yes("   continue anyway?", False): print("aborted; add streamers first ([s] in the menu)"); sys.exit(1)
    print("\n== writing configs")
    write_env(watch_host, discord_token, discord_guild, git_host, bot_host, live_host, domain, live_webhook, files_host, sync_host, clips_host, diag_host)
    write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); write_mediamtx(domain); write_copyparty()
    Path("stream-token.secret").write_text(read_env()["STREAM_TOKEN_SECRET"] + "\n"); os.chmod("stream-token.secret", 0o600)
    run(["sudo", "install", "-d", "-o", os.environ.get("SUDO_USER") or os.environ.get("USER") or "debian", "/srv/live"])
    print("== app data folders")
    run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/jellyfin/config", "/srv/appdata/jellyfin/cache",
         "/srv/appdata/authentik/media", "/srv/appdata/authentik/templates", "/srv/appdata/authentik/certs"])
    run(["sudo", "install", "-d", "/srv/appdata/authentik/db", "/srv/appdata/authentik/redis", "/srv/appdata/mariadb"])
    run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/discord-bot", "/srv/appdata/gitea", "/srv/appdata/act_runner"])
    print("== folders for legacy streamer entries (MediaMTX creates folders on first publish anyway)")
    for u in streamers:
        run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", f"/srv/media/streams/{u}", f"/srv/media/clips/{u}"])
        if not DRY and not Path(f"/srv/media/clips/{u}/.stignore").exists():
            run(["sudo", "install", "-m", "0644", "-o", "1000", "-g", "1000", "syncthing/stignore", f"/srv/media/clips/{u}/.stignore"])
    print("== dump users + restart dumpd")
    run(["sudo", "install", "-d", "-m", "0750", "/etc/vanguard"])
    run(["sudo", "install", "-m", "0600", "stream-token.secret", "/etc/vanguard/stream-token.secret"])
    run(["sudo", "install", "-m", "0644", "syncthing/stignore", "/etc/vanguard/stignore"])   # dumpd copies it into new clips folders
    e = read_env()
    write_deploy_env(e)                     # one copy of the key list, shared with --set
    Path("alert-webhook").write_text((e.get("ALERT_WEBHOOK_URL") or "") + "\n"); os.chmod("alert-webhook", 0o600)
    # URL is a credential for that channel, so it is not world-readable.
    # 0600 root: the only readers are root timers (vanguard-deadman.sh, vanguard-alerts.sh). It used to be
    # group-readable by nobody(65534) for the Alertmanager container, which no longer exists.
    run(["sudo", "install", "-m", "0600", "-o", "root", "-g", "root", "alert-webhook", "/etc/vanguard/alert-webhook"])
    write_alerts_env(e)                     # the timers' own knobs; a systemd timer never reads .env
    if all(Path(f).exists() for f in ("deploy.sh", "systemd/deploy-pending.service", "systemd/deploy-pending.timer")):
        run(["sudo", "install", "-m", "0755", "deploy.sh", "/usr/local/bin/vanguard-deploy"])
        run(["sudo", "install", "-m", "0644", "systemd/deploy-pending.service", "systemd/deploy-pending.timer", "/etc/systemd/system/"])
        run(["sudo", "systemctl", "daemon-reload"]); run(["sudo", "systemctl", "enable", "--now", "deploy-pending.timer"])
    else:
        print("   (deploy pipeline files missing — deploy.sh / systemd/deploy-pending.* not uploaded; skipping vanguard-deploy install)")
    run(["sudo", "install", "-m", "0755", "deployBotBundle.sh", "/usr/local/bin/vanguard-deploy-bot"])   # dumpd runs this for POST /deploy-bot
    run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/bot", "/srv/bot/app"])               # the bot's released bundle -> /app
    run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/elasticsearch"])           # the ES image runs as uid 1000
    # Elasticsearch refuses to start below this; it is a host setting, not a container one.
    run(["sudo", "sysctl", "-q", "-w", "vm.max_map_count=262144"])
    Path("99-vanguard-elasticsearch.conf").write_text("vm.max_map_count=262144\n")
    run(["sudo", "install", "-m", "0644", "99-vanguard-elasticsearch.conf", "/etc/sysctl.d/"])
    run(["sudo", "install", "-d", "/var/lib/vanguard"])                                                  # deadman state
    run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/media/.markers/metrics"])            # script counters -> dumpd /metrics
    # Everything bin/install.list names. Without this an --apply updated the containers but left STALE
    # binaries in /usr/local/bin while the repo said otherwise — a trap that has cost real debugging time.
    for mode, path in bin_manifest():
        run(["sudo", "install", "-m", mode, path, "/usr/local/bin/"])
    if all(Path(f).exists() for f in ("bin/vanguard-deadman.sh", "systemd/vanguard-deadman.timer")):
        run(["sudo", "install", "-m", "0644", "systemd/vanguard-deadman.service", "systemd/vanguard-deadman.timer", "/etc/systemd/system/"])
        run(["sudo", "systemctl", "daemon-reload"]); run(["sudo", "systemctl", "enable", "--now", "vanguard-deadman.timer"])
    # The other half: Kibana writes alerts into Elasticsearch, this takes them to Discord.
    if all(Path(f).exists() for f in ("bin/vanguard-alerts.sh", "systemd/vanguard-alerts.timer")):
        run(["sudo", "install", "-m", "0644", "systemd/vanguard-alerts.service", "systemd/vanguard-alerts.timer", "/etc/systemd/system/"])
        run(["sudo", "systemctl", "daemon-reload"]); run(["sudo", "systemctl", "enable", "--now", "vanguard-alerts.timer"])
    run(["sudo", "systemctl", "restart", "dumpd"])
    print("== containers")
    dc = compose_cmd()
    # --remove-orphans: a service deleted from the compose file should stop existing, not linger. This is how
    # the eight Victoria/vmalert/Alertmanager/exporter containers go away when the file no longer lists them.
    run(dc + ["up", "-d", "--build", "--remove-orphans"])   # recreates only services whose definition changed
    run(dc + ["restart", "copyparty"])                      # copyparty re-reads its config on start
    run(dc + ["exec", "caddy", "caddy", "reload", "--config", "/etc/caddy/Caddyfile"])   # zero-downtime
    post_start_fixups(dc)
    print("  mediamtx: config changes are NOT picked up live on this setup (bind-mounted file). If mediamtx.yml changed,")
    print("            run  docker compose restart mediamtx  between streams (cuts a live recording). Adding/removing")
    print("            streamers or tokens never touches mediamtx.yml, so day-to-day nothing restarts.")

def bin_manifest():
    """(mode, path) for everything that belongs in /usr/local/bin, from bin/install.list.

    One list, shared with install.sh and deploy.sh. Three hand-kept copies had already drifted: a
    push-to-deploy was installing five of the ten, so vgstuck.sh, both sourced helpers and both timer
    scripts never reached the host unless somebody ran --apply by hand.
    """
    out = []
    try:
        for line in Path("bin/install.list").read_text().splitlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            mode, path = line.split(None, 1)
            if Path(path).exists():
                out.append((mode, path))
    except FileNotFoundError:
        pass
    return out

def compose_cmd():
    """docker compose, with sudo only if this user cannot reach the daemon."""
    if DRY or subprocess.call(["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0:
        return ["docker", "compose"]
    return ["sudo", "docker", "compose"]

def es_security(dc):
    """Create Elasticsearch's native users after the cluster is up. Idempotent; safe to re-run.

    Security is on, so three accounts have to exist before Kibana or the Beats can work:
      kibana_system    Kibana's internal account (built in; only its password needs setting)
      VG_BEATS_USER    what Filebeat and Metricbeat authenticate as
      VG_DIAG_USER     who every Kibana visitor silently becomes (kibana.yml's anonymous provider)
    The elastic superuser itself is bootstrapped by the container from ELASTIC_PASSWORD, which only works
    while the cluster has never had security on. If that did not take, this says exactly what to run rather
    than guessing: a reset would desync the password the compose healthcheck uses.
    """
    env = read_env()
    elastic_pw = env.get("ES_ELASTIC_PASSWORD", "")
    if not elastic_pw:
        print("   elasticsearch: no ES_ELASTIC_PASSWORD in .env (write_env not run?) — skipped"); return
    if DRY:
        print("   (dry) elasticsearch: set kibana_system + create "
              f"{env.get('VG_BEATS_USER')} and {env.get('VG_DIAG_USER')}"); return

    base = "http://127.0.0.1:9200"
    def call(method, path, body=None):
        request = urllib.request.Request(base + path, method=method,
                                         data=None if body is None else json.dumps(body).encode())
        request.add_header("Content-Type", "application/json")
        request.add_header("Authorization", "Basic " + base64.b64encode(f"elastic:{elastic_pw}".encode()).decode())
        with urllib.request.urlopen(request, timeout=20) as response:
            raw = response.read()
            return json.loads(raw) if raw else {}

    status = None
    for _ in range(45):                      # the cluster can take a while on a box busy with x264
        try:
            call("GET", "/_cluster/health"); status = "ok"; break
        except urllib.error.HTTPError as error:
            status = error.code
            if error.code == 401: break      # answering, but our password is not the one it bootstrapped
            time.sleep(4)
        except Exception:
            time.sleep(4)
    if status == 401:
        print("   elasticsearch: REFUSED the password in .env. The cluster already had security enabled, so")
        print("                  ELASTIC_PASSWORD no longer bootstraps anything. Set it to the real one:")
        print("                    docker compose exec elasticsearch bin/elasticsearch-reset-password -u elastic -b -s")
        print("                  put that value in .env as ES_ELASTIC_PASSWORD, then re-run --apply.")
        return
    if status != "ok":
        print("   elasticsearch: not answering yet — re-run ./configure.py --apply once it is up"); return

    try:
        # If security is somehow still off, every call below 400s with "no handler found for uri". Check once
        # and say so plainly, rather than reporting a confusing failure on the first password write.
        call("POST", "/_security/user/_has_privileges", {"cluster": ["monitor"]})
    except urllib.error.HTTPError as error:
        if error.code == 400:
            print("   elasticsearch: the security API is absent — this cluster is still running with")
            print("                  xpack.security.enabled=false. Recreate it:  docker compose up -d elasticsearch")
            return
    try:
        call("POST", "/_security/user/kibana_system/_password", {"password": env["ES_KIBANA_PASSWORD"]})
        print("   elasticsearch: kibana_system password set")
        for user, password, roles in (
            (env["VG_BEATS_USER"], env["VG_BEATS_PASSWORD"], ["superuser"]),
            (env["VG_DIAG_USER"], env["VG_DIAG_PASSWORD"],
             [r.strip() for r in env.get("DIAG_KIBANA_ROLES", "superuser").split(",") if r.strip()]),
        ):
            call("PUT", f"/_security/user/{user}", {"password": password, "roles": roles,
                                                   "full_name": "Vanguard service account"})
            print(f"   elasticsearch: user {user} -> {', '.join(roles)}")
    except urllib.error.HTTPError as error:
        print(f"   elasticsearch: user setup failed ({error.code}) — {error.read()[:200].decode(errors='replace')}")
        return

    # The alert bus. Kibana's rules write here through the .index connector (the only outward connector a
    # basic licence allows), bin/vanguard-alerts.sh delivers from here to Discord, and the deadman watches
    # for the heartbeat document. Pre-created with an explicit mapping so "level" and "rule" are keywords
    # and therefore aggregatable, rather than whatever dynamic mapping the first document implies.
    alert_index = env.get("ALERT_INDEX", "vanguard-alerts")
    try:
        call("PUT", f"/{alert_index}", {
            "settings": {"number_of_replicas": 0},
            "mappings": {"properties": {
                "@timestamp": {"type": "date"},
                "kind": {"type": "keyword"},
                "rule": {"type": "keyword"},
                "level": {"type": "keyword"},
                "service": {"type": "keyword"},
                "message": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 1024}}},
            }}})
        print(f"   elasticsearch: alert index {alert_index} created")
    except urllib.error.HTTPError as error:
        body = error.read().decode(errors="replace")
        print(f"   elasticsearch: alert index {alert_index} already there"
              if "resource_already_exists" in body else f"   elasticsearch: alert index failed — {body[:160]}")

    # Retention. The Beats create their own ILM policies with a rollover but NO delete phase, so indices
    # grow until the disk watermark turns them read-only — which the deadman reports but cannot undo. Inject
    # a delete phase instead of writing a whole policy, so whatever the Beats chose for rollover is kept.
    keep_days = env.get("LOG_RETENTION_DAYS", "30")
    for policy in ("filebeat", "metricbeat"):
        try:
            current = call("GET", f"/_ilm/policy/{policy}")[policy]["policy"]
        except (urllib.error.HTTPError, KeyError):
            print(f"   elasticsearch: ILM policy {policy} not created yet — re-run --apply after the beat has started")
            continue
        phases = current.setdefault("phases", {})
        if phases.get("delete", {}).get("min_age") == f"{keep_days}d":
            print(f"   elasticsearch: {policy} already deletes after {keep_days}d")
            continue
        phases["delete"] = {"min_age": f"{keep_days}d", "actions": {"delete": {}}}
        call("PUT", f"/_ilm/policy/{policy}", {"policy": current})
        print(f"   elasticsearch: {policy} indices now deleted after {keep_days}d")

    # The deadman runs on the host as root and must authenticate too. netrc, not -u, so the password never
    # shows up in anyone's process list.
    Path("es-netrc").write_text(f"machine 127.0.0.1 login elastic password {elastic_pw}\n")
    run(["sudo", "install", "-m", "0600", "-o", "root", "-g", "root", "es-netrc", "/etc/vanguard/es-netrc"])
    Path("es-netrc").unlink(missing_ok=True)
    # Kibana and the Beats were started before those accounts existed, so they are sitting in a retry loop.
    run(dc + ["restart", "kibana", "filebeat", "metricbeat"])
    print("   elasticsearch: netrc written for the deadman; kibana + beats restarted")

def ak_shell(dc, script, label):
    """Run a short script inside authentik-server's Django shell and print only its own output.

    `ak shell` is an interactive console: it reads stdin line by line and rejects indented blocks, so the
    script goes in as a file and is executed by one unindented statement. Authentik also prints a wall of
    JSON boot logs to the same stream, which is why the output is filtered.
    """
    if DRY:
        print(f"   (dry) authentik: {label}"); return
    path = Path("ak-fixup.py")
    path.write_text(script)
    try:
        if subprocess.call(dc + ["cp", str(path), "authentik-server:/tmp/ak-fixup.py"],
                           stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) != 0:
            print(f"   authentik: cannot reach the container — {label} skipped"); return
        result = subprocess.run(dc + ["exec", "-T", "authentik-server", "ak", "shell",
                                      "-c", "exec(open('/tmp/ak-fixup.py').read())"],
                                capture_output=True, text=True)
        for line in (result.stdout + result.stderr).splitlines():
            if line.startswith(("{", "#", ">>>", "...")) or not line.strip():
                continue
            print("  ", line.rstrip())
    except Exception as error:
        print(f"   authentik: {label} skipped ({error})")
    finally:
        path.unlink(missing_ok=True)

# Proxy-provider state the Authentik UI can quietly put back. Enforced on every apply so a save in the UI
# cannot leave logout half-working for weeks without anyone noticing.
AK_LOGOUT_FIXUP = """
from authentik.providers.proxy.models import ProxyProvider
from authentik.flows.models import Flow
target = Flow.objects.filter(slug="default-invalidation-flow").first()
if target is None:
    print("authentik: no default-invalidation-flow found; logout flow left alone")
else:
    for provider in ProxyProvider.objects.all():
        before = getattr(provider.invalidation_flow, "slug", None)
        if before == target.slug:
            print(f"authentik: {provider.name!r} logout flow ok ({before})")
        else:
            provider.invalidation_flow = target
            provider.save(update_fields=["invalidation_flow"])
            print(f"authentik: {provider.name!r} logout flow {before} -> {target.slug}")
"""

def post_start_fixups(dc):
    """Idempotent first-run state that otherwise needs clicking: Syncthing GUI auth off, Gitea admin user, Jellyfin plugin."""
    env = read_env()
    print("== post-start fixups (idempotent)")
    es_security(dc)              # before anything else: Kibana and the Beats cannot authenticate until this runs
    # /logout must end the whole Authentik session, not just the application's. The provider-scoped flow
    # ("Logged out of application") leaves the SSO session alive, so logout looks like a page refresh.
    ak_shell(dc, AK_LOGOUT_FIXUP, "proxy provider logout flow")
    # Syncthing: no GUI login (the Authentik gate is the login); needs insecureAdminAccess since Caddy is not "localhost" to it
    cfg = Path("/srv/media/.syncthing/config.xml")
    if not DRY and cfg.exists():
        import time as _t
        for _ in range(20):
            if "<gui" in cfg.read_text(): break
            _t.sleep(2)
        x = cfg.read_text()
        if "<gui" in x and ('insecureAdminAccess="true"' not in x or "<user>" in x or "<password>" in x):
            x = re.sub(r"^\s*<user>.*</user>\s*$\n?", "", x, flags=re.M); x = re.sub(r"^\s*<password>.*</password>\s*$\n?", "", x, flags=re.M)
            x = re.sub(r'<gui ([^>]*?)insecureAdminAccess="false"', r'<gui \1insecureAdminAccess="true"', x)
            if 'insecureAdminAccess=' not in x: x = x.replace("<gui ", '<gui insecureAdminAccess="true" ', 1)
            subprocess.run(["sudo", "tee", str(cfg)], input=x, text=True, capture_output=True)
            run(dc + ["restart", "syncthing"]); print("   syncthing: GUI auth disabled, insecureAdminAccess on")
        else: print("   syncthing: ok")
    elif DRY: print("   (dry) syncthing config patch")
    # Gitea: create the admin user once (INSTALL_LOCK skips the install page)
    if not DRY:
        import time as _t
        for _ in range(30):
            r = subprocess.run(dc + ["exec", "-T", "-u", "1000", "gitea", "gitea", "admin", "user", "list", "--config", "/data/gitea/conf/app.ini"], capture_output=True, text=True)
            if r.returncode == 0: break
            _t.sleep(3)
        if r.returncode == 0 and env.get("GITEA_ADMIN_USER") and env["GITEA_ADMIN_USER"] not in r.stdout:
            subprocess.run(dc + ["exec", "-T", "-u", "1000", "gitea", "gitea", "admin", "user", "create", "--config", "/data/gitea/conf/app.ini", "--admin",
                            "--username", env["GITEA_ADMIN_USER"], "--password", env["GITEA_ADMIN_PASSWORD"], "--email", f"{env['GITEA_ADMIN_USER']}@{env.get('MAIN_HOST','localhost')}", "--must-change-password=false"], capture_output=True, text=True)
            print(f"   gitea: admin user {env['GITEA_ADMIN_USER']} created (password in .env)")
        else: print("   gitea: ok" if r.returncode == 0 else "   gitea: not ready (install page or db?) — run ./configure.py --apply again later")
    else: print("   (dry) gitea admin user")
    # Kibana first-run state: the alert connector, the heartbeat rule and the data view. Without this a
    # fresh install has no way to deliver an alert and the deadman alarms about its own missing heartbeat.
    # It needs an API key because Basic is not an accepted scheme for this Kibana (anonymous is the only
    # enabled provider), and it waits for Kibana itself, which es_security restarted a moment ago.
    setup = Path("observability/kibana-setup.py")
    if setup.exists() and env.get("ES_ELASTIC_PASSWORD"):
        if DRY:
            print("   (dry) kibana: connector, heartbeat rule, data view")
        else:
            try:
                request = urllib.request.Request(
                    "http://127.0.0.1:9200/_security/api_key", method="POST",
                    data=json.dumps({"name": "vanguard-setup", "expiration": "10m"}).encode())
                request.add_header("Content-Type", "application/json")
                request.add_header("Authorization", "Basic " + base64.b64encode(
                    f"elastic:{env['ES_ELASTIC_PASSWORD']}".encode()).decode())
                with urllib.request.urlopen(request, timeout=20) as response:
                    api_key = json.load(response)["encoded"]
                result = subprocess.run([sys.executable, str(setup)], text=True, capture_output=True,
                                        env={**os.environ, "KEY": api_key,
                                             "ALERT_INDEX": env.get("ALERT_INDEX", "vanguard-alerts")})
                for line in (result.stdout + result.stderr).strip().splitlines():
                    print("   kibana:", line)
            except Exception as error:
                print(f"   kibana: first-run setup skipped ({error}) — run it by hand, see its docstring")

    # Jellyfin plugin from the committed zip
    if Path("installJellyfinPlugin.sh").exists():
        if DRY: print("   (dry) installJellyfinPlugin.sh")
        else: subprocess.run(["bash", "installJellyfinPlugin.sh"], capture_output=True, text=True); print("   jellyfin plugin: ensured")

def show(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host="", diag_host=""):
    print(f"\n== gateway  : https://{domain or 'HOST'}/   (/whoami, /logout)")
    print(f"== files    : https://{files_host}/  (copyparty; editors / media-admins)" if files_host else "== files    : (not set — run --full)")
    print(f"== clips    : https://{clips_host}/  (self-service clip intake, any member)" if clips_host else "== clips    : (not set — run --full)")
    print(f"== sync     : https://{sync_host}/  (Syncthing UI, media-admins)" if sync_host else "== sync     : (not set — run --full)")
    _e = read_env()
    if _e.get("APEX_HOST") and _e.get("APEX_REDIRECT"): print(f"== apex     : https://{_e['APEX_HOST']}/ -> {_e['APEX_REDIRECT']}")
    print(f"== live     : https://{live_host}/  (portal; /dock/ for OBS; WebRTC needs 8189 open)" if live_host else "== live     : (not set — run --full)")
    alert_hook = read_env().get("ALERT_WEBHOOK_URL", "")
    print("== alerts   : " + (f"Discord webhook set ({alert_hook[:38]}…); vanguard-alerts.timer and the deadman post there" if alert_hook
          else "NO WEBHOOK — Kibana still records alerts in the vanguard-alerts index, but nothing delivers them: "
               "vanguard-alerts.sh and the deadman write to the journal and exit non-zero. Set ALERT_WEBHOOK_URL in .env."))
    print(f"== diag     : https://{diag_host}/  (Kibana: logs, dashboards, alerting; group '{read_env().get('DIAG_REQUIRE_GROUP','media-admins')}')" if diag_host else "== diag     : (not set — run --full)")
    print(f"== bot web  : https://{bot_host}/  (Authentik forward-auth; Caddy requires group '{read_env().get('BOT_REQUIRE_GROUP','bot-users') or 'any'}')" if bot_host else "== bot web  : (not set — run --full)")
    print(f"== gitea    : https://{git_host}/  ssh port 2222  admin {read_env().get('GITEA_ADMIN_USER','vanguard')} / {read_env().get('GITEA_ADMIN_PASSWORD','?')}  (runner: ./registerGiteaRunner.sh)" if git_host else "== gitea    : (not set — run --full)")
    print(f"== authentik: https://{auth_host}/   akadmin password = AUTHENTIK_BOOTSTRAP_PASSWORD in .env (set on first start; change it in the UI)" if auth_host else "== authentik: (not set — run --full)")
    print(f"== jellyfin : https://{watch_host}/" if watch_host else "== jellyfin : (not set — run --full)")
    env = read_env()
    print("== discord  : (./configure.py --set KEY=VALUE)")
    for key, why in DISCORD_KEYS:
        value = env.get(key, "")
        if not value:
            shown = "(not set)"
        elif key == "DISCORD_GUILD_ID":
            shown = value
        else:                                # tokens and webhook URLs are credentials: never print them whole
            shown = value[:34] + "…" if len(value) > 34 else value[:8] + "…"
        print(f"     {key:26} {shown:40} {why}")
    print(f"== mariadb  : {domain or 'HOST'}:3306 (public, root local-only)  root pass {env.get('MARIADB_ROOT_PASSWORD','(run apply)')}")
    print(f"     bot ci : Gitea -> VanguardBot -> Settings -> Actions -> Secrets: BOT_DEPLOY_SECRET={env.get('BOT_DEPLOY_SECRET','(run apply)')}")
    print(f"              pushes to main install themselves (BOT_DEPLOY_AUTO={env.get('BOT_DEPLOY_AUTO','1')}); by hand: ./deployBotBundle.sh <bundle>")
    print(f"              CI may set these .env keys (BOT_ENV_KEYS): {env.get('BOT_ENV_KEYS','DISCORD_BOT_TOKEN,DISCORD_GUILD_ID')}")
    print(f"     bot    : db {env.get('BOT_DB_NAME','vanguardbot')}  user {env.get('BOT_DB_USER','bot')}  pass {env.get('BOT_DB_PASSWORD','(run apply)')}")
    print(f"     gitea  : db gitea  user gitea  pass {env.get('GITEA_DB_PASSWORD','(run apply)')}   (created on MariaDB first init, else ./createGiteaDb.sh)")
    print("== access   : by Authentik group — editors (read streams/clips, write finals), media-admins (everything), admins (dump anyone)")
    print("              Discord login only — no WebDAV; downloads via the web UI")
    print("== streamers: anyone in the Authentik 'streamers' group (Discord role). No passwords, no keys to hand out.")
    print(f"    OBS: add the dock https://{live_host or 'LIVE'}/dock/ in OBS, log in with Discord, Go Live — it mints a 12 h token")
    print(f"    and sets the stream key itself. Manual setups: dock → 'Manual OBS setup' (rtmp://{domain or 'HOST'}:1935/ + key).")
    print(f"\n  editors: https://{files_host or 'FILES'}/   everyone: https://{domain or 'HOST'}/")
    print(f"  verify: curl -s -o /dev/null -w '%{{http_code}}\\n' https://{domain or 'HOST'}/   (expect 302 → Authentik)\n")

# ------------------------------------------------------------------ modes
def manage_users(kind, users):
    """add / remove / reset loop for one kind of user; mutates dict."""
    while True:
        print(f"\n  {kind}s: {', '.join(users) or '(none)'}")
        c = ask("  [a]dd  [r]emove  [p]assword reset  [d]one", "d").lower()[:1]
        if c == "d": return
        if c == "a":
            n = ask_name(kind, users)
            if n: users[n] = ask_pw(n)
        elif c in "rp":
            n = ask(f"  which {kind}").lower()
            if n not in users: print("  no such user"); continue
            if c == "r":
                if yes(f"  remove {n}? (their files stay on disk)", False): del users[n]
            else:
                users[n] = ask_pw(n, users[n])

def full(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host="", diag_host=""):
    print("== FULL SETUP (enter keeps the shown default)\n")
    while True:
        domain = ask("main hostname = the gateway page (DNS A record must point here)", domain)
        if domain and "." in domain and " " not in domain: break
        print("  need a hostname like media.example.com")
    files_host = ask("Files (copyparty) hostname (CNAME to the above)", files_host or default_sub(domain, "files"))
    clips_host = ask("Clips self-service hostname (CNAME to the above)", clips_host or default_sub(domain, "clips"))
    sync_host  = ask("Syncthing UI hostname (CNAME to the above)",     sync_host or default_sub(domain, "sync"))
    diag_host  = ask("Diagnostics hostname: logs, metrics, alerts (CNAME to the above)", diag_host or default_sub(domain, "diag"))
    auth_host  = ask("Authentik hostname (CNAME to the above)", auth_host or default_sub(domain, "auth"))
    watch_host = ask("Jellyfin hostname (CNAME to the above)",  watch_host or default_sub(domain, "watch"))
    git_host   = ask("Gitea hostname (CNAME to the above)",     git_host or default_sub(domain, "git"))
    bot_host   = ask("Bot web UI hostname (CNAME to the above)", bot_host or default_sub(domain, "bot"))
    live_host  = ask("Live portal hostname (CNAME to the above)", live_host or default_sub(domain, "live"))
    env0 = read_env(); parent = default_sub(domain, "x")[2:] if domain else ""
    apex = ask("Bare domain to serve too (A record must point here; blank = skip)", env0.get("APEX_HOST", parent))
    apex_to = ask("  bare domain redirects to", env0.get("APEX_REDIRECT") or f"https://{domain}/") if apex else ""
    print("\n-- Discord bot (optional; the Bot added to the Authentik Discord application). Enter keeps the current value.")
    discord_token, discord_guild = ask_discord()
    live_webhook = ask("  Discord webhook URL for 'is live' notices (optional)", read_env().get("LIVE_WEBHOOK_URL", ""))
    env0 = read_env()
    alert_webhook = ask("Discord webhook for ALERTS and the deadman (blank = log only, no Discord)",
                        env0.get("ALERT_WEBHOOK_URL", ""))
    print("\n-- streamers (OBS publishers; each gets streams/<name> and clips/<name>)")
    for u in list(streamers):
        k = ask(f"  {u}: [k]eep  [r]emove  [p]assword reset", "k").lower()[:1]
        if k == "r": del streamers[u]
        elif k == "p": streamers[u] = ask_pw(u, streamers[u])
    print("  add streamers:")
    while True:
        n = ask_name("streamer", streamers)
        if not n: break
        streamers[n] = ask_pw(n)
    for u in list(editors):
        k = ask(f"  {u}: [k]eep  [r]emove  [p]assword reset", "k").lower()[:1]
        if k == "r": del editors[u]
        elif k == "p": editors[u] = ask_pw(u, editors[u])
    print("  add editors:")
    while True:
        n = ask_name("editor", editors)
        if not n: break
        editors[n] = ask_pw(n)
    e = read_env(); e["APEX_HOST"] = apex; e["APEX_REDIRECT"] = apex_to
    Path(".env").write_text("".join(f"{k}={v}\n" for k, v in e.items())) if Path(".env").exists() else None
    if not Path(".env").exists():
        Path(".env").write_text(f"APEX_HOST={apex}\nAPEX_REDIRECT={apex_to}\n"); os.chmod(".env", 0o600)
    if alert_webhook != env0.get("ALERT_WEBHOOK_URL", ""):
        env = read_env(); env["ALERT_WEBHOOK_URL"] = alert_webhook
        Path(".env").write_text(render_env(env)); os.chmod(".env", 0o600)
    return domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host, diag_host

def ask_discord():
    env = read_env()
    cur_tok, cur_guild = env.get("DISCORD_BOT_TOKEN", ""), env.get("DISCORD_GUILD_ID", "")
    prompt = f"  bot token{' [keep ' + cur_tok[:6] + '…]' if cur_tok else ' [none]'}: "
    tok = (getpass.getpass(prompt) if sys.stdin.isatty() else input(prompt)).strip() or cur_tok
    guild = ask("  server (guild) ID", cur_guild)
    return tok, guild

def normal(domain, admin_pw, streamers, editors):
    if not domain:
        print("domain not set yet — run ./configure.py --full"); sys.exit(1)
    print(f"== MANAGE  (domain {domain}; run --full to change hostnames/admin)")
    discord = (None, None)
    while True:
        c = ask("\n[d]iscord bot token/guild  [v]iew  [a]pply & quit  [q]uit without applying", "a").lower()[:1]
        if c == "s": print("  streamers = the Authentik 'streamers' group (Discord role); nothing to manage here")
        elif c == "d": discord = ask_discord()
        elif c == "v": show(domain, admin_pw, streamers, editors)
        elif c == "a": return discord
        elif c == "q": return False

def main():
    mode = sys.argv[1] if len(sys.argv) > 1 else ""
    hosts = read_hosts()
    domain, auth_host, watch_host, git_host, bot_host, live_host = hosts.get("main", ""), hosts.get("auth", ""), hosts.get("watch", ""), hosts.get("git", ""), hosts.get("bot", ""), hosts.get("live", "")
    files_host, sync_host, clips_host = hosts.get("files", ""), hosts.get("sync", ""), hosts.get("clips", "")
    diag_host = hosts.get("diag", "")
    streamers, accounts = read_streamers(), read_accounts()
    admin_pw = accounts.pop("admin", "")
    editors = accounts
    if mode == "--set":
        pairs = [a for a in sys.argv[2:] if "=" in a]
        if not pairs:
            print("usage: configure.py --set KEY=VALUE [KEY=VALUE ...]\n\nSettable on their own:")
            for name, why in SETTABLE.items():
                print(f"   {name:22} {why}")
            print('\nexample: ./configure.py --set ALERT_WEBHOOK_URL=https://discord.com/api/webhooks/…')
            sys.exit(1)
        set_env(dict(p.split("=", 1) for p in pairs)); return
    if mode == "--es-security":
        # Just the Elasticsearch user bootstrap. Worth having on its own: if it is skipped or fails during an
        # --apply, Kibana reports "unavailable", Metricbeat exits on Kibana's 503 and Filebeat sits up but
        # publishes nothing — three confusing symptoms with one cause.
        es_security(compose_cmd()); return
    if mode == "--show":
        show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); return
    if mode == "--apply":
        if not domain: print("domain not set yet — run ./configure.py --full first"); sys.exit(1)
        apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host, live_host=live_host, files_host=files_host, sync_host=sync_host, clips_host=clips_host, diag_host=diag_host)
        show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); return
    if mode == "--hosts":
        if not 4 <= len(sys.argv) <= 11:
            print("usage: configure.py --hosts AUTH WATCH [GIT [BOT [LIVE [FILES [SYNC [CLIPS [DIAG]]]]]]]")
            print('  Pass "" to leave one unset. This is the ONLY non-interactive way to introduce a NEW host:')
            print("  --apply reads the hostnames back from the Caddyfile, so a role that is not in there yet")
            print("  cannot be written by it. --full asks for them all instead.")
            sys.exit(2)
        if not domain: print("domain not set yet — run ./configure.py --full first"); sys.exit(1)
        a = [x.strip() for x in sys.argv[2:]] + [None] * 9
        auth_host, watch_host = a[0], a[1]
        git_host, bot_host, live_host = (a[2] if a[2] is not None else git_host), (a[3] if a[3] is not None else bot_host), (a[4] if a[4] is not None else live_host)
        files_host, sync_host, clips_host = (a[5] if a[5] is not None else files_host), (a[6] if a[6] is not None else sync_host), (a[7] if a[7] is not None else clips_host)
        diag_host = a[8] if a[8] is not None else diag_host
        apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host, live_host=live_host, files_host=files_host, sync_host=sync_host, clips_host=clips_host, diag_host=diag_host)
        show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); return
    discord_token = discord_guild = live_webhook = None
    if mode == "--full":
        domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host, diag_host = full(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host)
        if not yes("\napply now (write configs, restart services)?"): print("nothing written"); return
    elif mode == "":
        r = normal(domain, admin_pw, streamers, editors)
        if r is False: print("nothing written"); return
        discord_token, discord_guild = r
    else:
        print(__doc__); sys.exit(2)
    apply(domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host, diag_host)
    show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host)

if __name__ == "__main__":
    try: main()
    except (KeyboardInterrupt, EOFError): print("\naborted, nothing written")