Phase 13 build: logs, metrics, alerting and a deadman on a diag. host
Six containers, all loopback-published, reachable only through the gated diag. host: victorialogs 30d retention, -http.pathPrefix=/logs so Caddy passes the prefix through unchanged vector docker_logs + journald -> VictoriaLogs (elasticsearch bulk, the documented path) victoriametrics 90d, scrapes the four taps that were already live plus the new services vmalert-metrics 11 metric rules: disk, load, memory, RAID, restart loops, a core container gone vmalert-logs 5 LogsQL rules: sweep failures, publish-deny bursts, failed bot deploys, cert trouble alertmanager groups and inhibits, posts to Discord, webhook read from a 0600 file not the config The deadman (bin/vanguard-deadman.sh, host timer, 5 min) is the part that does not trust any of the above: curl only, it checks each component answers, that the always-firing Watchdog alert is still arriving through vmalert -> Alertmanager -> dumpd /heartbeat, and the disks. It reports on change and announces recovery once. Verified here: failures listed, silence when nothing changes, disk threshold, recovery fired once. Collection uses the Docker socket (Phase 13 option A, chosen explicitly). That is root-equivalent on this host and is recorded as such in vector.yaml with the two upgrade paths. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
82
configure.py
82
configure.py
@@ -172,7 +172,7 @@ def gate(required_group=""):
|
||||
"""
|
||||
return block
|
||||
|
||||
def write_caddy(domain, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host=""):
|
||||
def write_caddy(domain, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host="", diag_host=""):
|
||||
env = read_env()
|
||||
gate_any = gate("")
|
||||
extra = ""
|
||||
@@ -223,6 +223,43 @@ def write_caddy(domain, auth_host="", watch_host="", git_host="", bot_host="", l
|
||||
file_server
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
if diag_host:
|
||||
extra += f"""
|
||||
# vanguard:diag
|
||||
{diag_host} {{
|
||||
encode zstd gzip
|
||||
{gate(env.get("DIAG_REQUIRE_GROUP", "media-admins"))} # VictoriaLogs runs with -http.pathPrefix=/logs and emits /logs/... URLs itself, so the prefix is
|
||||
# passed through unchanged. Stripping it would break the UI's own asset paths.
|
||||
redir /logs /logs/select/vmui/ 302
|
||||
redir /logs/ /logs/select/vmui/ 302
|
||||
handle /logs* {{
|
||||
reverse_proxy 127.0.0.1:9428
|
||||
}}
|
||||
handle /metrics-db* {{
|
||||
reverse_proxy 127.0.0.1:8428 # VictoriaMetrics UI + query API
|
||||
}}
|
||||
handle /alerts* {{
|
||||
reverse_proxy 127.0.0.1:9093 # Alertmanager: what is firing, silences
|
||||
}}
|
||||
handle /rules-metrics* {{
|
||||
reverse_proxy 127.0.0.1:8880 # vmalert over VictoriaMetrics
|
||||
}}
|
||||
handle /rules-logs* {{
|
||||
reverse_proxy 127.0.0.1:8881 # vmalert over VictoriaLogs
|
||||
}}
|
||||
handle {{
|
||||
respond "Vanguard diagnostics
|
||||
|
||||
/logs logs (VictoriaLogs)
|
||||
/metrics-db metrics (VictoriaMetrics)
|
||||
/alerts what is firing (Alertmanager)
|
||||
/rules-metrics metric rules (vmalert)
|
||||
/rules-logs log rules (vmalert)
|
||||
|
||||
Alerts go to Discord. The deadman runs on the host and watches this stack." 200
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
if live_host:
|
||||
extra += f"""
|
||||
@@ -333,11 +370,12 @@ name: {{http.request.header.X-Authentik-Name}}"
|
||||
}}
|
||||
""" + extra)
|
||||
|
||||
def write_env(watch_host, discord_token=None, discord_guild=None, git_host="", bot_host="", live_host="", main_host="", live_webhook=None, files_host="", sync_host="", clips_host=""):
|
||||
def write_env(watch_host, discord_token=None, discord_guild=None, git_host="", bot_host="", live_host="", main_host="", live_webhook=None, files_host="", sync_host="", clips_host="", diag_host=""):
|
||||
env = read_env()
|
||||
if files_host: env["FILES_HOST"] = files_host
|
||||
if sync_host: env["SYNC_HOST"] = sync_host
|
||||
if clips_host: env["CLIPS_HOST"] = clips_host
|
||||
if diag_host: env["DIAG_HOST"] = diag_host
|
||||
env["LIVE_HOST"] = live_host or env.get("LIVE_HOST", "live.example.com")
|
||||
if main_host: env["MAIN_HOST"] = main_host
|
||||
if live_webhook is not None: env["LIVE_WEBHOOK_URL"] = live_webhook
|
||||
@@ -373,6 +411,8 @@ def write_env(watch_host, discord_token=None, discord_guild=None, git_host="", b
|
||||
env.setdefault("BOT_DEPLOY_SECRET", secrets.token_hex(32)) # Gitea repo secret BOT_DEPLOY_SECRET: signs the bot's release bundle
|
||||
env.setdefault("BOT_DEPLOY_AUTO", "1") # 0 = refuse bundles posted by CI; install by hand instead
|
||||
env.setdefault("BOT_ENV_KEYS", "DISCORD_BOT_TOKEN,DISCORD_GUILD_ID") # the only .env keys the bot's CI may set
|
||||
env.setdefault("DIAG_REQUIRE_GROUP", "media-admins") # who may see logs, metrics and alerts
|
||||
env.setdefault("ALERT_WEBHOOK_URL", "") # Discord webhook for Alertmanager AND the deadman
|
||||
env.setdefault("DISCORD_BOT_TOKEN", "") # fill in: Discord dev portal -> the Authentik app -> Bot
|
||||
env.setdefault("DISCORD_GUILD_ID", "")
|
||||
env["WATCH_HOST"] = watch_host or "watch.example.com"
|
||||
@@ -512,15 +552,15 @@ def run(cmd, **kw):
|
||||
if DRY: return 0
|
||||
return subprocess.call(cmd, **kw)
|
||||
|
||||
def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", discord_token=None, discord_guild=None, git_host="", bot_host="", live_host="", live_webhook=None, files_host="", sync_host="", clips_host=""):
|
||||
def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", discord_token=None, discord_guild=None, git_host="", bot_host="", live_host="", live_webhook=None, files_host="", sync_host="", clips_host="", diag_host=""):
|
||||
# admin_pw / editors are legacy no-ops: access is by Authentik group now
|
||||
if not streamers and Path("mediamtx.yml").exists() and "CHANGEME" in Path("mediamtx.yml").read_text():
|
||||
print("\n!! mediamtx.yml contains CHANGEME placeholders — it looks like the repo template was copied over the")
|
||||
print(" generated config. Applying now would leave NO streamers (OBS ingest and the dump button stop working).")
|
||||
if not yes(" continue anyway?", False): print("aborted; add streamers first ([s] in the menu)"); sys.exit(1)
|
||||
print("\n== writing configs")
|
||||
write_env(watch_host, discord_token, discord_guild, git_host, bot_host, live_host, domain, live_webhook, files_host, sync_host, clips_host)
|
||||
write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host); write_mediamtx(domain); write_copyparty()
|
||||
write_env(watch_host, discord_token, discord_guild, git_host, bot_host, live_host, domain, live_webhook, files_host, sync_host, clips_host, diag_host)
|
||||
write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); write_mediamtx(domain); write_copyparty()
|
||||
Path("stream-token.secret").write_text(read_env()["STREAM_TOKEN_SECRET"] + "\n"); os.chmod("stream-token.secret", 0o600)
|
||||
run(["sudo", "install", "-d", "-o", os.environ.get("SUDO_USER") or os.environ.get("USER") or "debian", "/srv/live"])
|
||||
print("== app data folders")
|
||||
@@ -539,6 +579,8 @@ def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", dis
|
||||
run(["sudo", "install", "-m", "0644", "syncthing/stignore", "/etc/vanguard/stignore"]) # dumpd copies it into new clips folders
|
||||
e = read_env()
|
||||
Path("deploy.env").write_text("".join(f"{k}={e.get(k, '')}\n" for k in ("DEPLOY_AUTO", "DEPLOY_WEBHOOK_SECRET", "DEPLOY_BRANCH", "DEPLOY_NOTIFY_WEBHOOK_URL", "BOT_DEPLOY_AUTO", "BOT_DEPLOY_SECRET", "BOT_ENV_KEYS")) + f"DEPLOY_REPO_DIR={Path.cwd()}\n"); os.chmod("deploy.env", 0o600)
|
||||
Path("alert-webhook").write_text((e.get("ALERT_WEBHOOK_URL") or "") + "\n"); os.chmod("alert-webhook", 0o600)
|
||||
run(["sudo", "install", "-m", "0644", "alert-webhook", "/etc/vanguard/alert-webhook"]) # read by alertmanager (uid 65534) and the deadman
|
||||
run(["sudo", "install", "-m", "0600", "deploy.env", "/etc/vanguard/deploy.env"]) # read by dumpd (/deploy webhook) and deploy.sh
|
||||
if all(Path(f).exists() for f in ("deploy.sh", "systemd/deploy-pending.service", "systemd/deploy-pending.timer")):
|
||||
run(["sudo", "install", "-m", "0755", "deploy.sh", "/usr/local/bin/vanguard-deploy"])
|
||||
@@ -548,6 +590,11 @@ def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", dis
|
||||
print(" (deploy pipeline files missing — deploy.sh / systemd/deploy-pending.* not uploaded; skipping vanguard-deploy install)")
|
||||
run(["sudo", "install", "-m", "0755", "deployBotBundle.sh", "/usr/local/bin/vanguard-deploy-bot"]) # dumpd runs this for POST /deploy-bot
|
||||
run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/bot", "/srv/bot/app"]) # the bot's released bundle -> /app
|
||||
run(["sudo", "install", "-d", "/var/lib/vanguard"]) # deadman state
|
||||
if all(Path(f).exists() for f in ("bin/vanguard-deadman.sh", "systemd/vanguard-deadman.timer")):
|
||||
run(["sudo", "install", "-m", "0755", "bin/vanguard-deadman.sh", "/usr/local/bin/"])
|
||||
run(["sudo", "install", "-m", "0644", "systemd/vanguard-deadman.service", "systemd/vanguard-deadman.timer", "/etc/systemd/system/"])
|
||||
run(["sudo", "systemctl", "daemon-reload"]); run(["sudo", "systemctl", "enable", "--now", "vanguard-deadman.timer"])
|
||||
run(["sudo", "systemctl", "restart", "dumpd"])
|
||||
print("== containers")
|
||||
dc = ["docker", "compose"] if (DRY or subprocess.call(["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0) else ["sudo", "docker", "compose"]
|
||||
@@ -597,7 +644,7 @@ def post_start_fixups(dc):
|
||||
if DRY: print(" (dry) installJellyfinPlugin.sh")
|
||||
else: subprocess.run(["bash", "installJellyfinPlugin.sh"], capture_output=True, text=True); print(" jellyfin plugin: ensured")
|
||||
|
||||
def show(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host=""):
|
||||
def show(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host="", diag_host=""):
|
||||
print(f"\n== gateway : https://{domain or 'HOST'}/ (/whoami, /logout)")
|
||||
print(f"== files : https://{files_host}/ (copyparty; editors / media-admins)" if files_host else "== files : (not set — run --full)")
|
||||
print(f"== clips : https://{clips_host}/ (self-service clip intake, any member)" if clips_host else "== clips : (not set — run --full)")
|
||||
@@ -605,6 +652,7 @@ def show(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_
|
||||
_e = read_env()
|
||||
if _e.get("APEX_HOST") and _e.get("APEX_REDIRECT"): print(f"== apex : https://{_e['APEX_HOST']}/ -> {_e['APEX_REDIRECT']}")
|
||||
print(f"== live : https://{live_host}/ (portal; /dock/ for OBS; WebRTC needs 8189 open)" if live_host else "== live : (not set — run --full)")
|
||||
print(f"== diag : https://{diag_host}/logs (logs, metrics, alerts; group '{read_env().get('DIAG_REQUIRE_GROUP','media-admins')}')" if diag_host else "== diag : (not set — run --full)")
|
||||
print(f"== bot web : https://{bot_host}/ (Authentik forward-auth; Caddy requires group '{read_env().get('BOT_REQUIRE_GROUP','bot-users') or 'any'}')" if bot_host else "== bot web : (not set — run --full)")
|
||||
print(f"== gitea : https://{git_host}/ ssh port 2222 admin {read_env().get('GITEA_ADMIN_USER','vanguard')} / {read_env().get('GITEA_ADMIN_PASSWORD','?')} (runner: ./registerGiteaRunner.sh)" if git_host else "== gitea : (not set — run --full)")
|
||||
print(f"== authentik: https://{auth_host}/ akadmin password = AUTHENTIK_BOOTSTRAP_PASSWORD in .env (set on first start; change it in the UI)" if auth_host else "== authentik: (not set — run --full)")
|
||||
@@ -644,7 +692,7 @@ def manage_users(kind, users):
|
||||
else:
|
||||
users[n] = ask_pw(n, users[n])
|
||||
|
||||
def full(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host=""):
|
||||
def full(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host="", live_host="", files_host="", sync_host="", clips_host="", diag_host=""):
|
||||
print("== FULL SETUP (enter keeps the shown default)\n")
|
||||
while True:
|
||||
domain = ask("main hostname = the gateway page (DNS A record must point here)", domain)
|
||||
@@ -653,6 +701,7 @@ def full(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_
|
||||
files_host = ask("Files (copyparty) hostname (CNAME to the above)", files_host or default_sub(domain, "files"))
|
||||
clips_host = ask("Clips self-service hostname (CNAME to the above)", clips_host or default_sub(domain, "clips"))
|
||||
sync_host = ask("Syncthing UI hostname (CNAME to the above)", sync_host or default_sub(domain, "sync"))
|
||||
diag_host = ask("Diagnostics hostname: logs, metrics, alerts (CNAME to the above)", diag_host or default_sub(domain, "diag"))
|
||||
auth_host = ask("Authentik hostname (CNAME to the above)", auth_host or default_sub(domain, "auth"))
|
||||
watch_host = ask("Jellyfin hostname (CNAME to the above)", watch_host or default_sub(domain, "watch"))
|
||||
git_host = ask("Gitea hostname (CNAME to the above)", git_host or default_sub(domain, "git"))
|
||||
@@ -687,7 +736,7 @@ def full(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_
|
||||
Path(".env").write_text("".join(f"{k}={v}\n" for k, v in e.items())) if Path(".env").exists() else None
|
||||
if not Path(".env").exists():
|
||||
Path(".env").write_text(f"APEX_HOST={apex}\nAPEX_REDIRECT={apex_to}\n"); os.chmod(".env", 0o600)
|
||||
return domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host
|
||||
return domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host, diag_host
|
||||
|
||||
def ask_discord():
|
||||
env = read_env()
|
||||
@@ -715,15 +764,16 @@ def main():
|
||||
hosts = read_hosts()
|
||||
domain, auth_host, watch_host, git_host, bot_host, live_host = hosts.get("main", ""), hosts.get("auth", ""), hosts.get("watch", ""), hosts.get("git", ""), hosts.get("bot", ""), hosts.get("live", "")
|
||||
files_host, sync_host, clips_host = hosts.get("files", ""), hosts.get("sync", ""), hosts.get("clips", "")
|
||||
diag_host = hosts.get("diag", "")
|
||||
streamers, accounts = read_streamers(), read_accounts()
|
||||
admin_pw = accounts.pop("admin", "")
|
||||
editors = accounts
|
||||
if mode == "--show":
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host); return
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); return
|
||||
if mode == "--apply":
|
||||
if not domain: print("domain not set yet — run ./configure.py --full first"); sys.exit(1)
|
||||
apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host, live_host=live_host, files_host=files_host, sync_host=sync_host, clips_host=clips_host)
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host); return
|
||||
apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host, live_host=live_host, files_host=files_host, sync_host=sync_host, clips_host=clips_host, diag_host=diag_host)
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); return
|
||||
if mode == "--hosts":
|
||||
if not 4 <= len(sys.argv) <= 10: print("usage: configure.py --hosts AUTH WATCH [GIT [BOT [LIVE [FILES [SYNC [CLIPS]]]]]]"); sys.exit(2)
|
||||
if not domain: print("domain not set yet — run ./configure.py --full first"); sys.exit(1)
|
||||
@@ -731,11 +781,11 @@ def main():
|
||||
auth_host, watch_host = a[0], a[1]
|
||||
git_host, bot_host, live_host = (a[2] if a[2] is not None else git_host), (a[3] if a[3] is not None else bot_host), (a[4] if a[4] is not None else live_host)
|
||||
files_host, sync_host, clips_host = (a[5] if a[5] is not None else files_host), (a[6] if a[6] is not None else sync_host), (a[7] if a[7] is not None else clips_host)
|
||||
apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host, live_host=live_host, files_host=files_host, sync_host=sync_host, clips_host=clips_host)
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host); return
|
||||
apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host, live_host=live_host, files_host=files_host, sync_host=sync_host, clips_host=clips_host, diag_host=diag_host)
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); return
|
||||
discord_token = discord_guild = live_webhook = None
|
||||
if mode == "--full":
|
||||
domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host = full(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host)
|
||||
domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host, diag_host = full(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host)
|
||||
if not yes("\napply now (write configs, restart services)?"): print("nothing written"); return
|
||||
elif mode == "":
|
||||
r = normal(domain, admin_pw, streamers, editors)
|
||||
@@ -743,8 +793,8 @@ def main():
|
||||
discord_token, discord_guild = r
|
||||
else:
|
||||
print(__doc__); sys.exit(2)
|
||||
apply(domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host)
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host)
|
||||
apply(domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host, live_host, live_webhook, files_host, sync_host, clips_host, diag_host)
|
||||
show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host)
|
||||
|
||||
if __name__ == "__main__":
|
||||
try: main()
|
||||
|
||||
Reference in New Issue
Block a user