Rip out the Victoria stack for Elastic: four containers, Kibana as the single pane
Eight observability containers become four. Elasticsearch stores logs and metrics, Kibana is the one UI for Discover, dashboards and alerting, Filebeat ships container logs plus our journal units, and Metricbeat covers host and Docker metrics — which retires node_exporter and cAdvisor too — while still scraping MediaMTX's and Caddy's Prometheus endpoints. Pinned rather than left to chance: a fixed 2 GB heap (a self-sizing JVM would claim a quarter of a 62 GB box whose spare capacity belongs to x264), single-node discovery, security off because it only listens on loopback behind the Authentik gate, ML off, and vm.max_map_count set and persisted since ES will not start without it. Kibana gets a fixed encryption key or it loses its saved objects on every restart. diag.HOST is now just Kibana at the root: no path prefixes to get wrong and no index page of links. The deadman is the only thing kept from the old stack, repointed at Elastic. It checks that ES answers and is not red, that no index has been flipped read-only by the disk watermark (which does not clear itself when space is freed, so ingestion stays silently dead), that Kibana is available, that each Beat has actually written a document recently rather than merely running, that the Kibana heartbeat still arrives, and the disks. Verified against a mock Elasticsearch for all five failure modes plus the healthy case staying silent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
100
configure.py
100
configure.py
@@ -20,7 +20,7 @@ Reads : Caddyfile (hostnames), mediamtx.yml (streamers), .env
|
||||
(editors/admins are Authentik groups now: editors, media-admins, admins — nothing local)
|
||||
(these are GENERATED on the server and gitignored; *.example.* in the repo are reference only —
|
||||
never scp them over the live ones: that is how streamers got wiped once)
|
||||
Writes : Caddyfile, mediamtx.yml, copyparty.conf, observability/alertmanager.yml, stream-token.secret -> /etc/vanguard, .env (generated secrets +
|
||||
Writes : Caddyfile, mediamtx.yml, copyparty.conf, stream-token.secret -> /etc/vanguard, .env (generated secrets +
|
||||
Discord bot token/guild), folders, restarts
|
||||
Set VG_DRY=1 to write files only (no sudo / docker / folders).
|
||||
"""
|
||||
@@ -232,29 +232,10 @@ def write_caddy(domain, auth_host="", watch_host="", git_host="", bot_host="", l
|
||||
# vanguard:diag
|
||||
{diag_host} {{
|
||||
encode zstd gzip
|
||||
{gate(env.get("DIAG_REQUIRE_GROUP", "media-admins"))} # VictoriaLogs runs with -http.pathPrefix=/logs and emits /logs/... URLs itself, so the prefix is
|
||||
# passed through unchanged. Stripping it would break the UI's own asset paths.
|
||||
redir /logs /logs/select/vmui/ 302
|
||||
redir /logs/ /logs/select/vmui/ 302
|
||||
handle /logs* {{
|
||||
reverse_proxy 127.0.0.1:9428
|
||||
}}
|
||||
handle /metrics-db* {{
|
||||
reverse_proxy 127.0.0.1:8428 # VictoriaMetrics UI + query API
|
||||
}}
|
||||
handle /alerts* {{
|
||||
reverse_proxy 127.0.0.1:9093 # Alertmanager: what is firing, silences
|
||||
}}
|
||||
handle /rules-metrics* {{
|
||||
reverse_proxy 127.0.0.1:8880 # vmalert over VictoriaMetrics
|
||||
}}
|
||||
handle /rules-logs* {{
|
||||
reverse_proxy 127.0.0.1:8881 # vmalert over VictoriaLogs
|
||||
}}
|
||||
handle {{
|
||||
root * /srv/diag
|
||||
try_files {{path}} /index.html
|
||||
file_server # diag/index.html: what each path above is
|
||||
{gate(env.get("DIAG_REQUIRE_GROUP", "media-admins"))} # Kibana IS this host: logs (Discover), dashboards and alerting in one app, served at the root so there
|
||||
# is no path prefix to get wrong and no index page of links. Elasticsearch itself is never published.
|
||||
reverse_proxy 127.0.0.1:5601 {{
|
||||
flush_interval -1
|
||||
}}
|
||||
}}
|
||||
"""
|
||||
@@ -409,7 +390,8 @@ def write_env(watch_host, discord_token=None, discord_guild=None, git_host="", b
|
||||
env.setdefault("BOT_DEPLOY_AUTO", "1") # 0 = refuse bundles posted by CI; install by hand instead
|
||||
env.setdefault("BOT_ENV_KEYS", "DISCORD_BOT_TOKEN,DISCORD_GUILD_ID") # the only .env keys the bot's CI may set
|
||||
env.setdefault("DIAG_REQUIRE_GROUP", "media-admins") # who may see logs, metrics and alerts
|
||||
env.setdefault("ALERT_WEBHOOK_URL", "") # Discord webhook for Alertmanager AND the deadman
|
||||
env.setdefault("ALERT_WEBHOOK_URL", "") # Discord webhook for Kibana rules AND the deadman
|
||||
env.setdefault("KIBANA_ENCRYPTION_KEY", secrets.token_hex(32)) # without a fixed key Kibana loses saved objects on restart
|
||||
env.setdefault("DISCORD_BOT_TOKEN", "") # fill in: Discord dev portal -> the Authentik app -> Bot
|
||||
env.setdefault("DISCORD_GUILD_ID", "")
|
||||
env["WATCH_HOST"] = watch_host or "watch.example.com"
|
||||
@@ -497,59 +479,6 @@ paths:
|
||||
~^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$: {{}}
|
||||
""")
|
||||
|
||||
def write_alertmanager():
|
||||
"""Alertmanager config. Generated, because the Discord receiver must be ABSENT when no webhook is set:
|
||||
a receiver with no notifiers is valid and simply drops notifications, whereas pointing discord_configs at
|
||||
an empty webhook_url_file may stop Alertmanager from loading at all. The URL itself stays out of this file
|
||||
and in /etc/vanguard/alert-webhook (0640 root:65534), which is what webhook_url_file reads."""
|
||||
webhook = read_env().get("ALERT_WEBHOOK_URL", "").strip()
|
||||
discord = """ discord_configs:
|
||||
- webhook_url_file: /etc/vanguard/alert-webhook
|
||||
title: '{{ if eq .Status "firing" }}🔴{{ else }}✅{{ end }} {{ .CommonLabels.alertname }}'
|
||||
message: |
|
||||
{{ range .Alerts }}{{ .Annotations.summary }}
|
||||
{{ if .Annotations.detail }}{{ .Annotations.detail }}
|
||||
{{ end }}{{ end }}
|
||||
""" if webhook.startswith("https://") else """ # No ALERT_WEBHOOK_URL in .env, so this receiver sends nowhere: alerts still group and still show at
|
||||
# https://DIAG_HOST/alerts, they just are not delivered. Set the variable and re-apply to turn it on.
|
||||
"""
|
||||
Path("observability/alertmanager.yml").write_text(f"""# Alertmanager — GENERATED by configure.py from ALERT_WEBHOOK_URL. Reference copy: alertmanager.example.yml
|
||||
# The heartbeat route is half of the deadman: the Watchdog alert always fires, so Alertmanager posts to dumpd
|
||||
# every minute and bin/vanguard-deadman.sh treats a STALE heartbeat as the alarm.
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: discord
|
||||
group_by: [alertname, service]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
routes:
|
||||
- matchers: [severity="none"]
|
||||
receiver: heartbeat
|
||||
group_wait: 0s
|
||||
group_interval: 1m
|
||||
repeat_interval: 1m
|
||||
- matchers: [severity="critical"]
|
||||
receiver: discord
|
||||
group_wait: 10s
|
||||
repeat_interval: 1h
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers: [alertname="MediaDiskCritical"]
|
||||
target_matchers: [alertname="MediaDiskFilling"]
|
||||
- source_matchers: [alertname="ContainerGone"]
|
||||
target_matchers: [alertname="ContainerRestartLoop"]
|
||||
|
||||
receivers:
|
||||
- name: discord
|
||||
{discord} - name: heartbeat
|
||||
webhook_configs:
|
||||
- url: http://host.docker.internal:8081/heartbeat
|
||||
send_resolved: false
|
||||
""")
|
||||
|
||||
def write_copyparty():
|
||||
"""copyparty behind Caddy + Authentik: identity and groups arrive as headers; permissions are per group."""
|
||||
key = read_env().get("COPYPARTY_IDP_KEY", "key")
|
||||
@@ -610,7 +539,7 @@ def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", dis
|
||||
if not yes(" continue anyway?", False): print("aborted; add streamers first ([s] in the menu)"); sys.exit(1)
|
||||
print("\n== writing configs")
|
||||
write_env(watch_host, discord_token, discord_guild, git_host, bot_host, live_host, domain, live_webhook, files_host, sync_host, clips_host, diag_host)
|
||||
write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); write_mediamtx(domain); write_copyparty(); write_alertmanager()
|
||||
write_caddy(domain, auth_host, watch_host, git_host, bot_host, live_host, files_host, sync_host, clips_host, diag_host); write_mediamtx(domain); write_copyparty()
|
||||
Path("stream-token.secret").write_text(read_env()["STREAM_TOKEN_SECRET"] + "\n"); os.chmod("stream-token.secret", 0o600)
|
||||
run(["sudo", "install", "-d", "-o", os.environ.get("SUDO_USER") or os.environ.get("USER") or "debian", "/srv/live"])
|
||||
print("== app data folders")
|
||||
@@ -642,8 +571,11 @@ def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", dis
|
||||
print(" (deploy pipeline files missing — deploy.sh / systemd/deploy-pending.* not uploaded; skipping vanguard-deploy install)")
|
||||
run(["sudo", "install", "-m", "0755", "deployBotBundle.sh", "/usr/local/bin/vanguard-deploy-bot"]) # dumpd runs this for POST /deploy-bot
|
||||
run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/bot", "/srv/bot/app"]) # the bot's released bundle -> /app
|
||||
run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/victorialogs", "/srv/appdata/victoriametrics"])
|
||||
run(["sudo", "install", "-d", "-o", "65534", "-g", "65534", "/srv/appdata/alertmanager"]) # prom/alertmanager runs as nobody
|
||||
run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/elasticsearch"]) # the ES image runs as uid 1000
|
||||
# Elasticsearch refuses to start below this; it is a host setting, not a container one.
|
||||
run(["sudo", "sysctl", "-q", "-w", "vm.max_map_count=262144"])
|
||||
Path("99-vanguard-elasticsearch.conf").write_text("vm.max_map_count=262144\n")
|
||||
run(["sudo", "install", "-m", "0644", "99-vanguard-elasticsearch.conf", "/etc/sysctl.d/"])
|
||||
run(["sudo", "install", "-d", "/var/lib/vanguard"]) # deadman state
|
||||
if all(Path(f).exists() for f in ("bin/vanguard-deadman.sh", "systemd/vanguard-deadman.timer")):
|
||||
run(["sudo", "install", "-m", "0755", "bin/vanguard-deadman.sh", "/usr/local/bin/"])
|
||||
@@ -707,9 +639,9 @@ def show(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_
|
||||
if _e.get("APEX_HOST") and _e.get("APEX_REDIRECT"): print(f"== apex : https://{_e['APEX_HOST']}/ -> {_e['APEX_REDIRECT']}")
|
||||
print(f"== live : https://{live_host}/ (portal; /dock/ for OBS; WebRTC needs 8189 open)" if live_host else "== live : (not set — run --full)")
|
||||
alert_hook = read_env().get("ALERT_WEBHOOK_URL", "")
|
||||
print("== alerts : " + (f"Discord webhook set ({alert_hook[:38]}…); deadman posts there too" if alert_hook
|
||||
else "NO WEBHOOK — alerts and the deadman only write to the journal. Set ALERT_WEBHOOK_URL in .env."))
|
||||
print(f"== diag : https://{diag_host}/logs (logs, metrics, alerts; group '{read_env().get('DIAG_REQUIRE_GROUP','media-admins')}')" if diag_host else "== diag : (not set — run --full)")
|
||||
print("== alerts : " + (f"Discord webhook set ({alert_hook[:38]}…); Kibana rules and the deadman post there" if alert_hook
|
||||
else "NO WEBHOOK — Kibana has nowhere to send and the deadman only writes to the journal. Set ALERT_WEBHOOK_URL in .env."))
|
||||
print(f"== diag : https://{diag_host}/ (Kibana: logs, dashboards, alerting; group '{read_env().get('DIAG_REQUIRE_GROUP','media-admins')}')" if diag_host else "== diag : (not set — run --full)")
|
||||
print(f"== bot web : https://{bot_host}/ (Authentik forward-auth; Caddy requires group '{read_env().get('BOT_REQUIRE_GROUP','bot-users') or 'any'}')" if bot_host else "== bot web : (not set — run --full)")
|
||||
print(f"== gitea : https://{git_host}/ ssh port 2222 admin {read_env().get('GITEA_ADMIN_USER','vanguard')} / {read_env().get('GITEA_ADMIN_PASSWORD','?')} (runner: ./registerGiteaRunner.sh)" if git_host else "== gitea : (not set — run --full)")
|
||||
print(f"== authentik: https://{auth_host}/ akadmin password = AUTHENTIK_BOOTSTRAP_PASSWORD in .env (set on first start; change it in the UI)" if auth_host else "== authentik: (not set — run --full)")
|
||||
|
||||
Reference in New Issue
Block a user