Media-Server-Setup/observability/kibana-setup.py

#!/usr/bin/env python3
"""Kibana first-run state. Idempotent; safe to re-run. configure.py --apply runs this for you.

  connector  "Vanguard alert index"        — the .index connector, writing into the vanguard-alerts index
  rule       "Vanguard alerting heartbeat" — always-firing ES query rule, every minute, writing one doc
  data view  "Vanguard logs + metrics"     — filebeat-*,metricbeat-* so Discover and Lens have one source
  landing    "Vanguard ops" dashboard      — our own links, and defaultRoute so diag.BASE opens on it

Run one step on its own by naming it:  kibana-setup.py landing   (also: connector, rule, dataview, all)

WHY AN INDEX AND NOT A WEBHOOK
On a basic license the only connector types Kibana allows are .index and .server-log; every connector that
leaves the stack (webhook, Slack, email, PagerDuty…) is Gold or above and returns
    403 Action type .webhook is disabled because your basic license does not support it
So Kibana cannot talk to Discord. Instead every rule writes a document into ONE index, which is the alert
bus, and bin/vanguard-alerts.sh (a host timer) delivers from there to Discord and later to the bot. That is
strictly simpler than the webhook route: no Docker-bridge hop, no dumpd endpoint, and alerts are queryable
and replayable after the fact because they are just documents.

THE HEARTBEAT
bin/vanguard-deadman.sh alarms when no heartbeat document has arrived recently. It is the only check that
catches the ALERTING pipeline itself dying — if Kibana, its rule engine or the connector stops, no ordinary
alert can report it, because the thing that would report it is what broke. The rule queries metricbeat-*,
which always has recent documents, so it doubles as a Metricbeat liveness check.

Run it with an Elasticsearch API key: Basic is not an accepted scheme for this Kibana (the anonymous provider
is the only one enabled, so only ApiKey and Bearer are taken).

  pw=$(grep -m1 '^ES_ELASTIC_PASSWORD=' .env | cut -d= -f2-)
  KEY=$(curl -s -XPOST -u "elastic:$pw" localhost:9200/_security/api_key \
          -H 'Content-Type: application/json' -d '{"name":"vanguard-cli"}' \
        | python3 -c 'import json,sys; print(json.load(sys.stdin)["encoded"])')
  KEY="$KEY" python3 observability/kibana-setup.py

Order matters: run this AFTER security is on. A rule created while security was off holds no API key and
stops working the moment it is enabled, until it is disabled and re-enabled.

ADDING A REAL ALARM: author it in Kibana as usual, point its action at the "Vanguard alert index" connector,
and give the document a message plus a level (warn|error). Do NOT set kind=heartbeat — that value is how the
delivery script recognises the heartbeat and keeps it out of Discord. Use "On status change" or a throttle
for real alarms; only the heartbeat wants to fire every interval.
"""
import json, os, time, urllib.error, urllib.request
from pathlib import Path          # env_hosts() reads the stack's .env

import sys

STEPS = ("connector", "rule", "dataview", "landing")
WANTED = sys.argv[1] if len(sys.argv) > 1 else "all"
if WANTED not in STEPS + ("all",):
    raise SystemExit(f"unknown step {WANTED!r}; pick one of: {', '.join(STEPS)}, all")
def wanted(step):
    return WANTED in ("all", step)

KIBANA = "http://127.0.0.1:5601"
ALERT_INDEX = os.environ.get("ALERT_INDEX", "vanguard-alerts")
HEADERS = {"Content-Type": "application/json", "kbn-xsrf": "1",
           "Authorization": "ApiKey " + os.environ["KEY"],
           # Kibana 9 ships with server.restrictInternalApis on. /api/kibana/settings (defaultRoute) is an
           # internal API and refuses without this header: "exists but is not available with the current
           # configuration". Sending it on every call is harmless; loosening the restriction is not.
           "x-elastic-internal-origin": "Kibana"}

def wait_for_kibana(seconds=300):
    """Kibana has to be *available*, not merely listening.

    configure.py --es-security restarts Kibana as its last step, so anything that calls the Kibana API
    immediately afterwards races its boot and gets a connection reset. Kibana also answers /api/status while
    still starting, so the status level is what has to be checked, not the port.
    """
    deadline = time.time() + seconds
    announced = False
    while time.time() < deadline:
        try:
            with urllib.request.urlopen(KIBANA + "/api/status", timeout=10) as response:
                level = json.load(response).get("status", {}).get("overall", {}).get("level")
            if level == "available":
                print("kibana: available")
                return
            state = level or "no level"
        except Exception as error:
            state = type(error).__name__
        if not announced:
            print(f"kibana: waiting for it to become available ({state})…")
            announced = True
        time.sleep(5)
    raise SystemExit(f"kibana did not become available within {seconds}s — nothing was created")

def req(method, path, body=None):
    request = urllib.request.Request(KIBANA + path, method=method,
                                     data=None if body is None else json.dumps(body).encode())
    for key, value in HEADERS.items():
        request.add_header(key, value)
    try:
        with urllib.request.urlopen(request, timeout=30) as response:
            raw = response.read()
            return json.loads(raw) if raw else {}
    except urllib.error.HTTPError as error:
        print(f"  {method} {path} -> HTTP {error.code}")
        print("  " + error.read().decode(errors="replace")[:600])
        raise SystemExit(1)

wait_for_kibana()

# 1. the alert bus connector
CONNECTOR_NAME = "Vanguard alert index"
connector_id = None
if wanted("connector") or wanted("rule"):
  existing = {c["name"]: c["id"] for c in req("GET", "/api/actions/connectors")}
  if CONNECTOR_NAME in existing:
    connector_id = existing[CONNECTOR_NAME]
    print("connector already present:", connector_id)
  elif not wanted("connector"):
    raise SystemExit(f"connector {CONNECTOR_NAME!r} does not exist yet — run the connector step first")
  else:
      connector_id = req("POST", "/api/actions/connector", {
        "name": CONNECTOR_NAME,
        "connector_type_id": ".index",
        # executionTimeField stamps every document, so a rule body never has to get the timestamp right.
        "config": {"index": ALERT_INDEX, "refresh": True, "executionTimeField": "@timestamp"},
        "secrets": {}})["id"]
      print("connector created:", connector_id, "->", ALERT_INDEX)

# 2. the heartbeat rule. Handled independently of the data view below: an early exit here once looked
#    harmless and meant the data view was never created on any system where the rule already existed, which
#    is every re-run.
RULE_NAME = "Vanguard alerting heartbeat"
existing_rule = None if not wanted("rule") else next((r for r in req("GET", "/api/alerting/rules/_find?per_page=100").get("data", [])
                      if r["name"] == RULE_NAME), None)
if not wanted("rule"):
    pass
elif existing_rule:
    print("rule already present:", existing_rule["id"],
          "enabled" if existing_rule.get("enabled") else "DISABLED")
else:
    rule = req("POST", "/api/alerting/rule", {
        "name": RULE_NAME,
        "rule_type_id": ".es-query",
        "consumer": "stackAlerts",
        "enabled": True,
        "schedule": {"interval": "1m"},
        "params": {
            "searchType": "esQuery",
            "index": ["metricbeat-*"],
            "timeField": "@timestamp",
            "esQuery": json.dumps({"query": {"match_all": {}}}),
            "size": 1,
            "aggType": "count",
            "groupBy": "all",
            "thresholdComparator": ">",
            "threshold": [0],
            "timeWindowSize": 5,
            "timeWindowUnit": "m",
            # re-evaluate the whole window every run, so the rule stays active instead of going quiet once
            # it has already seen those documents
            "excludeHitsFromPreviousRun": False,
        },
        # onActiveAlert writes on EVERY check. Kibana's default only acts when a rule CHANGES status, which
        # would produce one heartbeat and then silence — the deadman would read that as a dead pipeline.
        "actions": [{
            "group": "query matched",
            "id": connector_id,
            "params": {"documents": [{
                "kind": "heartbeat",             # the delivery script drops these; never reach Discord
                "rule": "{{rule.name}}",
                "level": "info",
                "message": "kibana rule engine and the index connector are alive",
            }]},
            "frequency": {"summary": False, "notify_when": "onActiveAlert", "throttle": None},
        }],
    })
    print("rule created:", rule["id"], "| enabled:", rule.get("enabled"))

# 3. the data view. Without one, Discover and Lens have nothing to point at and every panel starts with a
#    detour through Stack Management. Both patterns in one view so a single dashboard can mix logs and
#    metrics, which is the whole reason they share a cluster.
VIEW_TITLE = "filebeat-*,metricbeat-*"
views = req("GET", "/api/data_views").get("data_view", []) if wanted("dataview") else []
if not wanted("dataview"):
    pass
elif any(v.get("title") == VIEW_TITLE for v in views):
    print("data view already present:", VIEW_TITLE)
else:
    view = req("POST", "/api/data_views/data_view", {
        "data_view": {"title": VIEW_TITLE, "name": "Vanguard logs + metrics", "timeFieldName": "@timestamp"}})
    print("data view created:", view["data_view"]["id"], "->", VIEW_TITLE)


# 4. the landing page. Kibana stays at the root of the diag host; this just makes arriving there open OUR
#    dashboard instead of Kibana's home. A markdown panel holds the links, and `defaultRoute` is the
#    advanced setting that does the redirect. That is the whole trick, and it avoids moving Kibana under a
#    /kibana prefix — which would have meant touching both Beats' setup.kibana, this script, the alarm
#    script and the deadman's status check, with a silent crash-loop waiting for whichever one was missed.
DASHBOARD_ID = "vanguard-ops"

def env_hosts():
    """Hostnames out of the stack's .env, so the links are real rather than examples."""
    values = {}
    for candidate in (Path(".env"), Path("/home/debian/vanguard-stream/.env")):
        if candidate.is_file():
            for line in candidate.read_text().splitlines():
                if "=" in line and not line.startswith("#"):
                    key, value = line.split("=", 1)
                    values[key.strip()] = value.strip()
            break
    return values

def landing_markdown(hosts):
    def url(key, path="/"):
        host = hosts.get(key)
        return f"https://{host}{path}" if host else None
    rows = [
        ("Stuck recordings", "/stuck", "finished but never published — why, and retry or bin"),
        ("Live portal", url("LIVE_HOST"), "watch every active stream"),
        ("OBS dock", url("LIVE_HOST", "/dock/"), "go live, titles, dump tonight"),
        ("Files", url("FILES_HOST"), "browse, download, upload finals"),
        ("Jellyfin", url("WATCH_HOST"), "recordings by streamer and season"),
        ("Gateway", url("MAIN_HOST"), "role-aware links and how-tos"),
        ("Gitea", url("GIT_HOST"), "code and CI"),
        ("Bot", url("BOT_HOST"), "the Discord bot's own pages"),
        ("Syncthing", url("SYNC_HOST"), "clip intake, device pairing"),
        ("Authentik", url("AUTH_HOST", "/if/admin/"), "users, groups, policies"),
    ]
    lines = ["### Vanguard", "", "| | |", "|---|---|"]
    for name, link, why in rows:
        if link:
            lines.append(f"| **[{name}]({link})** | {why} |")
    lines += [
        "", "### In here", "",
        "| | |", "|---|---|",
        "| **[Discover](/app/discover)** | search every log line; filter on `service.name` or `vanguard.category` |",
        "| **[Dashboards](/app/dashboards)** | the Metricbeat system and Docker dashboards |",
        "| **[Stack Rules](/app/management/insightsAndAlerting/triggersActions/rules)** | the six alarms, and whether each is firing |",
        "| **[Index Management](/app/management/data/index_management/indices)** | index sizes and ILM |",
        "",
        "_Alarms post to Discord through the `vanguard-alerts` index, not from Kibana directly: a basic "
        "licence has no webhook connector. The host deadman watches that whole path from outside._",
    ]
    return "\n".join(lines)

if wanted("landing"):
    hosts = env_hosts()
    panel = {
        "type": "visualization",
        "panelIndex": "1",
        "gridData": {"x": 0, "y": 0, "w": 24, "h": 18, "i": "1"},
        "embeddableConfig": {
            "savedVis": {
                "id": "", "title": "", "description": "", "type": "markdown",
                "params": {"fontSize": 12, "openLinksInNewTab": False,
                           "markdown": landing_markdown(hosts)},
                "uiState": {},
                "data": {"aggs": [], "searchSource": {"query": {"query": "", "language": "kuery"},
                                                      "filter": []}},
            },
            "enhancements": {},
        },
    }
    saved = req("POST", f"/api/saved_objects/dashboard/{DASHBOARD_ID}?overwrite=true", {
        "attributes": {
            "title": "Vanguard ops",
            "description": "Landing page for the diagnostics host. Edit freely: add metric panels beside the links.",
            "panelsJSON": json.dumps([panel]),
            "optionsJSON": json.dumps({"hidePanelTitles": True, "useMargins": True,
                                       "syncColors": False, "syncCursor": True, "syncTooltips": False}),
            "timeRestore": False,
            "kibanaSavedObjectMeta": {
                "searchSourceJSON": json.dumps({"query": {"query": "", "language": "kuery"}, "filter": []})},
        }})
    # Prove it rather than assume it: read the dashboard back and count the panels Kibana actually stored.
    stored = req("GET", f"/api/saved_objects/dashboard/{DASHBOARD_ID}")
    panels = json.loads(stored.get("attributes", {}).get("panelsJSON", "[]"))
    links = sum(1 for line in landing_markdown(hosts).splitlines() if line.startswith("| **["))
    print(f"dashboard: {stored.get('id')} | title {stored.get('attributes', {}).get('title')!r} | "
          f"{len(panels)} panel(s) stored | {links} links in the markdown")
    if not panels:
        raise SystemExit("  the dashboard saved but with no panels — Kibana rejected the panel shape")
    print("open:", f"/app/dashboards#/view/{DASHBOARD_ID}")

    route = f"/app/dashboards#/view/{DASHBOARD_ID}"
    req("POST", "/api/kibana/settings", {"changes": {"defaultRoute": route}})
    now = req("GET", "/api/kibana/settings").get("settings", {}).get("defaultRoute", {}).get("userValue")
    print("defaultRoute:", now or "(not set!)")
    if now != route:
        raise SystemExit("  Kibana did not keep that defaultRoute — set it by hand in Advanced Settings")