Media-Server-Setup/observability/kibana-setup.py

#!/usr/bin/env python3
"""Kibana first-run state. Idempotent; safe to re-run. configure.py --apply runs this for you.

  connector  "Vanguard alert index"        — the .index connector, writing into the vanguard-alerts index
  rule       "Vanguard alerting heartbeat" — always-firing ES query rule, every minute, writing one doc
  data view  "Vanguard logs + metrics"     — filebeat-*,metricbeat-* so Discover and Lens have one source

WHY AN INDEX AND NOT A WEBHOOK
On a basic license the only connector types Kibana allows are .index and .server-log; every connector that
leaves the stack (webhook, Slack, email, PagerDuty…) is Gold or above and returns
    403 Action type .webhook is disabled because your basic license does not support it
So Kibana cannot talk to Discord. Instead every rule writes a document into ONE index, which is the alert
bus, and bin/vanguard-alerts.sh (a host timer) delivers from there to Discord and later to the bot. That is
strictly simpler than the webhook route: no Docker-bridge hop, no dumpd endpoint, and alerts are queryable
and replayable after the fact because they are just documents.

THE HEARTBEAT
bin/vanguard-deadman.sh alarms when no heartbeat document has arrived recently. It is the only check that
catches the ALERTING pipeline itself dying — if Kibana, its rule engine or the connector stops, no ordinary
alert can report it, because the thing that would report it is what broke. The rule queries metricbeat-*,
which always has recent documents, so it doubles as a Metricbeat liveness check.

Run it with an Elasticsearch API key: Basic is not an accepted scheme for this Kibana (the anonymous provider
is the only one enabled, so only ApiKey and Bearer are taken).

  pw=$(grep -m1 '^ES_ELASTIC_PASSWORD=' .env | cut -d= -f2-)
  KEY=$(curl -s -XPOST -u "elastic:$pw" localhost:9200/_security/api_key \
          -H 'Content-Type: application/json' -d '{"name":"vanguard-cli"}' \
        | python3 -c 'import json,sys; print(json.load(sys.stdin)["encoded"])')
  KEY="$KEY" python3 observability/kibana-setup.py

Order matters: run this AFTER security is on. A rule created while security was off holds no API key and
stops working the moment it is enabled, until it is disabled and re-enabled.

ADDING A REAL ALARM: author it in Kibana as usual, point its action at the "Vanguard alert index" connector,
and give the document a message plus a level (warn|error). Do NOT set kind=heartbeat — that value is how the
delivery script recognises the heartbeat and keeps it out of Discord. Use "On status change" or a throttle
for real alarms; only the heartbeat wants to fire every interval.
"""
import json, os, time, urllib.error, urllib.request

KIBANA = "http://127.0.0.1:5601"
ALERT_INDEX = os.environ.get("ALERT_INDEX", "vanguard-alerts")
HEADERS = {"Content-Type": "application/json", "kbn-xsrf": "1",
           "Authorization": "ApiKey " + os.environ["KEY"]}

def wait_for_kibana(seconds=300):
    """Kibana has to be *available*, not merely listening.

    configure.py --es-security restarts Kibana as its last step, so anything that calls the Kibana API
    immediately afterwards races its boot and gets a connection reset. Kibana also answers /api/status while
    still starting, so the status level is what has to be checked, not the port.
    """
    deadline = time.time() + seconds
    announced = False
    while time.time() < deadline:
        try:
            with urllib.request.urlopen(KIBANA + "/api/status", timeout=10) as response:
                level = json.load(response).get("status", {}).get("overall", {}).get("level")
            if level == "available":
                print("kibana: available")
                return
            state = level or "no level"
        except Exception as error:
            state = type(error).__name__
        if not announced:
            print(f"kibana: waiting for it to become available ({state})…")
            announced = True
        time.sleep(5)
    raise SystemExit(f"kibana did not become available within {seconds}s — nothing was created")

def req(method, path, body=None):
    request = urllib.request.Request(KIBANA + path, method=method,
                                     data=None if body is None else json.dumps(body).encode())
    for key, value in HEADERS.items():
        request.add_header(key, value)
    try:
        with urllib.request.urlopen(request, timeout=30) as response:
            raw = response.read()
            return json.loads(raw) if raw else {}
    except urllib.error.HTTPError as error:
        print(f"  {method} {path} -> HTTP {error.code}")
        print("  " + error.read().decode(errors="replace")[:600])
        raise SystemExit(1)

wait_for_kibana()

# 1. the alert bus connector
CONNECTOR_NAME = "Vanguard alert index"
existing = {c["name"]: c["id"] for c in req("GET", "/api/actions/connectors")}
if CONNECTOR_NAME in existing:
    connector_id = existing[CONNECTOR_NAME]
    print("connector already present:", connector_id)
else:
    connector_id = req("POST", "/api/actions/connector", {
        "name": CONNECTOR_NAME,
        "connector_type_id": ".index",
        # executionTimeField stamps every document, so a rule body never has to get the timestamp right.
        "config": {"index": ALERT_INDEX, "refresh": True, "executionTimeField": "@timestamp"},
        "secrets": {}})["id"]
    print("connector created:", connector_id, "->", ALERT_INDEX)

# 2. the heartbeat rule. Handled independently of the data view below: an early exit here once looked
#    harmless and meant the data view was never created on any system where the rule already existed, which
#    is every re-run.
RULE_NAME = "Vanguard alerting heartbeat"
existing_rule = next((r for r in req("GET", "/api/alerting/rules/_find?per_page=100").get("data", [])
                      if r["name"] == RULE_NAME), None)
if existing_rule:
    print("rule already present:", existing_rule["id"],
          "enabled" if existing_rule.get("enabled") else "DISABLED")
else:
    rule = req("POST", "/api/alerting/rule", {
        "name": RULE_NAME,
        "rule_type_id": ".es-query",
        "consumer": "stackAlerts",
        "enabled": True,
        "schedule": {"interval": "1m"},
        "params": {
            "searchType": "esQuery",
            "index": ["metricbeat-*"],
            "timeField": "@timestamp",
            "esQuery": json.dumps({"query": {"match_all": {}}}),
            "size": 1,
            "aggType": "count",
            "groupBy": "all",
            "thresholdComparator": ">",
            "threshold": [0],
            "timeWindowSize": 5,
            "timeWindowUnit": "m",
            # re-evaluate the whole window every run, so the rule stays active instead of going quiet once
            # it has already seen those documents
            "excludeHitsFromPreviousRun": False,
        },
        # onActiveAlert writes on EVERY check. Kibana's default only acts when a rule CHANGES status, which
        # would produce one heartbeat and then silence — the deadman would read that as a dead pipeline.
        "actions": [{
            "group": "query matched",
            "id": connector_id,
            "params": {"documents": [{
                "kind": "heartbeat",             # the delivery script drops these; never reach Discord
                "rule": "{{rule.name}}",
                "level": "info",
                "message": "kibana rule engine and the index connector are alive",
            }]},
            "frequency": {"summary": False, "notify_when": "onActiveAlert", "throttle": None},
        }],
    })
    print("rule created:", rule["id"], "| enabled:", rule.get("enabled"))

# 3. the data view. Without one, Discover and Lens have nothing to point at and every panel starts with a
#    detour through Stack Management. Both patterns in one view so a single dashboard can mix logs and
#    metrics, which is the whole reason they share a cluster.
VIEW_TITLE = "filebeat-*,metricbeat-*"
views = req("GET", "/api/data_views").get("data_view", [])
if any(v.get("title") == VIEW_TITLE for v in views):
    print("data view already present:", VIEW_TITLE)
else:
    view = req("POST", "/api/data_views/data_view", {
        "data_view": {"title": VIEW_TITLE, "name": "Vanguard logs + metrics", "timeFieldName": "@timestamp"}})
    print("data view created:", view["data_view"]["id"], "->", VIEW_TITLE)