#!/usr/bin/env python3 """Kibana first-run state. Idempotent; safe to re-run. configure.py --apply runs this for you. connector "Vanguard alert index" — the .index connector, writing into the vanguard-alerts index rule "Vanguard alerting heartbeat" — always-firing ES query rule, every minute, writing one doc data view "Vanguard logs + metrics" — filebeat-*,metricbeat-* so Discover and Lens have one source WHY AN INDEX AND NOT A WEBHOOK On a basic license the only connector types Kibana allows are .index and .server-log; every connector that leaves the stack (webhook, Slack, email, PagerDuty…) is Gold or above and returns 403 Action type .webhook is disabled because your basic license does not support it So Kibana cannot talk to Discord. Instead every rule writes a document into ONE index, which is the alert bus, and bin/vanguard-alerts.sh (a host timer) delivers from there to Discord and later to the bot. That is strictly simpler than the webhook route: no Docker-bridge hop, no dumpd endpoint, and alerts are queryable and replayable after the fact because they are just documents. THE HEARTBEAT bin/vanguard-deadman.sh alarms when no heartbeat document has arrived recently. It is the only check that catches the ALERTING pipeline itself dying — if Kibana, its rule engine or the connector stops, no ordinary alert can report it, because the thing that would report it is what broke. The rule queries metricbeat-*, which always has recent documents, so it doubles as a Metricbeat liveness check. Run it with an Elasticsearch API key: Basic is not an accepted scheme for this Kibana (the anonymous provider is the only one enabled, so only ApiKey and Bearer are taken). pw=$(grep -m1 '^ES_ELASTIC_PASSWORD=' .env | cut -d= -f2-) KEY=$(curl -s -XPOST -u "elastic:$pw" localhost:9200/_security/api_key \ -H 'Content-Type: application/json' -d '{"name":"vanguard-cli"}' \ | python3 -c 'import json,sys; print(json.load(sys.stdin)["encoded"])') KEY="$KEY" python3 observability/kibana-setup.py Order matters: run this AFTER security is on. A rule created while security was off holds no API key and stops working the moment it is enabled, until it is disabled and re-enabled. ADDING A REAL ALARM: author it in Kibana as usual, point its action at the "Vanguard alert index" connector, and give the document a message plus a level (warn|error). Do NOT set kind=heartbeat — that value is how the delivery script recognises the heartbeat and keeps it out of Discord. Use "On status change" or a throttle for real alarms; only the heartbeat wants to fire every interval. """ import json, os, time, urllib.error, urllib.request KIBANA = "http://127.0.0.1:5601" ALERT_INDEX = os.environ.get("ALERT_INDEX", "vanguard-alerts") HEADERS = {"Content-Type": "application/json", "kbn-xsrf": "1", "Authorization": "ApiKey " + os.environ["KEY"]} def wait_for_kibana(seconds=300): """Kibana has to be *available*, not merely listening. configure.py --es-security restarts Kibana as its last step, so anything that calls the Kibana API immediately afterwards races its boot and gets a connection reset. Kibana also answers /api/status while still starting, so the status level is what has to be checked, not the port. """ deadline = time.time() + seconds announced = False while time.time() < deadline: try: with urllib.request.urlopen(KIBANA + "/api/status", timeout=10) as response: level = json.load(response).get("status", {}).get("overall", {}).get("level") if level == "available": print("kibana: available") return state = level or "no level" except Exception as error: state = type(error).__name__ if not announced: print(f"kibana: waiting for it to become available ({state})…") announced = True time.sleep(5) raise SystemExit(f"kibana did not become available within {seconds}s — nothing was created") def req(method, path, body=None): request = urllib.request.Request(KIBANA + path, method=method, data=None if body is None else json.dumps(body).encode()) for key, value in HEADERS.items(): request.add_header(key, value) try: with urllib.request.urlopen(request, timeout=30) as response: raw = response.read() return json.loads(raw) if raw else {} except urllib.error.HTTPError as error: print(f" {method} {path} -> HTTP {error.code}") print(" " + error.read().decode(errors="replace")[:600]) raise SystemExit(1) wait_for_kibana() # 1. the alert bus connector CONNECTOR_NAME = "Vanguard alert index" existing = {c["name"]: c["id"] for c in req("GET", "/api/actions/connectors")} if CONNECTOR_NAME in existing: connector_id = existing[CONNECTOR_NAME] print("connector already present:", connector_id) else: connector_id = req("POST", "/api/actions/connector", { "name": CONNECTOR_NAME, "connector_type_id": ".index", # executionTimeField stamps every document, so a rule body never has to get the timestamp right. "config": {"index": ALERT_INDEX, "refresh": True, "executionTimeField": "@timestamp"}, "secrets": {}})["id"] print("connector created:", connector_id, "->", ALERT_INDEX) # 2. the heartbeat rule. Handled independently of the data view below: an early exit here once looked # harmless and meant the data view was never created on any system where the rule already existed, which # is every re-run. RULE_NAME = "Vanguard alerting heartbeat" existing_rule = next((r for r in req("GET", "/api/alerting/rules/_find?per_page=100").get("data", []) if r["name"] == RULE_NAME), None) if existing_rule: print("rule already present:", existing_rule["id"], "enabled" if existing_rule.get("enabled") else "DISABLED") else: rule = req("POST", "/api/alerting/rule", { "name": RULE_NAME, "rule_type_id": ".es-query", "consumer": "stackAlerts", "enabled": True, "schedule": {"interval": "1m"}, "params": { "searchType": "esQuery", "index": ["metricbeat-*"], "timeField": "@timestamp", "esQuery": json.dumps({"query": {"match_all": {}}}), "size": 1, "aggType": "count", "groupBy": "all", "thresholdComparator": ">", "threshold": [0], "timeWindowSize": 5, "timeWindowUnit": "m", # re-evaluate the whole window every run, so the rule stays active instead of going quiet once # it has already seen those documents "excludeHitsFromPreviousRun": False, }, # onActiveAlert writes on EVERY check. Kibana's default only acts when a rule CHANGES status, which # would produce one heartbeat and then silence — the deadman would read that as a dead pipeline. "actions": [{ "group": "query matched", "id": connector_id, "params": {"documents": [{ "kind": "heartbeat", # the delivery script drops these; never reach Discord "rule": "{{rule.name}}", "level": "info", "message": "kibana rule engine and the index connector are alive", }]}, "frequency": {"summary": False, "notify_when": "onActiveAlert", "throttle": None}, }], }) print("rule created:", rule["id"], "| enabled:", rule.get("enabled")) # 3. the data view. Without one, Discover and Lens have nothing to point at and every panel starts with a # detour through Stack Management. Both patterns in one view so a single dashboard can mix logs and # metrics, which is the whole reason they share a cluster. VIEW_TITLE = "filebeat-*,metricbeat-*" views = req("GET", "/api/data_views").get("data_view", []) if any(v.get("title") == VIEW_TITLE for v in views): print("data view already present:", VIEW_TITLE) else: view = req("POST", "/api/data_views/data_view", { "data_view": {"title": VIEW_TITLE, "name": "Vanguard logs + metrics", "timeFieldName": "@timestamp"}}) print("data view created:", view["data_view"]["id"], "->", VIEW_TITLE)