Brandyn / Techy
b38d40ab0c
Observability was running but could not report anything. This closes that end to end and makes every piece of it reproducible from the installers instead of from a shell history. Logs - Filebeat 9 REMOVED the container input; it refuses the config per container and then runs with zero harvesters and no index. Container logs now use filestream + the container parser with a stable id each, and add_docker_metadata is told where the id is. - Every line gets event.dataset, or Kibana's Logs rate chart is one "unknown" bucket. - Beats pin replicas to 0: a single node cannot place one, so the cluster sat yellow forever. Security - xpack.security.enabled=true, because Kibana's Observability apps ask Elasticsearch whether the user holds a privilege and that API does not exist when security is off (the Infrastructure page failed on every view). Nobody sees a login: Kibana's anonymous provider signs every visitor in as one native user, and Caddy + Authentik + DIAG_REQUIRE_GROUP stays the only gate. configure.py generates the service passwords, creates the accounts and writes a root-only netrc for the deadman; --es-security retries just that step. Alerting - A basic licence has no webhook connector (403, Gold+), so Kibana's rules write documents into a vanguard-alerts index through the free .index connector and vanguard-alerts.timer delivers them to Discord. Alerts are therefore searchable beside the logs that caused them. - Six rules in observability/kibana-alarms.py: disk and memory below 15% free, bot not running, bot unhealthy, recordings stuck over an hour, normalise failures parked. Each repeats every 12 h while unresolved. - The deadman watches both halves of that pipeline, since nothing inside the stack can report that the stack's own alerting died. Our own metrics - bin/vgmetrics.sh counters from the sweep and vgdeband, plus live gauges on dumpd /metrics (loopback only, unrouted by Caddy), scraped by Metricbeat. vanguard_media_pending is the measured answer to whether normalising keeps up, which was previously only a benchmark guess. - The sweep now captures ffmpeg's stderr: rc= alone was not diagnosable. - Its file loop reads a process substitution, not a pipe, or the subshell discards every counter. Housekeeping - configure.py --apply installs the media scripts (it never did, so applies left stale binaries), creates the alert index, injects an ILM delete phase and runs Kibana's first-run setup. - The discord-bot container asks the bot about itself via /health rather than probing a port. - Removed the dead diag landing page: Kibana occupies the root of that host. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
70 lines
3.0 KiB
YAML
70 lines
3.0 KiB
YAML
# Metricbeat: host and per-container metrics, replacing node_exporter and cAdvisor, plus the Prometheus
|
|
# endpoints the stack already exposes. Runs on the host network so the system module sees the real interfaces
|
|
# and so it can reach MediaMTX, Caddy, Elasticsearch and Kibana on loopback.
|
|
|
|
metricbeat.modules:
|
|
- module: system
|
|
period: 30s
|
|
metricsets: [cpu, load, memory, network, process, process_summary, socket_summary, filesystem, fsstat, diskio, uptime]
|
|
processes: ['.*']
|
|
process.include_top_n:
|
|
by_cpu: 10
|
|
by_memory: 10
|
|
system.hostfs: /hostfs # the host's real filesystems, not the container's
|
|
|
|
- module: docker
|
|
period: 30s
|
|
hosts: ["unix:///var/run/docker.sock"]
|
|
metricsets: [container, cpu, diskio, healthcheck, info, memory, network]
|
|
|
|
- module: prometheus # MediaMTX: per path bytes/s, readers, publishers, dropped packets
|
|
period: 30s
|
|
metrics_path: /metrics
|
|
hosts: ["http://127.0.0.1:9998"]
|
|
|
|
- module: prometheus # Caddy: requests by host and status, durations, TLS
|
|
period: 60s
|
|
metrics_path: /metrics
|
|
hosts: ["http://127.0.0.1:2019"]
|
|
|
|
- module: prometheus # dumpd: OUR metrics — normalise queue, encode time, live streams
|
|
period: 30s
|
|
metrics_path: /metrics
|
|
hosts: ["http://127.0.0.1:8080"] # dumpd's loopback listener; Caddy routes no host to /metrics
|
|
# These are real counters, so let them arrive typed: Kibana then gets .rate (per-second) as well as the
|
|
# raw total, which is what makes "encode seconds per hour" a chart rather than a staircase.
|
|
use_types: true
|
|
rate_counters: true
|
|
|
|
processors:
|
|
- add_host_metadata: ~
|
|
# The same correlation fields Filebeat sets, so ONE dashboard filter spans logs and metrics. Without these a
|
|
# metric event carries container.name while a log event carries service.name, and nothing joins them.
|
|
- add_docker_metadata: ~
|
|
- copy_fields:
|
|
fields:
|
|
- from: container.labels.vanguard_service
|
|
to: service.name
|
|
- from: container.labels.vanguard_category
|
|
to: vanguard.category
|
|
fail_on_error: false
|
|
ignore_missing: true
|
|
|
|
output.elasticsearch:
|
|
hosts: ["http://127.0.0.1:9200"]
|
|
# Elasticsearch requires authentication now (see observability/kibana.yml for why security is on). These
|
|
# come from the container environment, which compose fills from .env — no secret is committed here.
|
|
username: "${ES_BEATS_USER}"
|
|
password: "${ES_BEATS_PASSWORD}"
|
|
|
|
setup.kibana: # no credentials: Kibana's anonymous provider signs these in
|
|
host: "http://127.0.0.1:5601"
|
|
setup.dashboards.enabled: true # the system and docker dashboards, so there is something to look at
|
|
# Single node: a replica has nowhere to go, so every beats index would sit unassigned and hold the
|
|
# cluster at yellow forever. Zero replicas is the correct setting here, not a workaround.
|
|
setup.template.overwrite: true
|
|
setup.template.settings:
|
|
index.number_of_replicas: 0
|
|
|
|
logging.level: info
|