Brandyn / Techy
8ae43905dc
Six containers, all loopback-published, reachable only through the gated diag. host: victorialogs 30d retention, -http.pathPrefix=/logs so Caddy passes the prefix through unchanged vector docker_logs + journald -> VictoriaLogs (elasticsearch bulk, the documented path) victoriametrics 90d, scrapes the four taps that were already live plus the new services vmalert-metrics 11 metric rules: disk, load, memory, RAID, restart loops, a core container gone vmalert-logs 5 LogsQL rules: sweep failures, publish-deny bursts, failed bot deploys, cert trouble alertmanager groups and inhibits, posts to Discord, webhook read from a 0600 file not the config The deadman (bin/vanguard-deadman.sh, host timer, 5 min) is the part that does not trust any of the above: curl only, it checks each component answers, that the always-firing Watchdog alert is still arriving through vmalert -> Alertmanager -> dumpd /heartbeat, and the disks. It reports on change and announces recovery once. Verified here: failures listed, silence when nothing changes, disk threshold, recovery fired once. Collection uses the Docker socket (Phase 13 option A, chosen explicitly). That is root-equivalent on this host and is recorded as such in vector.yaml with the two upgrade paths. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
85 lines
3.9 KiB
YAML
85 lines
3.9 KiB
YAML
# Rules vmalert evaluates against VictoriaMetrics. Metric conditions only; log conditions are in alerts-logs.yml.
|
|
groups:
|
|
- name: disk
|
|
interval: 1m
|
|
rules:
|
|
- alert: MediaDiskFilling
|
|
# The most likely quiet death on this box: recordings fill /srv and the sweep starts failing.
|
|
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 85
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations:
|
|
summary: "/srv is {{ $value | printf \"%.0f\" }}% full"
|
|
detail: "Recordings and clips live here. Check /srv/media/streams and the 15-day archive window."
|
|
- alert: MediaDiskCritical
|
|
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 94
|
|
for: 2m
|
|
labels: {severity: critical}
|
|
annotations:
|
|
summary: "/srv is {{ $value | printf \"%.0f\" }}% full — recording will fail"
|
|
- alert: RootDiskFilling
|
|
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100) > 85
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "/ is {{ $value | printf \"%.0f\" }}% full (docker images, journal, appdata)"}
|
|
|
|
- name: host
|
|
interval: 1m
|
|
rules:
|
|
- alert: LoadSustained
|
|
# The sweep is niced, so a sustained load above core count means something is not yielding.
|
|
expr: node_load15 > count(count by (cpu) (node_cpu_seconds_total))
|
|
for: 20m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "load15 {{ $value | printf \"%.1f\" }} above core count for 20m"}
|
|
- alert: MemoryLow
|
|
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "only {{ $value | printf \"%.0f\" }}% memory available"}
|
|
- alert: RaidDegraded
|
|
expr: node_md_disks{state!="active"} > 0
|
|
for: 5m
|
|
labels: {severity: critical}
|
|
annotations: {summary: "md RAID has a disk that is not active"}
|
|
|
|
- name: containers
|
|
interval: 1m
|
|
rules:
|
|
- alert: ContainerRestartLoop
|
|
expr: increase(container_start_time_seconds{name!=""}[15m]) > 0 and changes(container_start_time_seconds{name!=""}[1h]) > 2
|
|
for: 5m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "{{ $labels.name }} restarted more than twice in an hour"}
|
|
- alert: ContainerGone
|
|
# cAdvisor stops reporting a container that is not running at all.
|
|
expr: absent(container_last_seen{name="mediamtx"}) or absent(container_last_seen{name="caddy"}) or absent(container_last_seen{name="discord-bot"})
|
|
for: 5m
|
|
labels: {severity: critical}
|
|
annotations: {summary: "a core container is not running (mediamtx, caddy or discord-bot)"}
|
|
|
|
- name: streaming
|
|
interval: 1m
|
|
rules:
|
|
- alert: StreamDroppingPackets
|
|
expr: increase(mediamtx_paths_bytes_received{}[5m]) == 0 and mediamtx_paths{state="ready"} > 0
|
|
for: 10m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "a MediaMTX path is ready but has received nothing for 10m"}
|
|
|
|
- name: observability
|
|
interval: 1m
|
|
rules:
|
|
- alert: LogIngestionStalled
|
|
# If this fires, the log half is blind — and if it cannot fire, the deadman covers it.
|
|
expr: increase(vl_rows_ingested_total[15m]) == 0
|
|
for: 15m
|
|
labels: {severity: warning}
|
|
annotations: {summary: "VictoriaLogs has ingested nothing for 15m — is Vector running?"}
|
|
- alert: Watchdog
|
|
# Always firing on purpose: Alertmanager routes it to the heartbeat receiver, and the deadman script
|
|
# on the host notices when it stops arriving. This is how the monitoring stack reports its own death.
|
|
expr: vector(1)
|
|
labels: {severity: none}
|
|
annotations: {summary: "alerting pipeline is alive"}
|