Phase 13 build: logs, metrics, alerting and a deadman on a diag. host
Six containers, all loopback-published, reachable only through the gated diag. host: victorialogs 30d retention, -http.pathPrefix=/logs so Caddy passes the prefix through unchanged vector docker_logs + journald -> VictoriaLogs (elasticsearch bulk, the documented path) victoriametrics 90d, scrapes the four taps that were already live plus the new services vmalert-metrics 11 metric rules: disk, load, memory, RAID, restart loops, a core container gone vmalert-logs 5 LogsQL rules: sweep failures, publish-deny bursts, failed bot deploys, cert trouble alertmanager groups and inhibits, posts to Discord, webhook read from a 0600 file not the config The deadman (bin/vanguard-deadman.sh, host timer, 5 min) is the part that does not trust any of the above: curl only, it checks each component answers, that the always-firing Watchdog alert is still arriving through vmalert -> Alertmanager -> dumpd /heartbeat, and the disks. It reports on change and announces recovery once. Verified here: failures listed, silence when nothing changes, disk threshold, recovery fired once. Collection uses the Docker socket (Phase 13 option A, chosen explicitly). That is root-equivalent on this host and is recorded as such in vector.yaml with the two upgrade paths. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
50
observability/alertmanager.yml
Normal file
50
observability/alertmanager.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
# Alertmanager: groups what vmalert fires and posts it to Discord. The webhook URL is NOT in here — it is a
|
||||
# secret, so configure.py writes it to /etc/vanguard/alert-webhook with mode 0600 and this reads it from there.
|
||||
#
|
||||
# The "heartbeat" route is the other half of the deadman. The Watchdog alert in alerts-metrics.yml always fires,
|
||||
# so Alertmanager posts a heartbeat to a file-watching receiver every interval; bin/vanguard-deadman.sh on the
|
||||
# host notices when that stops and tells Discord itself. Nothing inside the stack can report its own death.
|
||||
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: discord
|
||||
group_by: [alertname, service]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
routes:
|
||||
- matchers: [severity="none"]
|
||||
receiver: heartbeat
|
||||
group_wait: 0s
|
||||
group_interval: 1m
|
||||
repeat_interval: 1m
|
||||
- matchers: [severity="critical"]
|
||||
receiver: discord
|
||||
group_wait: 10s
|
||||
repeat_interval: 1h
|
||||
|
||||
inhibit_rules:
|
||||
# While the disk is critical, do not also shout that it is merely filling.
|
||||
- source_matchers: [alertname="MediaDiskCritical"]
|
||||
target_matchers: [alertname="MediaDiskFilling"]
|
||||
# If a core container is gone, its restart-loop alert is noise.
|
||||
- source_matchers: [alertname="ContainerGone"]
|
||||
target_matchers: [alertname="ContainerRestartLoop"]
|
||||
|
||||
receivers:
|
||||
- name: discord
|
||||
discord_configs:
|
||||
- webhook_url_file: /etc/vanguard/alert-webhook
|
||||
title: '{{ if eq .Status "firing" }}🔴{{ else }}✅{{ end }} {{ .CommonLabels.alertname }}'
|
||||
message: |
|
||||
{{ range .Alerts }}{{ .Annotations.summary }}
|
||||
{{ if .Annotations.detail }}{{ .Annotations.detail }}
|
||||
{{ end }}{{ end }}
|
||||
- name: heartbeat
|
||||
# Goes to dumpd on the host (bridge-only port), which touches a file. bin/vanguard-deadman.sh checks that
|
||||
# file is fresh, so a heartbeat never reaches Discord but its ABSENCE does.
|
||||
webhook_configs:
|
||||
- url: http://127.0.0.1:9095/heartbeat
|
||||
send_resolved: false
|
||||
33
observability/alerts-logs.yml
Normal file
33
observability/alerts-logs.yml
Normal file
@@ -0,0 +1,33 @@
|
||||
# Rules vmalert evaluates against VictoriaLogs' /select/logsql/stats_query endpoints (LogsQL, not PromQL).
|
||||
groups:
|
||||
- name: logs
|
||||
interval: 2m
|
||||
type: vlogs
|
||||
rules:
|
||||
- alert: SweepFailing
|
||||
# normalize-sweep writes event=failed and leaves a .normalize-failed.log beside the file.
|
||||
expr: 'component:=sweep level:=error event:=failed | stats count() as hits'
|
||||
for: 1m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "the normalize sweep failed {{ $value }} time(s) in the last 2m"
|
||||
detail: "A recording or clip did not convert. Look for *.normalize-failed.log under /srv/media."
|
||||
- alert: PublishDenied
|
||||
# A few are normal (expired token, wrong prefix). A burst means a streamer is locked out.
|
||||
expr: 'component:=dumpd event:=publish_deny | stats count() as hits'
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $value }} publish denials in 2m — a streamer may be unable to go live"}
|
||||
- alert: BotDeployFailed
|
||||
expr: 'component:=dumpd event:=bot_deploy_exit level:=error | stats count() as hits'
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "a Discord bot release failed to install"}
|
||||
- alert: ServiceErrors
|
||||
expr: 'level:=error | stats by (service) count() as hits'
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $labels.service }} logged {{ $value }} errors in 2m"}
|
||||
- alert: CertificateTrouble
|
||||
expr: 'service:=caddy _msg:"obtain" _msg:"error" | stats count() as hits'
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "Caddy could not obtain or renew a certificate"}
|
||||
84
observability/alerts-metrics.yml
Normal file
84
observability/alerts-metrics.yml
Normal file
@@ -0,0 +1,84 @@
|
||||
# Rules vmalert evaluates against VictoriaMetrics. Metric conditions only; log conditions are in alerts-logs.yml.
|
||||
groups:
|
||||
- name: disk
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: MediaDiskFilling
|
||||
# The most likely quiet death on this box: recordings fill /srv and the sweep starts failing.
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 85
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "/srv is {{ $value | printf \"%.0f\" }}% full"
|
||||
detail: "Recordings and clips live here. Check /srv/media/streams and the 15-day archive window."
|
||||
- alert: MediaDiskCritical
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 94
|
||||
for: 2m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "/srv is {{ $value | printf \"%.0f\" }}% full — recording will fail"
|
||||
- alert: RootDiskFilling
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100) > 85
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "/ is {{ $value | printf \"%.0f\" }}% full (docker images, journal, appdata)"}
|
||||
|
||||
- name: host
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: LoadSustained
|
||||
# The sweep is niced, so a sustained load above core count means something is not yielding.
|
||||
expr: node_load15 > count(count by (cpu) (node_cpu_seconds_total))
|
||||
for: 20m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "load15 {{ $value | printf \"%.1f\" }} above core count for 20m"}
|
||||
- alert: MemoryLow
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "only {{ $value | printf \"%.0f\" }}% memory available"}
|
||||
- alert: RaidDegraded
|
||||
expr: node_md_disks{state!="active"} > 0
|
||||
for: 5m
|
||||
labels: {severity: critical}
|
||||
annotations: {summary: "md RAID has a disk that is not active"}
|
||||
|
||||
- name: containers
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: ContainerRestartLoop
|
||||
expr: increase(container_start_time_seconds{name!=""}[15m]) > 0 and changes(container_start_time_seconds{name!=""}[1h]) > 2
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $labels.name }} restarted more than twice in an hour"}
|
||||
- alert: ContainerGone
|
||||
# cAdvisor stops reporting a container that is not running at all.
|
||||
expr: absent(container_last_seen{name="mediamtx"}) or absent(container_last_seen{name="caddy"}) or absent(container_last_seen{name="discord-bot"})
|
||||
for: 5m
|
||||
labels: {severity: critical}
|
||||
annotations: {summary: "a core container is not running (mediamtx, caddy or discord-bot)"}
|
||||
|
||||
- name: streaming
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: StreamDroppingPackets
|
||||
expr: increase(mediamtx_paths_bytes_received{}[5m]) == 0 and mediamtx_paths{state="ready"} > 0
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "a MediaMTX path is ready but has received nothing for 10m"}
|
||||
|
||||
- name: observability
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: LogIngestionStalled
|
||||
# If this fires, the log half is blind — and if it cannot fire, the deadman covers it.
|
||||
expr: increase(vl_rows_ingested_total[15m]) == 0
|
||||
for: 15m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "VictoriaLogs has ingested nothing for 15m — is Vector running?"}
|
||||
- alert: Watchdog
|
||||
# Always firing on purpose: Alertmanager routes it to the heartbeat receiver, and the deadman script
|
||||
# on the host notices when it stops arriving. This is how the monitoring stack reports its own death.
|
||||
expr: vector(1)
|
||||
labels: {severity: none}
|
||||
annotations: {summary: "alerting pipeline is alive"}
|
||||
22
observability/scrape.yml
Normal file
22
observability/scrape.yml
Normal file
@@ -0,0 +1,22 @@
|
||||
# What VictoriaMetrics scrapes. Every target is loopback-only on this host (LOGGING.md "Metrics taps").
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
external_labels:
|
||||
host: vanguard
|
||||
|
||||
scrape_configs:
|
||||
- job_name: host # CPU, RAM, disk space + I/O, network, load, md RAID
|
||||
static_configs: [{targets: ["host.docker.internal:9100"]}]
|
||||
- job_name: containers # per container CPU/memory/network/block IO, labelled vanguard.*
|
||||
static_configs: [{targets: ["host.docker.internal:9101"]}]
|
||||
- job_name: mediamtx # per path bytes/s, readers, publishers, dropped packets
|
||||
static_configs: [{targets: ["host.docker.internal:9998"]}]
|
||||
- job_name: caddy # requests by host and status, durations, TLS
|
||||
static_configs: [{targets: ["host.docker.internal:2019"]}]
|
||||
- job_name: victorialogs
|
||||
static_configs: [{targets: ["victorialogs:9428"]}]
|
||||
metrics_path: /logs/metrics # it runs behind -http.pathPrefix=/logs
|
||||
- job_name: victoriametrics
|
||||
static_configs: [{targets: ["127.0.0.1:8428"]}]
|
||||
- job_name: alertmanager
|
||||
static_configs: [{targets: ["alertmanager:9093"]}]
|
||||
99
observability/vector.yaml
Normal file
99
observability/vector.yaml
Normal file
@@ -0,0 +1,99 @@
|
||||
# Vector: collects from the two taps in LOGGING.md and ships to VictoriaLogs.
|
||||
#
|
||||
# docker_logs every container's stdout/stderr, with the vanguard.service / vanguard.category labels
|
||||
# we put on each one, so filtering by category needs no parsing
|
||||
# journald our own systemd units (dumpd, the sweep, trash purge, deploy pending, vgdeband)
|
||||
#
|
||||
# TRADE-OFF, chosen deliberately (CommunityDiscordMedia.md Phase 13): docker_logs needs the Docker API, so this
|
||||
# container holds the socket and therefore root-equivalent access to the host — the same exposure act_runner
|
||||
# already has. Accepted "for now". The upgrade paths, in order of preference, are the journald log driver for
|
||||
# every container (one tap, no socket, labels survive as journal fields) or a read-only socket proxy in front
|
||||
# of this container. Neither changes anything downstream of here.
|
||||
#
|
||||
# Every event ends up with: service, category, source_kind, message, level where it could be determined.
|
||||
|
||||
data_dir: /var/lib/vector
|
||||
|
||||
sources:
|
||||
containers:
|
||||
type: docker_logs
|
||||
docker_host: unix:///var/run/docker.sock
|
||||
exclude_containers: [vector] # never ship our own output: it would feed itself
|
||||
units:
|
||||
type: journald
|
||||
current_boot_only: false
|
||||
journal_directory: /var/log/journal
|
||||
include_units: [dumpd, normalize-sweep, trash-purge, deploy-pending, vgdeband]
|
||||
|
||||
transforms:
|
||||
# ---- one shape, whichever tap it came from
|
||||
normalised:
|
||||
type: remap
|
||||
inputs: [containers, units]
|
||||
source: |
|
||||
if exists(.container_name) {
|
||||
.service = .label."vanguard.service" || .container_name
|
||||
.category = .label."vanguard.category" || "infra"
|
||||
.source_kind = "container"
|
||||
del(.label); del(.container_id); del(.image); del(.container_created_at); del(.stream)
|
||||
} else {
|
||||
unit = ._SYSTEMD_UNIT || .unit || "host"
|
||||
.service = replace(string!(unit), ".service", "")
|
||||
.category = if .service == "dumpd" { "stream" } else { "media" }
|
||||
.source_kind = "unit"
|
||||
if !exists(.message) && exists(.MESSAGE) { .message = del(.MESSAGE) }
|
||||
# the journal carries a lot of bookkeeping; keep what a query or a human needs
|
||||
del(.PRIORITY); del(.SYSLOG_IDENTIFIER); del(.SYSLOG_FACILITY); del(._TRANSPORT)
|
||||
del(._PID); del(._UID); del(._GID); del(._COMM); del(._EXE); del(._CMDLINE)
|
||||
del(._CAP_EFFECTIVE); del(._SELINUX_CONTEXT); del(._AUDIT_SESSION); del(._AUDIT_LOGINUID)
|
||||
del(._SYSTEMD_CGROUP); del(._SYSTEMD_SLICE); del(._SYSTEMD_INVOCATION_ID)
|
||||
del(._BOOT_ID); del(._MACHINE_ID); del(._SOURCE_REALTIME_TIMESTAMP); del(._SYSTEMD_UNIT)
|
||||
}
|
||||
|
||||
# ---- parse the shapes LOGGING.md documents; anything else stays a plain message
|
||||
parsed:
|
||||
type: remap
|
||||
inputs: [normalised]
|
||||
source: |
|
||||
text = string(.message) ?? ""
|
||||
if starts_with(text, "{") {
|
||||
# caddy and authentik emit JSON
|
||||
fields, err = parse_json(text)
|
||||
if err == null && is_object(fields) {
|
||||
.json = fields
|
||||
.level = get(object!(fields), ["level"]) ?? get(object!(fields), ["severity"]) ?? null
|
||||
.message = get(object!(fields), ["msg"]) ?? get(object!(fields), ["event"]) ?? text
|
||||
}
|
||||
} else if match(text, r'(^|\s)(ts|level|component|event)=') {
|
||||
# our own scripts, and syncthing's key=value tail
|
||||
fields, err = parse_logfmt(text)
|
||||
if err == null {
|
||||
.kv = fields
|
||||
.level = get(object!(fields), ["level"]) ?? null
|
||||
.event = get(object!(fields), ["event"]) ?? null
|
||||
}
|
||||
} else if match(text, r'^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} ') {
|
||||
# mediamtx: 2026/11/03 05:00:00 INF [component] message
|
||||
fields, err = parse_regex(text, r'^\S+ \S+ (?P<lvl>\w+) (?P<rest>.*)$')
|
||||
if err == null { .level = fields.lvl; .message = fields.rest }
|
||||
} else if match(text, r'^\[\d{2}:\d{2}:\d{2}') {
|
||||
# jellyfin: [00:00:00.000] [LVL] [thread] Logger: message (stack traces arrive as their own events)
|
||||
fields, err = parse_regex(text, r'^\[[\d:.]+\] \[(?P<lvl>\w+)\] \[\d+\] (?P<rest>.*)$')
|
||||
if err == null { .level = fields.lvl; .message = fields.rest }
|
||||
}
|
||||
if exists(.level) { .level = downcase(string!(.level)) }
|
||||
|
||||
sinks:
|
||||
victorialogs:
|
||||
type: elasticsearch
|
||||
inputs: [parsed]
|
||||
endpoints: ["http://victorialogs:9428/insert/elasticsearch/"]
|
||||
mode: bulk
|
||||
api_version: v8
|
||||
compression: gzip
|
||||
healthcheck:
|
||||
enabled: false
|
||||
query:
|
||||
_msg_field: message
|
||||
_time_field: timestamp
|
||||
_stream_fields: service,category,source_kind
|
||||
Reference in New Issue
Block a user