Rip out the Victoria stack for Elastic: four containers, Kibana as the single pane
Eight observability containers become four. Elasticsearch stores logs and metrics, Kibana is the one UI for Discover, dashboards and alerting, Filebeat ships container logs plus our journal units, and Metricbeat covers host and Docker metrics — which retires node_exporter and cAdvisor too — while still scraping MediaMTX's and Caddy's Prometheus endpoints. Pinned rather than left to chance: a fixed 2 GB heap (a self-sizing JVM would claim a quarter of a 62 GB box whose spare capacity belongs to x264), single-node discovery, security off because it only listens on loopback behind the Authentik gate, ML off, and vm.max_map_count set and persisted since ES will not start without it. Kibana gets a fixed encryption key or it loses its saved objects on every restart. diag.HOST is now just Kibana at the root: no path prefixes to get wrong and no index page of links. The deadman is the only thing kept from the old stack, repointed at Elastic. It checks that ES answers and is not red, that no index has been flipped read-only by the disk watermark (which does not clear itself when space is freed, so ingestion stays silently dead), that Kibana is available, that each Beat has actually written a document recently rather than merely running, that the Kibana heartbeat still arrives, and the disks. Verified against a mock Elasticsearch for all five failure modes plus the healthy case staying silent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,54 +0,0 @@
|
||||
# REFERENCE COPY ONLY — configure.py generates observability/alertmanager.yml from ALERT_WEBHOOK_URL and that
|
||||
# generated file is what runs (it omits the Discord receiver's notifier entirely when no webhook is set, so
|
||||
# the config stays valid and alerts simply are not delivered). Never scp this over the generated one.
|
||||
#
|
||||
# Alertmanager: groups what vmalert fires and posts it to Discord. The webhook URL is NOT in here — it is a
|
||||
# secret, so configure.py writes it to /etc/vanguard/alert-webhook with mode 0600 and this reads it from there.
|
||||
#
|
||||
# The "heartbeat" route is the other half of the deadman. The Watchdog alert in alerts-metrics.yml always fires,
|
||||
# so Alertmanager posts a heartbeat to a file-watching receiver every interval; bin/vanguard-deadman.sh on the
|
||||
# host notices when that stops and tells Discord itself. Nothing inside the stack can report its own death.
|
||||
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: discord
|
||||
group_by: [alertname, service]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
routes:
|
||||
- matchers: [severity="none"]
|
||||
receiver: heartbeat
|
||||
group_wait: 0s
|
||||
group_interval: 1m
|
||||
repeat_interval: 1m
|
||||
- matchers: [severity="critical"]
|
||||
receiver: discord
|
||||
group_wait: 10s
|
||||
repeat_interval: 1h
|
||||
|
||||
inhibit_rules:
|
||||
# While the disk is critical, do not also shout that it is merely filling.
|
||||
- source_matchers: [alertname="MediaDiskCritical"]
|
||||
target_matchers: [alertname="MediaDiskFilling"]
|
||||
# If a core container is gone, its restart-loop alert is noise.
|
||||
- source_matchers: [alertname="ContainerGone"]
|
||||
target_matchers: [alertname="ContainerRestartLoop"]
|
||||
|
||||
receivers:
|
||||
- name: discord
|
||||
discord_configs:
|
||||
- webhook_url_file: /etc/vanguard/alert-webhook
|
||||
title: '{{ if eq .Status "firing" }}🔴{{ else }}✅{{ end }} {{ .CommonLabels.alertname }}'
|
||||
message: |
|
||||
{{ range .Alerts }}{{ .Annotations.summary }}
|
||||
{{ if .Annotations.detail }}{{ .Annotations.detail }}
|
||||
{{ end }}{{ end }}
|
||||
- name: heartbeat
|
||||
# Goes to dumpd on the host (bridge-only port), which touches a file. bin/vanguard-deadman.sh checks that
|
||||
# file is fresh, so a heartbeat never reaches Discord but its ABSENCE does.
|
||||
webhook_configs:
|
||||
- url: http://127.0.0.1:9095/heartbeat
|
||||
send_resolved: false
|
||||
@@ -1,33 +0,0 @@
|
||||
# Rules vmalert evaluates against VictoriaLogs' /select/logsql/stats_query endpoints (LogsQL, not PromQL).
|
||||
groups:
|
||||
- name: logs
|
||||
interval: 2m
|
||||
type: vlogs
|
||||
rules:
|
||||
- alert: SweepFailing
|
||||
# normalize-sweep writes event=failed and leaves a .normalize-failed.log beside the file.
|
||||
expr: 'component:=sweep level:=error event:=failed | stats count() as hits'
|
||||
for: 1m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "the normalize sweep failed {{ $value }} time(s) in the last 2m"
|
||||
detail: "A recording or clip did not convert. Look for *.normalize-failed.log under /srv/media."
|
||||
- alert: PublishDenied
|
||||
# A few are normal (expired token, wrong prefix). A burst means a streamer is locked out.
|
||||
expr: 'component:=dumpd event:=publish_deny | stats count() as hits'
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $value }} publish denials in 2m — a streamer may be unable to go live"}
|
||||
- alert: BotDeployFailed
|
||||
expr: 'component:=dumpd event:=bot_deploy_exit level:=error | stats count() as hits'
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "a Discord bot release failed to install"}
|
||||
- alert: ServiceErrors
|
||||
expr: 'level:=error | stats by (service) count() as hits'
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $labels.service }} logged {{ $value }} errors in 2m"}
|
||||
- alert: CertificateTrouble
|
||||
expr: 'service:=caddy _msg:"obtain" _msg:"error" | stats count() as hits'
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "Caddy could not obtain or renew a certificate"}
|
||||
@@ -1,84 +0,0 @@
|
||||
# Rules vmalert evaluates against VictoriaMetrics. Metric conditions only; log conditions are in alerts-logs.yml.
|
||||
groups:
|
||||
- name: disk
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: MediaDiskFilling
|
||||
# The most likely quiet death on this box: recordings fill /srv and the sweep starts failing.
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 85
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations:
|
||||
summary: "/srv is {{ $value | printf \"%.0f\" }}% full"
|
||||
detail: "Recordings and clips live here. Check /srv/media/streams and the 15-day archive window."
|
||||
- alert: MediaDiskCritical
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 94
|
||||
for: 2m
|
||||
labels: {severity: critical}
|
||||
annotations:
|
||||
summary: "/srv is {{ $value | printf \"%.0f\" }}% full — recording will fail"
|
||||
- alert: RootDiskFilling
|
||||
expr: 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100) > 85
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "/ is {{ $value | printf \"%.0f\" }}% full (docker images, journal, appdata)"}
|
||||
|
||||
- name: host
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: LoadSustained
|
||||
# The sweep is niced, so a sustained load above core count means something is not yielding.
|
||||
expr: node_load15 > count(count by (cpu) (node_cpu_seconds_total))
|
||||
for: 20m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "load15 {{ $value | printf \"%.1f\" }} above core count for 20m"}
|
||||
- alert: MemoryLow
|
||||
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "only {{ $value | printf \"%.0f\" }}% memory available"}
|
||||
- alert: RaidDegraded
|
||||
expr: node_md_disks{state!="active"} > 0
|
||||
for: 5m
|
||||
labels: {severity: critical}
|
||||
annotations: {summary: "md RAID has a disk that is not active"}
|
||||
|
||||
- name: containers
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: ContainerRestartLoop
|
||||
expr: increase(container_start_time_seconds{name!=""}[15m]) > 0 and changes(container_start_time_seconds{name!=""}[1h]) > 2
|
||||
for: 5m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "{{ $labels.name }} restarted more than twice in an hour"}
|
||||
- alert: ContainerGone
|
||||
# cAdvisor stops reporting a container that is not running at all.
|
||||
expr: absent(container_last_seen{name="mediamtx"}) or absent(container_last_seen{name="caddy"}) or absent(container_last_seen{name="discord-bot"})
|
||||
for: 5m
|
||||
labels: {severity: critical}
|
||||
annotations: {summary: "a core container is not running (mediamtx, caddy or discord-bot)"}
|
||||
|
||||
- name: streaming
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: StreamDroppingPackets
|
||||
expr: increase(mediamtx_paths_bytes_received{}[5m]) == 0 and mediamtx_paths{state="ready"} > 0
|
||||
for: 10m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "a MediaMTX path is ready but has received nothing for 10m"}
|
||||
|
||||
- name: observability
|
||||
interval: 1m
|
||||
rules:
|
||||
- alert: LogIngestionStalled
|
||||
# If this fires, the log half is blind — and if it cannot fire, the deadman covers it.
|
||||
expr: increase(vl_rows_ingested_total[15m]) == 0
|
||||
for: 15m
|
||||
labels: {severity: warning}
|
||||
annotations: {summary: "VictoriaLogs has ingested nothing for 15m — is Vector running?"}
|
||||
- alert: Watchdog
|
||||
# Always firing on purpose: Alertmanager routes it to the heartbeat receiver, and the deadman script
|
||||
# on the host notices when it stops arriving. This is how the monitoring stack reports its own death.
|
||||
expr: vector(1)
|
||||
labels: {severity: none}
|
||||
annotations: {summary: "alerting pipeline is alive"}
|
||||
50
observability/filebeat.yml
Normal file
50
observability/filebeat.yml
Normal file
@@ -0,0 +1,50 @@
|
||||
# Filebeat: container logs + our own systemd units -> Elasticsearch.
|
||||
# Autodiscover picks up every container, and add_docker_metadata brings the vanguard.service /
|
||||
# vanguard.category labels with it (Beats turn the dots into underscores), so filtering by category in Kibana
|
||||
# needs no parsing. See LOGGING.md for what each service's lines look like.
|
||||
|
||||
filebeat.autodiscover:
|
||||
providers:
|
||||
- type: docker
|
||||
hints.enabled: false
|
||||
templates:
|
||||
- condition: ~ # every container
|
||||
config:
|
||||
- type: container
|
||||
paths: ["/var/lib/docker/containers/${data.docker.container.id}/*.log"]
|
||||
|
||||
filebeat.inputs:
|
||||
- type: journald
|
||||
id: vanguard-units
|
||||
include_matches:
|
||||
match:
|
||||
- "_SYSTEMD_UNIT=dumpd.service"
|
||||
- "_SYSTEMD_UNIT=normalize-sweep.service"
|
||||
- "_SYSTEMD_UNIT=trash-purge.service"
|
||||
- "_SYSTEMD_UNIT=vgdeband.service"
|
||||
- "_SYSTEMD_UNIT=vanguard-deadman.service"
|
||||
|
||||
processors:
|
||||
- add_docker_metadata: ~
|
||||
- add_host_metadata: ~
|
||||
# promote our labels to real fields so Kibana filters on service.name and vanguard.category
|
||||
- copy_fields:
|
||||
fields:
|
||||
- from: container.labels.vanguard_service
|
||||
to: service.name
|
||||
- from: container.labels.vanguard_category
|
||||
to: vanguard.category
|
||||
fail_on_error: false
|
||||
ignore_missing: true
|
||||
- drop_fields:
|
||||
fields: ["agent.ephemeral_id", "ecs.version", "input.type", "log.offset"]
|
||||
ignore_missing: true
|
||||
|
||||
output.elasticsearch:
|
||||
hosts: ["http://elasticsearch:9200"]
|
||||
|
||||
setup.kibana:
|
||||
host: "http://kibana:5601"
|
||||
setup.ilm.enabled: true # retention: set the delete phase once in Kibana (README 5h)
|
||||
|
||||
logging.level: info
|
||||
40
observability/metricbeat.yml
Normal file
40
observability/metricbeat.yml
Normal file
@@ -0,0 +1,40 @@
|
||||
# Metricbeat: host and per-container metrics, replacing node_exporter and cAdvisor, plus the Prometheus
|
||||
# endpoints the stack already exposes. Runs on the host network so the system module sees the real interfaces
|
||||
# and so it can reach MediaMTX, Caddy, Elasticsearch and Kibana on loopback.
|
||||
|
||||
metricbeat.modules:
|
||||
- module: system
|
||||
period: 30s
|
||||
metricsets: [cpu, load, memory, network, process, process_summary, socket_summary, filesystem, fsstat, diskio, uptime]
|
||||
processes: ['.*']
|
||||
process.include_top_n:
|
||||
by_cpu: 10
|
||||
by_memory: 10
|
||||
system.hostfs: /hostfs # the host's real filesystems, not the container's
|
||||
|
||||
- module: docker
|
||||
period: 30s
|
||||
hosts: ["unix:///var/run/docker.sock"]
|
||||
metricsets: [container, cpu, diskio, healthcheck, info, memory, network]
|
||||
|
||||
- module: prometheus # MediaMTX: per path bytes/s, readers, publishers, dropped packets
|
||||
period: 30s
|
||||
metrics_path: /metrics
|
||||
hosts: ["http://127.0.0.1:9998"]
|
||||
|
||||
- module: prometheus # Caddy: requests by host and status, durations, TLS
|
||||
period: 60s
|
||||
metrics_path: /metrics
|
||||
hosts: ["http://127.0.0.1:2019"]
|
||||
|
||||
processors:
|
||||
- add_host_metadata: ~
|
||||
|
||||
output.elasticsearch:
|
||||
hosts: ["http://127.0.0.1:9200"]
|
||||
|
||||
setup.kibana:
|
||||
host: "http://127.0.0.1:5601"
|
||||
setup.dashboards.enabled: true # the system and docker dashboards, so there is something to look at
|
||||
|
||||
logging.level: info
|
||||
@@ -1,30 +0,0 @@
|
||||
# What VictoriaMetrics scrapes. Every target is loopback-only on this host (LOGGING.md "Metrics taps").
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
external_labels:
|
||||
host: vanguard
|
||||
|
||||
scrape_configs:
|
||||
- job_name: host # CPU, RAM, disk space + I/O, network, load, md RAID
|
||||
static_configs: [{targets: ["host.docker.internal:9100"]}]
|
||||
- job_name: containers # per container CPU/memory/network/block IO, labelled vanguard.*
|
||||
static_configs: [{targets: ["host.docker.internal:9101"]}]
|
||||
- job_name: mediamtx # per path bytes/s, readers, publishers, dropped packets
|
||||
static_configs: [{targets: ["host.docker.internal:9998"]}]
|
||||
- job_name: caddy # requests by host and status, durations, TLS
|
||||
static_configs: [{targets: ["host.docker.internal:2019"]}]
|
||||
- job_name: victorialogs
|
||||
static_configs: [{targets: ["victorialogs:9428"]}]
|
||||
metrics_path: /logs/metrics # it runs behind -http.pathPrefix=/logs
|
||||
- job_name: victoriametrics
|
||||
static_configs: [{targets: ["127.0.0.1:8428"]}]
|
||||
metrics_path: /metrics-db/metrics
|
||||
- job_name: alertmanager
|
||||
static_configs: [{targets: ["alertmanager:9093"]}]
|
||||
metrics_path: /alerts/metrics
|
||||
- job_name: vmalert
|
||||
static_configs: [{targets: ["vmalert-metrics:8880"], labels: {rules: metrics}}]
|
||||
metrics_path: /rules-metrics/metrics
|
||||
- job_name: vmalert-logs
|
||||
static_configs: [{targets: ["vmalert-logs:8881"], labels: {rules: logs}}]
|
||||
metrics_path: /rules-logs/metrics
|
||||
@@ -1,102 +0,0 @@
|
||||
# Vector: collects from the two taps in LOGGING.md and ships to VictoriaLogs.
|
||||
#
|
||||
# docker_logs every container's stdout/stderr, with the vanguard.service / vanguard.category labels
|
||||
# we put on each one, so filtering by category needs no parsing
|
||||
# journald our own systemd units (dumpd, the sweep, trash purge, deploy pending, vgdeband)
|
||||
#
|
||||
# TRADE-OFF, chosen deliberately (CommunityDiscordMedia.md Phase 13): docker_logs needs the Docker API, so this
|
||||
# container holds the socket and therefore root-equivalent access to the host — the same exposure act_runner
|
||||
# already has. Accepted "for now". The upgrade paths, in order of preference, are the journald log driver for
|
||||
# every container (one tap, no socket, labels survive as journal fields) or a read-only socket proxy in front
|
||||
# of this container. Neither changes anything downstream of here.
|
||||
#
|
||||
# Every event ends up with: service, category, source_kind, message, level where it could be determined.
|
||||
|
||||
data_dir: /var/lib/vector
|
||||
|
||||
sources:
|
||||
containers:
|
||||
type: docker_logs
|
||||
docker_host: unix:///var/run/docker.sock
|
||||
exclude_containers: [vector] # never ship our own output: it would feed itself
|
||||
units:
|
||||
type: journald
|
||||
# systemd 250-257 reject reading past boots through journalctl, so this stays true: only the current boot
|
||||
# is shipped. Older entries are still on the host for `journalctl --boot=-1`.
|
||||
current_boot_only: true
|
||||
journal_directory: /var/log/journal
|
||||
include_units: [dumpd, normalize-sweep, trash-purge, deploy-pending, vgdeband]
|
||||
|
||||
transforms:
|
||||
# ---- one shape, whichever tap it came from
|
||||
normalised:
|
||||
type: remap
|
||||
inputs: [containers, units]
|
||||
source: |
|
||||
if exists(.container_name) {
|
||||
.service = .label."vanguard.service" || .container_name
|
||||
.category = .label."vanguard.category" || "infra"
|
||||
.source_kind = "container"
|
||||
del(.label); del(.container_id); del(.image); del(.container_created_at); del(.stream)
|
||||
} else {
|
||||
unit = ._SYSTEMD_UNIT || .unit || "host"
|
||||
.service = replace(string!(unit), ".service", "")
|
||||
.category = if .service == "dumpd" { "stream" } else { "media" }
|
||||
.source_kind = "unit"
|
||||
if !exists(.message) && exists(.MESSAGE) { .message = del(.MESSAGE) }
|
||||
# the journal carries a lot of bookkeeping; keep what a query or a human needs
|
||||
del(.PRIORITY); del(.SYSLOG_IDENTIFIER); del(.SYSLOG_FACILITY); del(._TRANSPORT)
|
||||
del(._PID); del(._UID); del(._GID); del(._COMM); del(._EXE); del(._CMDLINE)
|
||||
del(._CAP_EFFECTIVE); del(._SELINUX_CONTEXT); del(._AUDIT_SESSION); del(._AUDIT_LOGINUID)
|
||||
del(._SYSTEMD_CGROUP); del(._SYSTEMD_SLICE); del(._SYSTEMD_INVOCATION_ID)
|
||||
del(._BOOT_ID); del(._MACHINE_ID); del(._SOURCE_REALTIME_TIMESTAMP); del(._SYSTEMD_UNIT)
|
||||
}
|
||||
|
||||
# ---- parse the shapes LOGGING.md documents; anything else stays a plain message
|
||||
parsed:
|
||||
type: remap
|
||||
inputs: [normalised]
|
||||
source: |
|
||||
text = string(.message) ?? ""
|
||||
if starts_with(text, "{") {
|
||||
# caddy and authentik emit JSON
|
||||
fields, err = parse_json(text)
|
||||
if err == null {
|
||||
.json = fields
|
||||
.level = get(fields, ["level"]) ?? get(fields, ["severity"]) ?? null
|
||||
.message = get(fields, ["msg"]) ?? get(fields, ["event"]) ?? text
|
||||
}
|
||||
} else if match(text, r'(^|\s)(ts|level|component|event)=') {
|
||||
# our own scripts, and syncthing's key=value tail
|
||||
fields, err = parse_logfmt(text)
|
||||
if err == null {
|
||||
.kv = fields
|
||||
.level = get(fields, ["level"]) ?? null
|
||||
.event = get(fields, ["event"]) ?? null
|
||||
}
|
||||
} else if match(text, r'^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} ') {
|
||||
# mediamtx: 2026/11/03 05:00:00 INF [component] message
|
||||
fields, err = parse_regex(text, r'^\S+ \S+ (?P<lvl>\w+) (?P<rest>.*)$')
|
||||
if err == null { .level = fields.lvl; .message = fields.rest }
|
||||
} else if match(text, r'^\[\d{2}:\d{2}:\d{2}') {
|
||||
# jellyfin: [00:00:00.000] [LVL] [thread] Logger: message (stack traces arrive as their own events)
|
||||
fields, err = parse_regex(text, r'^\[[\d:.]+\] \[(?P<lvl>\w+)\] \[\d+\] (?P<rest>.*)$')
|
||||
if err == null { .level = fields.lvl; .message = fields.rest }
|
||||
}
|
||||
if exists(.level) { .level = downcase(string!(.level)) }
|
||||
|
||||
sinks:
|
||||
victorialogs:
|
||||
type: elasticsearch
|
||||
inputs: [parsed]
|
||||
# -http.pathPrefix=/logs applies to EVERY endpoint, ingestion included, so the bulk path lives under it
|
||||
endpoints: ["http://victorialogs:9428/logs/insert/elasticsearch/"]
|
||||
mode: bulk
|
||||
api_version: v8
|
||||
compression: gzip
|
||||
healthcheck:
|
||||
enabled: false
|
||||
query:
|
||||
_msg_field: message
|
||||
_time_field: timestamp
|
||||
_stream_fields: service,category,source_kind
|
||||
Reference in New Issue
Block a user