Rip out the Victoria stack for Elastic: four containers, Kibana as the single pane
Eight observability containers become four. Elasticsearch stores logs and metrics, Kibana is the one UI for Discover, dashboards and alerting, Filebeat ships container logs plus our journal units, and Metricbeat covers host and Docker metrics — which retires node_exporter and cAdvisor too — while still scraping MediaMTX's and Caddy's Prometheus endpoints. Pinned rather than left to chance: a fixed 2 GB heap (a self-sizing JVM would claim a quarter of a 62 GB box whose spare capacity belongs to x264), single-node discovery, security off because it only listens on loopback behind the Authentik gate, ML off, and vm.max_map_count set and persisted since ES will not start without it. Kibana gets a fixed encryption key or it loses its saved objects on every restart. diag.HOST is now just Kibana at the root: no path prefixes to get wrong and no index page of links. The deadman is the only thing kept from the old stack, repointed at Elastic. It checks that ES answers and is not red, that no index has been flipped read-only by the disk watermark (which does not clear itself when space is freed, so ingestion stays silently dead), that Kibana is available, that each Beat has actually written a document recently rather than merely running, that the Kibana heartbeat still arrives, and the disks. Verified against a mock Elasticsearch for all five failure modes plus the healthy case staying silent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -314,165 +314,99 @@ services:
|
||||
- gitea
|
||||
restart: unless-stopped
|
||||
|
||||
# ---- metrics taps (LOGGING.md): Prometheus text format on loopback, scraped by whichever store is chosen later
|
||||
node_exporter:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: node_exporter, vanguard.category: infra }
|
||||
image: quay.io/prometheus/node-exporter:latest
|
||||
container_name: node_exporter
|
||||
network_mode: host # listens on 127.0.0.1:9100 only
|
||||
pid: host
|
||||
command: ["--path.rootfs=/host", "--web.listen-address=127.0.0.1:9100", "--collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run|var/lib/docker/.+)($$|/)"]
|
||||
volumes:
|
||||
- /:/host:ro,rslave
|
||||
restart: unless-stopped
|
||||
|
||||
cadvisor:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: cadvisor, vanguard.category: infra }
|
||||
image: gcr.io/cadvisor/cadvisor:latest
|
||||
container_name: cadvisor
|
||||
ports:
|
||||
- "127.0.0.1:9101:8080" # per-container CPU / memory / network / disk
|
||||
command: ["--housekeeping_interval=10s", "--docker_only=true", "--store_container_labels=false", "--whitelisted_container_labels=vanguard.service,vanguard.category"]
|
||||
volumes:
|
||||
- /:/rootfs:ro
|
||||
- /var/run:/var/run:ro
|
||||
- /sys:/sys:ro
|
||||
- /var/lib/docker/:/var/lib/docker:ro
|
||||
- /dev/disk/:/dev/disk:ro
|
||||
devices:
|
||||
- /dev/kmsg
|
||||
privileged: true
|
||||
restart: unless-stopped
|
||||
|
||||
# ================================================================ observability (Phase 13)
|
||||
# Logs: Vector -> VictoriaLogs. Metrics: VictoriaMetrics scrapes the four taps.
|
||||
# Alerting: a vmalert per datasource -> Alertmanager -> Discord. All loopback; published via diag.HOST.
|
||||
# The deadman (bin/vanguard-deadman.sh, a host timer) watches THIS stack, because nothing in it can.
|
||||
victorialogs:
|
||||
# ================================================================ observability — Elastic (Phase 13)
|
||||
# One store, one UI, one collector pair. Replaces VictoriaLogs+Vector+VictoriaMetrics+2×vmalert+Alertmanager
|
||||
# +node_exporter+cAdvisor: eight containers for four, and Kibana is the single pane for logs, dashboards and
|
||||
# alerting. Everything binds loopback and is published only through diag.HOST behind the Authentik gate.
|
||||
elasticsearch:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: victorialogs, vanguard.category: infra }
|
||||
image: docker.io/victoriametrics/victoria-logs:latest
|
||||
container_name: victorialogs
|
||||
user: "1000:1000"
|
||||
command:
|
||||
- -storageDataPath=/data
|
||||
- -retentionPeriod=30d
|
||||
- -httpListenAddr=:9428
|
||||
- -http.pathPrefix=/logs # its own UI then emits /logs/... so Caddy must NOT strip the prefix
|
||||
ports:
|
||||
- "127.0.0.1:9428:9428"
|
||||
volumes:
|
||||
- /srv/appdata/victorialogs:/data
|
||||
restart: unless-stopped
|
||||
|
||||
vector:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: vector, vanguard.category: infra }
|
||||
image: docker.io/timberio/vector:latest-debian
|
||||
container_name: vector
|
||||
labels: { vanguard.service: elasticsearch, vanguard.category: infra }
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:9.2.0
|
||||
container_name: elasticsearch
|
||||
environment:
|
||||
VECTOR_CONFIG: /etc/vector/vector.yaml
|
||||
# Pinned deliberately. A JVM that sizes its own heap on a 62 GB box would take a quarter of it, on a
|
||||
# machine whose spare CPU and RAM belong to x264. 2 GB is ample for one host's logs and metrics.
|
||||
ES_JAVA_OPTS: -Xms2g -Xmx2g
|
||||
discovery.type: single-node
|
||||
xpack.security.enabled: "false" # loopback only, behind Caddy + Authentik; no second login
|
||||
xpack.ml.enabled: "false" # not used, and it reserves memory
|
||||
cluster.routing.allocation.disk.watermark.low: 85%
|
||||
cluster.routing.allocation.disk.watermark.high: 90%
|
||||
cluster.routing.allocation.disk.watermark.flood_stage: 95%
|
||||
bootstrap.memory_lock: "true"
|
||||
ulimits:
|
||||
memlock: { soft: -1, hard: -1 }
|
||||
nofile: { soft: 65536, hard: 65536 }
|
||||
ports:
|
||||
- "127.0.0.1:9200:9200"
|
||||
volumes:
|
||||
- ./observability/vector.yaml:/etc/vector/vector.yaml:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock # Phase 13 decision "A": container logs need the Docker
|
||||
# API, so this is root-equivalent on the host. Upgrade
|
||||
# paths: journald log driver, or a read-only socket proxy.
|
||||
- /var/log/journal:/var/log/journal:ro # our own systemd units
|
||||
- /etc/machine-id:/etc/machine-id:ro # journald needs it to find the right journal
|
||||
- vector_data:/var/lib/vector
|
||||
- /srv/appdata/elasticsearch:/usr/share/elasticsearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s || exit 1"]
|
||||
interval: 20s
|
||||
timeout: 10s
|
||||
retries: 12
|
||||
start_period: 90s
|
||||
restart: unless-stopped
|
||||
|
||||
kibana:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: kibana, vanguard.category: infra }
|
||||
image: docker.elastic.co/kibana/kibana:9.2.0
|
||||
container_name: kibana
|
||||
environment:
|
||||
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||
SERVER_PUBLICBASEURL: https://${DIAG_HOST:-diag.example.com}
|
||||
SERVER_BASEPATH: "" # served at the root of diag.HOST, so no prefix rewriting to get wrong
|
||||
TELEMETRY_OPTIN: "false"
|
||||
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY}
|
||||
XPACK_REPORTING_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY}
|
||||
XPACK_SECURITY_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY}
|
||||
ports:
|
||||
- "127.0.0.1:5601:5601"
|
||||
depends_on:
|
||||
- victorialogs
|
||||
elasticsearch: { condition: service_healthy }
|
||||
restart: unless-stopped
|
||||
|
||||
victoriametrics:
|
||||
filebeat:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: victoriametrics, vanguard.category: infra }
|
||||
image: docker.io/victoriametrics/victoria-metrics:latest
|
||||
container_name: victoriametrics
|
||||
user: "1000:1000"
|
||||
command:
|
||||
- -storageDataPath=/data
|
||||
- -retentionPeriod=90d
|
||||
- -httpListenAddr=:8428
|
||||
- -http.pathPrefix=/metrics-db # served at diag.HOST/metrics-db, so its own UI must emit that prefix
|
||||
- -promscrape.config=/etc/vm/scrape.yml
|
||||
- -promscrape.config.strictParse=false
|
||||
ports:
|
||||
- "127.0.0.1:8428:8428"
|
||||
labels: { vanguard.service: filebeat, vanguard.category: infra }
|
||||
image: docker.elastic.co/beats/filebeat:9.2.0
|
||||
container_name: filebeat
|
||||
user: root
|
||||
command: ["filebeat", "-e", "--strict.perms=false"]
|
||||
volumes:
|
||||
- /srv/appdata/victoriametrics:/data
|
||||
- ./observability/scrape.yml:/etc/vm/scrape.yml:ro
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway" # the four taps bind the host's loopback
|
||||
restart: unless-stopped
|
||||
|
||||
vmalert-metrics:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: vmalert-metrics, vanguard.category: infra }
|
||||
image: docker.io/victoriametrics/vmalert:latest
|
||||
container_name: vmalert-metrics
|
||||
command:
|
||||
- -datasource.url=http://victoriametrics:8428/metrics-db
|
||||
- -notifier.url=http://alertmanager:9093/alerts
|
||||
- -remoteWrite.url=http://victoriametrics:8428/metrics-db
|
||||
- -rule=/etc/alerts/alerts-metrics.yml
|
||||
- -httpListenAddr=:8880
|
||||
- -http.pathPrefix=/rules-metrics
|
||||
- -evaluationInterval=1m
|
||||
ports:
|
||||
- "127.0.0.1:8880:8880"
|
||||
volumes:
|
||||
- ./observability/alerts-metrics.yml:/etc/alerts/alerts-metrics.yml:ro
|
||||
- ./observability/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
|
||||
- /var/lib/docker/containers:/var/lib/docker/containers:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock # autodiscover reads container names and our vanguard.* labels
|
||||
- /var/log/journal:/var/log/journal:ro # our own systemd units
|
||||
- /etc/machine-id:/etc/machine-id:ro
|
||||
- filebeat_data:/usr/share/filebeat/data
|
||||
depends_on:
|
||||
- victoriametrics
|
||||
- alertmanager
|
||||
elasticsearch: { condition: service_healthy }
|
||||
restart: unless-stopped
|
||||
|
||||
vmalert-logs:
|
||||
metricbeat:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: vmalert-logs, vanguard.category: infra }
|
||||
image: docker.io/victoriametrics/vmalert:latest
|
||||
container_name: vmalert-logs
|
||||
# A vmalert instance has one datasource, so log rules need their own: this one queries VictoriaLogs'
|
||||
# LogsQL stats endpoints, which answer in the Prometheus format vmalert expects.
|
||||
command:
|
||||
- -datasource.url=http://victorialogs:9428/logs
|
||||
- -notifier.url=http://alertmanager:9093/alerts
|
||||
- -remoteWrite.url=http://victoriametrics:8428/metrics-db
|
||||
- -rule=/etc/alerts/alerts-logs.yml
|
||||
- -httpListenAddr=:8881
|
||||
- -http.pathPrefix=/rules-logs
|
||||
- -evaluationInterval=2m
|
||||
ports:
|
||||
- "127.0.0.1:8881:8881"
|
||||
labels: { vanguard.service: metricbeat, vanguard.category: infra }
|
||||
image: docker.elastic.co/beats/metricbeat:9.2.0
|
||||
container_name: metricbeat
|
||||
user: root
|
||||
command: ["metricbeat", "-e", "--strict.perms=false"]
|
||||
# system + docker modules replace node_exporter and cAdvisor, and ship their own Kibana dashboards.
|
||||
network_mode: host # real interfaces, and loopback reach to MediaMTX/Caddy/ES/Kibana
|
||||
pid: host
|
||||
volumes:
|
||||
- ./observability/alerts-logs.yml:/etc/alerts/alerts-logs.yml:ro
|
||||
- ./observability/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- /proc:/hostfs/proc:ro
|
||||
- /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
|
||||
- /:/hostfs:ro
|
||||
- metricbeat_data:/usr/share/metricbeat/data
|
||||
depends_on:
|
||||
- victorialogs
|
||||
- alertmanager
|
||||
restart: unless-stopped
|
||||
|
||||
alertmanager:
|
||||
logging: *vglog
|
||||
labels: { vanguard.service: alertmanager, vanguard.category: infra }
|
||||
image: docker.io/prom/alertmanager:latest
|
||||
container_name: alertmanager
|
||||
command:
|
||||
- --config.file=/etc/alertmanager/alertmanager.yml
|
||||
- --storage.path=/data
|
||||
- --web.listen-address=:9093
|
||||
- --web.route-prefix=/alerts
|
||||
- --web.external-url=https://${DIAG_HOST:-diag.example.com}/alerts
|
||||
ports:
|
||||
- "127.0.0.1:9093:9093"
|
||||
volumes:
|
||||
- ./observability/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
|
||||
- /etc/vanguard/alert-webhook:/etc/vanguard/alert-webhook:ro # written by configure.py, mode 0600
|
||||
- /srv/appdata/alertmanager:/data
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway" # the Watchdog heartbeat goes to dumpd on the bridge port
|
||||
elasticsearch: { condition: service_healthy }
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
@@ -480,6 +414,7 @@ networks:
|
||||
|
||||
volumes:
|
||||
copyparty_hist:
|
||||
vector_data:
|
||||
filebeat_data:
|
||||
metricbeat_data:
|
||||
caddy_data:
|
||||
caddy_config:
|
||||
|
||||
Reference in New Issue
Block a user