# Rules vmalert evaluates against VictoriaMetrics. Metric conditions only; log conditions are in alerts-logs.yml.
groups:
  - name: disk
    interval: 1m
    rules:
      - alert: MediaDiskFilling
        # The most likely quiet death on this box: recordings fill /srv and the sweep starts failing.
        expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 85
        for: 10m
        labels: {severity: warning}
        annotations:
          summary: "/srv is {{ $value | printf \"%.0f\" }}% full"
          detail: "Recordings and clips live here. Check /srv/media/streams and the 15-day archive window."
      - alert: MediaDiskCritical
        expr: 100 - (node_filesystem_avail_bytes{mountpoint="/srv"} / node_filesystem_size_bytes{mountpoint="/srv"} * 100) > 94
        for: 2m
        labels: {severity: critical}
        annotations:
          summary: "/srv is {{ $value | printf \"%.0f\" }}% full — recording will fail"
      - alert: RootDiskFilling
        expr: 100 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100) > 85
        for: 10m
        labels: {severity: warning}
        annotations: {summary: "/ is {{ $value | printf \"%.0f\" }}% full (docker images, journal, appdata)"}

  - name: host
    interval: 1m
    rules:
      - alert: LoadSustained
        # The sweep is niced, so a sustained load above core count means something is not yielding.
        expr: node_load15 > count(count by (cpu) (node_cpu_seconds_total))
        for: 20m
        labels: {severity: warning}
        annotations: {summary: "load15 {{ $value | printf \"%.1f\" }} above core count for 20m"}
      - alert: MemoryLow
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
        for: 10m
        labels: {severity: warning}
        annotations: {summary: "only {{ $value | printf \"%.0f\" }}% memory available"}
      - alert: RaidDegraded
        expr: node_md_disks{state!="active"} > 0
        for: 5m
        labels: {severity: critical}
        annotations: {summary: "md RAID has a disk that is not active"}

  - name: containers
    interval: 1m
    rules:
      - alert: ContainerRestartLoop
        expr: increase(container_start_time_seconds{name!=""}[15m]) > 0 and changes(container_start_time_seconds{name!=""}[1h]) > 2
        for: 5m
        labels: {severity: warning}
        annotations: {summary: "{{ $labels.name }} restarted more than twice in an hour"}
      - alert: ContainerGone
        # cAdvisor stops reporting a container that is not running at all.
        expr: absent(container_last_seen{name="mediamtx"}) or absent(container_last_seen{name="caddy"}) or absent(container_last_seen{name="discord-bot"})
        for: 5m
        labels: {severity: critical}
        annotations: {summary: "a core container is not running (mediamtx, caddy or discord-bot)"}

  - name: streaming
    interval: 1m
    rules:
      - alert: StreamDroppingPackets
        expr: increase(mediamtx_paths_bytes_received{}[5m]) == 0 and mediamtx_paths{state="ready"} > 0
        for: 10m
        labels: {severity: warning}
        annotations: {summary: "a MediaMTX path is ready but has received nothing for 10m"}

  - name: observability
    interval: 1m
    rules:
      - alert: LogIngestionStalled
        # If this fires, the log half is blind — and if it cannot fire, the deadman covers it.
        expr: increase(vl_rows_ingested_total[15m]) == 0
        for: 15m
        labels: {severity: warning}
        annotations: {summary: "VictoriaLogs has ingested nothing for 15m — is Vector running?"}
      - alert: Watchdog
        # Always firing on purpose: Alertmanager routes it to the heartbeat receiver, and the deadman script
        # on the host notices when it stops arriving. This is how the monitoring stack reports its own death.
        expr: vector(1)
        labels: {severity: none}
        annotations: {summary: "alerting pipeline is alive"}