#!/bin/bash
# Triage recordings that never published: see why each one is stuck, then retry it or bin it.
#
#   vgstuck.sh                     list everything stuck, with the reason        (read-only, the default)
#   vgstuck.sh list --durations    ...and measure each one's length (slow: fragmented MP4 is read end to end)
#   vgstuck.sh retry <match>       clear what is blocking it so the sweep tries again
#   vgstuck.sh retry --all         ...for everything that is parked
#   vgstuck.sh drop  <match>       move the recording to trash: 3-day undo, same as "dump tonight"
#   vgstuck.sh list  --under 5     only recordings smaller than 5 MB (reconnect fragments)
#   vgstuck.sh drop  --under 5     bin all of those; still-recording files are never touched
#   vgstuck.sh wipe                bin EVERYTHING unprocessed: stops the sweep (killing any conversion in flight),
#                                  then moves every .live/ file — parked, waiting, published-collision, fresh — with
#                                  its sidecars, failure logs and .part leftovers to trash, then drops the emptied
#                                  folders. The ONLY thing left alone is a file MediaMTX has open, i.e. a stream
#                                  being recorded this minute. The Dump tab's "Bin all unprocessed" button runs this.
#   vgstuck.sh sweep               kick the sweep once, without waiting on it
#   vgstuck.sh json                the same survey as JSON (what dumpd serves to the web page)
#
# <match> is any substring of the path (a streamer name, a title, a timestamp). It must match at least one
# recording, and `drop` refuses --all so a stray pattern cannot bin everything by accident: the deliberate
# version is spelled out as its own word, `wipe`.
#
# WHY A RECORDING STAYS IN .live/
#   parked     a <Title>__<ts>.normalize-failed.log sits next to the intended output. The sweep skips the
#              file entirely until that log is deleted, so it will never retry on its own.
#   published  an output <Title>__<ts>.mp4 already exists, so the sweep leaves the source alone.
#   waiting    nothing is blocking it; it is simply newer than MINS_IDLE or the sweep has not run yet.
#   partial    a leftover <output>.mp4.part. This does NOT block anything (ffmpeg overwrites it with -y);
#              it only wastes disk, and retry removes it.
#
# Nothing here deletes media. `drop` MOVES to /srv/media/trash/<today>/, which trash-purge clears after
# three days, so a wrong call is recoverable for the same window as a dumped night.
set -uo pipefail

ROOT="${MEDIA_ROOT:-/srv/media}"
TRASH="$ROOT/trash"
MINS_IDLE="${MINS_IDLE:-3}"

vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=vgstuck %s\n' "$(date -Is)" "$lvl" "$*"; }

# Everything about one .live source, as: src|out|state|age_s|bytes|dur_s|reason
survey() {
  local src titledir user title stamp base out state reason dur
  while IFS= read -r -d '' src; do
    titledir="$(dirname "$(dirname "$src")")"
    user="$(basename "$(dirname "$titledir")")"
    title="$(basename "$titledir")"
    stamp="$(basename "${src%.*}")"
    base="$(dirname "$titledir")/${title}__${stamp}"
    out="$base.mp4"
    reason=""
    if [[ -e "$base.normalize-failed.log" ]]; then
      state=parked
      reason="$(head -1 "$base.normalize-failed.log" 2>/dev/null)"
      local why; why=$(grep -A2 '^--- ffmpeg stderr ---$' "$base.normalize-failed.log" 2>/dev/null | sed -n '2p')
      [[ -n ${why:-} ]] && reason="$reason :: $why"
    elif [[ -e "$out" ]]; then
      state=published
      reason="output already exists: $(basename "$out")"
    elif [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; then
      state=writing
      reason="touched in the last ${MINS_IDLE}m — still recording"
    else
      state=waiting
      reason="nothing blocking it; the sweep has not got to it"
    fi
    [[ -e "$out.part" ]] && reason="$reason [leftover .part]"
    # Duration is OPT-IN. Everything in .live/ is fragmented MP4 with no duration in its header, so ffprobe
    # has to read the whole file to answer — gigabytes per recording. With many stuck files the survey took
    # over three minutes, dumpd's request timed out, and the web page broke. Size and age come from stat.
    dur=""
    if [[ ${VGSTUCK_DURATIONS:-0} == 1 ]]; then
      dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
    fi
    printf '%s|%s|%s|%s|%s|%s|%s\n' "$src" "$out" "$state" \
      "$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:--}" "$reason"
  done < <(find "$ROOT/streams" -path '*/.live/*' -name '*.mp4' -print0 2>/dev/null)
}

show() {
  local any=0 total=0 src out state age bytes dur reason
  printf '\n  %-9s %-9s %9s %8s  %s\n' state waiting length size recording
  while IFS='|' read -r src out state age bytes dur reason; do
    total=$(( total + bytes ))
    any=1
    if [[ $dur == - || -z $dur ]]; then length="-"; else
      length=$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 ))); fi
    printf '  %-9s %-9s %9s %7.0fM  %s\n' "$state" \
      "$(( age / 3600 ))h$(printf '%02d' $(( (age % 3600) / 60 )))m" \
      "$length" \
      "$(awk -v b="$bytes" 'BEGIN{print b/1000000}')" "${src#"$ROOT"/streams/}"
    printf '            %s\n' "$reason"
  done < <(selected "${1:---all}" "${2:-}" | sort -t'|' -k3,3 -k4,4nr)
  (( any )) || printf '  nothing matches.\n'
  (( any )) && printf '  %s item(s), %s GB\n' "$(selected "${1:---all}" "${2:-}" | wc -l)" "$(awk -v b="$total" 'BEGIN{printf "%.1f", b/1e9}')"
  [[ ${VGSTUCK_DURATIONS:-0} == 1 ]] || printf '\n  lengths not measured (slow on fragmented MP4): vgstuck.sh list --durations\n'
  printf '  retry one:  vgstuck.sh retry <substring>      bin one:  vgstuck.sh drop <substring>\n\n'
}

# rows matching the pattern, everything when --all, or everything smaller than N MB with --under N.
# --under exists for reconnect fragments: every time a publisher drops and comes back MediaMTX opens a new
# recording, so a bad night leaves hundreds of one- and two-second files. Still-recording files never match.
selected() {
  local pattern="$1" limit="${2:-}"
  if [[ $pattern == --under ]]; then
    [[ $limit =~ ^[0-9]+$ ]] || { echo "usage: --under <megabytes>" >&2; return 2; }
    survey | awk -F'|' -v max="$(( limit * 1000000 ))" '$3 != "writing" && $5 < max'
  elif [[ $pattern == --all ]]; then survey
  else survey | grep -F -- "$pattern" || true
  fi
}

do_retry() {
  local pattern="$1" limit="${2:-}" n=0 src out state age bytes dur reason
  while IFS='|' read -r src out state age bytes dur reason; do
    [[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
    rm -f "$out.part"                                  # leftover partial output: harmless, but no reason to keep
    rm -f "${out%.mp4}.normalize-failed.log"           # THIS is what makes the sweep skip the file forever
    n=$((n + 1))
    vglog info "event=retry file=\"$src\" was=$state"
  done < <(selected "$pattern" "$limit")
  (( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
  vglog info "event=retry_done count=$n msg=\"run vgstuck.sh sweep, or wait for the timer\""
}

# one row per line on stdin (survey format) -> trash. Prints the count it moved.
drop_rows() {
  local night="$1" n=0 src out state age bytes dur reason rel dst victim
  while IFS='|' read -r src out state age bytes dur reason; do
    # The survey labels a file by its blocker first (a parked file stays "parked" even while being written), so the
    # safety check here does not trust the label: open by any process, or touched inside MINS_IDLE, means leave it.
    # In wipe mode (WIPE=1) only "open" counts — the sweep has been stopped, so an open file is MediaMTX recording.
    if fuser -s "$src" 2>/dev/null || { [[ ${WIPE:-0} != 1 ]] && { [[ $state == writing ]] || [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; }; }; then
      vglog info "event=skip reason=still_recording file=\"$src\"" >&2; continue
    fi
    for victim in "$src" "${src%.*}.json" "${src%.*}.stitch.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
      [[ -e $victim ]] || continue
      rel="${victim#"$ROOT"/}"
      dst="$TRASH/$night/$rel"
      mkdir -p "$(dirname "$dst")"
      mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3" >&2
    done
    rmdir "$(dirname "$src")" "$(dirname "$(dirname "$src")")" 2>/dev/null || true
    n=$((n + 1))
  done
  echo "$n"
}

do_drop() {
  local pattern="$1" limit="${2:-}" night n
  [[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them, use --under <MB>, or wipe"' >&2; return 2; }
  night="$(date +%F)"
  n=$(selected "$pattern" "$limit" | drop_rows "$night")          # per-file lines go to stderr, the count to stdout
  (( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
  vglog info "event=drop_done count=$n trash=\"$TRASH/$night\""
}

# Everything unprocessed, deliberately. First the sweep is stopped, which kills any ffmpeg conversion it has in
# flight (the timer starts it again within five minutes, to an empty queue). Then every .live/ file goes, parked
# or fresh, and so do failure logs whose recording is already gone (they would keep the Stuck tab listing ghosts)
# and leftover .part outputs. The one exception: a file some process still holds open after the sweep is down,
# which is MediaMTX recording a live stream — pulling that from under it would lose the stream in progress.
do_wipe() {
  local night n stale=0 log rel dst
  night="$(date +%F)"
  if systemctl is-active --quiet normalize-sweep.service; then
    systemctl stop normalize-sweep.service && vglog info "event=cancelled unit=normalize-sweep.service msg=\"conversion in flight killed\""
  fi
  systemctl is-active --quiet vgdeband.service && systemctl stop vgdeband.service && vglog info "event=cancelled unit=vgdeband.service"
  n=$(survey | WIPE=1 drop_rows "$night")
  while IFS= read -r -d '' log; do
    rel="${log#"$ROOT"/}"; dst="$TRASH/$night/$rel"; mkdir -p "$(dirname "$dst")"
    mv -f "$log" "$dst" && stale=$((stale + 1))
  done < <(find "$ROOT/streams" -maxdepth 2 \( -name '*.normalize-failed.log' -o -name '*.mp4.part' \) -print0 2>/dev/null)
  find "$ROOT/streams" -mindepth 2 -type d -name .live -empty -delete 2>/dev/null
  find "$ROOT/streams" -mindepth 2 -maxdepth 2 -type d -empty -delete 2>/dev/null
  vglog info "event=wipe_done recordings=${n:-0} stale_files=$stale trash=\"$TRASH/$night\" undo_days=3"
}

# Same survey, as JSON. bin/dumpd.py serves this to the web page, so the classification lives in exactly
# one place and the browser and the CLI can never disagree about why something is stuck.
emit_json() {
  survey | python3 -c '
import json, sys
rows = []
for line in sys.stdin:
    parts = line.rstrip("\n").split("|", 6)
    if len(parts) != 7:
        continue
    src, out, state, age, size, dur, reason = parts
    rows.append({"src": src, "out": out, "state": state, "age_s": int(age or 0),
                 "bytes": int(size or 0),
                 "duration_s": int(dur) if dur not in ("", "-") else None, "reason": reason})
print(json.dumps(rows))'
}

for a in "$@"; do [[ $a == --durations ]] && export VGSTUCK_DURATIONS=1; done

case "${1:-list}" in
  list|"")  show "${2:---all}" "${3:-}" ;;
  json)     emit_json ;;
  retry)    [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all|--under <MB>" >&2; exit 1; }; do_retry "$2" "${3:-}" ;;
  drop)     [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>|--under <MB>" >&2; exit 1; }; do_drop "$2" "${3:-}" ;;
  wipe)     do_wipe ;;
  sweep)    systemctl start --no-block normalize-sweep.service && vglog info "event=sweep_started" ;;
  *)        sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
esac