Media-Server-Setup/bin/vgstuck.sh

#!/bin/bash
# Triage recordings that never published: see why each one is stuck, then retry it or bin it.
#
#   vgstuck.sh                     list everything stuck, with the reason        (read-only, the default)
#   vgstuck.sh retry <match>       clear what is blocking it so the sweep tries again
#   vgstuck.sh retry --all         ...for everything that is parked
#   vgstuck.sh drop  <match>       move the recording to trash: 3-day undo, same as "dump tonight"
#   vgstuck.sh sweep               kick the sweep once, without waiting on it
#   vgstuck.sh json                the same survey as JSON (what dumpd serves to the web page)
#
# <match> is any substring of the path (a streamer name, a title, a timestamp). It must match at least one
# recording, and `drop` refuses --all: binning every stuck recording in one keystroke is never a thing
# somebody meant to do.
#
# WHY A RECORDING STAYS IN .live/
#   parked     a <Title>__<ts>.normalize-failed.log sits next to the intended output. The sweep skips the
#              file entirely until that log is deleted, so it will never retry on its own.
#   published  an output <Title>__<ts>.mp4 already exists, so the sweep leaves the source alone.
#   waiting    nothing is blocking it; it is simply newer than MINS_IDLE or the sweep has not run yet.
#   partial    a leftover <output>.mp4.part. This does NOT block anything (ffmpeg overwrites it with -y);
#              it only wastes disk, and retry removes it.
#
# Nothing here deletes media. `drop` MOVES to /srv/media/trash/<today>/, which trash-purge clears after
# three days, so a wrong call is recoverable for the same window as a dumped night.
set -uo pipefail

ROOT="${MEDIA_ROOT:-/srv/media}"
TRASH="$ROOT/trash"
MINS_IDLE="${MINS_IDLE:-3}"

vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=vgstuck %s\n' "$(date -Is)" "$lvl" "$*"; }

# Everything about one .live source, as: src|out|state|age_s|bytes|dur_s|reason
survey() {
  local src titledir user title stamp base out state reason dur
  while IFS= read -r -d '' src; do
    titledir="$(dirname "$(dirname "$src")")"
    user="$(basename "$(dirname "$titledir")")"
    title="$(basename "$titledir")"
    stamp="$(basename "${src%.*}")"
    base="$(dirname "$titledir")/${title}__${stamp}"
    out="$base.mp4"
    reason=""
    if [[ -e "$base.normalize-failed.log" ]]; then
      state=parked
      reason="$(head -1 "$base.normalize-failed.log" 2>/dev/null)"
      local why; why=$(grep -A2 '^--- ffmpeg stderr ---$' "$base.normalize-failed.log" 2>/dev/null | sed -n '2p')
      [[ -n ${why:-} ]] && reason="$reason :: $why"
    elif [[ -e "$out" ]]; then
      state=published
      reason="output already exists: $(basename "$out")"
    elif [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; then
      state=writing
      reason="touched in the last ${MINS_IDLE}m — still recording"
    else
      state=waiting
      reason="nothing blocking it; the sweep has not got to it"
    fi
    [[ -e "$out.part" ]] && reason="$reason [leftover .part]"
    dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
    printf '%s|%s|%s|%s|%s|%s|%s\n' "$src" "$out" "$state" \
      "$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:-0}" "$reason"
  done < <(find "$ROOT/streams" -path '*/.live/*' -name '*.mp4' -print0 2>/dev/null)
}

show() {
  local any=0 src out state age bytes dur reason
  printf '\n  %-9s %-9s %9s %8s  %s\n' state waiting length size recording
  while IFS='|' read -r src out state age bytes dur reason; do
    any=1
    printf '  %-9s %-9s %9s %7.0fM  %s\n' "$state" \
      "$(( age / 3600 ))h$(printf '%02d' $(( (age % 3600) / 60 )))m" \
      "$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 )))" \
      "$(awk -v b="$bytes" 'BEGIN{print b/1000000}')" "${src#"$ROOT"/streams/}"
    printf '            %s\n' "$reason"
  done < <(survey | sort -t'|' -k3,3 -k4,4nr)
  (( any )) || printf '  nothing is stuck.\n'
  printf '\n  retry one:  vgstuck.sh retry <substring>      bin one:  vgstuck.sh drop <substring>\n\n'
}

# rows matching the pattern, or everything when --all
selected() {
  local pattern="$1"
  if [[ $pattern == --all ]]; then survey; else survey | grep -F -- "$pattern" || true; fi
}

do_retry() {
  local pattern="$1" n=0 src out state age bytes dur reason
  while IFS='|' read -r src out state age bytes dur reason; do
    [[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
    rm -f "$out.part"                                  # leftover partial output: harmless, but no reason to keep
    rm -f "${out%.mp4}.normalize-failed.log"           # THIS is what makes the sweep skip the file forever
    n=$((n + 1))
    vglog info "event=retry file=\"$src\" was=$state"
  done < <(selected "$pattern")
  (( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
  vglog info "event=retry_done count=$n msg=\"run vgstuck.sh sweep, or wait for the timer\""
}

do_drop() {
  local pattern="$1" night n=0 src out state age bytes dur reason rel dst
  [[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them"' >&2; return 2; }
  night="$(date +%F)"
  while IFS='|' read -r src out state age bytes dur reason; do
    [[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
    for victim in "$src" "${src%.*}.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
      [[ -e $victim ]] || continue
      rel="${victim#"$ROOT"/}"
      dst="$TRASH/$night/$rel"
      mkdir -p "$(dirname "$dst")"
      mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3"
    done
    rmdir "$(dirname "$src")" "$(dirname "$(dirname "$src")")" 2>/dev/null || true
    n=$((n + 1))
  done < <(selected "$pattern")
  (( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
  vglog info "event=drop_done count=$n trash=\"$TRASH/$night\""
}

# Same survey, as JSON. bin/dumpd.py serves this to the web page, so the classification lives in exactly
# one place and the browser and the CLI can never disagree about why something is stuck.
emit_json() {
  survey | python3 -c '
import json, sys
rows = []
for line in sys.stdin:
    parts = line.rstrip("\n").split("|", 6)
    if len(parts) != 7:
        continue
    src, out, state, age, size, dur, reason = parts
    rows.append({"src": src, "out": out, "state": state, "age_s": int(age or 0),
                 "bytes": int(size or 0), "duration_s": int(dur or 0), "reason": reason})
print(json.dumps(rows))'
}

case "${1:-list}" in
  list|"")  show ;;
  json)     emit_json ;;
  retry)    [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all" >&2; exit 1; }; do_retry "$2" ;;
  drop)     [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>" >&2; exit 1; }; do_drop "$2" ;;
  sweep)    systemctl start --no-block normalize-sweep.service && vglog info "event=sweep_started" ;;
  *)        sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
esac