208 lines
12 KiB
Bash
Executable File
208 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# Triage recordings that never published: see why each one is stuck, then retry it or bin it.
|
|
#
|
|
# vgstuck.sh list everything stuck, with the reason (read-only, the default)
|
|
# vgstuck.sh list --durations ...and measure each one's length (slow: fragmented MP4 is read end to end)
|
|
# vgstuck.sh retry <match> clear what is blocking it so the sweep tries again
|
|
# vgstuck.sh retry --all ...for everything that is parked
|
|
# vgstuck.sh drop <match> move the recording to trash: 3-day undo, same as "dump tonight"
|
|
# vgstuck.sh list --under 5 only recordings smaller than 5 MB (reconnect fragments)
|
|
# vgstuck.sh drop --under 5 bin all of those; still-recording files are never touched
|
|
# vgstuck.sh wipe bin EVERYTHING unprocessed: stops the sweep (killing any conversion in flight),
|
|
# then moves every .live/ file — parked, waiting, published-collision, fresh — with
|
|
# its sidecars, failure logs and .part leftovers to trash, then drops the emptied
|
|
# folders. The ONLY thing left alone is a file MediaMTX has open, i.e. a stream
|
|
# being recorded this minute. The Dump tab's "Bin all unprocessed" button runs this.
|
|
# vgstuck.sh sweep kick the sweep once, without waiting on it
|
|
# vgstuck.sh json the same survey as JSON (what dumpd serves to the web page)
|
|
#
|
|
# <match> is any substring of the path (a streamer name, a title, a timestamp). It must match at least one
|
|
# recording, and `drop` refuses --all so a stray pattern cannot bin everything by accident: the deliberate
|
|
# version is spelled out as its own word, `wipe`.
|
|
#
|
|
# WHY A RECORDING STAYS IN .live/
|
|
# parked a <Title>__<ts>.normalize-failed.log sits next to the intended output. The sweep skips the
|
|
# file entirely until that log is deleted, so it will never retry on its own.
|
|
# published an output <Title>__<ts>.mp4 already exists, so the sweep leaves the source alone.
|
|
# waiting nothing is blocking it; it is simply newer than MINS_IDLE or the sweep has not run yet.
|
|
# partial a leftover <output>.mp4.part. This does NOT block anything (ffmpeg overwrites it with -y);
|
|
# it only wastes disk, and retry removes it.
|
|
#
|
|
# Nothing here deletes media. `drop` MOVES to /srv/media/trash/<today>/, which trash-purge clears after
|
|
# three days, so a wrong call is recoverable for the same window as a dumped night.
|
|
set -uo pipefail
|
|
|
|
ROOT="${MEDIA_ROOT:-/srv/media}"
|
|
TRASH="$ROOT/trash"
|
|
MINS_IDLE="${MINS_IDLE:-3}"
|
|
|
|
vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=vgstuck %s\n' "$(date -Is)" "$lvl" "$*"; }
|
|
|
|
# Everything about one .live source, as: src|out|state|age_s|bytes|dur_s|reason
|
|
survey() {
|
|
local src titledir user title stamp base out state reason dur
|
|
while IFS= read -r -d '' src; do
|
|
titledir="$(dirname "$(dirname "$src")")"
|
|
user="$(basename "$(dirname "$titledir")")"
|
|
title="$(basename "$titledir")"
|
|
stamp="$(basename "${src%.*}")"
|
|
base="$(dirname "$titledir")/${title}__${stamp}"
|
|
out="$base.mp4"
|
|
reason=""
|
|
if [[ -e "$base.normalize-failed.log" ]]; then
|
|
state=parked
|
|
reason="$(head -1 "$base.normalize-failed.log" 2>/dev/null)"
|
|
local why; why=$(grep -A2 '^--- ffmpeg stderr ---$' "$base.normalize-failed.log" 2>/dev/null | sed -n '2p')
|
|
[[ -n ${why:-} ]] && reason="$reason :: $why"
|
|
elif [[ -e "$out" ]]; then
|
|
state=published
|
|
reason="output already exists: $(basename "$out")"
|
|
elif [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; then
|
|
state=writing
|
|
reason="touched in the last ${MINS_IDLE}m — still recording"
|
|
else
|
|
state=waiting
|
|
reason="nothing blocking it; the sweep has not got to it"
|
|
fi
|
|
[[ -e "$out.part" ]] && reason="$reason [leftover .part]"
|
|
# Duration is OPT-IN. Everything in .live/ is fragmented MP4 with no duration in its header, so ffprobe
|
|
# has to read the whole file to answer — gigabytes per recording. With many stuck files the survey took
|
|
# over three minutes, dumpd's request timed out, and the web page broke. Size and age come from stat.
|
|
dur=""
|
|
if [[ ${VGSTUCK_DURATIONS:-0} == 1 ]]; then
|
|
dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
|
|
fi
|
|
printf '%s|%s|%s|%s|%s|%s|%s\n' "$src" "$out" "$state" \
|
|
"$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:--}" "$reason"
|
|
done < <(find "$ROOT/streams" -path '*/.live/*' -name '*.mp4' -print0 2>/dev/null)
|
|
}
|
|
|
|
show() {
|
|
local any=0 total=0 src out state age bytes dur reason
|
|
printf '\n %-9s %-9s %9s %8s %s\n' state waiting length size recording
|
|
while IFS='|' read -r src out state age bytes dur reason; do
|
|
total=$(( total + bytes ))
|
|
any=1
|
|
if [[ $dur == - || -z $dur ]]; then length="-"; else
|
|
length=$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 ))); fi
|
|
printf ' %-9s %-9s %9s %7.0fM %s\n' "$state" \
|
|
"$(( age / 3600 ))h$(printf '%02d' $(( (age % 3600) / 60 )))m" \
|
|
"$length" \
|
|
"$(awk -v b="$bytes" 'BEGIN{print b/1000000}')" "${src#"$ROOT"/streams/}"
|
|
printf ' %s\n' "$reason"
|
|
done < <(selected "${1:---all}" "${2:-}" | sort -t'|' -k3,3 -k4,4nr)
|
|
(( any )) || printf ' nothing matches.\n'
|
|
(( any )) && printf ' %s item(s), %s GB\n' "$(selected "${1:---all}" "${2:-}" | wc -l)" "$(awk -v b="$total" 'BEGIN{printf "%.1f", b/1e9}')"
|
|
[[ ${VGSTUCK_DURATIONS:-0} == 1 ]] || printf '\n lengths not measured (slow on fragmented MP4): vgstuck.sh list --durations\n'
|
|
printf ' retry one: vgstuck.sh retry <substring> bin one: vgstuck.sh drop <substring>\n\n'
|
|
}
|
|
|
|
# rows matching the pattern, everything when --all, or everything smaller than N MB with --under N.
|
|
# --under exists for reconnect fragments: every time a publisher drops and comes back MediaMTX opens a new
|
|
# recording, so a bad night leaves hundreds of one- and two-second files. Still-recording files never match.
|
|
selected() {
|
|
local pattern="$1" limit="${2:-}"
|
|
if [[ $pattern == --under ]]; then
|
|
[[ $limit =~ ^[0-9]+$ ]] || { echo "usage: --under <megabytes>" >&2; return 2; }
|
|
survey | awk -F'|' -v max="$(( limit * 1000000 ))" '$3 != "writing" && $5 < max'
|
|
elif [[ $pattern == --all ]]; then survey
|
|
else survey | grep -F -- "$pattern" || true
|
|
fi
|
|
}
|
|
|
|
do_retry() {
|
|
local pattern="$1" limit="${2:-}" n=0 src out state age bytes dur reason
|
|
while IFS='|' read -r src out state age bytes dur reason; do
|
|
[[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
|
|
rm -f "$out.part" # leftover partial output: harmless, but no reason to keep
|
|
rm -f "${out%.mp4}.normalize-failed.log" # THIS is what makes the sweep skip the file forever
|
|
n=$((n + 1))
|
|
vglog info "event=retry file=\"$src\" was=$state"
|
|
done < <(selected "$pattern" "$limit")
|
|
(( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
|
|
vglog info "event=retry_done count=$n msg=\"run vgstuck.sh sweep, or wait for the timer\""
|
|
}
|
|
|
|
# one row per line on stdin (survey format) -> trash. Prints the count it moved.
|
|
drop_rows() {
|
|
local night="$1" n=0 src out state age bytes dur reason rel dst victim
|
|
while IFS='|' read -r src out state age bytes dur reason; do
|
|
# The survey labels a file by its blocker first (a parked file stays "parked" even while being written), so the
|
|
# safety check here does not trust the label: open by any process, or touched inside MINS_IDLE, means leave it.
|
|
# In wipe mode (WIPE=1) only "open" counts — the sweep has been stopped, so an open file is MediaMTX recording.
|
|
if fuser -s "$src" 2>/dev/null || { [[ ${WIPE:-0} != 1 ]] && { [[ $state == writing ]] || [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; }; }; then
|
|
vglog info "event=skip reason=still_recording file=\"$src\"" >&2; continue
|
|
fi
|
|
for victim in "$src" "${src%.*}.json" "${src%.*}.stitch.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
|
|
[[ -e $victim ]] || continue
|
|
rel="${victim#"$ROOT"/}"
|
|
dst="$TRASH/$night/$rel"
|
|
mkdir -p "$(dirname "$dst")"
|
|
mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3" >&2
|
|
done
|
|
rmdir "$(dirname "$src")" "$(dirname "$(dirname "$src")")" 2>/dev/null || true
|
|
n=$((n + 1))
|
|
done
|
|
echo "$n"
|
|
}
|
|
|
|
do_drop() {
|
|
local pattern="$1" limit="${2:-}" night n
|
|
[[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them, use --under <MB>, or wipe"' >&2; return 2; }
|
|
night="$(date +%F)"
|
|
n=$(selected "$pattern" "$limit" | drop_rows "$night") # per-file lines go to stderr, the count to stdout
|
|
(( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
|
|
vglog info "event=drop_done count=$n trash=\"$TRASH/$night\""
|
|
}
|
|
|
|
# Everything unprocessed, deliberately. First the sweep is stopped, which kills any ffmpeg conversion it has in
|
|
# flight (the timer starts it again within five minutes, to an empty queue). Then every .live/ file goes, parked
|
|
# or fresh, and so do failure logs whose recording is already gone (they would keep the Stuck tab listing ghosts)
|
|
# and leftover .part outputs. The one exception: a file some process still holds open after the sweep is down,
|
|
# which is MediaMTX recording a live stream — pulling that from under it would lose the stream in progress.
|
|
do_wipe() {
|
|
local night n stale=0 log rel dst
|
|
night="$(date +%F)"
|
|
if systemctl is-active --quiet normalize-sweep.service; then
|
|
systemctl stop normalize-sweep.service && vglog info "event=cancelled unit=normalize-sweep.service msg=\"conversion in flight killed\""
|
|
fi
|
|
systemctl is-active --quiet vgdeband.service && systemctl stop vgdeband.service && vglog info "event=cancelled unit=vgdeband.service"
|
|
n=$(survey | WIPE=1 drop_rows "$night")
|
|
while IFS= read -r -d '' log; do
|
|
rel="${log#"$ROOT"/}"; dst="$TRASH/$night/$rel"; mkdir -p "$(dirname "$dst")"
|
|
mv -f "$log" "$dst" && stale=$((stale + 1))
|
|
done < <(find "$ROOT/streams" -maxdepth 2 \( -name '*.normalize-failed.log' -o -name '*.mp4.part' \) -print0 2>/dev/null)
|
|
find "$ROOT/streams" -mindepth 2 -type d -name .live -empty -delete 2>/dev/null
|
|
find "$ROOT/streams" -mindepth 2 -maxdepth 2 -type d -empty -delete 2>/dev/null
|
|
vglog info "event=wipe_done recordings=${n:-0} stale_files=$stale trash=\"$TRASH/$night\" undo_days=3"
|
|
}
|
|
|
|
# Same survey, as JSON. bin/dumpd.py serves this to the web page, so the classification lives in exactly
|
|
# one place and the browser and the CLI can never disagree about why something is stuck.
|
|
emit_json() {
|
|
survey | python3 -c '
|
|
import json, sys
|
|
rows = []
|
|
for line in sys.stdin:
|
|
parts = line.rstrip("\n").split("|", 6)
|
|
if len(parts) != 7:
|
|
continue
|
|
src, out, state, age, size, dur, reason = parts
|
|
rows.append({"src": src, "out": out, "state": state, "age_s": int(age or 0),
|
|
"bytes": int(size or 0),
|
|
"duration_s": int(dur) if dur not in ("", "-") else None, "reason": reason})
|
|
print(json.dumps(rows))'
|
|
}
|
|
|
|
for a in "$@"; do [[ $a == --durations ]] && export VGSTUCK_DURATIONS=1; done
|
|
|
|
case "${1:-list}" in
|
|
list|"") show "${2:---all}" "${3:-}" ;;
|
|
json) emit_json ;;
|
|
retry) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all|--under <MB>" >&2; exit 1; }; do_retry "$2" "${3:-}" ;;
|
|
drop) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>|--under <MB>" >&2; exit 1; }; do_drop "$2" "${3:-}" ;;
|
|
wipe) do_wipe ;;
|
|
sweep) systemctl start --no-block normalize-sweep.service && vglog info "event=sweep_started" ;;
|
|
*) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
|
|
esac
|