techy/Media-Server-Setup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bunch of shit

Browse Source
This commit is contained in:
Brandyn / Techy
2026-11-06 19:05:47 -06:00
parent ca01516d42
commit 630b253278
29 changed files with 2793 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
bin/vgstuck.sh
View File

@@ -2,15 +2,23 @@
# Triage recordings that never published: see why each one is stuck, then retry it or bin it.
#
# vgstuck.sh list everything stuck, with the reason (read-only, the default)
# vgstuck.sh list --durations ...and measure each one's length (slow: fragmented MP4 is read end to end)
# vgstuck.sh retry <match> clear what is blocking it so the sweep tries again
# vgstuck.sh retry --all ...for everything that is parked
# vgstuck.sh drop <match> move the recording to trash: 3-day undo, same as "dump tonight"
# vgstuck.sh list --under 5 only recordings smaller than 5 MB (reconnect fragments)
# vgstuck.sh drop --under 5 bin all of those; still-recording files are never touched
# vgstuck.sh wipe bin EVERYTHING unprocessed: stops the sweep (killing any conversion in flight),
# then moves every .live/ file — parked, waiting, published-collision, fresh — with
# its sidecars, failure logs and .part leftovers to trash, then drops the emptied
# folders. The ONLY thing left alone is a file MediaMTX has open, i.e. a stream
# being recorded this minute. The Dump tab's "Bin all unprocessed" button runs this.
# vgstuck.sh sweep kick the sweep once, without waiting on it
# vgstuck.sh json the same survey as JSON (what dumpd serves to the web page)
#
# <match> is any substring of the path (a streamer name, a title, a timestamp). It must match at least one
# recording, and `drop` refuses --all: binning every stuck recording in one keystroke is never a thing
# somebody meant to do.
# recording, and `drop` refuses --all so a stray pattern cannot bin everything by accident: the deliberate
# version is spelled out as its own word, `wipe`.
#
# WHY A RECORDING STAYS IN .live/
# parked a <Title>__<ts>.normalize-failed.log sits next to the intended output. The sweep skips the
@@ -57,66 +65,118 @@ survey() {
reason="nothing blocking it; the sweep has not got to it"
fi
[[ -e "$out.part" ]] && reason="$reason [leftover .part]"
dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
# Duration is OPT-IN. Everything in .live/ is fragmented MP4 with no duration in its header, so ffprobe
# has to read the whole file to answer — gigabytes per recording. With many stuck files the survey took
# over three minutes, dumpd's request timed out, and the web page broke. Size and age come from stat.
dur=""
if [[ ${VGSTUCK_DURATIONS:-0} == 1 ]]; then
dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
fi
printf '%s|%s|%s|%s|%s|%s|%s\n' "$src" "$out" "$state" \
"$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:-0}" "$reason"
"$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:--}" "$reason"
done < <(find "$ROOT/streams" -path '*/.live/*' -name '*.mp4' -print0 2>/dev/null)
}
show() {
local any=0 src out state age bytes dur reason
local any=0 total=0 src out state age bytes dur reason
printf '\n %-9s %-9s %9s %8s %s\n' state waiting length size recording
while IFS='|' read -r src out state age bytes dur reason; do
total=$(( total + bytes ))
any=1
if [[ $dur == - || -z $dur ]]; then length="-"; else
length=$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 ))); fi
printf ' %-9s %-9s %9s %7.0fM %s\n' "$state" \
"$(( age / 3600 ))h$(printf '%02d' $(( (age % 3600) / 60 )))m" \
"$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 )))" \
"$length" \
"$(awk -v b="$bytes" 'BEGIN{print b/1000000}')" "${src#"$ROOT"/streams/}"
printf ' %s\n' "$reason"
done < <(survey | sort -t'|' -k3,3 -k4,4nr)
(( any )) || printf ' nothing is stuck.\n'
printf '\n retry one: vgstuck.sh retry <substring> bin one: vgstuck.sh drop <substring>\n\n'
done < <(selected "${1:---all}" "${2:-}" | sort -t'|' -k3,3 -k4,4nr)
(( any )) || printf ' nothing matches.\n'
(( any )) && printf ' %s item(s), %s GB\n' "$(selected "${1:---all}" "${2:-}" | wc -l)" "$(awk -v b="$total" 'BEGIN{printf "%.1f", b/1e9}')"
[[ ${VGSTUCK_DURATIONS:-0} == 1 ]] || printf '\n lengths not measured (slow on fragmented MP4): vgstuck.sh list --durations\n'
printf ' retry one: vgstuck.sh retry <substring> bin one: vgstuck.sh drop <substring>\n\n'
}
# rows matching the pattern, or everything when --all
# rows matching the pattern, everything when --all, or everything smaller than N MB with --under N.
# --under exists for reconnect fragments: every time a publisher drops and comes back MediaMTX opens a new
# recording, so a bad night leaves hundreds of one- and two-second files. Still-recording files never match.
selected() {
local pattern="$1"
if [[ $pattern == --all ]]; then survey; else survey | grep -F -- "$pattern" || true; fi
local pattern="$1" limit="${2:-}"
if [[ $pattern == --under ]]; then
[[ $limit =~ ^[0-9]+$ ]] || { echo "usage: --under <megabytes>" >&2; return 2; }
survey | awk -F'|' -v max="$(( limit * 1000000 ))" '$3 != "writing" && $5 < max'
elif [[ $pattern == --all ]]; then survey
else survey | grep -F -- "$pattern" || true
fi
}
do_retry() {
local pattern="$1" n=0 src out state age bytes dur reason
local pattern="$1" limit="${2:-}" n=0 src out state age bytes dur reason
while IFS='|' read -r src out state age bytes dur reason; do
[[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
rm -f "$out.part" # leftover partial output: harmless, but no reason to keep
rm -f "${out%.mp4}.normalize-failed.log" # THIS is what makes the sweep skip the file forever
n=$((n + 1))
vglog info "event=retry file=\"$src\" was=$state"
done < <(selected "$pattern")
done < <(selected "$pattern" "$limit")
(( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
vglog info "event=retry_done count=$n msg=\"run vgstuck.sh sweep, or wait for the timer\""
}
do_drop() {
local pattern="$1" night n=0 src out state age bytes dur reason rel dst
[[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them"' >&2; return 2; }
night="$(date +%F)"
# one row per line on stdin (survey format) -> trash. Prints the count it moved.
drop_rows() {
local night="$1" n=0 src out state age bytes dur reason rel dst victim
while IFS='|' read -r src out state age bytes dur reason; do
[[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
for victim in "$src" "${src%.*}.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
# The survey labels a file by its blocker first (a parked file stays "parked" even while being written), so the
# safety check here does not trust the label: open by any process, or touched inside MINS_IDLE, means leave it.
# In wipe mode (WIPE=1) only "open" counts — the sweep has been stopped, so an open file is MediaMTX recording.
if fuser -s "$src" 2>/dev/null || { [[ ${WIPE:-0} != 1 ]] && { [[ $state == writing ]] || [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; }; }; then
vglog info "event=skip reason=still_recording file=\"$src\"" >&2; continue
fi
for victim in "$src" "${src%.*}.json" "${src%.*}.stitch.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
[[ -e $victim ]] || continue
rel="${victim#"$ROOT"/}"
dst="$TRASH/$night/$rel"
mkdir -p "$(dirname "$dst")"
mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3"
mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3" >&2
done
rmdir "$(dirname "$src")" "$(dirname "$(dirname "$src")")" 2>/dev/null || true
n=$((n + 1))
done < <(selected "$pattern")
done
echo "$n"
}
do_drop() {
local pattern="$1" limit="${2:-}" night n
[[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them, use --under <MB>, or wipe"' >&2; return 2; }
night="$(date +%F)"
n=$(selected "$pattern" "$limit" | drop_rows "$night") # per-file lines go to stderr, the count to stdout
(( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
vglog info "event=drop_done count=$n trash=\"$TRASH/$night\""
}
# Everything unprocessed, deliberately. First the sweep is stopped, which kills any ffmpeg conversion it has in
# flight (the timer starts it again within five minutes, to an empty queue). Then every .live/ file goes, parked
# or fresh, and so do failure logs whose recording is already gone (they would keep the Stuck tab listing ghosts)
# and leftover .part outputs. The one exception: a file some process still holds open after the sweep is down,
# which is MediaMTX recording a live stream — pulling that from under it would lose the stream in progress.
do_wipe() {
local night n stale=0 log rel dst
night="$(date +%F)"
if systemctl is-active --quiet normalize-sweep.service; then
systemctl stop normalize-sweep.service && vglog info "event=cancelled unit=normalize-sweep.service msg=\"conversion in flight killed\""
fi
systemctl is-active --quiet vgdeband.service && systemctl stop vgdeband.service && vglog info "event=cancelled unit=vgdeband.service"
n=$(survey | WIPE=1 drop_rows "$night")
while IFS= read -r -d '' log; do
rel="${log#"$ROOT"/}"; dst="$TRASH/$night/$rel"; mkdir -p "$(dirname "$dst")"
mv -f "$log" "$dst" && stale=$((stale + 1))
done < <(find "$ROOT/streams" -maxdepth 2 \( -name '*.normalize-failed.log' -o -name '*.mp4.part' \) -print0 2>/dev/null)
find "$ROOT/streams" -mindepth 2 -type d -name .live -empty -delete 2>/dev/null
find "$ROOT/streams" -mindepth 2 -maxdepth 2 -type d -empty -delete 2>/dev/null
vglog info "event=wipe_done recordings=${n:-0} stale_files=$stale trash=\"$TRASH/$night\" undo_days=3"
}
# Same survey, as JSON. bin/dumpd.py serves this to the web page, so the classification lives in exactly
# one place and the browser and the CLI can never disagree about why something is stuck.
emit_json() {
@@ -129,15 +189,19 @@ for line in sys.stdin:
continue
src, out, state, age, size, dur, reason = parts
rows.append({"src": src, "out": out, "state": state, "age_s": int(age or 0),
"bytes": int(size or 0), "duration_s": int(dur or 0), "reason": reason})
"bytes": int(size or 0),
"duration_s": int(dur) if dur not in ("", "-") else None, "reason": reason})
print(json.dumps(rows))'
}
for a in "$@"; do [[ $a == --durations ]] && export VGSTUCK_DURATIONS=1; done
case "${1:-list}" in
list|"") show ;;
list|"") show "${2:---all}" "${3:-}" ;;
json) emit_json ;;
retry) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all" >&2; exit 1; }; do_retry "$2" ;;
drop) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>" >&2; exit 1; }; do_drop "$2" ;;
retry) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all|--under <MB>" >&2; exit 1; }; do_retry "$2" "${3:-}" ;;
drop) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>|--under <MB>" >&2; exit 1; }; do_drop "$2" "${3:-}" ;;
wipe) do_wipe ;;
sweep) systemctl start --no-block normalize-sweep.service && vglog info "event=sweep_started" ;;
*) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
esac