Sweep: crop the timecode band off published streams (re-encode after decode); vgdeband.sh catch-up
The published recording is the game picture + audio + tmcd track, nothing else. Streams without the band are still remuxed losslessly. vgstrip plan exports TC_BAND and marks the sidecar band_removed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
@@ -6,7 +6,10 @@
|
||||
# - anything else (HEVC, 10-bit, VFR, MKV, >1440p, >60fps): full-res re-encode -> name.mp4,
|
||||
# raw source kept in a hidden .src/ subfolder next to it (Jellyfin/copyparty hide dot-dirs)
|
||||
# - streams/<user>/<Title>/.live/<ts>.mp4 (MediaMTX writes here, hidden from Jellyfin/copyparty while live):
|
||||
# when finished -> flat MP4 published as streams/<user>/<Title>__<ts>.mp4 (one file per recording directly
|
||||
# when finished -> the timecode overscan band (if the streamer had it on) is decoded and then CROPPED OFF in a
|
||||
# re-encode (libx264, frame timing preserved, audio copied) so the published file is the game image + audio +
|
||||
# a tmcd timecode track, nothing else; streams without the band are remuxed losslessly. Published as flat
|
||||
# MP4 streams/<user>/<Title>__<ts>.mp4 (one file per recording directly
|
||||
# in the streamer's folder: Jellyfin would merge several files inside one title folder into ONE episode
|
||||
# with "versions"), the .live source deleted, the empty title folder removed.
|
||||
# - Syncthing folders (marker dir .stfolder in the folder root, receive-only): compliant files are left
|
||||
@@ -23,6 +26,9 @@ MINS_IDLE=3
|
||||
# the total missing time is >= this many ms. 0 = never (default; a 3 h re-encode is ~6 h of CPU on the E-2136).
|
||||
# Set via systemd Environment=FILL_GAPS_MS=... ; viewers already see held frames either way.
|
||||
FILL_GAPS_MS="${FILL_GAPS_MS:-0}"
|
||||
# Band removal re-encodes every stream that carried the timecode band. The raw (with band) is deleted after a
|
||||
# successful encode; KEEP_RAW_STREAMS=1 keeps it in the hidden .src/ instead (doubles disk per night).
|
||||
KEEP_RAW_STREAMS="${KEEP_RAW_STREAMS:-0}"
|
||||
MAX_W=2560; MAX_H=1440; MAX_FPS=60
|
||||
exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}"
|
||||
flock -n 9 || exit 0
|
||||
@@ -110,7 +116,7 @@ while IFS= read -r -d '' src; do
|
||||
fi
|
||||
tmp="$out.part"
|
||||
log "$verdict $src ($reason)"
|
||||
tcargs=(); tcjson=""
|
||||
tcargs=(); tcjson=""; band=0; deband=0
|
||||
if (( live )); then
|
||||
# timecode: decode the overscan strip (server clock) or fall back to the server arrival time in the filename
|
||||
stem="$(basename "${src%.*}")"
|
||||
@@ -132,6 +138,11 @@ PYEOF
|
||||
tcjson="${TC_JSON:-}"
|
||||
log "tc $src source=$TC_SOURCE timecode=$TC_TIMECODE"
|
||||
fi
|
||||
# the band has done its job once decoded: remove it from the picture (re-encode with a crop)
|
||||
if [[ ${TC_SOURCE:-} == strip && ${TC_BAND:-0} -gt 0 ]]; then
|
||||
band=$TC_BAND; deband=1; verdict=encode; reason="remove ${band}px timecode band"
|
||||
log "deband $src cropping ${band} rows (re-encode, timing preserved)"
|
||||
fi
|
||||
# opt-in gap filling: switch this stream from remux to a CFR re-encode when the holes add up to enough
|
||||
if (( FILL_GAPS_MS > 0 )) && [[ $verdict == remux && -n $tcjson ]]; then
|
||||
missing=$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get("gaps",{}).get("total_missing_ms",0))' "$tcjson" 2>/dev/null || echo 0)
|
||||
@@ -146,13 +157,18 @@ PYEOF
|
||||
else
|
||||
# scale down into the cap (never up), keep aspect, even dimensions; cap fps; CFR; all audio -> AAC
|
||||
vf="scale='min(iw,$MAX_W)':'min(ih,$MAX_H)':force_original_aspect_ratio=decrease:force_divisible_by=2"
|
||||
(( deband )) && vf="crop=iw:ih-${band}:0:0,$vf" # drop the band rows first (band is even, so sizes stay even)
|
||||
fpsargs=(); fpsnow=$(awk -F/ '{ if ($2>0) printf "%.3f", $1/$2; else print 0 }' <<<"$(ffprobe -v error -select_streams v:0 -show_entries stream=avg_frame_rate -of csv=p=0 "$src")")
|
||||
awk "BEGIN{exit !($fpsnow > $MAX_FPS + 0.01)}" && fpsargs=(-r "$MAX_FPS")
|
||||
# streams keep their exact frame timing (gaps stay gaps unless FILL_GAPS_MS asked for CFR) and their AAC audio as-is
|
||||
fpsmode=cfr; aargs=(-c:a aac -b:a 160k)
|
||||
if (( live )) && [[ $reason != fill-gaps* ]]; then fpsmode=passthrough; fi
|
||||
if (( live )) && [[ $reason != *audio=* ]]; then aargs=(-c:a copy); fi
|
||||
nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \
|
||||
-map 0:v:0 -map '0:a?' \
|
||||
-c:v libx264 -preset veryfast -crf 18 -maxrate 30M -bufsize 60M -pix_fmt yuv420p -profile:v high \
|
||||
-vf "$vf" -fps_mode cfr "${fpsargs[@]}" \
|
||||
-c:a aac -b:a 160k \
|
||||
-vf "$vf" -fps_mode "$fpsmode" "${fpsargs[@]}" \
|
||||
"${aargs[@]}" \
|
||||
"${tcargs[@]}" -movflags +faststart -f mp4 "$tmp"
|
||||
fi
|
||||
rc=$?
|
||||
@@ -163,11 +179,11 @@ PYEOF
|
||||
if [[ -n $tcjson ]]; then # sidecar: exact capture time (read by the Jellyfin plugin) + drift samples
|
||||
printf '%s\n' "$tcjson" > "${out%.*}.json"; chown 1000:1000 "${out%.*}.json" 2>/dev/null
|
||||
fi
|
||||
if [[ $verdict == encode ]]; then # gap-filled re-encode: keep the raw recording in the hidden .src/
|
||||
if [[ $verdict == encode ]] && (( ! deband || KEEP_RAW_STREAMS )); then # gap-filled re-encode (or asked to): keep the raw in the hidden .src/
|
||||
srcdir="$(dirname "$out")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null
|
||||
mv -f "$src" "$srcdir/$(basename "$out" .mp4).raw.mp4"
|
||||
else
|
||||
rm -f "$src"
|
||||
rm -f "$src" # band removal: the cropped file IS the recording; raw-with-band is not kept
|
||||
fi
|
||||
rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true # drop .live and the empty title folder
|
||||
log "done $out"; continue
|
||||
|
||||
43
bin/vgdeband.sh
Executable file
43
bin/vgdeband.sh
Executable file
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
# One-off / catch-up: remove the timecode overscan band from stream recordings that were published BEFORE the
|
||||
# sweep learned to crop it. Re-encodes (libx264, frame timing preserved, audio copied), re-applies the timecode
|
||||
# from the sidecar, replaces the file, marks the sidecar band_removed. Skips files whose sidecar already says so.
|
||||
# vgdeband.sh all of /srv/media/streams/*/*.mp4 with a strip sidecar
|
||||
# vgdeband.sh FILE.mp4 ... just these
|
||||
# Long: a 3 h 1440p60 night is a few hours of CPU. Run detached: sudo systemd-run --unit vgdeband -p Nice=15 \
|
||||
# -p IOSchedulingClass=idle /usr/local/bin/vgdeband.sh ; follow with journalctl -fu vgdeband
|
||||
set -uo pipefail
|
||||
ROOT="${MEDIA_ROOT:-/srv/media}"
|
||||
log() { echo "$(date '+%F %T') $*"; }
|
||||
dur() { ffprobe -v error -show_entries format=duration -of csv=p=0 "$1" 2>/dev/null | cut -d. -f1; }
|
||||
if (( $# )); then files=("$@"); else mapfile -t files < <(find "$ROOT/streams" -mindepth 2 -maxdepth 2 -type f -name '*.mp4' | sort); fi
|
||||
for f in "${files[@]}"; do
|
||||
side="${f%.*}.json"
|
||||
[[ -f $side ]] || { log "skip $f (no sidecar)"; continue; }
|
||||
read -r src removed tc ctime < <(python3 -c '
|
||||
import json,sys; d=json.load(open(sys.argv[1]))
|
||||
print(d.get("source","none"), 1 if d.get("band_removed") else 0, d.get("timecode",""), d.get("timestamp",""))' "$side")
|
||||
[[ $src == strip ]] || { log "skip $f (source=$src)"; continue; }
|
||||
(( removed )) && { log "skip $f (band already removed)"; continue; }
|
||||
fuser -s "$f" 2>/dev/null && { log "skip $f (in use)"; continue; }
|
||||
w=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of csv=p=0 "$f")
|
||||
band=$(python3 /usr/local/bin/vgstrip.py band "$w")
|
||||
tmp="$f.part"
|
||||
tcargs=(); [[ -n $tc ]] && tcargs=(-timecode "$tc" -write_tmcd 1); [[ -n $ctime ]] && tcargs+=(-metadata "creation_time=$ctime")
|
||||
log "deband $f (${w}px wide -> crop ${band} rows; tc=$tc)"
|
||||
nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$f" \
|
||||
-map 0:v:0 -map '0:a?' \
|
||||
-c:v libx264 -preset veryfast -crf 18 -maxrate 30M -bufsize 60M -pix_fmt yuv420p -profile:v high \
|
||||
-vf "crop=iw:ih-${band}:0:0" -fps_mode passthrough -c:a copy \
|
||||
"${tcargs[@]}" -movflags +faststart -f mp4 "$tmp"
|
||||
rc=$?; d1=$(dur "$f"); d2=$(dur "$tmp")
|
||||
if (( rc == 0 )) && [[ -n $d1 && -n $d2 ]] && (( d2 >= d1 - 2 && d2 <= d1 + 2 )); then
|
||||
mv -f "$tmp" "$f"; chown 1000:1000 "$f" 2>/dev/null
|
||||
python3 -c '
|
||||
import json,sys; p=sys.argv[1]; d=json.load(open(p)); d["band_rows"]=int(sys.argv[2]); d["band_removed"]=True
|
||||
json.dump(d, open(p,"w"))' "$side" "$band"; chown 1000:1000 "$side" 2>/dev/null
|
||||
log "done $f"
|
||||
else
|
||||
rm -f "$tmp"; log "FAILED $f rc=$rc src_dur=${d1:-?} out_dur=${d2:-?} (file left untouched)" >&2
|
||||
fi
|
||||
done
|
||||
@@ -162,6 +162,11 @@ def plan(path, arrival_ms=None, tz="America/New_York"):
|
||||
else:
|
||||
t0 = None; src = "none"
|
||||
out = {"source": src, "fps": r.get("fps"), "drift_ms": r.get("drift_ms", 0), "samples": r.get("samples", [])}
|
||||
if src == "strip":
|
||||
# the band is removed (cropped off in a re-encode) by the sweep once it has been decoded; record it
|
||||
_, band = geometry(r["width"]) if r.get("width") else geometry(probe(path)[0])
|
||||
out.update({"band_rows": band, "band_removed": True})
|
||||
print(f"TC_BAND={band}")
|
||||
try:
|
||||
out["gaps"] = find_gaps(path, r.get("fps") or 60.0)
|
||||
except Exception:
|
||||
@@ -181,4 +186,6 @@ if __name__ == "__main__":
|
||||
if len(sys.argv) < 2: print(__doc__); sys.exit(2)
|
||||
if sys.argv[1] == "plan":
|
||||
plan(sys.argv[2], int(sys.argv[3]) if len(sys.argv) > 3 and sys.argv[3] else None); sys.exit(0)
|
||||
if sys.argv[1] == "band": # band rows for a canvas width (vgdeband.sh)
|
||||
print(geometry(int(sys.argv[2]))[1]); sys.exit(0)
|
||||
print(json.dumps(analyse(sys.argv[1]), indent=2))
|
||||
|
||||
Reference in New Issue
Block a user