From 473247e7109ba44ada4bf4720df320dd3d035e9f Mon Sep 17 00:00:00 2001 From: Brandyn / Techy Date: Sat, 31 Oct 2026 16:19:25 -0500 Subject: [PATCH] =?UTF-8?q?Sweep:=20opt-in=20gap=20filling=20for=20streams?= =?UTF-8?q?=20(FILL=5FGAPS=5FMS)=20=E2=80=94=20CFR=20re-encode=20with=20du?= =?UTF-8?q?plicated=20frames,=20raw=20kept=20in=20.src/,=20timecode=20carr?= =?UTF-8?q?ied;=20default=20off?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- README.md | 13 ++++++++----- bin/normalize-sweep.sh | 19 ++++++++++++++++++- systemd/normalize-sweep.service | 1 + 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c07d189..b25c062 100644 --- a/README.md +++ b/README.md @@ -315,11 +315,14 @@ Viewing: the live portal crops the band; Jellyfin shows it as a thin strip at th Cross-check when it matters: everyone is in the same voice chat, so audio-waveform sync is sample-accurate. Why continuous and not a start burst: a PC clock drifts vs the server (10 ppm ≈ 4 frames over 2 h). -Dropped frames / gaps: OBS keeps stamping frames from its clock, so after a drop the file simply has a hole and -every frame that exists is still at the right time — the timecode stays correct. The sweep **never re-encodes a -stream** to fill holes (hours of CPU); the sidecar lists the gaps (`gaps.count`, `total_missing_ms`, first 50 -positions) so editors know where a clip may need a nudge. Premiere preserves timing on such files by default; -in Resolve check "Timecode" interpretation on gappy clips. Clips (short) are still converted to constant frame rate. +Dropped frames / gaps: OBS keeps stamping frames from its clock, so after a drop the file has a hole and every +frame that exists is still at the right time — the timecode stays correct, and players (Jellyfin, the portal) +hold the last frame across the hole with audio in sync. By default the sweep does **not** re-encode a stream to +fill holes (hours of CPU on this box); the sidecar lists them (`gaps.count`, `total_missing_ms`, first 50 +positions). Premiere preserves timing on such files; in Resolve check the clip's frame-rate interpretation. +Opt-in filling: `Environment=FILL_GAPS_MS=1000` in `normalize-sweep.service` makes the sweep re-encode any +stream whose holes total ≥ 1 s to constant frame rate with duplicated frames (raw kept in `.src/`, timecode +carried over). Sensible once the box has a hardware encoder (SYS-3). Clips are always converted to CFR. ## Layout on the server diff --git a/bin/normalize-sweep.sh b/bin/normalize-sweep.sh index 3f662d1..56ebaf1 100755 --- a/bin/normalize-sweep.sh +++ b/bin/normalize-sweep.sh @@ -19,6 +19,10 @@ set -uo pipefail ROOT="${MEDIA_ROOT:-/srv/media}" MINS_IDLE=3 +# Gap filling for STREAMS: re-encode to constant frame rate (duplicating frames across dropped-frame holes) when +# the total missing time is >= this many ms. 0 = never (default; a 3 h re-encode is ~6 h of CPU on the E-2136). +# Set via systemd Environment=FILL_GAPS_MS=... ; viewers already see held frames either way. +FILL_GAPS_MS="${FILL_GAPS_MS:-0}" MAX_W=2560; MAX_H=1440; MAX_FPS=60 exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}" flock -n 9 || exit 0 @@ -128,6 +132,13 @@ PYEOF tcjson="${TC_JSON:-}" log "tc $src source=$TC_SOURCE timecode=$TC_TIMECODE" fi + # opt-in gap filling: switch this stream from remux to a CFR re-encode when the holes add up to enough + if (( FILL_GAPS_MS > 0 )) && [[ $verdict == remux && -n $tcjson ]]; then + missing=$(python3 -c 'import json,sys; d=json.loads(json.loads(sys.argv[1])); print(d.get("gaps",{}).get("total_missing_ms",0))' "$tcjson" 2>/dev/null || echo 0) + if (( missing >= FILL_GAPS_MS )); then + verdict=encode; reason="fill-gaps ${missing}ms"; log "gaps $src ${missing} ms missing -> CFR re-encode (FILL_GAPS_MS=$FILL_GAPS_MS)" + fi + fi fi if [[ $verdict == remux ]]; then nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \ @@ -152,7 +163,13 @@ PYEOF if [[ -n $tcjson ]]; then # sidecar: exact capture time (read by the Jellyfin plugin) + drift samples printf '%s\n' "$tcjson" > "${out%.*}.json"; chown 1000:1000 "${out%.*}.json" 2>/dev/null fi - rm -f "$src"; rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true # drop .live and the empty title folder + if [[ $verdict == encode ]]; then # gap-filled re-encode: keep the raw recording in the hidden .src/ + srcdir="$(dirname "$out")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null + mv -f "$src" "$srcdir/$(basename "$out" .mp4).raw.mp4" + else + rm -f "$src" + fi + rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true # drop .live and the empty title folder log "done $out"; continue elif (( synced )); then rel_src="${src#"$stroot"/}"; rel_out="${out#"$stroot"/}" diff --git a/systemd/normalize-sweep.service b/systemd/normalize-sweep.service index 58d4c9e..4c9c0a4 100644 --- a/systemd/normalize-sweep.service +++ b/systemd/normalize-sweep.service @@ -8,3 +8,4 @@ Type=oneshot ExecStart=/usr/local/bin/normalize-sweep.sh Nice=15 IOSchedulingClass=idle +# Environment=FILL_GAPS_MS=1000 # opt-in: CFR re-encode streams whose dropped-frame holes total >= 1 s (hours of CPU on this box)