#!/bin/bash # Normalize every finished original so editors always get the same kind of file: # MP4 · H.264 8-bit 4:2:0 · constant frame rate · AAC audio · index up front · ≤2560x1440 · ≤60 fps # - compliant files: untouched # - fragmented/unindexed MP4 (MediaMTX output) with good codecs: lossless remux, replaced in place # - anything else (HEVC, 10-bit, VFR, MKV, >1440p, >60fps): full-res re-encode -> name.mp4, # raw source kept in a hidden .src/ subfolder next to it (Jellyfin/copyparty hide dot-dirs) # - streams///.live/<ts>.mp4 (MediaMTX writes here, hidden from Jellyfin/copyparty while live): # when finished -> the timecode overscan band (if the streamer had it on) is decoded and then CROPPED OFF in a # re-encode (libx264, frame timing preserved, audio copied) so the published file is the game image + audio + # a tmcd timecode track, nothing else; streams without the band are remuxed losslessly. Published as flat # MP4 streams/<user>/<Title>__<ts>.mp4 (one file per recording directly # in the streamer's folder: Jellyfin would merge several files inside one title folder into ONE episode # with "versions"), the .live source deleted, the empty title folder removed. # - Syncthing folders (marker dir .stfolder in the folder root, receive-only): compliant files are left # alone (zero copies); a non-compliant file is converted to name.mp4 (name.h264.mp4 if the raw is # already .mp4), the raw's name is appended to .stlocal (an #include'd ignore list) so Syncthing # drops it without flagging or re-pulling, then the raw is deleted. One copy on the server. # - reconnect fragments of ONE stream (same user, same title, fewer than STITCH_MAX_PARTS (500) files, gaps under # STITCH_MAX_GAP_S) are first stitched back into one file by vgstitch.py (lossless concat; the arrival-time # file names say where every piece sits), then published as one recording. Joins are recorded in the sidecar. # Nothing in a stream's .live/ is touched until the stream is over (no open or recently written file). # - a LONE stream recording under MIN_LIVE_SECONDS (10) is a reconnect stub with nothing to join to: moved to # trash, never published. Only .live/ files — clips are frequently a few seconds long and are never subject to this. # - failure: name.normalize-failed.log is written and the file is skipped until that log is deleted # Idempotent; safe every few minutes. set -uo pipefail ROOT="${MEDIA_ROOT:-/srv/media}" MINS_IDLE=3 # Gap filling for STREAMS: re-encode to constant frame rate (duplicating frames across dropped-frame holes) when # the total missing time is >= this many ms. 0 = never (default; a 3 h re-encode is ~6 h of CPU on the E-2136). # Set via systemd Environment=FILL_GAPS_MS=... ; viewers already see held frames either way. FILL_GAPS_MS="${FILL_GAPS_MS:-0}" # Band removal re-encodes every stream that carried the timecode band. The raw (with band) is deleted after a # successful encode; KEEP_RAW_STREAMS=1 keeps it in the hidden .src/ instead (doubles disk per night). KEEP_RAW_STREAMS="${KEEP_RAW_STREAMS:-0}" # Encoder threads for every re-encode (clips and streams). 2 leaves the box responsive for Jellyfin/live viewers; # conversions take longer. systemd Environment=ENCODE_THREADS=... to change. ENCODE_THREADS="${ENCODE_THREADS:-2}" # STREAM recordings shorter than this are reconnect stubs, not recordings: every time a publisher drops and # comes back, MediaMTX opens a new file, and one bad night left 471 one- and two-second files in .live/. # They go to trash (3-day undo) instead of being published. Applies ONLY to .live/ — clips are often a few # seconds long on purpose and are never touched by this. 0 disables. systemd Environment=MIN_LIVE_SECONDS=… MIN_LIVE_SECONDS="${MIN_LIVE_SECONDS:-10}" # Stitching (vgstitch.py): fewer than this many fragments in one stream's .live/ are joined into one recording. # 500, not 20: on a lossy uplink MediaMTX restarts the recorder every ~10 s (a keyframe whose slices were lost # leaves an access unit with only SPS/PPS, which the recorder refuses), so a half-hour stream is ~180 same- # parameter files two seconds apart — exactly the case for stitching. The gap rule (STITCH_MAX_GAP_S seconds # between fragments = separate recordings) is the real guard. 0 parts disables stitching. systemd Environment=… STITCH_MAX_PARTS="${STITCH_MAX_PARTS:-500}" STITCH_MAX_GAP_S="${STITCH_MAX_GAP_S:-600}" MAX_W=2560; MAX_H=1440; MAX_FPS=60 exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}" flock -n 9 || exit 0 # Metrics (LOGGING.md): counters for dumpd's /metrics. Optional on purpose — if the helper is not installed # yet, these become no-ops and normalising carries on exactly as before. if [[ -r "$(dirname "$0")/vgmetrics.sh" ]]; then VG_MET_COMPONENT=sweep . "$(dirname "$0")/vgmetrics.sh" && vgmet_init else vgmet_add() { :; }; vgmet_set() { :; }; vgmet_time() { echo 0; }; vgmet_since() { echo 0; } fi # logfmt (LOGGING.md): vglog <level> event=<verb> key=value ... values with spaces must arrive pre-quoted vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=sweep %s\n' "$(date -Is)" "$lvl" "$*"; } log() { vglog info "msg=\"$*\""; } dur() { ffprobe -v error -show_entries format=duration -of csv=p=0 "$1" 2>/dev/null | cut -d. -f1; } # A stream's fragments are one recording: join them before anything below looks at them one by one. if (( STITCH_MAX_PARTS > 0 )) && [[ -r "$(dirname "$0")/vgstitch.py" ]]; then # installed side by side python3 "$(dirname "$0")/vgstitch.py" "$ROOT" --max-parts "$STITCH_MAX_PARTS" --max-gap-s "$STITCH_MAX_GAP_S" --idle-min "$MINS_IDLE" fi # --- probe: prints "verdict reason" where verdict ∈ ok|remux|encode|skip ---------------------- probe() { local f="$1" ext="${1##*.}" local json; json=$(ffprobe -v error -show_entries stream=codec_type,codec_name,pix_fmt,width,height,r_frame_rate,avg_frame_rate -of json "$f" 2>/dev/null) || { echo "skip unreadable"; return; } local vcodec pix w h rfr afr acodecs vcodec=$(jq -r '[.streams[]|select(.codec_type=="video")][0].codec_name // "none"' <<<"$json") pix=$(jq -r '[.streams[]|select(.codec_type=="video")][0].pix_fmt // ""' <<<"$json") w=$(jq -r '[.streams[]|select(.codec_type=="video")][0].width // 0' <<<"$json") h=$(jq -r '[.streams[]|select(.codec_type=="video")][0].height // 0' <<<"$json") rfr=$(jq -r '[.streams[]|select(.codec_type=="video")][0].r_frame_rate // "0/1"' <<<"$json") afr=$(jq -r '[.streams[]|select(.codec_type=="video")][0].avg_frame_rate // "0/1"' <<<"$json") acodecs=$(jq -r '[.streams[]|select(.codec_type=="audio")|.codec_name]|unique|join(",")' <<<"$json") [[ $vcodec == none ]] && { echo "skip no-video"; return; } # Take the LOWER of r_frame_rate and avg_frame_rate. Either is unreliable alone: avg is garbage on a # fragmented or looped recording (one reported 95 fps for a 30 fps source, which sent it down the fps-cap # path and broke the encode), and r is a huge nominal value on genuine VFR. The lower of the two is the # honest answer to "is this really above 60". local fps; fps=$(awk -v rfr="$rfr" -v afr="$afr" 'BEGIN{ split(rfr, a, "/"); split(afr, b, "/") r = (a[2] > 0) ? a[1]/a[2] : 0; v = (b[2] > 0) ? b[1]/b[2] : 0 if (r > 0 && v > 0) printf "%.3f", (r < v) ? r : v; else printf "%.3f", (r > 0) ? r : v }') local reasons=() [[ $vcodec == h264 ]] || reasons+=("codec=$vcodec") [[ $pix == yuv420p ]] || reasons+=("pix=$pix") (( w <= MAX_W && h <= MAX_H )) || reasons+=("size=${w}x${h}") awk "BEGIN{exit !($fps > $MAX_FPS + 0.01)}" && reasons+=("fps=$fps") [[ -z $acodecs || $acodecs == aac ]] || reasons+=("audio=$acodecs") [[ ${ext,,} == mp4 ]] || reasons+=("container=$ext") if (( ${#reasons[@]} )); then echo "encode ${reasons[*]}"; return; fi # codecs fine. Fragmented MP4 (MediaMTX output) -> lossless remux. ffprobe's frame-rate figures are # meaningless on fragmented files, so the VFR check only runs on flat files (i.e. after the remux). local head; head=$(head -c 2097152 "$f" | grep -aob -m3 -E 'moov|mdat|moof' | tr '\n' ' ') if grep -q moof <<<"$head"; then echo "remux fragmented"; return; fi # real VFR test: do the gaps between the first ~400 frames vary? (dts is monotonic even with B-frames) # portable awk (Debian's mawk has no asort): deltas -> sort -n -> median + spread local vfr; vfr=$(ffprobe -v error -select_streams v:0 -show_entries packet=dts_time -of csv=p=0 -read_intervals '%+#400' "$f" 2>/dev/null | awk 'NF && $1!="N/A" { if (p!="") print $1-p; p=$1 }' | sort -n | awk '{ d[n++]=$1 } END { if (n<20) { print "cfr"; exit } med=d[int(n/2)]; if (med<=0) { print "cfr"; exit } bad=0; for(i=0;i<n;i++){ x=d[i]-med; if(x<0)x=-x; if (x > med*0.25) bad++ } print (bad > n*0.02) ? "vfr" : "cfr" }') # streams: gaps from dropped frames are legitimate and re-encoding a 3 h file is hours of CPU — never VFR-encode # them (the timecode sidecar lists the gaps instead). Clips (short, often genuinely VFR) still get CFR. if [[ $vfr == vfr && $f != "$ROOT/streams/"* ]]; then echo "encode vfr"; return; fi local moov mdat; moov=$(grep -o '[0-9]*:moov' <<<"$head" | cut -d: -f1 | head -1); mdat=$(grep -o '[0-9]*:mdat' <<<"$head" | cut -d: -f1 | head -1) if [[ -z $moov || ( -n $mdat && $mdat -lt $moov ) ]]; then echo "remux index-last"; return; fi echo "ok" } while IFS= read -r -d '' src; do base="${src%.*}"; ext="${src##*.}" live=0; stream="" if [[ $(basename "$(dirname "$src")") == .live ]]; then live=1 titledir="$(dirname "$(dirname "$src")")" # streams/<user>/<Title> stream="$(basename "$(dirname "$titledir")")/$(basename "$titledir")" # user/Title, the log correlation key base="$(dirname "$titledir")/$(basename "$titledir")__$(basename "${src%.*}")" # streams/<user>/<Title>__<ts> fi synced=0; stroot="" d="$(dirname "$src")" while [[ $d == "$ROOT"/* ]]; do # find the Syncthing folder root, if any if [[ -d "$d/.stfolder" ]]; then synced=1; stroot="$d"; break; fi d="$(dirname "$d")" done if [[ -e "$base.normalize-failed.log" ]]; then # Say so every run. Skipping silently meant a parked file produced no log line after its original # failure, so Kibana could only ever show a count and never which recording. bin/vgstuck.sh triages it. vglog info "event=parked file=\"$src\" stream=\"$stream\" log=\"$base.normalize-failed.log\"" continue fi fuser -s "$src" 2>/dev/null && continue # still being written [[ -e "$src" ]] || continue # dumped meanwhile if (( live )); then # The stream is still going while any sibling fragment is open or fresh: publishing this piece now would # make it a separate recording that the stitcher can never join. Wait for the stream to end. going=0 for sib in "$(dirname "$src")"/*.mp4; do [[ -e $sib && $sib != "$src" ]] || continue if fuser -s "$sib" 2>/dev/null || [[ -n $(find "$sib" -mmin -"$MINS_IDLE" 2>/dev/null) ]]; then going=1; break; fi done if (( going )); then vglog info "event=defer reason=\"stream still going\" file=\"$src\" stream=\"$stream\""; continue; fi fi if (( live && MIN_LIVE_SECONDS > 0 )); then secs=$(dur "$src"); secs=${secs:-0} if (( secs < MIN_LIVE_SECONDS )); then night="$(date +%F)"; rel="${src#"$ROOT"/}"; dst="$ROOT/trash/$night/$rel" mkdir -p "$(dirname "$dst")" mv -f "$src" "$dst" [[ -e "${src%.*}.json" ]] && mv -f "${src%.*}.json" "$(dirname "$dst")/" rm -f "${src%.*}.stitch.json" rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true vgmet_add "jobs:discard:done:stream" vglog info "event=discarded reason=too_short seconds=$secs min=$MIN_LIVE_SECONDS file=\"$src\" stream=\"$stream\" trash=\"$dst\"" continue fi fi read -r verdict reason <<<"$(probe "$src")" # Keep what the PROBE said: the deband branch below overwrites $reason with its own text, which silently # loses the "audio=<codec>" marker. That marker is the only thing telling the encode to convert the audio, # so without this a Go Live stream with the timecode band AND Opus audio got -c:a copy and shipped Opus # inside an MP4 — legal, but editors choke on it. probe_reason="$reason" if (( live )); then [[ $verdict == skip ]] && continue [[ $verdict == ok ]] && verdict=remux # must leave .live/ either way out="$base.mp4"; [[ -e "$out" ]] && { vglog info "event=skip reason=exists file=\"$out\" src=\"$src\""; continue; } else case "$verdict" in ok|skip) continue ;; esac out="$base.mp4" fi if (( synced )); then [[ $verdict == remux ]] && continue # synced + only an index issue: not worth touching [[ ${ext,,} == mp4 ]] && out="$base.h264.mp4" # raw is already name.mp4: don't collide [[ -e "$out" ]] && continue fi tmp="$out.part" kind=clip; (( live )) && kind=stream # metric label: stream encodes are the expensive ones vglog info "event=verdict verdict=$verdict file=\"$src\" reason=\"$reason\" stream=\"$stream\"" tcargs=(); tcjson=""; band=0; deband=0 if (( live )); then # timecode: decode the overscan strip (server clock) or fall back to the server arrival time in the filename stem="$(basename "${src%.*}")" arrival_ms=$(python3 - "$stem" <<'PYEOF' import re, sys from datetime import datetime from zoneinfo import ZoneInfo m = re.search(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})(?:-(\d{1,6}))?", sys.argv[1]) if not m: sys.exit(0) y, mo, d, h, mi, s, us = m.groups() us = int((us or "0").ljust(6, "0")) t = datetime(int(y), int(mo), int(d), int(h), int(mi), int(s), us, tzinfo=ZoneInfo("America/New_York")) print(int(t.timestamp() * 1000)) PYEOF ) eval "$(python3 /usr/local/bin/vgstrip.py plan "$src" "${arrival_ms:-}" 2>/dev/null || true)" if [[ -n ${TC_TIMECODE:-} ]]; then tcargs=(-timecode "$TC_TIMECODE" -metadata "creation_time=$TC_CREATION" -write_tmcd 1) tcjson="${TC_JSON:-}" vglog info "event=timecode file=\"$src\" stream=\"$stream\" source=$TC_SOURCE timecode=\"$TC_TIMECODE\"" fi # the band has done its job once decoded: remove it from the picture (re-encode with a crop) if [[ ${TC_SOURCE:-} == strip && ${TC_BAND:-0} -gt 0 ]]; then band=$TC_BAND; deband=1; verdict=encode; reason="remove ${band}px timecode band" vglog info "event=deband file=\"$src\" stream=\"$stream\" band_rows=$band" fi # opt-in gap filling: switch this stream from remux to a CFR re-encode when the holes add up to enough if (( FILL_GAPS_MS > 0 )) && [[ $verdict == remux && -n $tcjson ]]; then missing=$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get("gaps",{}).get("total_missing_ms",0))' "$tcjson" 2>/dev/null || echo 0) if (( missing >= FILL_GAPS_MS )); then verdict=encode; reason="fill-gaps ${missing}ms"; vglog info "event=fill_gaps file=\"$src\" stream=\"$stream\" missing_ms=$missing" fi fi fi job_t0=$(vgmet_time) err=$(mktemp /tmp/vgsweep-err.XXXXXX) # ffmpeg's own words; rc= alone is not diagnosable if [[ $verdict == remux ]]; then nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \ -map 0 -c copy "${tcargs[@]}" -movflags +faststart -f mp4 "$tmp" 2>"$err" else # scale down into the cap (never up), keep aspect, even dimensions; cap fps; CFR; all audio -> AAC vf="scale='min(iw,$MAX_W)':'min(ih,$MAX_H)':force_original_aspect_ratio=decrease:force_divisible_by=2" (( deband )) && vf="crop=iw:ih-${band}:0:0,$vf" # drop the band rows first (band is even, so sizes stay even) fpsargs=() read -r fps_r fps_a < <(ffprobe -v error -select_streams v:0 -show_entries stream=r_frame_rate,avg_frame_rate -of csv=p=0 "$src" | tr ',' ' ') fpsnow=$(awk -v rfr="${fps_r:-0/1}" -v afr="${fps_a:-0/1}" 'BEGIN{ split(rfr, a, "/"); split(afr, b, "/") r = (a[2] > 0) ? a[1]/a[2] : 0; v = (b[2] > 0) ? b[1]/b[2] : 0 if (r > 0 && v > 0) printf "%.3f", (r < v) ? r : v; else printf "%.3f", (r > 0) ? r : v }') awk "BEGIN{exit !($fpsnow > $MAX_FPS + 0.01)}" && fpsargs=(-r "$MAX_FPS") # streams keep their exact frame timing (gaps stay gaps unless FILL_GAPS_MS asked for CFR) and their AAC audio as-is fpsmode=cfr; aargs=(-c:a aac -b:a 160k) if (( live )) && [[ $reason != fill-gaps* ]]; then fpsmode=passthrough; fi # Capping the rate means re-timing, which is by definition not passthrough. ffmpeg refuses the pair # outright ("One of -r/-fpsmax was specified together a non-CFR -vsync/-fps_mode") and fails with # "Error opening output files: Invalid argument", so the cap has to win. (( ${#fpsargs[@]} )) && fpsmode=cfr if (( live )) && [[ $probe_reason != *audio=* ]]; then aargs=(-c:a copy); fi nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \ -map 0:v:0 -map '0:a?' \ -c:v libx264 -preset veryfast -crf 18 -maxrate 30M -bufsize 60M -pix_fmt yuv420p -profile:v high -threads "$ENCODE_THREADS" \ -vf "$vf" -fps_mode "$fpsmode" "${fpsargs[@]}" \ "${aargs[@]}" \ "${tcargs[@]}" -movflags +faststart -f mp4 "$tmp" 2>"$err" fi rc=$? job_secs=$(vgmet_since "$job_t0") d1=$(dur "$src"); d2=$(dur "$tmp") if (( rc == 0 )) && [[ -n $d1 && -n $d2 ]] && (( d2 >= d1 - 2 && d2 <= d1 + 2 )); then vgmet_add "jobs:$verdict:done:$kind"; vgmet_add "seconds:$verdict:$kind" "$job_secs" (( deband )) && vgmet_add deband 1 if (( live )); then mv -f "$tmp" "$out"; chown 1000:1000 "$out" 2>/dev/null stitchjson="${src%.*}.stitch.json" # vgstitch.py: parts, offsets and the time missing at each join if [[ -n $tcjson || -e $stitchjson ]]; then # sidecar: exact capture time (read by the Jellyfin plugin) + drift samples python3 - "$tcjson" "$stitchjson" > "${out%.*}.json" <<'PYEOF' import json, sys, os tc, sj = sys.argv[1], sys.argv[2] d = json.loads(tc) if tc else {} if os.path.exists(sj): d["stitched"] = json.load(open(sj)) # the strip decode's drift figure spans the joins too, so it is not clock drift on a stitched file if "drift_ms" in d: d["drift_ms_note"] = "includes collapsed joins; see stitched.joins" print(json.dumps(d)) PYEOF chown 1000:1000 "${out%.*}.json" 2>/dev/null; rm -f "$stitchjson" fi if [[ $verdict == encode ]] && (( ! deband || KEEP_RAW_STREAMS )); then # gap-filled re-encode (or asked to): keep the raw in the hidden .src/ srcdir="$(dirname "$out")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null mv -f "$src" "$srcdir/$(basename "$out" .mp4).raw.mp4" else rm -f "$src" # band removal: the cropped file IS the recording; raw-with-band is not kept fi rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true # drop .live and the empty title folder rm -f "$err" vglog info "event=published file=\"$out\" stream=\"$stream\" verdict=$verdict"; continue elif (( synced )); then rel_src="${src#"$stroot"/}"; rel_out="${out#"$stroot"/}" { echo "/$rel_out"; echo "/$rel_src"; } >> "$stroot/.stlocal" # ignore our output and the raw chown 1000:1000 "$stroot/.stlocal" 2>/dev/null sleep 2 # let Syncthing re-read ignores before the delete mv -f "$tmp" "$out" rm -f "$src" # raw stays on the gamer's PC; one copy here elif [[ $verdict == remux ]]; then mv -f "$tmp" "$src" # lossless: replace in place else srcdir="$(dirname "$src")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null mv -f "$src" "$srcdir/$(basename "$src")" # keep raw source, hidden from editors/Jellyfin mv -f "$tmp" "$out" fi chown 1000:1000 "$out" 2>/dev/null vglog info "event=done file=\"$out\" verdict=$verdict" else rm -f "$tmp" vgmet_add "jobs:$verdict:failed:$kind"; vgmet_add "seconds:$verdict:$kind" "$job_secs" { echo "$(date '+%F %T') $verdict failed rc=$rc src_dur=${d1:-?} out_dur=${d2:-?} ($reason)" echo "--- ffmpeg stderr ---"; tail -40 "$err" 2>/dev/null; } > "$base.normalize-failed.log" why=$(tail -1 "$err" 2>/dev/null | tr -d '"' | cut -c1-200) vglog error "event=failed file=\"$src\" stream=\"$stream\" verdict=$verdict rc=$rc reason=\"${why:-no stderr}\" log=\"$base.normalize-failed.log\"" >&2 fi rm -f "$err" done < <(find "$ROOT/streams" "$ROOT/clips" -type f \ \( -iname '*.mp4' -o -iname '*.mkv' -o -iname '*.mov' -o -iname '*.ts' -o -iname '*.m4v' -o -iname '*.webm' \) \ ! -path '*/.src/*' ! -path '*/.stfolder/*' ! -path '*/.stversions/*' ! -name '*.part' -mmin +"$MINS_IDLE" -print0 2>/dev/null)