techy/Media-Server-Setup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Logging framework: logfmt events in our scripts, container labels + rotated json-file logs, Caddy JSON access logs; LOGGING.md

Browse Source 
Store-agnostic: journald + Docker logs are the only taps any collector (Alloy/Loki, VictoriaLogs, Netdata) needs.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Brandyn / Techy
2026-11-01 19:53:05 -06:00
parent 88bfda4c9d
commit 704bc8c4d3
10 changed files with 186 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
bin/normalize-sweep.sh
View File

@@ -36,7 +36,9 @@ MAX_W=2560; MAX_H=1440; MAX_FPS=60
exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}"
flock -n 9 || exit 0
log() { echo "$(date '+%F %T') $*"; }
# logfmt (LOGGING.md): vglog <level> event=<verb> key=value ... values with spaces must arrive pre-quoted
vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=sweep %s\n' "$(date -Is)" "$lvl" "$*"; }
log() { vglog info "msg=\"$*\""; }
dur() { ffprobe -v error -show_entries format=duration -of csv=p=0 "$1" 2>/dev/null | cut -d. -f1; }
# --- probe: prints "verdict reason" where verdict ∈ ok|remux|encode|skip ----------------------
@@ -87,10 +89,11 @@ find "$ROOT/streams" "$ROOT/clips" -type f \
! -path '*/.src/*' ! -path '*/.stfolder/*' ! -path '*/.stversions/*' ! -name '*.part' -mmin +"$MINS_IDLE" -print0 2>/dev/null |
while IFS= read -r -d '' src; do
base="${src%.*}"; ext="${src##*.}"
live=0
live=0; stream=""
if [[ $(basename "$(dirname "$src")") == .live ]]; then
live=1
titledir="$(dirname "$(dirname "$src")")" # streams/<user>/<Title>
stream="$(basename "$(dirname "$titledir")")/$(basename "$titledir")" # user/Title, the log correlation key
base="$(dirname "$titledir")/$(basename "$titledir")__$(basename "${src%.*}")" # streams/<user>/<Title>__<ts>
fi
synced=0; stroot=""
@@ -107,7 +110,7 @@ while IFS= read -r -d '' src; do
if (( live )); then
[[ $verdict == skip ]] && continue
[[ $verdict == ok ]] && verdict=remux # must leave .live/ either way
out="$base.mp4"; [[ -e "$out" ]] && { log "exists $out (leaving $src)"; continue; }
out="$base.mp4"; [[ -e "$out" ]] && { vglog info "event=skip reason=exists file=\"$out\" src=\"$src\""; continue; }
else
case "$verdict" in ok|skip) continue ;; esac
out="$base.mp4"
@@ -118,7 +121,7 @@ while IFS= read -r -d '' src; do
[[ -e "$out" ]] && continue
fi
tmp="$out.part"
log "$verdict $src ($reason)"
vglog info "event=verdict verdict=$verdict file=\"$src\" reason=\"$reason\" stream=\"$stream\""
tcargs=(); tcjson=""; band=0; deband=0
if (( live )); then
# timecode: decode the overscan strip (server clock) or fall back to the server arrival time in the filename
@@ -139,18 +142,18 @@ PYEOF
if [[ -n ${TC_TIMECODE:-} ]]; then
tcargs=(-timecode "$TC_TIMECODE" -metadata "creation_time=$TC_CREATION" -write_tmcd 1)
tcjson="${TC_JSON:-}"
log "tc $src source=$TC_SOURCE timecode=$TC_TIMECODE"
vglog info "event=timecode file=\"$src\" stream=\"$stream\" source=$TC_SOURCE timecode=\"$TC_TIMECODE\""
fi
# the band has done its job once decoded: remove it from the picture (re-encode with a crop)
if [[ ${TC_SOURCE:-} == strip && ${TC_BAND:-0} -gt 0 ]]; then
band=$TC_BAND; deband=1; verdict=encode; reason="remove ${band}px timecode band"
log "deband $src cropping ${band} rows (re-encode, timing preserved)"
vglog info "event=deband file=\"$src\" stream=\"$stream\" band_rows=$band"
fi
# opt-in gap filling: switch this stream from remux to a CFR re-encode when the holes add up to enough
if (( FILL_GAPS_MS > 0 )) && [[ $verdict == remux && -n $tcjson ]]; then
missing=$(python3 -c 'import json,sys; d=json.loads(sys.argv[1]); print(d.get("gaps",{}).get("total_missing_ms",0))' "$tcjson" 2>/dev/null || echo 0)
if (( missing >= FILL_GAPS_MS )); then
verdict=encode; reason="fill-gaps ${missing}ms"; log "gaps $src ${missing} ms missing -> CFR re-encode (FILL_GAPS_MS=$FILL_GAPS_MS)"
verdict=encode; reason="fill-gaps ${missing}ms"; vglog info "event=fill_gaps file=\"$src\" stream=\"$stream\" missing_ms=$missing"
fi
fi
fi
@@ -189,7 +192,7 @@ PYEOF
rm -f "$src" # band removal: the cropped file IS the recording; raw-with-band is not kept
fi
rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true # drop .live and the empty title folder
log "done $out"; continue
vglog info "event=published file=\"$out\" stream=\"$stream\" verdict=$verdict"; continue
elif (( synced )); then
rel_src="${src#"$stroot"/}"; rel_out="${out#"$stroot"/}"
{ echo "/$rel_out"; echo "/$rel_src"; } >> "$stroot/.stlocal" # ignore our output and the raw
@@ -205,10 +208,10 @@ PYEOF
mv -f "$tmp" "$out"
fi
chown 1000:1000 "$out" 2>/dev/null
log "done $out"
vglog info "event=done file=\"$out\" verdict=$verdict"
else
rm -f "$tmp"
{ echo "$(date '+%F %T') $verdict failed rc=$rc src_dur=${d1:-?} out_dur=${d2:-?} ($reason)"; } > "$base.normalize-failed.log"
log "FAILED $src (see $base.normalize-failed.log)" >&2
vglog error "event=failed file=\"$src\" stream=\"$stream\" verdict=$verdict rc=$rc log=\"$base.normalize-failed.log\"" >&2
fi
done