Vanguard media stack: ingest, copyparty, Syncthing, normalize sweep, Authentik, Jellyfin + Vanguard Metadata plugin, MariaDB, Gitea, bot runtime
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
167
bin/dumpd.py
Executable file
167
bin/dumpd.py
Executable file
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dump-tonight endpoint. Moves one streamer's night into trash (soft delete).
|
||||
|
||||
POST /dump Authorization: Basic user:pass body (optional JSON): {"night":"YYYY-MM-DD","user":"name"}
|
||||
GET /dump -> lists what a dump would move (dry run)
|
||||
|
||||
"Night" = everything the user recorded from 17:00 Eastern on <night> until 02:00 Eastern the next day.
|
||||
Default night: today if it is 17:00 or later, else yesterday (so 00:00-02:00 still counts as last night).
|
||||
Users may dump only their own prefix; 'admin' may pass "user" to dump anyone, or "all".
|
||||
"""
|
||||
import base64, hashlib, json, os, shutil, sys
|
||||
from datetime import datetime, timedelta, time
|
||||
from zoneinfo import ZoneInfo
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get("MEDIA_ROOT", "/srv/media"))
|
||||
SRC_DIRS = [ROOT / "streams", ROOT / "clips"]
|
||||
TRASH = ROOT / "trash"
|
||||
MARK = ROOT / ".markers"
|
||||
USERS_FILE = Path(os.environ.get("DUMP_USERS", "/etc/vanguard/dump-users.txt"))
|
||||
LISTEN = ("127.0.0.1", 8080)
|
||||
TZ = ZoneInfo(os.environ.get("DUMP_TZ", "America/New_York"))
|
||||
NIGHT_START = time(17, 0) # 5 pm
|
||||
NIGHT_END = time(2, 0) # 2 am next day
|
||||
|
||||
def load_users():
|
||||
users = {}
|
||||
for line in USERS_FILE.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or ":" not in line:
|
||||
continue
|
||||
u, h = line.split(":", 1)
|
||||
users[u.strip()] = h.strip().lower()
|
||||
return users
|
||||
|
||||
def check_auth(header):
|
||||
if not header or not header.startswith("Basic "):
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(header[6:]).decode()
|
||||
u, p = raw.split(":", 1)
|
||||
except Exception:
|
||||
return None
|
||||
users = load_users()
|
||||
if users.get(u) == hashlib.sha256(p.encode()).hexdigest():
|
||||
return u
|
||||
return None
|
||||
|
||||
def default_night(now=None):
|
||||
now = now or datetime.now(TZ)
|
||||
d = now.date() if now.time() >= NIGHT_START else (now - timedelta(days=1)).date()
|
||||
return d.isoformat()
|
||||
|
||||
def night_window(night):
|
||||
d = datetime.strptime(night, "%Y-%m-%d").date()
|
||||
return (datetime.combine(d, NIGHT_START, tzinfo=TZ),
|
||||
datetime.combine(d + timedelta(days=1), NIGHT_END, tzinfo=TZ))
|
||||
|
||||
def candidates(user, night):
|
||||
start, end = night_window(night)
|
||||
out = []
|
||||
for base in SRC_DIRS:
|
||||
udirs = [p for p in base.iterdir() if p.is_dir()] if user == "all" else [base / user]
|
||||
for ud in udirs:
|
||||
if not ud.is_dir():
|
||||
continue
|
||||
for f in ud.rglob("*"):
|
||||
if not f.is_file():
|
||||
continue
|
||||
mt = datetime.fromtimestamp(f.stat().st_mtime, TZ)
|
||||
if start <= mt <= end:
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
def do_move(files, night):
|
||||
moved, size = 0, 0
|
||||
for f in files:
|
||||
rel = f.relative_to(ROOT)
|
||||
dst = TRASH / night / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
size += f.stat().st_size
|
||||
shutil.move(str(f), str(dst))
|
||||
moved += 1
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
# drop now-empty title folders
|
||||
for base in SRC_DIRS:
|
||||
for d in sorted((p for p in base.rglob("*") if p.is_dir()), key=lambda p: -len(p.parts)):
|
||||
if d.parent != base and not any(d.iterdir()):
|
||||
d.rmdir()
|
||||
return moved, size
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
server_version = "dumpd/1"
|
||||
def _json(self, code, obj):
|
||||
body = json.dumps(obj).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
def _deny(self):
|
||||
self.send_response(401)
|
||||
self.send_header("WWW-Authenticate", 'Basic realm="dump"')
|
||||
self.end_headers()
|
||||
def _parse(self):
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
body = json.loads(self.rfile.read(n) or b"{}") if n else {}
|
||||
night = body.get("night")
|
||||
if night:
|
||||
datetime.strptime(night, "%Y-%m-%d") # validate
|
||||
return night, body.get("user")
|
||||
def _resolve(self, night, tgt):
|
||||
"""Explicit night wins. Otherwise: tonight, falling back to last night if tonight is empty."""
|
||||
if night:
|
||||
return night, candidates(tgt, night)
|
||||
n = default_night()
|
||||
files = candidates(tgt, n)
|
||||
if not files:
|
||||
prev = (datetime.strptime(n, "%Y-%m-%d") - timedelta(days=1)).strftime("%Y-%m-%d")
|
||||
pf = candidates(tgt, prev)
|
||||
if pf:
|
||||
return prev, pf
|
||||
return n, files
|
||||
def _target(self, auth_user, req_user):
|
||||
if auth_user == "admin":
|
||||
return req_user or "all"
|
||||
return auth_user
|
||||
def do_GET(self):
|
||||
if not self.path.startswith("/dump"):
|
||||
return self._json(404, {"error": "not found"})
|
||||
u = check_auth(self.headers.get("Authorization"))
|
||||
if not u:
|
||||
return self._deny()
|
||||
night, req_user = self._parse()
|
||||
tgt = self._target(u, req_user)
|
||||
night, files = self._resolve(night, tgt)
|
||||
self._json(200, {"night": night, "user": tgt, "dry_run": True,
|
||||
"files": [str(f.relative_to(ROOT)) for f in files],
|
||||
"gb": round(sum(f.stat().st_size for f in files) / 1e9, 2)})
|
||||
def do_POST(self):
|
||||
if not self.path.startswith("/dump"):
|
||||
return self._json(404, {"error": "not found"})
|
||||
u = check_auth(self.headers.get("Authorization"))
|
||||
if not u:
|
||||
return self._deny()
|
||||
try:
|
||||
night, req_user = self._parse()
|
||||
except Exception as e:
|
||||
return self._json(400, {"error": str(e)})
|
||||
tgt = self._target(u, req_user)
|
||||
night, files = self._resolve(night, tgt)
|
||||
moved, size = do_move(files, night)
|
||||
MARK.mkdir(exist_ok=True)
|
||||
(MARK / f"{night}-{tgt}.dumped").write_text(f"{datetime.now(TZ).isoformat()} by {u}\n")
|
||||
print(f"{datetime.now(TZ):%F %T} dump night={night} user={tgt} by={u} files={moved} gb={size/1e9:.2f}", flush=True)
|
||||
self._json(200, {"night": night, "user": tgt, "moved": moved, "gb": round(size / 1e9, 2),
|
||||
"undo_until": (datetime.now(TZ) + timedelta(days=3)).strftime("%F")})
|
||||
def log_message(self, *a): # quiet
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
TRASH.mkdir(exist_ok=True); MARK.mkdir(exist_ok=True)
|
||||
print(f"dumpd listening on {LISTEN[0]}:{LISTEN[1]} tz={TZ.key} night={NIGHT_START:%H:%M}-{NIGHT_END:%H:%M}", flush=True)
|
||||
ThreadingHTTPServer(LISTEN, H).serve_forever()
|
||||
128
bin/normalize-sweep.sh
Executable file
128
bin/normalize-sweep.sh
Executable file
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
# Normalize every finished original so editors always get the same kind of file:
|
||||
# MP4 · H.264 8-bit 4:2:0 · constant frame rate · AAC audio · index up front · ≤2560x1440 · ≤60 fps
|
||||
# - compliant files: untouched
|
||||
# - fragmented/unindexed MP4 (MediaMTX output) with good codecs: lossless remux, replaced in place
|
||||
# - anything else (HEVC, 10-bit, VFR, MKV, >1440p, >60fps): full-res re-encode -> name.mp4,
|
||||
# raw source kept in a hidden .src/ subfolder next to it (Jellyfin/copyparty hide dot-dirs)
|
||||
# - Syncthing folders (marker dir .stfolder in the folder root, receive-only): compliant files are left
|
||||
# alone (zero copies); a non-compliant file is converted to name.mp4 (name.h264.mp4 if the raw is
|
||||
# already .mp4), the raw's name is appended to .stlocal (an #include'd ignore list) so Syncthing
|
||||
# drops it without flagging or re-pulling, then the raw is deleted. One copy on the server.
|
||||
# - failure: name.normalize-failed.log is written and the file is skipped until that log is deleted
|
||||
# Idempotent; safe every few minutes.
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="${MEDIA_ROOT:-/srv/media}"
|
||||
MINS_IDLE=3
|
||||
MAX_W=2560; MAX_H=1440; MAX_FPS=60
|
||||
exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}"
|
||||
flock -n 9 || exit 0
|
||||
|
||||
log() { echo "$(date '+%F %T') $*"; }
|
||||
dur() { ffprobe -v error -show_entries format=duration -of csv=p=0 "$1" 2>/dev/null | cut -d. -f1; }
|
||||
|
||||
# --- probe: prints "verdict reason" where verdict ∈ ok|remux|encode|skip ----------------------
|
||||
probe() {
|
||||
local f="$1" ext="${1##*.}"
|
||||
local json; json=$(ffprobe -v error -show_entries stream=codec_type,codec_name,pix_fmt,width,height,r_frame_rate,avg_frame_rate -of json "$f" 2>/dev/null) || { echo "skip unreadable"; return; }
|
||||
local vcodec pix w h rfr afr acodecs
|
||||
vcodec=$(jq -r '[.streams[]|select(.codec_type=="video")][0].codec_name // "none"' <<<"$json")
|
||||
pix=$(jq -r '[.streams[]|select(.codec_type=="video")][0].pix_fmt // ""' <<<"$json")
|
||||
w=$(jq -r '[.streams[]|select(.codec_type=="video")][0].width // 0' <<<"$json")
|
||||
h=$(jq -r '[.streams[]|select(.codec_type=="video")][0].height // 0' <<<"$json")
|
||||
rfr=$(jq -r '[.streams[]|select(.codec_type=="video")][0].r_frame_rate // "0/1"' <<<"$json")
|
||||
afr=$(jq -r '[.streams[]|select(.codec_type=="video")][0].avg_frame_rate // "0/1"' <<<"$json")
|
||||
acodecs=$(jq -r '[.streams[]|select(.codec_type=="audio")|.codec_name]|unique|join(",")' <<<"$json")
|
||||
[[ $vcodec == none ]] && { echo "skip no-video"; return; }
|
||||
local fps; fps=$(awk -F/ '{ if ($2>0) printf "%.3f", $1/$2; else print 0 }' <<<"$afr")
|
||||
local reasons=()
|
||||
[[ $vcodec == h264 ]] || reasons+=("codec=$vcodec")
|
||||
[[ $pix == yuv420p ]] || reasons+=("pix=$pix")
|
||||
(( w <= MAX_W && h <= MAX_H )) || reasons+=("size=${w}x${h}")
|
||||
awk "BEGIN{exit !($fps > $MAX_FPS + 0.01)}" && reasons+=("fps=$fps")
|
||||
[[ -z $acodecs || $acodecs == aac ]] || reasons+=("audio=$acodecs")
|
||||
[[ ${ext,,} == mp4 ]] || reasons+=("container=$ext")
|
||||
if (( ${#reasons[@]} )); then echo "encode ${reasons[*]}"; return; fi
|
||||
# codecs fine. Fragmented MP4 (MediaMTX output) -> lossless remux. ffprobe's frame-rate figures are
|
||||
# meaningless on fragmented files, so the VFR check only runs on flat files (i.e. after the remux).
|
||||
local head; head=$(head -c 2097152 "$f" | grep -aob -m3 -E 'moov|mdat|moof' | tr '\n' ' ')
|
||||
if grep -q moof <<<"$head"; then echo "remux fragmented"; return; fi
|
||||
# real VFR test: do the gaps between the first ~400 frames vary? (dts is monotonic even with B-frames)
|
||||
local vfr; vfr=$(ffprobe -v error -select_streams v:0 -show_entries packet=dts_time -of csv=p=0 -read_intervals '%+#400' "$f" 2>/dev/null |
|
||||
awk 'NF && $1!="N/A" { if (p!="") d[n++]=$1-p; p=$1 }
|
||||
END { if (n<20) { print "cfr"; exit }
|
||||
for(i=0;i<n;i++) a[i]=d[i]; asort(a); med=a[int(n/2)]; if (med<=0) { print "cfr"; exit }
|
||||
bad=0; for(i=0;i<n;i++){ x=d[i]-med; if(x<0)x=-x; if (x > med*0.25) bad++ }
|
||||
print (bad > n*0.02) ? "vfr" : "cfr" }')
|
||||
[[ $vfr == vfr ]] && { echo "encode vfr"; return; }
|
||||
local moov mdat; moov=$(grep -o '[0-9]*:moov' <<<"$head" | cut -d: -f1 | head -1); mdat=$(grep -o '[0-9]*:mdat' <<<"$head" | cut -d: -f1 | head -1)
|
||||
if [[ -z $moov || ( -n $mdat && $mdat -lt $moov ) ]]; then echo "remux index-last"; return; fi
|
||||
echo "ok"
|
||||
}
|
||||
|
||||
find "$ROOT/streams" "$ROOT/clips" -type f \
|
||||
\( -iname '*.mp4' -o -iname '*.mkv' -o -iname '*.mov' -o -iname '*.ts' -o -iname '*.m4v' -o -iname '*.webm' \) \
|
||||
! -path '*/.src/*' ! -path '*/.stfolder/*' ! -path '*/.stversions/*' ! -name '*.part' -mmin +"$MINS_IDLE" -print0 2>/dev/null |
|
||||
while IFS= read -r -d '' src; do
|
||||
base="${src%.*}"; ext="${src##*.}"
|
||||
synced=0; stroot=""
|
||||
d="$(dirname "$src")"
|
||||
while [[ $d == "$ROOT"/* ]]; do # find the Syncthing folder root, if any
|
||||
if [[ -d "$d/.stfolder" ]]; then synced=1; stroot="$d"; break; fi
|
||||
d="$(dirname "$d")"
|
||||
done
|
||||
[[ -e "$base.normalize-failed.log" ]] && continue
|
||||
fuser -s "$src" 2>/dev/null && continue # still being written
|
||||
[[ -e "$src" ]] || continue # dumped meanwhile
|
||||
|
||||
read -r verdict reason <<<"$(probe "$src")"
|
||||
case "$verdict" in ok|skip) continue ;; esac
|
||||
out="$base.mp4"
|
||||
if (( synced )); then
|
||||
[[ $verdict == remux ]] && continue # synced + only an index issue: not worth touching
|
||||
[[ ${ext,,} == mp4 ]] && out="$base.h264.mp4" # raw is already name.mp4: don't collide
|
||||
[[ -e "$out" ]] && continue
|
||||
fi
|
||||
tmp="$out.part"
|
||||
log "$verdict $src ($reason)"
|
||||
if [[ $verdict == remux ]]; then
|
||||
nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \
|
||||
-map 0 -c copy -movflags +faststart -f mp4 "$tmp"
|
||||
else
|
||||
# scale down into the cap (never up), keep aspect, even dimensions; cap fps; CFR; all audio -> AAC
|
||||
vf="scale='min(iw,$MAX_W)':'min(ih,$MAX_H)':force_original_aspect_ratio=decrease:force_divisible_by=2"
|
||||
fpsargs=(); fpsnow=$(awk -F/ '{ if ($2>0) printf "%.3f", $1/$2; else print 0 }' <<<"$(ffprobe -v error -select_streams v:0 -show_entries stream=avg_frame_rate -of csv=p=0 "$src")")
|
||||
awk "BEGIN{exit !($fpsnow > $MAX_FPS + 0.01)}" && fpsargs=(-r "$MAX_FPS")
|
||||
nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \
|
||||
-map 0:v:0 -map '0:a?' \
|
||||
-c:v libx264 -preset veryfast -crf 18 -maxrate 30M -bufsize 60M -pix_fmt yuv420p -profile:v high \
|
||||
-vf "$vf" -fps_mode cfr "${fpsargs[@]}" \
|
||||
-c:a aac -b:a 160k \
|
||||
-movflags +faststart -f mp4 "$tmp"
|
||||
fi
|
||||
rc=$?
|
||||
d1=$(dur "$src"); d2=$(dur "$tmp")
|
||||
if (( rc == 0 )) && [[ -n $d1 && -n $d2 ]] && (( d2 >= d1 - 2 && d2 <= d1 + 2 )); then
|
||||
if (( synced )); then
|
||||
rel_src="${src#"$stroot"/}"; rel_out="${out#"$stroot"/}"
|
||||
{ echo "/$rel_out"; echo "/$rel_src"; } >> "$stroot/.stlocal" # ignore our output and the raw
|
||||
chown 1000:1000 "$stroot/.stlocal" 2>/dev/null
|
||||
sleep 2 # let Syncthing re-read ignores before the delete
|
||||
mv -f "$tmp" "$out"
|
||||
rm -f "$src" # raw stays on the gamer's PC; one copy here
|
||||
elif [[ $verdict == remux ]]; then
|
||||
mv -f "$tmp" "$src" # lossless: replace in place
|
||||
else
|
||||
srcdir="$(dirname "$src")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null
|
||||
mv -f "$src" "$srcdir/$(basename "$src")" # keep raw source, hidden from editors/Jellyfin
|
||||
mv -f "$tmp" "$out"
|
||||
fi
|
||||
chown 1000:1000 "$out" 2>/dev/null
|
||||
log "done $out"
|
||||
else
|
||||
rm -f "$tmp"
|
||||
{ echo "$(date '+%F %T') $verdict failed rc=$rc src_dur=${d1:-?} out_dur=${d2:-?} ($reason)"; } > "$base.normalize-failed.log"
|
||||
log "FAILED $src (see $base.normalize-failed.log)" >&2
|
||||
fi
|
||||
done
|
||||
10
bin/trash-purge.sh
Executable file
10
bin/trash-purge.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
# Delete dumped nights older than 3 days; drop empty dirs; clean stale markers.
|
||||
set -uo pipefail
|
||||
TRASH=/srv/media/trash
|
||||
MARK=/srv/media/.markers
|
||||
[[ -d $TRASH ]] || exit 0
|
||||
find "$TRASH" -mindepth 1 -type f -mtime +3 -delete
|
||||
find "$TRASH" -mindepth 1 -type d -empty -delete
|
||||
find "$MARK" -type f -mtime +30 -delete 2>/dev/null
|
||||
echo "$(date '+%F %T') trash purge done"
|
||||
Reference in New Issue
Block a user