bunch of shit
This commit is contained in:
609
bin/dumpd.py
609
bin/dumpd.py
@@ -3,7 +3,10 @@
|
||||
|
||||
POST /dump identity from X-Authentik-Username / X-Authentik-Groups (set by Caddy after Authentik)
|
||||
body (optional JSON): {"night":"YYYY-MM-DD","user":"name"}
|
||||
GET /dump -> lists what a dump would move (dry run)
|
||||
GET /dump -> lists what a dump would move (dry run); ?user=all&night=YYYY-MM-DD select whose and when
|
||||
Two doors, different reach: /dump on live. (the dock's Dump tonight) is ALWAYS the caller's own night,
|
||||
admin or not — "user" is ignored there. /streaming/api/dump on diag. (the streaming page's Dump tab) is
|
||||
where admins pick a streamer or "all"; non-admins still get only their own there.
|
||||
GET /me/stream-token -> short-lived publish token for the caller (member of streamers/admins); the dock
|
||||
and the live portal set it as the OBS/WHIP credential. No stored passwords anywhere.
|
||||
GET /me/clips -> Syncthing self-service status for the caller: our device id, their folder/device state
|
||||
@@ -21,10 +24,18 @@ POST /deploy-bot -> Gitea Actions (job container -> host gateway) hands over a D
|
||||
HMAC-SHA256 signed with BOT_DEPLOY_SECRET; installs it with vanguard-deploy-bot.
|
||||
May carry X-Vanguard-Env (base64 JSON): the bot's own settings, kept in Gitea, written
|
||||
into the stack .env for the keys BOT_ENV_KEYS allows. Signature covers header+body.
|
||||
GET /stuck/api -> every unpublished recording and why it is stuck (any gated user)
|
||||
POST /stuck/api -> {"action":"retry"|"drop","src":...}: clear the blocker, or move it to trash with the
|
||||
usual 3-day undo. Admin groups only; both actions run bin/vgstuck.sh so the web page and the
|
||||
CLI share one implementation.
|
||||
GET /me/stream-health -> the caller's live streams with B-frame depth: the dock warns when WebRTC will refuse
|
||||
GET /streaming/api/live -> what is publishing, from where, loss on the way in, recorder restarts, transport history, blocks
|
||||
POST /streaming/api/live -> {"action":"stop","path","kind","session","minutes"}: kick that publish session via MediaMTX's
|
||||
API and block the account's publishing for N minutes (0 = kick only); {"action":"unblock","user"}. Admins only.
|
||||
/mtx-auth denies a blocked account, so neither OBS's auto-reconnect nor a Go Live restart gets around it.
|
||||
GET /streaming/api/stuck -> every unpublished recording and why it is stuck (any gated user)
|
||||
GET /streaming/api/disk -> disk use by streamer / clipper / trash by month, system dirs, Docker, and logs (ES indices
|
||||
by month, journald, Docker json logs). Cached 10 min; ?refresh=1 rewalks.
|
||||
POST /streaming/api/stuck -> {"action":"retry"|"drop","src":...}: clear the blocker, or move it to trash with the
|
||||
usual 3-day undo; {"action":"drop_under","mb":N} bins every stream fragment under N MB; {"action":"wipe"}
|
||||
bins everything unprocessed (not being written). Admin groups only; every action runs bin/vgstuck.sh
|
||||
so the web page and the CLI share one implementation.
|
||||
GET /metrics -> Prometheus text for Metricbeat: the media scripts' counters (bin/vgmetrics.sh) plus live
|
||||
gauges for the normalise queue and MediaMTX. Loopback listener only; Caddy routes nothing here.
|
||||
POST /mtx-auth -> MediaMTX authHTTP hook: allows read/playback; allows publish iff user/pass carries a valid
|
||||
@@ -32,10 +43,12 @@ POST /mtx-auth -> MediaMTX authHTTP hook: allows read/playback; allows publish i
|
||||
|
||||
"Night" = everything the user recorded from 17:00 Eastern on <night> until 02:00 Eastern the next day.
|
||||
Default night: today if it is 17:00 or later, else yesterday (so 00:00-02:00 still counts as last night).
|
||||
Users may dump only their own prefix; members of the admins / media-admins groups may pass "user" to dump anyone, or "all".
|
||||
Users may dump only their own prefix; members of the admins / media-admins groups may pass "user" to dump anyone, or "all",
|
||||
but only through /streaming/api/dump (the diag triage page). On live.'s /dump everyone dumps their own night only.
|
||||
"""
|
||||
import base64, hashlib, hmac, json, os, re, shutil, subprocess, sys, threading
|
||||
import time as _time
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, time
|
||||
from zoneinfo import ZoneInfo
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
@@ -51,7 +64,7 @@ PUBLISH_GROUPS = set(g.strip() for g in os.environ.get("PUBLISH_GROUPS", "stream
|
||||
AUTH_USER_HEADER = os.environ.get("AUTH_USER_HEADER", "X-Authentik-Username")
|
||||
AUTH_GROUPS_HEADER = os.environ.get("AUTH_GROUPS_HEADER", "X-Authentik-Groups")
|
||||
ADMIN_GROUPS = set(g.strip() for g in os.environ.get("DUMP_ADMIN_GROUPS", "admins,media-admins").split(",") if g.strip())
|
||||
LISTEN = ("127.0.0.1", 8080) # /dump, /me/* — only via Caddy (identity headers)
|
||||
LISTEN = ("127.0.0.1", 8080) # /dump, /streaming/api/*, /me/* — only via Caddy (identity headers)
|
||||
AUTH_LISTEN = (os.environ.get("MTX_AUTH_BIND", "0.0.0.0"), int(os.environ.get("MTX_AUTH_PORT", "8081"))) # /mtx-auth only — reachable from Docker's bridge (firewall), used by MediaMTX
|
||||
TZ = ZoneInfo(os.environ.get("DUMP_TZ", "America/New_York"))
|
||||
NIGHT_START = time(17, 0) # 5 pm
|
||||
@@ -429,20 +442,441 @@ def verify_token(token):
|
||||
return None
|
||||
return user.lower()
|
||||
|
||||
|
||||
|
||||
# ---- stream health: B-frames ------------------------------------------------------------------------
|
||||
# MediaMTX refuses to serve H.264 with B-frames over WebRTC ("WebRTC doesn't support H264 streams with
|
||||
# B-frames"), so an OBS encoder left on its default of 2 B-frames is unwatchable on the portal. Nothing on the
|
||||
# publish path can see this: obs-websocket cannot set encoder settings in OBS's Advanced output mode, and
|
||||
# MediaMTX only notices when a viewer tries. The recording can: ffprobe reads the reorder depth off the first
|
||||
# fragments of the growing .live file in ~50 ms with a small probesize. Cached per path for 30 s.
|
||||
_health_cache = {}
|
||||
|
||||
def stream_health(path):
|
||||
"""{'bframes', 'profile', 'res', 'fps', 'file'} for a live path, or None if nothing is on disk yet."""
|
||||
now = _time.time()
|
||||
hit = _health_cache.get(path)
|
||||
if hit and now - hit["at"] < 30:
|
||||
return hit["data"]
|
||||
live_dir = ROOT / "streams" / path / ".live"
|
||||
try:
|
||||
newest = max((f for f in live_dir.glob("*.mp4")), key=lambda f: f.stat().st_mtime, default=None)
|
||||
except OSError:
|
||||
newest = None
|
||||
data = None
|
||||
if newest is not None:
|
||||
# One probe, fields by name: csv column order follows ffprobe's internal field order, not the list.
|
||||
done = subprocess.run(["ffprobe", "-v", "error", "-probesize", "2M", "-analyzeduration", "2M",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=has_b_frames,profile,width,height,r_frame_rate",
|
||||
"-of", "default=noprint_wrappers=1", str(newest)],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
fields = dict(line.split("=", 1) for line in done.stdout.splitlines() if "=" in line)
|
||||
if fields:
|
||||
try:
|
||||
num, den = (fields.get("r_frame_rate", "0/1").split("/") + ["1"])[:2]
|
||||
fps = round(float(num) / float(den), 2) if float(den) else None
|
||||
except (ValueError, ZeroDivisionError):
|
||||
fps = None
|
||||
data = {"bframes": int(fields.get("has_b_frames") or 0), "profile": fields.get("profile", ""),
|
||||
"res": f"{fields.get('width', '?')}x{fields.get('height', '?')}", "fps": fps, "file": newest.name}
|
||||
_health_cache[path] = {"at": now, "data": data}
|
||||
return data
|
||||
|
||||
# ---- live streaming status ------------------------------------------------------------------------
|
||||
# GET /streaming/api/live: what is publishing, from where, how much it has lost on the way in, and why
|
||||
# MediaMTX has been opening recording files. Cached 5 s: the Live tab polls every 10 s and several admins
|
||||
# may have it open. Reads MediaMTX's API and metrics on loopback, and its container log for the counters.
|
||||
_live_cache = {"at": 0.0, "data": None}
|
||||
MTX_METRICS = os.environ.get("MTX_METRICS_URL", "http://127.0.0.1:9998/metrics")
|
||||
|
||||
# WebRTC publish sessions and their transport. MediaMTX's API labels the selected ICE pair as
|
||||
# "<type>/<protocol>/<addr>/<port>" (host/udp/… or host/tcp/…), so the protocol a publisher is really on is
|
||||
# right there — that is how the Live tab tells a Go Live on TCP (the lossy-uplink switch) from one on UDP.
|
||||
# A session history is kept in MARK/webrtc-history.json so "when did he switch" survives the page being closed:
|
||||
# it is updated on every Live-tab poll AND every metrics scrape (Metricbeat, every few seconds), and a session
|
||||
# that is no longer listed is stamped ended. 24 h, 50 sessions per path.
|
||||
HISTORY_FILE = MARK / "webrtc-history.json"
|
||||
_history_lock = threading.Lock()
|
||||
|
||||
def webrtc_sessions():
|
||||
with urllib.request.urlopen(f"{MTX_API}/v3/webrtcsessions/list?itemsPerPage=1000", timeout=3) as response:
|
||||
return json.load(response).get("items") or []
|
||||
|
||||
def session_transport(item):
|
||||
label = item.get("localCandidate") or item.get("remoteCandidate") or ""
|
||||
parts = label.split("/")
|
||||
return parts[1].lower() if len(parts) >= 2 and parts[1] else ""
|
||||
|
||||
def track_sessions(items):
|
||||
"""Fold the current publish sessions into the history file; returns the history (path -> sessions)."""
|
||||
now = datetime.now(TZ)
|
||||
with _history_lock:
|
||||
try:
|
||||
hist = json.loads(HISTORY_FILE.read_text())
|
||||
except (OSError, ValueError):
|
||||
hist = {}
|
||||
live_ids = set()
|
||||
for it in items:
|
||||
if str(it.get("state", "")).lower() != "publish":
|
||||
continue
|
||||
sid = it.get("id", ""); live_ids.add(sid)
|
||||
rows = hist.setdefault(it.get("path", "?"), [])
|
||||
row = next((r for r in rows if r["id"] == sid), None)
|
||||
if row is None:
|
||||
row = {"id": sid, "created": it.get("created"), "remote": it.get("remoteAddr"), "transport": "", "ended": None}
|
||||
rows.append(row)
|
||||
if session_transport(it): row["transport"] = session_transport(it)
|
||||
row["received"] = it.get("rtpPacketsReceived", 0); row["lost"] = it.get("rtpPacketsLost", 0)
|
||||
row["bytes"] = it.get("bytesReceived", 0); row["last_seen"] = now.isoformat(timespec="seconds")
|
||||
cutoff = now - timedelta(hours=24)
|
||||
for path in list(hist):
|
||||
kept = []
|
||||
for r in hist[path]:
|
||||
if r["id"] not in live_ids and not r.get("ended"):
|
||||
r["ended"] = r.get("last_seen") or now.isoformat(timespec="seconds")
|
||||
try:
|
||||
when = datetime.fromisoformat((r.get("ended") or r.get("last_seen") or "").replace("Z", "+00:00"))
|
||||
if when.tzinfo is None: when = when.replace(tzinfo=TZ)
|
||||
except ValueError:
|
||||
when = now
|
||||
if when >= cutoff: kept.append(r)
|
||||
kept.sort(key=lambda r: r.get("created") or "")
|
||||
hist[path] = kept[-50:]
|
||||
if not hist[path]: del hist[path]
|
||||
try:
|
||||
MARK.mkdir(exist_ok=True)
|
||||
tmp = HISTORY_FILE.with_suffix(".tmp"); tmp.write_text(json.dumps(hist)); os.replace(tmp, HISTORY_FILE)
|
||||
except OSError as error:
|
||||
log("history_write_failed", level="warn", error=str(error))
|
||||
return hist
|
||||
|
||||
def session_history():
|
||||
"""History as the page wants it: one list, newest first, with switch markers."""
|
||||
try:
|
||||
hist = json.loads(HISTORY_FILE.read_text())
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
out = []
|
||||
for path, rows in hist.items():
|
||||
prev = None
|
||||
for r in rows:
|
||||
entry = {"path": path, **{k: r.get(k) for k in ("id", "created", "ended", "remote", "transport", "received", "lost", "bytes")}}
|
||||
entry["id"] = (entry["id"] or "")[:8]
|
||||
if prev is not None and prev.get("transport") and r.get("transport") and prev["transport"] != r["transport"]:
|
||||
entry["switched_from"] = prev["transport"]
|
||||
prev = r
|
||||
out.append(entry)
|
||||
out.sort(key=lambda e: e.get("created") or "", reverse=True)
|
||||
return out
|
||||
|
||||
# Turning a stream off from the Live tab. Two parts, because a kick alone is a hiccup: OBS reconnects on its
|
||||
# own within seconds and Go Live can be restarted with one click. So "stop" = kick the publish session through
|
||||
# MediaMTX's API AND put the streamer on a block list that /mtx-auth consults for every publish, for a chosen
|
||||
# number of minutes (0 = kick only). Blocks live in MARK/publish-blocks.json, survive a dumpd restart, expire on
|
||||
# their own, and the Live tab lists them with an unblock button.
|
||||
BLOCKS_FILE = MARK / "publish-blocks.json"
|
||||
_blocks_lock = threading.Lock()
|
||||
KICK_ENDPOINT = {"webrtc": "webrtcsessions", "rtmp": "rtmpconns", "srt": "srtconns", "rtsp": "rtspsessions"}
|
||||
|
||||
def read_blocks():
|
||||
try:
|
||||
blocks = json.loads(BLOCKS_FILE.read_text())
|
||||
except (OSError, ValueError):
|
||||
blocks = {}
|
||||
now = _time.time()
|
||||
live = {u: b for u, b in blocks.items() if b.get("until", 0) > now}
|
||||
if len(live) != len(blocks):
|
||||
write_blocks(live)
|
||||
return live
|
||||
|
||||
def write_blocks(blocks):
|
||||
try:
|
||||
MARK.mkdir(exist_ok=True)
|
||||
tmp = BLOCKS_FILE.with_suffix(".tmp"); tmp.write_text(json.dumps(blocks)); os.replace(tmp, BLOCKS_FILE)
|
||||
except OSError as error:
|
||||
log("blocks_write_failed", level="error", error=str(error))
|
||||
|
||||
def block_user(user, minutes, by, reason=""):
|
||||
with _blocks_lock:
|
||||
blocks = read_blocks()
|
||||
blocks[user] = {"until": _time.time() + minutes * 60, "by": by, "reason": reason,
|
||||
"since": datetime.now(TZ).isoformat(timespec="seconds")}
|
||||
write_blocks(blocks)
|
||||
log("publish_block", user=user, minutes=minutes, by=by)
|
||||
|
||||
def unblock_user(user, by):
|
||||
with _blocks_lock:
|
||||
blocks = read_blocks()
|
||||
if blocks.pop(user, None) is not None:
|
||||
write_blocks(blocks); log("publish_unblock", user=user, by=by); return True
|
||||
return False
|
||||
|
||||
def blocked(user):
|
||||
return read_blocks().get(user)
|
||||
|
||||
def kick_publisher(kind, session_id):
|
||||
endpoint = KICK_ENDPOINT.get(kind)
|
||||
if not endpoint:
|
||||
raise ValueError(f"cannot kick a {kind} publisher")
|
||||
request = urllib.request.Request(f"{MTX_API}/v3/{endpoint}/kick/{urllib.parse.quote(session_id, safe='')}", method="POST")
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status
|
||||
|
||||
def stop_stream(path, kind, session_id, minutes, who):
|
||||
user = path.split("/", 1)[0].lower()
|
||||
out = {"path": path, "user": user, "kicked": False, "blocked_minutes": minutes}
|
||||
if minutes > 0:
|
||||
block_user(user, minutes, who, reason=f"stopped from the Live tab ({path})") # block first: no reconnect window
|
||||
try:
|
||||
kick_publisher(kind, session_id); out["kicked"] = True
|
||||
except Exception as error:
|
||||
out["error"] = f"kick failed: {error}"
|
||||
# the Opus mirror of an OBS stream dies with its source (runOnReady exits), so nothing else to stop
|
||||
_live_cache["data"] = None
|
||||
log("stream_stop", level="info" if out["kicked"] else "error", stream=path, kind=kind, by=who, minutes=minutes,
|
||||
result=out.get("error", "kicked"))
|
||||
return out
|
||||
|
||||
def live_status():
|
||||
now = _time.time()
|
||||
if _live_cache["data"] is not None and now - _live_cache["at"] < 5:
|
||||
return _live_cache["data"]
|
||||
out = {"paths": [], "publishers": [], "counts": {}, "log": [], "errors": [], "history": []}
|
||||
sessions = {}
|
||||
try:
|
||||
items = webrtc_sessions()
|
||||
track_sessions(items)
|
||||
for it in items:
|
||||
sessions[it.get("id", "")] = {"transport": session_transport(it), "since": it.get("created"),
|
||||
"established": it.get("peerConnectionEstablished")}
|
||||
out["history"] = session_history()
|
||||
except Exception as error:
|
||||
out["errors"].append(f"MediaMTX WebRTC sessions: {error}")
|
||||
try:
|
||||
with urllib.request.urlopen(f"{MTX_API}/v3/paths/list?itemsPerPage=1000", timeout=3) as response:
|
||||
for item in json.load(response).get("items") or []:
|
||||
if item.get("ready"):
|
||||
row = {"name": item["name"], "tracks": item.get("tracks") or [],
|
||||
"readers": len(item.get("readers") or []),
|
||||
"since": item.get("readyTime"), "bytes": item.get("bytesReceived", 0),
|
||||
"mirror": str(item["name"]).endswith(".webrtc")}
|
||||
if not row["mirror"]:
|
||||
try:
|
||||
row["health"] = stream_health(item["name"])
|
||||
except Exception as error:
|
||||
row["health_error"] = str(error)
|
||||
out["paths"].append(row)
|
||||
except Exception as error:
|
||||
out["errors"].append(f"MediaMTX API: {error}")
|
||||
try:
|
||||
with urllib.request.urlopen(MTX_METRICS, timeout=3) as response:
|
||||
text = response.read().decode(errors="replace")
|
||||
rows = {}
|
||||
for match in re.finditer(r"^(webrtc_sessions|rtmp_conns|srt_conns)(_rtp_packets_received|_rtp_packets_lost|_bytes_received)?\{([^}]*)\} (\d+)$", text, re.M):
|
||||
kind, field, labels_text, value = match.groups()
|
||||
labels = dict(re.findall(r'(\w+)="([^"]*)"', labels_text))
|
||||
if labels.get("state") != "publish":
|
||||
continue
|
||||
key = (labels.get("path", "?"), labels.get("id", ""), labels.get("remoteAddr", "?"))
|
||||
row = rows.setdefault(key, {"kind": kind.split("_")[0]})
|
||||
if field == "_rtp_packets_received": row["received"] = int(value)
|
||||
elif field == "_rtp_packets_lost": row["lost"] = int(value)
|
||||
elif field == "_bytes_received": row["bytes"] = int(value)
|
||||
for (path, sid, addr), row in rows.items():
|
||||
out["publishers"].append({"path": path, "id": sid[:8], "session": sid, "from": addr, **row, **sessions.get(sid, {})})
|
||||
out["blocks"] = [{"user": u, **b, "until_iso": datetime.fromtimestamp(b["until"], TZ).isoformat(timespec="seconds")}
|
||||
for u, b in sorted(read_blocks().items())]
|
||||
except Exception as error:
|
||||
out["errors"].append(f"MediaMTX metrics: {error}")
|
||||
try:
|
||||
done = subprocess.run(["docker", "logs", "mediamtx", "--since", "10m"],
|
||||
capture_output=True, text=True, timeout=15)
|
||||
log_text = done.stdout + done.stderr
|
||||
out["counts"] = {
|
||||
"publish_sessions": log_text.count("is publishing"),
|
||||
"recorder_restarts": len(re.findall(r"recording \d+ tracks", log_text)),
|
||||
"processing_errors": log_text.count("processing errors"),
|
||||
"access_unit_errors": log_text.count("recorder] access unit"),
|
||||
"window": "10m",
|
||||
}
|
||||
out["log"] = [l for l in log_text.splitlines() if re.search(r"recorder|is publishing|closed|ERR", l)][-14:]
|
||||
except Exception as error:
|
||||
out["errors"].append(f"MediaMTX log: {error}")
|
||||
_live_cache.update(at=now, data=out)
|
||||
return out
|
||||
|
||||
# ---- disk report ------------------------------------------------------------------------------------
|
||||
# GET /streaming/api/disk: where the bytes are. Streamers (streams/<user>, month from the recording's own
|
||||
# "__YYYY-MM-DD" stamp, else mtime), clippers (clips/<user>, mtime), trash (its night folders), system (appdata
|
||||
# per app, /srv/live, /srv/bot, Docker's own accounting) and logs (Elasticsearch indices by the date in their
|
||||
# name, the ES data dir as the on-disk truth, journald, Docker's json logs). Walking a few TB of files takes
|
||||
# seconds, so the report is cached for ten minutes and ?refresh=1 forces it. Sizes are allocated blocks, i.e.
|
||||
# what the disk actually loses, not apparent length.
|
||||
_disk_cache = {"at": 0.0, "data": None}
|
||||
_disk_lock = threading.Lock()
|
||||
DISK_CACHE_S = 600
|
||||
APPDATA = Path("/srv/appdata")
|
||||
ES_NETRC = Path(os.environ.get("ES_NETRC_FILE", "/etc/vanguard/es-netrc"))
|
||||
ES_URL = os.environ.get("ES_URL", "http://127.0.0.1:9200")
|
||||
MONTH_IN_NAME = re.compile(r"__(\d{4})-(\d{2})-\d{2}_")
|
||||
MONTH_IN_INDEX = re.compile(r"(\d{4})\.(\d{2})\.\d{2}")
|
||||
|
||||
def _blocks(st):
|
||||
return st.st_blocks * 512 if hasattr(st, "st_blocks") else st.st_size
|
||||
|
||||
def walk_sizes(root, month_of):
|
||||
"""{month: bytes} and total for everything under root; month_of(path, stat) -> 'YYYY-MM'."""
|
||||
months, total = {}, 0
|
||||
stack = [root]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
try:
|
||||
with os.scandir(current) as entries:
|
||||
for entry in entries:
|
||||
try:
|
||||
if entry.is_symlink(): continue
|
||||
if entry.is_dir(follow_symlinks=False): stack.append(entry.path); continue
|
||||
st = entry.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
continue
|
||||
size = _blocks(st); total += size
|
||||
month = month_of(entry.path, st)
|
||||
months[month] = months.get(month, 0) + size
|
||||
except OSError:
|
||||
continue
|
||||
return {"total": total, "months": months}
|
||||
|
||||
def month_from_mtime(_path, st):
|
||||
return datetime.fromtimestamp(st.st_mtime, TZ).strftime("%Y-%m")
|
||||
|
||||
def month_from_name(path, st):
|
||||
m = MONTH_IN_NAME.search(os.path.basename(path))
|
||||
return f"{m.group(1)}-{m.group(2)}" if m else month_from_mtime(path, st)
|
||||
|
||||
def dir_total(path):
|
||||
return walk_sizes(path, lambda p, s: "all")["total"] if Path(path).is_dir() else 0
|
||||
|
||||
def es_indices():
|
||||
"""[{index, bytes, month}] from _cat/indices, with the netrc the deadman also uses."""
|
||||
import netrc as _netrc
|
||||
request = urllib.request.Request(f"{ES_URL}/_cat/indices?bytes=b&format=json&h=index,store.size,creation.date.string&expand_wildcards=all")
|
||||
try:
|
||||
creds = _netrc.netrc(str(ES_NETRC)).authenticators(urllib.parse.urlparse(ES_URL).hostname or "127.0.0.1")
|
||||
except (OSError, _netrc.NetrcParseError):
|
||||
creds = None
|
||||
if creds:
|
||||
request.add_header("Authorization", "Basic " + base64.b64encode(f"{creds[0]}:{creds[2]}".encode()).decode())
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
rows = json.load(response)
|
||||
out = []
|
||||
for row in rows:
|
||||
name = row.get("index", "")
|
||||
m = MONTH_IN_INDEX.search(name)
|
||||
month = f"{m.group(1)}-{m.group(2)}" if m else (row.get("creation.date.string") or "")[:7] or "other"
|
||||
out.append({"index": name, "bytes": int(row.get("store.size") or 0), "month": month})
|
||||
return out
|
||||
|
||||
def docker_df():
|
||||
done = subprocess.run(["docker", "system", "df", "--format", "{{json .}}"], capture_output=True, text=True, timeout=60)
|
||||
rows = []
|
||||
for line in done.stdout.splitlines():
|
||||
try: rows.append(json.loads(line))
|
||||
except ValueError: pass
|
||||
return rows
|
||||
|
||||
def journal_bytes():
|
||||
done = subprocess.run(["journalctl", "--disk-usage"], capture_output=True, text=True, timeout=20)
|
||||
m = re.search(r"([\d.]+)\s*([KMGT]?)B?", done.stdout.replace(",", ""))
|
||||
if not m: return 0
|
||||
return int(float(m.group(1)) * {"": 1, "K": 1e3, "M": 1e6, "G": 1e9, "T": 1e12}[m.group(2)])
|
||||
|
||||
def disk_report(force=False):
|
||||
now = _time.time()
|
||||
with _disk_lock:
|
||||
if not force and _disk_cache["data"] is not None and now - _disk_cache["at"] < DISK_CACHE_S:
|
||||
return _disk_cache["data"]
|
||||
t0 = _time.time()
|
||||
out = {"generated": datetime.now(TZ).isoformat(timespec="seconds"), "fs": {}, "streamers": {}, "clippers": {},
|
||||
"trash": {}, "system": {}, "logs": {}, "errors": []}
|
||||
for label, path in (("media", ROOT), ("root", Path("/"))):
|
||||
try:
|
||||
u = shutil.disk_usage(path); out["fs"][label] = {"path": str(path), "total": u.total, "used": u.used, "free": u.free}
|
||||
except OSError as error:
|
||||
out["errors"].append(f"{path}: {error}")
|
||||
same_fs = False
|
||||
try: same_fs = os.stat(ROOT).st_dev == os.stat("/").st_dev
|
||||
except OSError: pass
|
||||
out["fs"]["same_filesystem"] = same_fs
|
||||
for user_dir in sorted(p for p in (ROOT / "streams").glob("*") if p.is_dir()):
|
||||
r = walk_sizes(user_dir, month_from_name)
|
||||
r["raw"] = dir_total(user_dir / ".src"); r["unpublished"] = sum(dir_total(d) for d in user_dir.glob("*/.live"))
|
||||
out["streamers"][user_dir.name] = r
|
||||
for user_dir in sorted(p for p in (ROOT / "clips").glob("*") if p.is_dir()):
|
||||
out["clippers"][user_dir.name] = walk_sizes(user_dir, month_from_mtime)
|
||||
out["trash"] = walk_sizes(ROOT / "trash", lambda p, s: (re.search(r"/trash/(\d{4}-\d{2})-\d{2}/", p) or [None, month_from_mtime(p, s)])[1]) if (ROOT / "trash").is_dir() else {"total": 0, "months": {}}
|
||||
other_media = 0
|
||||
for entry in ROOT.iterdir():
|
||||
if entry.name not in ("streams", "clips", "trash") and entry.is_dir() and not entry.is_symlink():
|
||||
other_media += dir_total(entry)
|
||||
out["system"]["media_other"] = other_media # .markers, .syncthing, .stversions, …
|
||||
for app_dir in sorted(p for p in APPDATA.glob("*") if p.is_dir()) if APPDATA.is_dir() else []:
|
||||
size = dir_total(app_dir)
|
||||
if app_dir.name == "elasticsearch": out["logs"]["es_data_dir"] = size
|
||||
else: out["system"][f"appdata/{app_dir.name}"] = size
|
||||
for label, path in (("live portal", "/srv/live"), ("bot", "/srv/bot"), ("etc/vanguard", "/etc/vanguard")):
|
||||
out["system"][label] = dir_total(path)
|
||||
try:
|
||||
for row in docker_df():
|
||||
out["system"][f"docker {row.get('Type', '?').lower()}"] = row.get("Size", "?") # docker's own figures (human strings)
|
||||
except Exception as error:
|
||||
out["errors"].append(f"docker system df: {error}")
|
||||
try:
|
||||
out["logs"]["indices"] = es_indices()
|
||||
out["logs"]["es_total"] = sum(i["bytes"] for i in out["logs"]["indices"])
|
||||
by_month = {}
|
||||
for i in out["logs"]["indices"]: by_month[i["month"]] = by_month.get(i["month"], 0) + i["bytes"]
|
||||
out["logs"]["months"] = by_month
|
||||
except Exception as error:
|
||||
out["errors"].append(f"Elasticsearch: {error}")
|
||||
try: out["logs"]["journald"] = journal_bytes()
|
||||
except Exception as error: out["errors"].append(f"journalctl: {error}")
|
||||
out["logs"]["docker_json_logs"] = sum(_blocks(p.stat()) for p in Path("/var/lib/docker/containers").glob("*/*-json.log*")) if Path("/var/lib/docker/containers").is_dir() else 0
|
||||
out["seconds"] = round(_time.time() - t0, 1)
|
||||
_disk_cache.update(at=_time.time(), data=out)
|
||||
return out
|
||||
|
||||
# ---- stuck recordings -----------------------------------------------------------------------------
|
||||
# GET /stuck/api every unpublished recording and why, for anyone the gate let through
|
||||
# POST /stuck/api {"action":"retry"|"drop","src":"<path>"} — admins only
|
||||
# GET /streaming/api/stuck every unpublished recording and why, for anyone the gate let through
|
||||
# POST /streaming/api/stuck {"action":"retry"|"drop","src":"<path>"} — admins only
|
||||
# The classification and the file moves both live in bin/vgstuck.sh, so the web page and the CLI can never
|
||||
# disagree about why something is stuck, and "delete" means the same three-day trash in both.
|
||||
STUCK_TOOL = os.environ.get("VGSTUCK_TOOL", "/usr/local/bin/vgstuck.sh")
|
||||
|
||||
_stuck_cache = {"at": 0.0, "items": None}
|
||||
STUCK_CACHE_S = 20 # the page refreshes every 30 s and several admins may have it open
|
||||
|
||||
def stuck_list():
|
||||
done = subprocess.run([STUCK_TOOL, "json"], capture_output=True, text=True, timeout=180)
|
||||
"""The survey, cached briefly. It runs ffprobe on every stuck file, so with many of them it is slow, and
|
||||
without a cache every open tab re-ran it every 30 s. The cache is also invalidated by any action."""
|
||||
now = _time.time()
|
||||
if _stuck_cache["items"] is not None and now - _stuck_cache["at"] < STUCK_CACHE_S:
|
||||
return _stuck_cache["items"]
|
||||
try:
|
||||
return json.loads(done.stdout or "[]")
|
||||
except ValueError:
|
||||
log("stuck_list_failed", level="error", rc=done.returncode, err=(done.stderr or "")[:200])
|
||||
return []
|
||||
done = subprocess.run([STUCK_TOOL, "json"], capture_output=True, text=True, timeout=120)
|
||||
except subprocess.TimeoutExpired:
|
||||
log("stuck_list_timeout", level="error", seconds=120)
|
||||
raise RuntimeError("the survey took over 120 s; too many files, or ffprobe is stuck") from None
|
||||
if done.returncode != 0:
|
||||
log("stuck_list_failed", level="error", rc=done.returncode, err=(done.stderr or "")[:300])
|
||||
raise RuntimeError(f"vgstuck.sh exited {done.returncode}: {(done.stderr or '').strip()[-200:]}")
|
||||
try:
|
||||
items = json.loads(done.stdout or "[]")
|
||||
except ValueError as error:
|
||||
log("stuck_list_badjson", level="error", err=str(error), head=(done.stdout or "")[:120])
|
||||
raise RuntimeError(f"vgstuck.sh did not return JSON: {error}") from None
|
||||
_stuck_cache.update(at=now, items=items)
|
||||
return items
|
||||
|
||||
def stuck_target(src):
|
||||
"""Resolve what the browser asked us to act on, or raise. This is the only thing between a request body
|
||||
@@ -461,6 +895,7 @@ def stuck_target(src):
|
||||
def stuck_action(action, src, who):
|
||||
path = stuck_target(src)
|
||||
done = subprocess.run([STUCK_TOOL, action, str(path)], capture_output=True, text=True, timeout=600)
|
||||
_stuck_cache["items"] = None # the list just changed; the next read must re-survey
|
||||
lines = [l for l in (done.stdout + done.stderr).splitlines() if l.strip()]
|
||||
log(f"stuck_{action}", level="info" if done.returncode == 0 else "error",
|
||||
file=str(path), by=who, rc=done.returncode)
|
||||
@@ -579,6 +1014,10 @@ def metrics_text():
|
||||
# 4. MediaMTX. The Prometheus scrape of :9998 already has per-path detail; this is the count the portal
|
||||
# and the dock actually mean by "live", with the derived .webrtc audio mirrors separated out.
|
||||
paths = mtx_paths()
|
||||
try:
|
||||
track_sessions(webrtc_sessions()) # keeps the transport history current between page views
|
||||
except Exception:
|
||||
pass
|
||||
add("vanguard_mediamtx_up", "gauge", "did the MediaMTX API answer this scrape", {}, 0 if paths is None else 1)
|
||||
if paths is not None:
|
||||
ready = [p for p in paths if p.get("ready")]
|
||||
@@ -621,6 +1060,11 @@ class H(BaseHTTPRequestHandler):
|
||||
def _parse(self):
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
body = json.loads(self.rfile.read(n) or b"{}") if n else {}
|
||||
# ?user=all&night=YYYY-MM-DD also work (the Dump tab's dry run). GET never moves anything: a link
|
||||
# unfurled by Discord must not be able to dump a night.
|
||||
query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query)
|
||||
for key in ("night", "user"):
|
||||
if key not in body and query.get(key): body[key] = query[key][0]
|
||||
night = body.get("night")
|
||||
if night:
|
||||
datetime.strptime(night, "%Y-%m-%d") # validate
|
||||
@@ -637,8 +1081,15 @@ class H(BaseHTTPRequestHandler):
|
||||
if pf:
|
||||
return prev, pf
|
||||
return n, files
|
||||
def _is_dump(self):
|
||||
path = self.path.split("?", 1)[0]
|
||||
return path in ("/dump", "/streaming/api/dump") # live. dock button / diag. Dump tab
|
||||
def _admin_door(self):
|
||||
return self.path.split("?", 1)[0] == "/streaming/api/dump"
|
||||
def _target(self, auth_user, is_admin, req_user):
|
||||
if is_admin:
|
||||
# Dumping someone else's night is triage, so it is only offered where triage lives: the diag host's
|
||||
# streaming page. The dock's Dump tonight (live. /dump) is personal for everyone, admins included.
|
||||
if is_admin and self._admin_door():
|
||||
return req_user or auth_user # admins default to their own night; pass "all" or a name for others
|
||||
return auth_user
|
||||
def do_GET(self):
|
||||
@@ -660,10 +1111,51 @@ class H(BaseHTTPRequestHandler):
|
||||
return self._json(200, {"user": who, "groups": groups, "admin": is_admin,
|
||||
"name": self.headers.get("X-Authentik-Name") or who, "email": self.headers.get("X-Authentik-Email") or "",
|
||||
"discord_id": proxy_claim(self.headers, "discord_id")})
|
||||
if self.path == "/stuck/api":
|
||||
if self.path == "/me/stream-health":
|
||||
# The dock polls this after Go Live. Only the caller's own paths, and only what is live now.
|
||||
who, _ = identity(self.headers)
|
||||
if not who: return self._json(401, {"error": "not authenticated"})
|
||||
streams = []
|
||||
try:
|
||||
with urllib.request.urlopen(f"{MTX_API}/v3/paths/list?itemsPerPage=1000", timeout=3) as response:
|
||||
for item in json.load(response).get("items") or []:
|
||||
name = str(item.get("name", ""))
|
||||
if item.get("ready") and name.lower().startswith(who.lower() + "/") and not name.endswith(".webrtc"):
|
||||
health = stream_health(name)
|
||||
streams.append({"path": name, "tracks": item.get("tracks") or [], **(health or {}),
|
||||
"webrtc_ok": bool(health) and health["bframes"] == 0})
|
||||
except Exception as error:
|
||||
return self._json(502, {"error": f"MediaMTX: {error}"})
|
||||
# Go Live polls this while live: an empty list means the server no longer has the stream (kicked, or
|
||||
# the session died), and "blocked" says why it must not simply reconnect.
|
||||
hold = blocked(who.lower())
|
||||
return self._json(200, {"user": who, "streams": streams,
|
||||
"blocked": ({"until": datetime.fromtimestamp(hold["until"], TZ).isoformat(timespec="seconds"),
|
||||
"by": hold.get("by")} if hold else None)})
|
||||
if self.path.split("?", 1)[0] == "/streaming/api/disk":
|
||||
who, is_admin = identity(self.headers)
|
||||
if not who: return self._json(401, {"error": "not authenticated"})
|
||||
return self._json(200, {"admin": is_admin, "user": who, "items": stuck_list()})
|
||||
force = "refresh=1" in (self.path.split("?", 1) + [""])[1]
|
||||
try:
|
||||
return self._json(200, {"admin": is_admin, "user": who, **disk_report(force)})
|
||||
except Exception as error:
|
||||
return self._json(500, {"error": str(error)})
|
||||
if self.path == "/streaming/api/live":
|
||||
who, is_admin = identity(self.headers)
|
||||
if not who: return self._json(401, {"error": "not authenticated"})
|
||||
try:
|
||||
return self._json(200, {"admin": is_admin, "user": who, **live_status()})
|
||||
except Exception as error:
|
||||
return self._json(500, {"error": str(error)})
|
||||
if self.path == "/streaming/api/stuck":
|
||||
who, is_admin = identity(self.headers)
|
||||
if not who: return self._json(401, {"error": "not authenticated"})
|
||||
try:
|
||||
return self._json(200, {"admin": is_admin, "user": who, "items": stuck_list()})
|
||||
except Exception as error:
|
||||
# Always answer. A handler that raises sends nothing, Caddy turns that into an empty 502,
|
||||
# and the page can only say "unexpected end of JSON" — which is exactly what happened.
|
||||
return self._json(500, {"error": str(error)})
|
||||
if self.path.startswith("/me/clips"):
|
||||
who, _ = identity(self.headers)
|
||||
if not who: return self._json(401, {"error": "not authenticated"})
|
||||
@@ -682,17 +1174,24 @@ class H(BaseHTTPRequestHandler):
|
||||
return self._json(500, {"error": str(e)})
|
||||
# "pass" kept for the dock/portal: the token IS the password for MediaMTX
|
||||
return self._json(200, {"user": who, "pass": tok, "token": tok, "expires": exp, "ttl_hours": TOKEN_TTL_H})
|
||||
if not self.path.startswith("/dump"):
|
||||
if not self._is_dump():
|
||||
return self._json(404, {"error": "not found"})
|
||||
u, is_admin = identity(self.headers)
|
||||
if not u:
|
||||
return self._deny()
|
||||
night, req_user = self._parse()
|
||||
try:
|
||||
night, req_user = self._parse()
|
||||
except Exception as e:
|
||||
return self._json(400, {"error": str(e)})
|
||||
tgt = self._target(u, is_admin, req_user)
|
||||
night, files = self._resolve(night, tgt)
|
||||
self._json(200, {"night": night, "user": tgt, "dry_run": True,
|
||||
"files": [str(f.relative_to(ROOT)) for f in files],
|
||||
"gb": round(sum(f.stat().st_size for f in files) / 1e9, 2)})
|
||||
start, end = night_window(night)
|
||||
entries = [{"path": str(f.relative_to(ROOT)), "bytes": f.stat().st_size} for f in files]
|
||||
self._json(200, {"night": night, "user": tgt, "by": u, "admin": is_admin, "dry_run": True,
|
||||
"window": [start.isoformat(timespec="minutes"), end.isoformat(timespec="minutes")],
|
||||
"default_night": default_night(),
|
||||
"files": [e["path"] for e in entries], "entries": entries,
|
||||
"gb": round(sum(e["bytes"] for e in entries) / 1e9, 2)})
|
||||
def do_POST(self):
|
||||
if self.server.server_port == AUTH_LISTEN[1] and self.path.startswith("/heartbeat"):
|
||||
# Alertmanager posts here every minute because the Watchdog alert always fires. Nothing is read from
|
||||
@@ -776,7 +1275,7 @@ class H(BaseHTTPRequestHandler):
|
||||
path = (req.get("path") or "").lower()
|
||||
if path.endswith(".webrtc"):
|
||||
# audio mirrors (<user>/<Title>.webrtc) are published by the hook inside the mediamtx container only
|
||||
if req.get("ip") in ("127.0.0.1", "::1") and re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+\.webrtc", path):
|
||||
if req.get("ip") in ("127.0.0.1", "::1") and re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_-]+\.webrtc", path):
|
||||
return self._json(200, {"ok": True})
|
||||
log("publish_deny", level="warn", stream=path, ip=req.get("ip"), reason="mirror paths are internal")
|
||||
return self._json(401, {"error": ".webrtc paths are reserved"})
|
||||
@@ -787,12 +1286,43 @@ class H(BaseHTTPRequestHandler):
|
||||
if not user:
|
||||
log("publish_deny", level="warn", stream=path, ip=req.get("ip"), reason="bad or expired token")
|
||||
return self._json(401, {"error": "invalid or expired stream token"})
|
||||
if not re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+", path) or not path.startswith(user + "/"):
|
||||
# title segment is dot-free (mirrors are <title>.webrtc and must never collide with a title)
|
||||
if not re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_-]+", path) or not path.startswith(user + "/"):
|
||||
log("publish_deny", level="warn", user=user, stream=path, reason="not their prefix")
|
||||
return self._json(401, {"error": "path must be <your-username>/<Title>"})
|
||||
hold = blocked(user.lower())
|
||||
if hold: # stopped from the Live tab: no reconnecting around it
|
||||
log("publish_deny", level="warn", user=user, stream=path, ip=req.get("ip"), reason="blocked", by=hold.get("by"))
|
||||
return self._json(401, {"error": "publishing is paused for this account by an admin"})
|
||||
log("publish_allow", user=user, stream=path, ip=req.get("ip"))
|
||||
return self._json(200, {"ok": True})
|
||||
if self.path == "/stuck/api":
|
||||
if self.path == "/streaming/api/live":
|
||||
# {"action":"stop","path":..,"kind":..,"session":..,"minutes":N} or {"action":"unblock","user":..} — admins only
|
||||
who, is_admin = identity(self.headers)
|
||||
if not who: return self._deny()
|
||||
if not is_admin:
|
||||
return self._json(403, {"error": f"stopping streams is for {', '.join(sorted(ADMIN_GROUPS))}"})
|
||||
try:
|
||||
body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}")
|
||||
except ValueError as error:
|
||||
return self._json(400, {"error": str(error)})
|
||||
action = body.get("action")
|
||||
if action == "unblock":
|
||||
user = str(body.get("user", "")).lower()
|
||||
if not re.fullmatch(r"[a-z0-9_.-]+", user): return self._json(400, {"error": "bad user"})
|
||||
_live_cache["data"] = None
|
||||
return self._json(200, {"ok": unblock_user(user, who), "user": user})
|
||||
if action == "stop":
|
||||
path, kind, session = str(body.get("path", "")), str(body.get("kind", "")), str(body.get("session", ""))
|
||||
if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", path) or kind not in KICK_ENDPOINT or not re.fullmatch(r"[A-Za-z0-9-]{8,64}", session):
|
||||
return self._json(400, {"error": "need path, kind and session as the live list reports them"})
|
||||
try:
|
||||
minutes = max(0, min(int(body.get("minutes", 30)), 24 * 60))
|
||||
except (TypeError, ValueError):
|
||||
minutes = 30
|
||||
return self._json(200, stop_stream(path, kind, session, minutes, who))
|
||||
return self._json(400, {"error": "action must be stop or unblock"})
|
||||
if self.path == "/streaming/api/stuck":
|
||||
who, is_admin = identity(self.headers)
|
||||
if not who: return self._deny()
|
||||
if not is_admin:
|
||||
@@ -802,8 +1332,33 @@ class H(BaseHTTPRequestHandler):
|
||||
except ValueError as error:
|
||||
return self._json(400, {"error": str(error)})
|
||||
action = body.get("action")
|
||||
if action == "wipe":
|
||||
# Everything unprocessed, deliberately (the Dump tab's button): stops the sweep, then every .live/
|
||||
# file except one MediaMTX still has open, plus stale failure logs and .part leftovers. vgstuck.sh
|
||||
# does the moving, into trash.
|
||||
done = subprocess.run([STUCK_TOOL, "wipe"], capture_output=True, text=True, timeout=900)
|
||||
_stuck_cache["items"] = None
|
||||
lines = [l for l in (done.stdout + done.stderr).splitlines() if l.strip()]
|
||||
log("stuck_wipe", level="info" if done.returncode == 0 else "error", by=who, rc=done.returncode,
|
||||
result=(lines[-1] if lines else ""))
|
||||
return self._json(200, {"ok": done.returncode == 0, "rc": done.returncode, "output": lines[-6:]})
|
||||
if action == "drop_under":
|
||||
# Bulk: every unpublished STREAM recording smaller than N MB — reconnect stubs. Bounded so a
|
||||
# typo cannot bin real recordings; clips never appear here because they never pass through
|
||||
# .live/. Still-recording files are skipped by the tool itself.
|
||||
try:
|
||||
mb = int(body.get("mb", 0))
|
||||
except (TypeError, ValueError):
|
||||
mb = 0
|
||||
if not 1 <= mb <= 50:
|
||||
return self._json(400, {"error": "mb must be between 1 and 50"})
|
||||
done = subprocess.run([STUCK_TOOL, "drop", "--under", str(mb)], capture_output=True, text=True, timeout=900)
|
||||
_stuck_cache["items"] = None
|
||||
lines = [l for l in (done.stdout + done.stderr).splitlines() if l.strip()]
|
||||
log("stuck_drop_under", level="info" if done.returncode == 0 else "error", mb=mb, by=who, rc=done.returncode)
|
||||
return self._json(200, {"ok": done.returncode == 0, "rc": done.returncode, "output": lines[-6:]})
|
||||
if action not in ("retry", "drop"):
|
||||
return self._json(400, {"error": "action must be retry or drop"})
|
||||
return self._json(400, {"error": "action must be retry, drop, drop_under or wipe"})
|
||||
try:
|
||||
return self._json(200, stuck_action(action, str(body.get("src", "")), who))
|
||||
except ValueError as error:
|
||||
@@ -821,7 +1376,7 @@ class H(BaseHTTPRequestHandler):
|
||||
return self._json(400, {"error": str(e)})
|
||||
except Exception as e:
|
||||
return self._json(502, {"error": f"syncthing: {e}"})
|
||||
if not self.path.startswith("/dump"):
|
||||
if not self._is_dump():
|
||||
return self._json(404, {"error": "not found"})
|
||||
u, is_admin = identity(self.headers)
|
||||
if not u:
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
0755 bin/trash-purge.sh
|
||||
0755 bin/dumpd.py
|
||||
0755 bin/vgstrip.py
|
||||
0755 bin/vgstitch.py
|
||||
0755 bin/vghosts-check.py
|
||||
0755 bin/vgdeband.sh
|
||||
0755 bin/vgstuck.sh
|
||||
0755 bin/vanguard-deadman.sh
|
||||
|
||||
@@ -16,6 +16,12 @@
|
||||
# alone (zero copies); a non-compliant file is converted to name.mp4 (name.h264.mp4 if the raw is
|
||||
# already .mp4), the raw's name is appended to .stlocal (an #include'd ignore list) so Syncthing
|
||||
# drops it without flagging or re-pulling, then the raw is deleted. One copy on the server.
|
||||
# - reconnect fragments of ONE stream (same user, same title, fewer than STITCH_MAX_PARTS (500) files, gaps under
|
||||
# STITCH_MAX_GAP_S) are first stitched back into one file by vgstitch.py (lossless concat; the arrival-time
|
||||
# file names say where every piece sits), then published as one recording. Joins are recorded in the sidecar.
|
||||
# Nothing in a stream's .live/ is touched until the stream is over (no open or recently written file).
|
||||
# - a LONE stream recording under MIN_LIVE_SECONDS (10) is a reconnect stub with nothing to join to: moved to
|
||||
# trash, never published. Only .live/ files — clips are frequently a few seconds long and are never subject to this.
|
||||
# - failure: name.normalize-failed.log is written and the file is skipped until that log is deleted
|
||||
# Idempotent; safe every few minutes.
|
||||
set -uo pipefail
|
||||
@@ -32,6 +38,18 @@ KEEP_RAW_STREAMS="${KEEP_RAW_STREAMS:-0}"
|
||||
# Encoder threads for every re-encode (clips and streams). 2 leaves the box responsive for Jellyfin/live viewers;
|
||||
# conversions take longer. systemd Environment=ENCODE_THREADS=... to change.
|
||||
ENCODE_THREADS="${ENCODE_THREADS:-2}"
|
||||
# STREAM recordings shorter than this are reconnect stubs, not recordings: every time a publisher drops and
|
||||
# comes back, MediaMTX opens a new file, and one bad night left 471 one- and two-second files in .live/.
|
||||
# They go to trash (3-day undo) instead of being published. Applies ONLY to .live/ — clips are often a few
|
||||
# seconds long on purpose and are never touched by this. 0 disables. systemd Environment=MIN_LIVE_SECONDS=…
|
||||
MIN_LIVE_SECONDS="${MIN_LIVE_SECONDS:-10}"
|
||||
# Stitching (vgstitch.py): fewer than this many fragments in one stream's .live/ are joined into one recording.
|
||||
# 500, not 20: on a lossy uplink MediaMTX restarts the recorder every ~10 s (a keyframe whose slices were lost
|
||||
# leaves an access unit with only SPS/PPS, which the recorder refuses), so a half-hour stream is ~180 same-
|
||||
# parameter files two seconds apart — exactly the case for stitching. The gap rule (STITCH_MAX_GAP_S seconds
|
||||
# between fragments = separate recordings) is the real guard. 0 parts disables stitching. systemd Environment=…
|
||||
STITCH_MAX_PARTS="${STITCH_MAX_PARTS:-500}"
|
||||
STITCH_MAX_GAP_S="${STITCH_MAX_GAP_S:-600}"
|
||||
MAX_W=2560; MAX_H=1440; MAX_FPS=60
|
||||
exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}"
|
||||
flock -n 9 || exit 0
|
||||
@@ -50,6 +68,11 @@ vglog() { local lvl="$1"; shift; printf 'ts=%s level=%s component=sweep %s\n' "$
|
||||
log() { vglog info "msg=\"$*\""; }
|
||||
dur() { ffprobe -v error -show_entries format=duration -of csv=p=0 "$1" 2>/dev/null | cut -d. -f1; }
|
||||
|
||||
# A stream's fragments are one recording: join them before anything below looks at them one by one.
|
||||
if (( STITCH_MAX_PARTS > 0 )) && [[ -r "$(dirname "$0")/vgstitch.py" ]]; then # installed side by side
|
||||
python3 "$(dirname "$0")/vgstitch.py" "$ROOT" --max-parts "$STITCH_MAX_PARTS" --max-gap-s "$STITCH_MAX_GAP_S" --idle-min "$MINS_IDLE"
|
||||
fi
|
||||
|
||||
# --- probe: prints "verdict reason" where verdict ∈ ok|remux|encode|skip ----------------------
|
||||
probe() {
|
||||
local f="$1" ext="${1##*.}"
|
||||
@@ -123,6 +146,31 @@ while IFS= read -r -d '' src; do
|
||||
fi
|
||||
fuser -s "$src" 2>/dev/null && continue # still being written
|
||||
[[ -e "$src" ]] || continue # dumped meanwhile
|
||||
if (( live )); then
|
||||
# The stream is still going while any sibling fragment is open or fresh: publishing this piece now would
|
||||
# make it a separate recording that the stitcher can never join. Wait for the stream to end.
|
||||
going=0
|
||||
for sib in "$(dirname "$src")"/*.mp4; do
|
||||
[[ -e $sib && $sib != "$src" ]] || continue
|
||||
if fuser -s "$sib" 2>/dev/null || [[ -n $(find "$sib" -mmin -"$MINS_IDLE" 2>/dev/null) ]]; then going=1; break; fi
|
||||
done
|
||||
if (( going )); then vglog info "event=defer reason=\"stream still going\" file=\"$src\" stream=\"$stream\""; continue; fi
|
||||
fi
|
||||
|
||||
if (( live && MIN_LIVE_SECONDS > 0 )); then
|
||||
secs=$(dur "$src"); secs=${secs:-0}
|
||||
if (( secs < MIN_LIVE_SECONDS )); then
|
||||
night="$(date +%F)"; rel="${src#"$ROOT"/}"; dst="$ROOT/trash/$night/$rel"
|
||||
mkdir -p "$(dirname "$dst")"
|
||||
mv -f "$src" "$dst"
|
||||
[[ -e "${src%.*}.json" ]] && mv -f "${src%.*}.json" "$(dirname "$dst")/"
|
||||
rm -f "${src%.*}.stitch.json"
|
||||
rmdir "$(dirname "$src")" "$titledir" 2>/dev/null || true
|
||||
vgmet_add "jobs:discard:done:stream"
|
||||
vglog info "event=discarded reason=too_short seconds=$secs min=$MIN_LIVE_SECONDS file=\"$src\" stream=\"$stream\" trash=\"$dst\""
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
read -r verdict reason <<<"$(probe "$src")"
|
||||
# Keep what the PROBE said: the deband branch below overwrites $reason with its own text, which silently
|
||||
@@ -220,8 +268,19 @@ PYEOF
|
||||
(( deband )) && vgmet_add deband 1
|
||||
if (( live )); then
|
||||
mv -f "$tmp" "$out"; chown 1000:1000 "$out" 2>/dev/null
|
||||
if [[ -n $tcjson ]]; then # sidecar: exact capture time (read by the Jellyfin plugin) + drift samples
|
||||
printf '%s\n' "$tcjson" > "${out%.*}.json"; chown 1000:1000 "${out%.*}.json" 2>/dev/null
|
||||
stitchjson="${src%.*}.stitch.json" # vgstitch.py: parts, offsets and the time missing at each join
|
||||
if [[ -n $tcjson || -e $stitchjson ]]; then # sidecar: exact capture time (read by the Jellyfin plugin) + drift samples
|
||||
python3 - "$tcjson" "$stitchjson" > "${out%.*}.json" <<'PYEOF'
|
||||
import json, sys, os
|
||||
tc, sj = sys.argv[1], sys.argv[2]
|
||||
d = json.loads(tc) if tc else {}
|
||||
if os.path.exists(sj):
|
||||
d["stitched"] = json.load(open(sj))
|
||||
# the strip decode's drift figure spans the joins too, so it is not clock drift on a stitched file
|
||||
if "drift_ms" in d: d["drift_ms_note"] = "includes collapsed joins; see stitched.joins"
|
||||
print(json.dumps(d))
|
||||
PYEOF
|
||||
chown 1000:1000 "${out%.*}.json" 2>/dev/null; rm -f "$stitchjson"
|
||||
fi
|
||||
if [[ $verdict == encode ]] && (( ! deband || KEEP_RAW_STREAMS )); then # gap-filled re-encode (or asked to): keep the raw in the hidden .src/
|
||||
srcdir="$(dirname "$out")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null
|
||||
|
||||
407
bin/vghosts-check.py
Executable file
407
bin/vghosts-check.py
Executable file
@@ -0,0 +1,407 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a hostname change end to end: what is on disk, what is running, what the UIs were told, what DNS says.
|
||||
|
||||
sudo vghosts-check.py on the server: everything
|
||||
vghosts-check.py --repo on a dev box: only the checks that need the repo (defaults, docs, pages)
|
||||
sudo vghosts-check.py --json machine-readable
|
||||
|
||||
Source of truth is the Caddyfile (`# vanguard:<role>` markers), the same place configure.py reads the hostnames
|
||||
back from. Every other place a hostname lives is checked AGAINST it: a host that is not one of Caddy's is stale.
|
||||
"ovh1.<base>" is the box's ssh name and must appear nowhere a browser is sent.
|
||||
|
||||
ORDER MATTERS, and the sections below run in the order a change must be made, because each depends on the one
|
||||
before it (HOSTNAME-CHANGE.md, section E):
|
||||
1 DNS nothing can be issued a certificate or reached until the name resolves to the box
|
||||
2 Caddyfile the roles and their names: what configure.py --full / --hosts wrote; certificates follow
|
||||
3 .env the same names as compose variables; a mismatch here means a half-done --full
|
||||
4 mediamtx.yml webrtcAdditionalHosts = main host (WebRTC ICE); needs a MediaMTX restart to take effect
|
||||
5 containers the env each running container was CREATED with (a .env edit without a recreate shows here)
|
||||
6 HTTPS every host answers with a valid certificate; gated hosts send browsers to the auth host
|
||||
7 Jellyfin SSO plugin OID endpoint, branding disclaimer, Vanguard plugin URLs (on disk) + the SSO start
|
||||
8 Gitea app.ini, the auth source (live redirect), the runner, the server clone's remote
|
||||
9 Authentik outpost host, proxy provider external host + cookie domain, Jellyfin/Gitea redirect URIs.
|
||||
LAST among the services, on purpose: an app changed before Authentik is merely unreachable on
|
||||
its new name until Authentik catches up; Authentik changed before the app locks you out of it
|
||||
10 Discord the OAuth2 redirect in the developer portal — cannot be read; the exact value is printed
|
||||
11 repo deployLive.sh / switchToClone.sh defaults, about page labels = Caddy roles, docs
|
||||
12 streamers OBS dock URL and clock source: cannot be read; listed so nobody forgets
|
||||
A failure in an early section usually explains failures in later ones: fix top-down, re-run. The sweep runs the
|
||||
Jellyfin and Gitea checks before Authentik, and the report lists them that way, because that is the order to make
|
||||
the change in.
|
||||
"""
|
||||
import argparse, json, os, re, socket, ssl, subprocess, sys, urllib.request, urllib.error
|
||||
from pathlib import Path
|
||||
|
||||
ROLES = ("main", "auth", "watch", "git", "bot", "live", "files", "sync", "clips", "diag")
|
||||
ENV_KEY = {"main": "MAIN_HOST", "watch": "WATCH_HOST", "git": "GIT_HOST", "bot": "BOT_HOST", "live": "LIVE_HOST",
|
||||
"files": "FILES_HOST", "sync": "SYNC_HOST", "clips": "CLIPS_HOST", "diag": "DIAG_HOST"}
|
||||
GATED = ("main", "files", "sync", "clips", "live", "bot", "diag") # forward-auth: anonymous -> auth host
|
||||
REPO = Path(os.environ.get("VANGUARD_REPO", str(Path.home() / "vanguard-stream") if Path.home().joinpath("vanguard-stream").is_dir() else "."))
|
||||
APPDATA = Path("/srv/appdata")
|
||||
|
||||
results = [] # (section, status, text)
|
||||
def rec(section, status, text): results.append((section, status, text))
|
||||
def ok(section, text): rec(section, "PASS", text)
|
||||
def bad(section, text): rec(section, "FAIL", text)
|
||||
def warn(section, text): rec(section, "WARN", text)
|
||||
def manual(section, text): rec(section, "MANUAL", text)
|
||||
|
||||
def sh(cmd, timeout=30, **kw):
|
||||
try:
|
||||
return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, **kw)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
class R: returncode, stdout, stderr = 1, "", str(error)
|
||||
return R()
|
||||
|
||||
def read_hosts(caddyfile):
|
||||
hosts, role = {}, None
|
||||
for line in caddyfile.read_text().splitlines():
|
||||
if line[:1].isspace(): continue
|
||||
t = line.strip()
|
||||
m = re.match(r"#\s*vanguard:(\w+)", t)
|
||||
if m: role = m.group(1); continue
|
||||
if t and not t.startswith(("#", "}", "{")) and t.endswith("{") and " " in t:
|
||||
host = t.split()[0].rstrip(",") # "a, b {": an alias; the first name is the role's
|
||||
if "example.com" in host or role == "redirect": role = None; continue
|
||||
hosts[role or ("main" if "main" not in hosts else host)] = host
|
||||
role = None
|
||||
return hosts
|
||||
|
||||
def read_env(path):
|
||||
env = {}
|
||||
try:
|
||||
for line in path.read_text().splitlines():
|
||||
if "=" in line and not line.startswith("#"):
|
||||
k, v = line.split("=", 1); env[k.strip()] = v.strip()
|
||||
except OSError:
|
||||
pass
|
||||
return env
|
||||
|
||||
URL_RE = re.compile(r"https?://([A-Za-z0-9.-]+\.[A-Za-z]{2,})(?::\d+)?")
|
||||
def hosts_in(text):
|
||||
return sorted(set(h.lower() for h in URL_RE.findall(text)))
|
||||
|
||||
def classify(section, where, found, allowed, expect=None, base=""):
|
||||
"""found: hostnames seen in `where`. allowed: every current host. expect: the one role's host, if specific."""
|
||||
if not found:
|
||||
warn(section, f"{where}: no hostname found"); return
|
||||
for h in found:
|
||||
if expect and h == expect: ok(section, f"{where}: {h}")
|
||||
elif expect and h != expect and (h.endswith("." + base) or h.startswith("ovh1.")):
|
||||
bad(section, f"{where}: {h} — expected {expect}")
|
||||
elif h in allowed: ok(section, f"{where}: {h}")
|
||||
elif base and (h.endswith("." + base) or h == base or h.startswith("ovh1.")):
|
||||
bad(section, f"{where}: {h} — not a current host (stale)")
|
||||
else:
|
||||
pass # third-party hostnames (discord.com, github.com, …) are not ours to judge
|
||||
|
||||
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *a, **k): return None
|
||||
_opener = urllib.request.build_opener(_NoRedirect)
|
||||
|
||||
def curl_head(url, timeout=12):
|
||||
"""(status, location, error) for a browser-like GET, without following redirects. GET, not HEAD: the
|
||||
forward-auth gate answers HEAD with 200 and the SSO start endpoints are GET-only."""
|
||||
req = urllib.request.Request(url, method="GET", headers={"User-Agent": "Mozilla/5.0 vghosts-check", "Accept": "text/html,*/*"})
|
||||
try:
|
||||
with _opener.open(req, timeout=timeout) as r:
|
||||
r.read(2048)
|
||||
return r.status, r.headers.get("Location", ""), ""
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.headers.get("Location", ""), ""
|
||||
except Exception as e:
|
||||
return 0, "", str(e)
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--repo", action="store_true", help="repo-only checks (dev box)")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
ap.add_argument("--caddyfile", default=None, help="path to the generated Caddyfile (default: <repo>/Caddyfile)")
|
||||
args = ap.parse_args()
|
||||
repo = REPO
|
||||
caddyfile = Path(args.caddyfile) if args.caddyfile else repo / "Caddyfile"
|
||||
|
||||
# ---- 2 Caddyfile: the truth -------------------------------------------------------------------
|
||||
S = "2 Caddyfile"
|
||||
if not caddyfile.is_file():
|
||||
if args.repo:
|
||||
hosts = {}
|
||||
warn(S, f"no Caddyfile at {caddyfile} (dev box): repo checks use the names in deployLive.sh / switchToClone.sh")
|
||||
else:
|
||||
bad(S, f"no Caddyfile at {caddyfile}: nothing to validate against — run configure.py --full first"); return report(args)
|
||||
else:
|
||||
hosts = read_hosts(caddyfile)
|
||||
main_host = hosts.get("main", "")
|
||||
base = main_host.split(".", 1)[1] if "." in main_host else ""
|
||||
allowed = set(hosts.values())
|
||||
for role in (ROLES if hosts else ()):
|
||||
if role in hosts: ok(S, f"{role:6} {hosts[role]}")
|
||||
else: warn(S, f"{role:6} no site block (configure.py --hosts to add one)")
|
||||
if base and f"ovh1.{base}" in allowed: bad(S, f"ovh1.{base} still has a site block")
|
||||
|
||||
# ---- 1 DNS -------------------------------------------------------------------------------------
|
||||
S = "1 DNS"
|
||||
if hosts and not args.repo:
|
||||
ips = {}
|
||||
for role, h in hosts.items():
|
||||
try: ips[h] = sorted(set(a[4][0] for a in socket.getaddrinfo(h, 443, proto=socket.IPPROTO_TCP)))
|
||||
except OSError as e: bad(S, f"{h}: does not resolve ({e})")
|
||||
role_ips = {h: v for r, h in hosts.items() for hh, v in ips.items() if hh == h and r in ROLES}
|
||||
distinct = set(tuple(v) for v in role_ips.values())
|
||||
for h, v in ips.items():
|
||||
if h in role_ips: (ok if len(distinct) == 1 else bad)(S, f"{h} -> {', '.join(v)}")
|
||||
else: warn(S, f"{h} -> {', '.join(v)} (apex/other block: a redirect may live anywhere, but Caddy must hold its certificate to answer)")
|
||||
if len(distinct) > 1: bad(S, "the role hosts resolve to different addresses — a CNAME points elsewhere")
|
||||
if base:
|
||||
try:
|
||||
old = sorted(set(a[4][0] for a in socket.getaddrinfo(f"ovh1.{base}", 22, proto=socket.IPPROTO_TCP)))
|
||||
ok(S, f"ovh1.{base} -> {', '.join(old)} (ssh only; fine to keep)")
|
||||
except OSError: warn(S, f"ovh1.{base} no longer resolves — ssh by IP or another name")
|
||||
|
||||
# ---- 3 .env --------------------------------------------------------------------------------------
|
||||
S = "3 .env"
|
||||
env = read_env(repo / ".env")
|
||||
if not args.repo:
|
||||
if not env: bad(S, f"{repo / '.env'} unreadable (run with sudo, from the checkout)")
|
||||
for role, key in ENV_KEY.items():
|
||||
if role not in hosts: continue
|
||||
v = env.get(key, "")
|
||||
if v == hosts[role]: ok(S, f"{key}={v}")
|
||||
elif not v: bad(S, f"{key} unset — expected {hosts[role]}")
|
||||
else: bad(S, f"{key}={v} — Caddyfile says {hosts[role]}")
|
||||
apex, apex_to = env.get("APEX_HOST", ""), env.get("APEX_REDIRECT", "")
|
||||
if apex:
|
||||
found = hosts_in(apex_to); classify(S, f"APEX_REDIRECT ({apex} ->)", found, allowed, base=base)
|
||||
else: ok(S, "APEX_HOST empty (bare domain not served)")
|
||||
|
||||
# ---- 4 mediamtx.yml ------------------------------------------------------------------------------
|
||||
S = "4 mediamtx.yml"
|
||||
if not args.repo:
|
||||
mtx = repo / "mediamtx.yml"
|
||||
if mtx.is_file():
|
||||
m = re.search(r"^webrtcAdditionalHosts:\s*\[([^\]]*)\]", mtx.read_text(), re.M)
|
||||
got = [x.strip().strip("'\"") for x in (m.group(1).split(",") if m else []) if x.strip()]
|
||||
if main_host in got: ok(S, f"webrtcAdditionalHosts contains {main_host}")
|
||||
else: bad(S, f"webrtcAdditionalHosts = {got or 'missing'} — expected {main_host} (then: docker compose restart mediamtx, between streams)")
|
||||
else: bad(S, "mediamtx.yml missing from the checkout")
|
||||
|
||||
# ---- 5 running containers ------------------------------------------------------------------------
|
||||
S = "5 containers"
|
||||
if not args.repo:
|
||||
want = {"jellyfin": ("JELLYFIN_PublishedServerUrl", "watch"), "gitea": ("GITEA__server__ROOT_URL", "git"),
|
||||
"act_runner": ("GITEA_INSTANCE_URL", "git"), "discord-bot": ("WEB_PUBLIC_URL", "bot"),
|
||||
"mediamtx": ("LIVE_HOST", "live"), "kibana": ("DIAG_HOST", "diag")}
|
||||
for container, (var, role) in want.items():
|
||||
if role not in hosts: continue
|
||||
r = sh(["docker", "inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container])
|
||||
if r.returncode != 0: warn(S, f"{container}: not running or no such container"); continue
|
||||
val = next((l.split("=", 1)[1] for l in r.stdout.splitlines() if l.startswith(var + "=")), None)
|
||||
if val is None: warn(S, f"{container}: {var} not in its environment"); continue
|
||||
if hosts[role] in val: ok(S, f"{container}: {var}={val}")
|
||||
else: bad(S, f"{container}: {var}={val} — expected {hosts[role]}; the container was created before .env changed: docker compose up -d {container}")
|
||||
gd = sh(["docker", "inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", "gitea"])
|
||||
for var in ("GITEA__server__DOMAIN", "GITEA__server__SSH_DOMAIN"):
|
||||
val = next((l.split("=", 1)[1] for l in gd.stdout.splitlines() if l.startswith(var + "=")), None)
|
||||
if val and "git" in hosts: (ok if val == hosts["git"] else bad)(S, f"gitea: {var}={val}" + ("" if val == hosts["git"] else f" — expected {hosts['git']}"))
|
||||
|
||||
# ---- 6 HTTPS -------------------------------------------------------------------------------------
|
||||
S = "6 HTTPS"
|
||||
if hosts and not args.repo:
|
||||
auth = hosts.get("auth", "")
|
||||
for role, h in hosts.items():
|
||||
status, loc, err = curl_head(f"https://{h}/")
|
||||
if err:
|
||||
if role in ROLES: bad(S, f"{h}: {err[:90]} (no certificate yet? Caddy block missing? DNS?)")
|
||||
else: warn(S, f"{h}: {err[:90]} — the bare domain's A record does not reach this box, so its Caddy block cannot get a certificate; fix the record or clear APEX_HOST")
|
||||
continue
|
||||
loc_host = (hosts_in(loc) or [""])[0]
|
||||
if role in GATED:
|
||||
if status in (301, 302, 303, 307, 308) and loc_host == auth: ok(S, f"{h}: {status} -> {auth} (gate)")
|
||||
elif status in (301, 302, 303, 307, 308): bad(S, f"{h}: {status} -> {loc} — gate sends to the wrong host")
|
||||
else: warn(S, f"{h}: {status} without a redirect to {auth} — is the forward-auth gate on?")
|
||||
else:
|
||||
(ok if status < 400 else bad)(S, f"{h}: {status}{(' -> ' + loc) if loc else ''}")
|
||||
# a move in progress: aliases must answer like the primary names; old names must redirect to the new ones
|
||||
alt, red = env.get("ALT_BASE", "").strip().lower(), env.get("REDIRECT_BASE", "").strip().lower()
|
||||
for role, h in hosts.items():
|
||||
if role not in ROLES or not h.endswith("." + base): continue
|
||||
if alt and alt != base:
|
||||
a = h[: -len(base)] + alt
|
||||
status, loc, err = curl_head(f"https://{a}/")
|
||||
(bad if err else ok)(S, f"alias {a}: {err[:80] if err else status}")
|
||||
if red and red != base:
|
||||
o = h[: -len(base)] + red
|
||||
status, loc, err = curl_head(f"https://{o}/")
|
||||
if err: warn(S, f"old name {o}: {err[:80]} (its certificate may still be issuing)")
|
||||
elif status == 308 and (hosts_in(loc) or [""])[0] == h: ok(S, f"old name {o}: 308 -> {h}")
|
||||
else: bad(S, f"old name {o}: {status} -> {loc or '(no Location)'} — expected 308 to {h}")
|
||||
if base:
|
||||
status, loc, err = curl_head(f"https://ovh1.{base}/", timeout=8)
|
||||
if err: ok(S, f"ovh1.{base}: no https (expected: ssh only)")
|
||||
else: bad(S, f"ovh1.{base} still answers https ({status}) — a Caddy block or another server is serving the old name")
|
||||
|
||||
# ---- 7 Authentik ----------------------------------------------------------------------------------
|
||||
S = "9 Authentik"
|
||||
if not args.repo and hosts:
|
||||
AK = r'''
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
from authentik.outposts.models import Outpost
|
||||
from authentik.providers.proxy.models import ProxyProvider
|
||||
from authentik.providers.oauth2.models import OAuth2Provider
|
||||
from authentik.sources.oauth.models import OAuthSource
|
||||
from authentik.brands.models import Brand
|
||||
out = {"outposts": [], "proxy": [], "oauth2": [], "sources": [], "brands": []}
|
||||
for o in Outpost.objects.all():
|
||||
out["outposts"].append({"name": o.name, "authentik_host": (o.config or {}).get("authentik_host", "") if isinstance(o.config, dict) else getattr(o.config, "authentik_host", "")})
|
||||
for p in ProxyProvider.objects.all():
|
||||
out["proxy"].append({"name": p.name, "external_host": p.external_host, "cookie_domain": p.cookie_domain, "mode": str(p.mode)})
|
||||
for p in OAuth2Provider.objects.all():
|
||||
uris = []
|
||||
try:
|
||||
uris = [u.url for u in p.redirect_uris]
|
||||
except Exception:
|
||||
try: uris = [u.get("url", "") for u in (p._redirect_uris or [])]
|
||||
except Exception: uris = [str(getattr(p, "redirect_uris", ""))]
|
||||
out["oauth2"].append({"name": p.name, "redirect_uris": uris})
|
||||
for s in OAuthSource.objects.all():
|
||||
out["sources"].append({"name": s.name, "slug": s.slug, "provider_type": s.provider_type})
|
||||
for b in Brand.objects.all():
|
||||
out["brands"].append({"domain": b.domain, "default": b.default, "logo": str(b.branding_logo), "default_app": str(b.default_application) if b.default_application_id else ""})
|
||||
print("VGHOSTS " + json.dumps(out))
|
||||
'''
|
||||
r = sh(["docker", "exec", "-i", "authentik-server", "ak", "shell", "-c", "import sys; exec(sys.stdin.read())"], input=AK, timeout=120)
|
||||
line = next((l for l in (r.stdout + r.stderr).splitlines() if l.startswith("VGHOSTS ")), None)
|
||||
if not line:
|
||||
bad(S, "could not read Authentik (" + (r.stderr.strip().splitlines() or ["no output"])[-1][:120] + ")")
|
||||
else:
|
||||
ak = json.loads(line[8:])
|
||||
auth = hosts.get("auth", "")
|
||||
for o in ak["outposts"]:
|
||||
h = hosts_in(o["authentik_host"])
|
||||
if h == [auth]: ok(S, f"outpost {o['name']!r}: authentik_host {o['authentik_host']}")
|
||||
elif not o["authentik_host"]: warn(S, f"outpost {o['name']!r}: authentik_host unset (the embedded outpost then uses its own address; set it to https://{auth})")
|
||||
else: bad(S, f"outpost {o['name']!r}: authentik_host {o['authentik_host']} — expected https://{auth}")
|
||||
for p in ak["proxy"]:
|
||||
h = hosts_in(p["external_host"])
|
||||
classify(S, f"proxy provider {p['name']!r} external_host", h, allowed, base=base)
|
||||
cd = p["cookie_domain"]
|
||||
if "domain" in p["mode"].lower():
|
||||
(ok if cd == base else bad)(S, f"proxy provider {p['name']!r} cookie_domain={cd or '(unset)'}" + ("" if cd == base else f" — expected {base}; sessions will not span the hosts"))
|
||||
for p in ak["oauth2"]:
|
||||
expect = hosts.get("watch") if "jellyfin" in p["name"].lower() else hosts.get("git") if "gitea" in p["name"].lower() else None
|
||||
for u in p["redirect_uris"]:
|
||||
classify(S, f"OAuth2 provider {p['name']!r} redirect", hosts_in(u), allowed, expect=expect, base=base)
|
||||
if not p["redirect_uris"]: warn(S, f"OAuth2 provider {p['name']!r}: no redirect URIs")
|
||||
for b in ak["brands"]:
|
||||
if b["domain"] in ("", "authentik-default"): ok(S, f"brand domain {b['domain']!r} (default)")
|
||||
elif b["domain"] == base or b["domain"].endswith(base): ok(S, f"brand domain {b['domain']}")
|
||||
else: warn(S, f"brand domain {b['domain']} — not under {base}")
|
||||
# ---- 8 Discord --------------------------------------------------------------------------
|
||||
for s in ak["sources"]:
|
||||
if s["provider_type"] == "discord":
|
||||
manual("10 Discord", f"developer portal → OAuth2 → Redirects must contain exactly: https://{auth}/source/oauth/callback/{s['slug']}/")
|
||||
if not any(s["provider_type"] == "discord" for s in ak["sources"]):
|
||||
bad("10 Discord", "no Discord OAuth source in Authentik")
|
||||
|
||||
# ---- 9 Jellyfin -----------------------------------------------------------------------------------
|
||||
S = "7 Jellyfin"
|
||||
if not args.repo and hosts:
|
||||
jf = APPDATA / "jellyfin" / "config"
|
||||
if jf.is_dir():
|
||||
for xml in sorted(jf.rglob("*.xml")):
|
||||
try: text = xml.read_text(errors="replace")
|
||||
except OSError: continue
|
||||
found = [h for h in hosts_in(text) if base and (h.endswith("." + base) or h == base or h.startswith("ovh1."))]
|
||||
if not found: continue
|
||||
name = xml.name.lower()
|
||||
expect = hosts.get("auth") if "sso" in name else None
|
||||
classify(S, f"{xml.relative_to(jf)}", found, allowed, expect=expect, base=base)
|
||||
else: warn(S, f"{jf} not found")
|
||||
if "watch" in hosts:
|
||||
status, loc, err = curl_head(f"https://{hosts['watch']}/sso/OID/start/authentik")
|
||||
lh = (hosts_in(loc) or [""])[0]
|
||||
if err: bad(S, f"SSO start: {err[:80]}")
|
||||
elif status in (301, 302, 303, 307) and lh == hosts.get("auth"): ok(S, f"SSO start -> {hosts['auth']} (plugin OID endpoint is right)")
|
||||
else: bad(S, f"SSO start answered {status} -> {loc or '(no Location)'} — SSO-Auth plugin OID endpoint should be https://{hosts.get('auth')}/application/o/jellyfin/")
|
||||
|
||||
# ---- 10 Gitea ---------------------------------------------------------------------------------------
|
||||
S = "8 Gitea"
|
||||
if not args.repo and hosts and "git" in hosts:
|
||||
ini = APPDATA / "gitea" / "gitea" / "conf" / "app.ini"
|
||||
if ini.is_file():
|
||||
text = ini.read_text(errors="replace")
|
||||
for key in ("ROOT_URL", "DOMAIN", "SSH_DOMAIN"):
|
||||
m = re.search(rf"^{key}\s*=\s*(\S+)", text, re.M)
|
||||
v = m.group(1) if m else ""
|
||||
(ok if hosts["git"] in v else bad)(S, f"app.ini {key}={v or '(missing)'}" + ("" if hosts["git"] in v else f" — expected {hosts['git']} (compose env sets it; recreate gitea)"))
|
||||
else: warn(S, f"{ini} not found")
|
||||
status, loc, err = curl_head(f"https://{hosts['git']}/user/oauth2/authentik")
|
||||
lh = (hosts_in(loc) or [""])[0]
|
||||
if err: bad(S, f"auth source: {err[:80]}")
|
||||
elif lh == hosts.get("auth"): ok(S, f"auth source 'authentik' -> {hosts['auth']} (discovery URL is right)")
|
||||
else: bad(S, f"auth source: {status} -> {loc or '(no Location)'} — Site administration → Authentication sources → authentik → discovery URL https://{hosts.get('auth')}/application/o/gitea/.well-known/openid-configuration")
|
||||
runner = Path("/srv/appdata/act_runner/.runner")
|
||||
if runner.is_file():
|
||||
classify(S, "runner registration (.runner)", hosts_in(runner.read_text(errors="replace")), allowed, expect=hosts["git"], base=base)
|
||||
else: warn(S, "runner: /srv/appdata/act_runner/.runner not found — ./registerGiteaRunner.sh")
|
||||
r = sh(["git", "-C", str(repo), "remote", "get-url", "origin"])
|
||||
if r.returncode == 0:
|
||||
url = r.stdout.strip(); m = re.search(r"@([A-Za-z0-9.-]+)", url)
|
||||
h = (m.group(1).lower() if m else "")
|
||||
(ok if h == hosts["git"] else bad)(S, f"server clone remote: {url}" + ("" if h == hosts["git"] else f" — expected {hosts['git']}"))
|
||||
else: warn(S, f"{repo}: not a git clone (deploy.sh pull needs one)")
|
||||
for cfg in (Path.home() / ".ssh" / "config.vanguard", Path("/home/debian/.ssh/config.vanguard")):
|
||||
if cfg.is_file():
|
||||
classify(S, f"{cfg}", [h for h in re.findall(r"(?im)^\s*host(?:name)?\s+(\S+)", cfg.read_text()) if "." in h], allowed, expect=hosts["git"], base=base); break
|
||||
|
||||
# ---- 11 repo ----------------------------------------------------------------------------------------
|
||||
S = "11 repo"
|
||||
for script, var in (("deployLive.sh", "HOST"), ("switchToClone.sh", "REPO_SSH")):
|
||||
p = repo / script
|
||||
if not p.is_file(): warn(S, f"{script} missing"); continue
|
||||
m = re.search(rf"^{var}=.*", p.read_text(), re.M)
|
||||
found = [h for h in re.findall(r"@([A-Za-z0-9.-]+\.[a-z]{2,})", m.group(0))] if m else []
|
||||
if hosts: classify(S, f"{script} {var}", [h.lower() for h in found], allowed, base=base)
|
||||
else: ok(S, f"{script} {var}: {', '.join(found) or '(none)'}")
|
||||
about = repo / "about" / "index.html"
|
||||
if about.is_file():
|
||||
labels = sorted(set(re.findall(r"sub\('([a-z]+)'\)", about.read_text())))
|
||||
if hosts:
|
||||
for label in labels:
|
||||
(ok if label in hosts else bad)(S, f"about page label '{label}'" + ("" if label in hosts else " — no such Caddy role; the gateway link will 404/TLS-fail"))
|
||||
else: ok(S, f"about page labels: {', '.join(labels)}")
|
||||
for doc in ("README.md", "FRESH-INSTALL.md", "STATUS.md", "LOGGING.md"):
|
||||
p = repo / doc
|
||||
if not p.is_file(): continue
|
||||
stale = [(n, l.strip()) for n, l in enumerate(p.read_text().splitlines(), 1)
|
||||
if re.search(r"https?://ovh1\.|ovh1\.[a-z.]+/", l) and not re.search(r"ssh|scp|ssh-only|ssh only", l)]
|
||||
if stale: bad(S, f"{doc}: {len(stale)} line(s) send a browser to ovh1 — first: {stale[0][0]}: {stale[0][1][:80]}")
|
||||
else: ok(S, f"{doc}: no browser links to ovh1")
|
||||
plan = repo / "CommunityDiscordMedia.md"
|
||||
if plan.is_file() and re.search(r"https?://ovh1\.", plan.read_text()):
|
||||
warn(S, "CommunityDiscordMedia.md still describes ovh1 as a web host (historical plan; mark the lines as superseded or leave)")
|
||||
|
||||
# ---- 12 streamers -------------------------------------------------------------------------------
|
||||
if hosts.get("live"):
|
||||
manual("12 streamers", f"OBS → Docks → Custom Browser Docks: https://{hosts['live']}/dock/ ; dock Settings re-saved; Timecode band → \"Add band + clock source\" again; manual RTMP/SRT configs re-copied from the dock")
|
||||
manual("12 streamers", f"Jellyfin apps on phones/TVs: server https://{hosts.get('watch','')} ; Discord pins: https://{main_host}/ (members), https://{hosts.get('clips','')}/ (gamers), https://{hosts.get('files','')}/ (editors)")
|
||||
return report(args)
|
||||
|
||||
def report(args):
|
||||
order = lambda s: int(s.split()[0])
|
||||
sections = sorted(set(s for s, _, _ in results), key=order)
|
||||
if args.json:
|
||||
print(json.dumps([{"section": s, "status": st, "text": t} for s, st, t in results], indent=1)); return 0
|
||||
mark = {"PASS": " ok ", "FAIL": " FAIL ", "WARN": " warn ", "MANUAL": " todo "}
|
||||
for s in sections:
|
||||
print(f"\n== {s}")
|
||||
for sec, st, t in results:
|
||||
if sec == s: print(f" [{mark[st]}] {t}")
|
||||
fails = sum(1 for _, st, _ in results if st == "FAIL"); warns = sum(1 for _, st, _ in results if st == "WARN")
|
||||
todo = sum(1 for _, st, _ in results if st == "MANUAL")
|
||||
print(f"\n{fails} failed, {warns} warnings, {todo} to check by hand. Fix top-down: an early failure explains the later ones.")
|
||||
return 1 if fails else 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
179
bin/vgstitch.py
Executable file
179
bin/vgstitch.py
Executable file
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stitch reconnect fragments of one stream back into one recording (run by normalize-sweep.sh).
|
||||
|
||||
Every time a publisher drops and comes back, MediaMTX opens a new file under streams/<user>/<Title>/.live/,
|
||||
so a night with a flaky uplink leaves one stream as a row of fragments: 20 min, 2 s, 14 min, 1 s, ... The
|
||||
fragments are the SAME stream (same user, same title) and the file names carry the server arrival time to
|
||||
the microsecond, so we know exactly where each piece sits on the clock. This joins them losslessly:
|
||||
|
||||
* only when the stream is over: nothing in .live/ open (fuser) or modified in the last IDLE_MIN minutes
|
||||
* only fewer than MAX_PARTS fragments (default 500 — a lossy uplink splits a half-hour stream into ~180
|
||||
files, see normalize-sweep.sh): beyond that is a broken night, left for vgstuck.sh / the streaming page
|
||||
* a chain breaks where the gap between fragments is longer than MAX_GAP_S (default 10 min: a reconnect
|
||||
is seconds, a second session the same evening is not), or where the video/audio parameters change
|
||||
(resolution, codec, rate): those cannot be copy-concatenated, so each side is stitched on its own
|
||||
* ffmpeg concat demuxer with -c copy — no re-encode, the sweep's own pass (timecode, band removal,
|
||||
Opus->AAC) runs on the result as on any other recording
|
||||
* the gaps between fragments are COLLAPSED (the picture jumps), and recorded in <stem>.stitch.json which
|
||||
the sweep merges into the published sidecar: part files, each part's arrival time, its offset in the
|
||||
stitched file, and the wall-clock time missing at every join
|
||||
* the stitched file takes the FIRST fragment's name (so the arrival-time fallback for the timecode still
|
||||
points at the real start), the fragments move to trash/<night>/ for the usual 3-day undo
|
||||
* a fragment ffprobe cannot read is binned (reason=unreadable) rather than blocking its stream forever
|
||||
|
||||
Prints logfmt lines (component=sweep) for Kibana. Exit 0 unless the arguments are wrong.
|
||||
vgstitch.py <media root> [--max-parts N] [--max-gap-s S] [--idle-min M] [--dry-run]
|
||||
"""
|
||||
import argparse, json, os, re, subprocess, sys, time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
TZ = ZoneInfo("America/New_York")
|
||||
NAME_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})(?:-(\d{1,6}))?")
|
||||
|
||||
def vglog(level, **kv):
|
||||
parts = [f"ts={datetime.now().astimezone().isoformat(timespec='seconds')}", f"level={level}", "component=sweep"]
|
||||
for k, v in kv.items():
|
||||
v = str(v)
|
||||
parts.append(f'{k}="{v}"' if (" " in v or "/" in v or v == "") else f"{k}={v}")
|
||||
print(" ".join(parts), flush=True)
|
||||
|
||||
def arrival_ms(stem):
|
||||
m = NAME_RE.search(stem)
|
||||
if not m: return None
|
||||
y, mo, d, h, mi, s, us = m.groups()
|
||||
us = int((us or "0").ljust(6, "0"))
|
||||
return int(datetime(int(y), int(mo), int(d), int(h), int(mi), int(s), us, tzinfo=TZ).timestamp() * 1000)
|
||||
|
||||
def ffprobe(path):
|
||||
"""(params tuple, duration_s) or None when unreadable."""
|
||||
try:
|
||||
out = subprocess.run(["ffprobe", "-v", "error", "-show_entries",
|
||||
"stream=codec_type,codec_name,width,height,pix_fmt,sample_rate,channels:format=duration",
|
||||
"-of", "json", str(path)], capture_output=True, text=True, timeout=120, check=True).stdout
|
||||
j = json.loads(out)
|
||||
v = [s for s in j["streams"] if s["codec_type"] == "video"]
|
||||
a = [s for s in j["streams"] if s["codec_type"] == "audio"]
|
||||
if not v: return None
|
||||
dur = float(j["format"].get("duration") or 0)
|
||||
if dur <= 0: return None
|
||||
params = (v[0].get("codec_name"), v[0].get("width"), v[0].get("height"), v[0].get("pix_fmt"),
|
||||
tuple((s.get("codec_name"), s.get("sample_rate"), s.get("channels")) for s in a))
|
||||
return params, dur
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def busy(path):
|
||||
return subprocess.run(["fuser", "-s", str(path)], capture_output=True).returncode == 0
|
||||
|
||||
def to_trash(root, path, night):
|
||||
rel = path.relative_to(root)
|
||||
dst = root / "trash" / night / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.replace(path, dst)
|
||||
return dst
|
||||
|
||||
def stitch_dir(root, live, args):
|
||||
titledir = live.parent
|
||||
stream = f"{titledir.parent.name}/{titledir.name}"
|
||||
files = sorted(p for p in live.iterdir() if p.is_file() and p.suffix.lower() == ".mp4" and not p.name.endswith(".part"))
|
||||
if len(files) < 2: return
|
||||
now = time.time()
|
||||
for p in files: # still streaming? leave the whole set alone
|
||||
if busy(p) or now - p.stat().st_mtime < args.idle_min * 60:
|
||||
vglog("info", event="stitch_wait", stream=stream, parts=len(files), reason="stream still going")
|
||||
return
|
||||
if len(files) >= args.max_parts:
|
||||
vglog("warn", event="stitch_skipped", stream=stream, parts=len(files), reason="too_many", max=args.max_parts)
|
||||
return
|
||||
night = datetime.now().strftime("%Y-%m-%d")
|
||||
parts = []
|
||||
for p in files:
|
||||
t = arrival_ms(p.stem)
|
||||
info = ffprobe(p)
|
||||
if t is None:
|
||||
vglog("warn", event="stitch_skipped", stream=stream, file=p, reason="name has no timestamp"); return
|
||||
if info is None:
|
||||
if not args.dry_run: dst = to_trash(root, p, night)
|
||||
else: dst = "(dry run)"
|
||||
vglog("warn", event="discarded", reason="unreadable", file=p, stream=stream, trash=dst)
|
||||
continue
|
||||
parts.append({"file": p, "arrival_ms": t, "params": info[0], "duration_ms": int(info[1] * 1000)})
|
||||
parts.sort(key=lambda x: x["arrival_ms"])
|
||||
# chains: consecutive, same parameters, gap under the limit
|
||||
chains, cur = [], []
|
||||
for part in parts:
|
||||
if cur:
|
||||
prev = cur[-1]
|
||||
gap = part["arrival_ms"] - (prev["arrival_ms"] + prev["duration_ms"])
|
||||
if part["params"] != prev["params"]:
|
||||
vglog("info", event="stitch_break", stream=stream, reason="params_changed", file=part["file"],
|
||||
before=f"{prev['params'][1]}x{prev['params'][2]}", after=f"{part['params'][1]}x{part['params'][2]}")
|
||||
chains.append(cur); cur = []
|
||||
elif gap > args.max_gap_s * 1000:
|
||||
vglog("info", event="stitch_break", stream=stream, reason="gap", gap_s=gap // 1000, file=part["file"])
|
||||
chains.append(cur); cur = []
|
||||
cur.append(part)
|
||||
if cur: chains.append(cur)
|
||||
for chain in chains:
|
||||
if len(chain) < 2: continue
|
||||
first = chain[0]["file"]
|
||||
offset, joins, plist = 0, [], []
|
||||
for i, part in enumerate(chain):
|
||||
if i:
|
||||
prev = chain[i - 1]
|
||||
missing = part["arrival_ms"] - (prev["arrival_ms"] + prev["duration_ms"])
|
||||
joins.append({"at_s": round(offset / 1000, 3), "missing_ms": max(0, missing)})
|
||||
plist.append({"file": part["file"].name, "arrival_ms": part["arrival_ms"],
|
||||
"duration_ms": part["duration_ms"], "offset_ms": offset})
|
||||
offset += part["duration_ms"]
|
||||
if args.dry_run:
|
||||
vglog("info", event="stitched", stream=stream, parts=len(chain), joins=len(joins), dry_run=1,
|
||||
missing_ms=sum(j["missing_ms"] for j in joins), out=first); continue
|
||||
listing = live / f"{first.stem}.concat.txt"
|
||||
tmp = live / f"{first.stem}.stitch.part"
|
||||
listing.write_text("".join("file '%s'\n" % str(p["file"]).replace("'", r"'\''") for p in chain))
|
||||
run = subprocess.run(["nice", "-n", "15", "ionice", "-c", "3", "ffmpeg", "-nostdin", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "concat", "-safe", "0", "-i", str(listing), "-map", "0", "-c", "copy",
|
||||
"-movflags", "+faststart", "-f", "mp4", str(tmp)], capture_output=True, text=True)
|
||||
listing.unlink(missing_ok=True)
|
||||
got = ffprobe(tmp) if run.returncode == 0 else None
|
||||
want = offset / 1000
|
||||
if got is None or abs(got[1] - want) > 2 + len(chain):
|
||||
tmp.unlink(missing_ok=True)
|
||||
why = (run.stderr.strip().splitlines() or ["no stderr"])[-1][:200]
|
||||
vglog("error", event="stitch_failed", stream=stream, parts=len(chain), rc=run.returncode,
|
||||
want_s=int(want), got_s=int(got[1]) if got else "?", reason=why)
|
||||
continue # fragments untouched: the sweep publishes them one by one
|
||||
last_mtime = max(p["file"].stat().st_mtime for p in chain)
|
||||
for p in chain: to_trash(root, p["file"], night)
|
||||
final = live / f"{first.stem}.mp4"
|
||||
os.replace(tmp, final)
|
||||
os.utime(final, (last_mtime, last_mtime)) # keep it "idle" so this very sweep publishes it
|
||||
meta = {"parts": plist, "joins": joins, "missing_ms": sum(j["missing_ms"] for j in joins),
|
||||
"trash": str(root / "trash" / night / live.relative_to(root)), "stitched_at": datetime.now(TZ).isoformat(timespec="seconds")}
|
||||
(live / f"{first.stem}.stitch.json").write_text(json.dumps(meta))
|
||||
vglog("info", event="stitched", stream=stream, parts=len(chain), joins=len(joins),
|
||||
missing_ms=meta["missing_ms"], seconds=int(want), out=final, trash=meta["trash"])
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("root")
|
||||
ap.add_argument("--max-parts", type=int, default=500)
|
||||
ap.add_argument("--max-gap-s", type=int, default=600)
|
||||
ap.add_argument("--idle-min", type=int, default=3)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
root = Path(args.root)
|
||||
streams = root / "streams"
|
||||
if not streams.is_dir(): return 0
|
||||
for live in sorted(streams.glob("*/*/.live")):
|
||||
if live.is_dir():
|
||||
try: stitch_dir(root, live, args)
|
||||
except Exception as error:
|
||||
vglog("error", event="stitch_failed", dir=live, reason=str(error)[:200])
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
116
bin/vgstuck.sh
116
bin/vgstuck.sh
@@ -2,15 +2,23 @@
|
||||
# Triage recordings that never published: see why each one is stuck, then retry it or bin it.
|
||||
#
|
||||
# vgstuck.sh list everything stuck, with the reason (read-only, the default)
|
||||
# vgstuck.sh list --durations ...and measure each one's length (slow: fragmented MP4 is read end to end)
|
||||
# vgstuck.sh retry <match> clear what is blocking it so the sweep tries again
|
||||
# vgstuck.sh retry --all ...for everything that is parked
|
||||
# vgstuck.sh drop <match> move the recording to trash: 3-day undo, same as "dump tonight"
|
||||
# vgstuck.sh list --under 5 only recordings smaller than 5 MB (reconnect fragments)
|
||||
# vgstuck.sh drop --under 5 bin all of those; still-recording files are never touched
|
||||
# vgstuck.sh wipe bin EVERYTHING unprocessed: stops the sweep (killing any conversion in flight),
|
||||
# then moves every .live/ file — parked, waiting, published-collision, fresh — with
|
||||
# its sidecars, failure logs and .part leftovers to trash, then drops the emptied
|
||||
# folders. The ONLY thing left alone is a file MediaMTX has open, i.e. a stream
|
||||
# being recorded this minute. The Dump tab's "Bin all unprocessed" button runs this.
|
||||
# vgstuck.sh sweep kick the sweep once, without waiting on it
|
||||
# vgstuck.sh json the same survey as JSON (what dumpd serves to the web page)
|
||||
#
|
||||
# <match> is any substring of the path (a streamer name, a title, a timestamp). It must match at least one
|
||||
# recording, and `drop` refuses --all: binning every stuck recording in one keystroke is never a thing
|
||||
# somebody meant to do.
|
||||
# recording, and `drop` refuses --all so a stray pattern cannot bin everything by accident: the deliberate
|
||||
# version is spelled out as its own word, `wipe`.
|
||||
#
|
||||
# WHY A RECORDING STAYS IN .live/
|
||||
# parked a <Title>__<ts>.normalize-failed.log sits next to the intended output. The sweep skips the
|
||||
@@ -57,66 +65,118 @@ survey() {
|
||||
reason="nothing blocking it; the sweep has not got to it"
|
||||
fi
|
||||
[[ -e "$out.part" ]] && reason="$reason [leftover .part]"
|
||||
dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
|
||||
# Duration is OPT-IN. Everything in .live/ is fragmented MP4 with no duration in its header, so ffprobe
|
||||
# has to read the whole file to answer — gigabytes per recording. With many stuck files the survey took
|
||||
# over three minutes, dumpd's request timed out, and the web page broke. Size and age come from stat.
|
||||
dur=""
|
||||
if [[ ${VGSTUCK_DURATIONS:-0} == 1 ]]; then
|
||||
dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$src" 2>/dev/null | cut -d. -f1)
|
||||
fi
|
||||
printf '%s|%s|%s|%s|%s|%s|%s\n' "$src" "$out" "$state" \
|
||||
"$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:-0}" "$reason"
|
||||
"$(( $(date +%s) - $(stat -c %Y "$src") ))" "$(stat -c %s "$src")" "${dur:--}" "$reason"
|
||||
done < <(find "$ROOT/streams" -path '*/.live/*' -name '*.mp4' -print0 2>/dev/null)
|
||||
}
|
||||
|
||||
show() {
|
||||
local any=0 src out state age bytes dur reason
|
||||
local any=0 total=0 src out state age bytes dur reason
|
||||
printf '\n %-9s %-9s %9s %8s %s\n' state waiting length size recording
|
||||
while IFS='|' read -r src out state age bytes dur reason; do
|
||||
total=$(( total + bytes ))
|
||||
any=1
|
||||
if [[ $dur == - || -z $dur ]]; then length="-"; else
|
||||
length=$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 ))); fi
|
||||
printf ' %-9s %-9s %9s %7.0fM %s\n' "$state" \
|
||||
"$(( age / 3600 ))h$(printf '%02d' $(( (age % 3600) / 60 )))m" \
|
||||
"$(printf '%d:%02d:%02d' $(( dur / 3600 )) $(( (dur % 3600) / 60 )) $(( dur % 60 )))" \
|
||||
"$length" \
|
||||
"$(awk -v b="$bytes" 'BEGIN{print b/1000000}')" "${src#"$ROOT"/streams/}"
|
||||
printf ' %s\n' "$reason"
|
||||
done < <(survey | sort -t'|' -k3,3 -k4,4nr)
|
||||
(( any )) || printf ' nothing is stuck.\n'
|
||||
printf '\n retry one: vgstuck.sh retry <substring> bin one: vgstuck.sh drop <substring>\n\n'
|
||||
done < <(selected "${1:---all}" "${2:-}" | sort -t'|' -k3,3 -k4,4nr)
|
||||
(( any )) || printf ' nothing matches.\n'
|
||||
(( any )) && printf ' %s item(s), %s GB\n' "$(selected "${1:---all}" "${2:-}" | wc -l)" "$(awk -v b="$total" 'BEGIN{printf "%.1f", b/1e9}')"
|
||||
[[ ${VGSTUCK_DURATIONS:-0} == 1 ]] || printf '\n lengths not measured (slow on fragmented MP4): vgstuck.sh list --durations\n'
|
||||
printf ' retry one: vgstuck.sh retry <substring> bin one: vgstuck.sh drop <substring>\n\n'
|
||||
}
|
||||
|
||||
# rows matching the pattern, or everything when --all
|
||||
# rows matching the pattern, everything when --all, or everything smaller than N MB with --under N.
|
||||
# --under exists for reconnect fragments: every time a publisher drops and comes back MediaMTX opens a new
|
||||
# recording, so a bad night leaves hundreds of one- and two-second files. Still-recording files never match.
|
||||
selected() {
|
||||
local pattern="$1"
|
||||
if [[ $pattern == --all ]]; then survey; else survey | grep -F -- "$pattern" || true; fi
|
||||
local pattern="$1" limit="${2:-}"
|
||||
if [[ $pattern == --under ]]; then
|
||||
[[ $limit =~ ^[0-9]+$ ]] || { echo "usage: --under <megabytes>" >&2; return 2; }
|
||||
survey | awk -F'|' -v max="$(( limit * 1000000 ))" '$3 != "writing" && $5 < max'
|
||||
elif [[ $pattern == --all ]]; then survey
|
||||
else survey | grep -F -- "$pattern" || true
|
||||
fi
|
||||
}
|
||||
|
||||
do_retry() {
|
||||
local pattern="$1" n=0 src out state age bytes dur reason
|
||||
local pattern="$1" limit="${2:-}" n=0 src out state age bytes dur reason
|
||||
while IFS='|' read -r src out state age bytes dur reason; do
|
||||
[[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
|
||||
rm -f "$out.part" # leftover partial output: harmless, but no reason to keep
|
||||
rm -f "${out%.mp4}.normalize-failed.log" # THIS is what makes the sweep skip the file forever
|
||||
n=$((n + 1))
|
||||
vglog info "event=retry file=\"$src\" was=$state"
|
||||
done < <(selected "$pattern")
|
||||
done < <(selected "$pattern" "$limit")
|
||||
(( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
|
||||
vglog info "event=retry_done count=$n msg=\"run vgstuck.sh sweep, or wait for the timer\""
|
||||
}
|
||||
|
||||
do_drop() {
|
||||
local pattern="$1" night n=0 src out state age bytes dur reason rel dst
|
||||
[[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them"' >&2; return 2; }
|
||||
night="$(date +%F)"
|
||||
# one row per line on stdin (survey format) -> trash. Prints the count it moved.
|
||||
drop_rows() {
|
||||
local night="$1" n=0 src out state age bytes dur reason rel dst victim
|
||||
while IFS='|' read -r src out state age bytes dur reason; do
|
||||
[[ $state == writing ]] && { vglog info "event=skip reason=still_recording file=\"$src\""; continue; }
|
||||
for victim in "$src" "${src%.*}.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
|
||||
# The survey labels a file by its blocker first (a parked file stays "parked" even while being written), so the
|
||||
# safety check here does not trust the label: open by any process, or touched inside MINS_IDLE, means leave it.
|
||||
# In wipe mode (WIPE=1) only "open" counts — the sweep has been stopped, so an open file is MediaMTX recording.
|
||||
if fuser -s "$src" 2>/dev/null || { [[ ${WIPE:-0} != 1 ]] && { [[ $state == writing ]] || [[ -n $(find "$src" -mmin "-$MINS_IDLE" -print -quit 2>/dev/null) ]]; }; }; then
|
||||
vglog info "event=skip reason=still_recording file=\"$src\"" >&2; continue
|
||||
fi
|
||||
for victim in "$src" "${src%.*}.json" "${src%.*}.stitch.json" "${out%.mp4}.normalize-failed.log" "$out.part"; do
|
||||
[[ -e $victim ]] || continue
|
||||
rel="${victim#"$ROOT"/}"
|
||||
dst="$TRASH/$night/$rel"
|
||||
mkdir -p "$(dirname "$dst")"
|
||||
mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3"
|
||||
mv -f "$victim" "$dst" && vglog info "event=trashed file=\"$rel\" undo_days=3" >&2
|
||||
done
|
||||
rmdir "$(dirname "$src")" "$(dirname "$(dirname "$src")")" 2>/dev/null || true
|
||||
n=$((n + 1))
|
||||
done < <(selected "$pattern")
|
||||
done
|
||||
echo "$n"
|
||||
}
|
||||
|
||||
do_drop() {
|
||||
local pattern="$1" limit="${2:-}" night n
|
||||
[[ $pattern == --all ]] && { vglog error 'event=refused msg="drop --all is not allowed; name them, use --under <MB>, or wipe"' >&2; return 2; }
|
||||
night="$(date +%F)"
|
||||
n=$(selected "$pattern" "$limit" | drop_rows "$night") # per-file lines go to stderr, the count to stdout
|
||||
(( n )) || { vglog warn "event=nothing_matched pattern=\"$pattern\""; return 1; }
|
||||
vglog info "event=drop_done count=$n trash=\"$TRASH/$night\""
|
||||
}
|
||||
|
||||
# Everything unprocessed, deliberately. First the sweep is stopped, which kills any ffmpeg conversion it has in
|
||||
# flight (the timer starts it again within five minutes, to an empty queue). Then every .live/ file goes, parked
|
||||
# or fresh, and so do failure logs whose recording is already gone (they would keep the Stuck tab listing ghosts)
|
||||
# and leftover .part outputs. The one exception: a file some process still holds open after the sweep is down,
|
||||
# which is MediaMTX recording a live stream — pulling that from under it would lose the stream in progress.
|
||||
do_wipe() {
|
||||
local night n stale=0 log rel dst
|
||||
night="$(date +%F)"
|
||||
if systemctl is-active --quiet normalize-sweep.service; then
|
||||
systemctl stop normalize-sweep.service && vglog info "event=cancelled unit=normalize-sweep.service msg=\"conversion in flight killed\""
|
||||
fi
|
||||
systemctl is-active --quiet vgdeband.service && systemctl stop vgdeband.service && vglog info "event=cancelled unit=vgdeband.service"
|
||||
n=$(survey | WIPE=1 drop_rows "$night")
|
||||
while IFS= read -r -d '' log; do
|
||||
rel="${log#"$ROOT"/}"; dst="$TRASH/$night/$rel"; mkdir -p "$(dirname "$dst")"
|
||||
mv -f "$log" "$dst" && stale=$((stale + 1))
|
||||
done < <(find "$ROOT/streams" -maxdepth 2 \( -name '*.normalize-failed.log' -o -name '*.mp4.part' \) -print0 2>/dev/null)
|
||||
find "$ROOT/streams" -mindepth 2 -type d -name .live -empty -delete 2>/dev/null
|
||||
find "$ROOT/streams" -mindepth 2 -maxdepth 2 -type d -empty -delete 2>/dev/null
|
||||
vglog info "event=wipe_done recordings=${n:-0} stale_files=$stale trash=\"$TRASH/$night\" undo_days=3"
|
||||
}
|
||||
|
||||
# Same survey, as JSON. bin/dumpd.py serves this to the web page, so the classification lives in exactly
|
||||
# one place and the browser and the CLI can never disagree about why something is stuck.
|
||||
emit_json() {
|
||||
@@ -129,15 +189,19 @@ for line in sys.stdin:
|
||||
continue
|
||||
src, out, state, age, size, dur, reason = parts
|
||||
rows.append({"src": src, "out": out, "state": state, "age_s": int(age or 0),
|
||||
"bytes": int(size or 0), "duration_s": int(dur or 0), "reason": reason})
|
||||
"bytes": int(size or 0),
|
||||
"duration_s": int(dur) if dur not in ("", "-") else None, "reason": reason})
|
||||
print(json.dumps(rows))'
|
||||
}
|
||||
|
||||
for a in "$@"; do [[ $a == --durations ]] && export VGSTUCK_DURATIONS=1; done
|
||||
|
||||
case "${1:-list}" in
|
||||
list|"") show ;;
|
||||
list|"") show "${2:---all}" "${3:-}" ;;
|
||||
json) emit_json ;;
|
||||
retry) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all" >&2; exit 1; }; do_retry "$2" ;;
|
||||
drop) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>" >&2; exit 1; }; do_drop "$2" ;;
|
||||
retry) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh retry <substring>|--all|--under <MB>" >&2; exit 1; }; do_retry "$2" "${3:-}" ;;
|
||||
drop) [[ -n ${2:-} ]] || { echo "usage: vgstuck.sh drop <substring>|--under <MB>" >&2; exit 1; }; do_drop "$2" "${3:-}" ;;
|
||||
wipe) do_wipe ;;
|
||||
sweep) systemctl start --no-block normalize-sweep.service && vglog info "event=sweep_started" ;;
|
||||
*) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user