bunch of shit

bin/dumpd.py

@@ -3,7 +3,10 @@
 
 POST /dump   identity from X-Authentik-Username / X-Authentik-Groups (set by Caddy after Authentik)
              body (optional JSON): {"night":"YYYY-MM-DD","user":"name"}
-GET  /dump   -> lists what a dump would move (dry run)
+GET  /dump   -> lists what a dump would move (dry run); ?user=all&night=YYYY-MM-DD select whose and when
+             Two doors, different reach: /dump on live. (the dock's Dump tonight) is ALWAYS the caller's own night,
+             admin or not — "user" is ignored there. /streaming/api/dump on diag. (the streaming page's Dump tab) is
+             where admins pick a streamer or "all"; non-admins still get only their own there.
 GET  /me/stream-token -> short-lived publish token for the caller (member of streamers/admins); the dock
              and the live portal set it as the OBS/WHIP credential. No stored passwords anywhere.
 GET  /me/clips  -> Syncthing self-service status for the caller: our device id, their folder/device state
@@ -21,10 +24,18 @@ POST /deploy-bot -> Gitea Actions (job container -> host gateway) hands over a D
                    HMAC-SHA256 signed with BOT_DEPLOY_SECRET; installs it with vanguard-deploy-bot.
                    May carry X-Vanguard-Env (base64 JSON): the bot's own settings, kept in Gitea, written
                    into the stack .env for the keys BOT_ENV_KEYS allows. Signature covers header+body.
-GET  /stuck/api  -> every unpublished recording and why it is stuck (any gated user)
-POST /stuck/api  -> {"action":"retry"|"drop","src":...}: clear the blocker, or move it to trash with the
-             usual 3-day undo. Admin groups only; both actions run bin/vgstuck.sh so the web page and the
-             CLI share one implementation.
+GET  /me/stream-health -> the caller's live streams with B-frame depth: the dock warns when WebRTC will refuse
+GET  /streaming/api/live  -> what is publishing, from where, loss on the way in, recorder restarts, transport history, blocks
+POST /streaming/api/live  -> {"action":"stop","path","kind","session","minutes"}: kick that publish session via MediaMTX's
+             API and block the account's publishing for N minutes (0 = kick only); {"action":"unblock","user"}. Admins only.
+             /mtx-auth denies a blocked account, so neither OBS's auto-reconnect nor a Go Live restart gets around it.
+GET  /streaming/api/stuck -> every unpublished recording and why it is stuck (any gated user)
+GET  /streaming/api/disk  -> disk use by streamer / clipper / trash by month, system dirs, Docker, and logs (ES indices
+             by month, journald, Docker json logs). Cached 10 min; ?refresh=1 rewalks.
+POST /streaming/api/stuck -> {"action":"retry"|"drop","src":...}: clear the blocker, or move it to trash with the
+             usual 3-day undo; {"action":"drop_under","mb":N} bins every stream fragment under N MB; {"action":"wipe"}
+             bins everything unprocessed (not being written). Admin groups only; every action runs bin/vgstuck.sh
+             so the web page and the CLI share one implementation.
 GET  /metrics -> Prometheus text for Metricbeat: the media scripts' counters (bin/vgmetrics.sh) plus live
              gauges for the normalise queue and MediaMTX. Loopback listener only; Caddy routes nothing here.
 POST /mtx-auth -> MediaMTX authHTTP hook: allows read/playback; allows publish iff user/pass carries a valid
@@ -32,10 +43,12 @@ POST /mtx-auth -> MediaMTX authHTTP hook: allows read/playback; allows publish i
 
 "Night" = everything the user recorded from 17:00 Eastern on <night> until 02:00 Eastern the next day.
 Default night: today if it is 17:00 or later, else yesterday (so 00:00-02:00 still counts as last night).
-Users may dump only their own prefix; members of the admins / media-admins groups may pass "user" to dump anyone, or "all".
+Users may dump only their own prefix; members of the admins / media-admins groups may pass "user" to dump anyone, or "all",
+but only through /streaming/api/dump (the diag triage page). On live.'s /dump everyone dumps their own night only.
 """
 import base64, hashlib, hmac, json, os, re, shutil, subprocess, sys, threading
 import time as _time
+import urllib.parse
 from datetime import datetime, timedelta, time
 from zoneinfo import ZoneInfo
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -51,7 +64,7 @@ PUBLISH_GROUPS = set(g.strip() for g in os.environ.get("PUBLISH_GROUPS", "stream
 AUTH_USER_HEADER = os.environ.get("AUTH_USER_HEADER", "X-Authentik-Username")
 AUTH_GROUPS_HEADER = os.environ.get("AUTH_GROUPS_HEADER", "X-Authentik-Groups")
 ADMIN_GROUPS = set(g.strip() for g in os.environ.get("DUMP_ADMIN_GROUPS", "admins,media-admins").split(",") if g.strip())
-LISTEN = ("127.0.0.1", 8080)                     # /dump, /me/* — only via Caddy (identity headers)
+LISTEN = ("127.0.0.1", 8080)                     # /dump, /streaming/api/*, /me/* — only via Caddy (identity headers)
 AUTH_LISTEN = (os.environ.get("MTX_AUTH_BIND", "0.0.0.0"), int(os.environ.get("MTX_AUTH_PORT", "8081")))   # /mtx-auth only — reachable from Docker's bridge (firewall), used by MediaMTX
 TZ = ZoneInfo(os.environ.get("DUMP_TZ", "America/New_York"))
 NIGHT_START = time(17, 0)   # 5 pm
@@ -429,20 +442,441 @@ def verify_token(token):
         return None
     return user.lower()
 
+
+
+# ---- stream health: B-frames ------------------------------------------------------------------------
+# MediaMTX refuses to serve H.264 with B-frames over WebRTC ("WebRTC doesn't support H264 streams with
+# B-frames"), so an OBS encoder left on its default of 2 B-frames is unwatchable on the portal. Nothing on the
+# publish path can see this: obs-websocket cannot set encoder settings in OBS's Advanced output mode, and
+# MediaMTX only notices when a viewer tries. The recording can: ffprobe reads the reorder depth off the first
+# fragments of the growing .live file in ~50 ms with a small probesize. Cached per path for 30 s.
+_health_cache = {}
+
+def stream_health(path):
+    """{'bframes', 'profile', 'res', 'fps', 'file'} for a live path, or None if nothing is on disk yet."""
+    now = _time.time()
+    hit = _health_cache.get(path)
+    if hit and now - hit["at"] < 30:
+        return hit["data"]
+    live_dir = ROOT / "streams" / path / ".live"
+    try:
+        newest = max((f for f in live_dir.glob("*.mp4")), key=lambda f: f.stat().st_mtime, default=None)
+    except OSError:
+        newest = None
+    data = None
+    if newest is not None:
+        # One probe, fields by name: csv column order follows ffprobe's internal field order, not the list.
+        done = subprocess.run(["ffprobe", "-v", "error", "-probesize", "2M", "-analyzeduration", "2M",
+                               "-select_streams", "v:0",
+                               "-show_entries", "stream=has_b_frames,profile,width,height,r_frame_rate",
+                               "-of", "default=noprint_wrappers=1", str(newest)],
+                              capture_output=True, text=True, timeout=10)
+        fields = dict(line.split("=", 1) for line in done.stdout.splitlines() if "=" in line)
+        if fields:
+            try:
+                num, den = (fields.get("r_frame_rate", "0/1").split("/") + ["1"])[:2]
+                fps = round(float(num) / float(den), 2) if float(den) else None
+            except (ValueError, ZeroDivisionError):
+                fps = None
+            data = {"bframes": int(fields.get("has_b_frames") or 0), "profile": fields.get("profile", ""),
+                    "res": f"{fields.get('width', '?')}x{fields.get('height', '?')}", "fps": fps, "file": newest.name}
+    _health_cache[path] = {"at": now, "data": data}
+    return data
+
+# ---- live streaming status ------------------------------------------------------------------------
+# GET /streaming/api/live: what is publishing, from where, how much it has lost on the way in, and why
+# MediaMTX has been opening recording files. Cached 5 s: the Live tab polls every 10 s and several admins
+# may have it open. Reads MediaMTX's API and metrics on loopback, and its container log for the counters.
+_live_cache = {"at": 0.0, "data": None}
+MTX_METRICS = os.environ.get("MTX_METRICS_URL", "http://127.0.0.1:9998/metrics")
+
+# WebRTC publish sessions and their transport. MediaMTX's API labels the selected ICE pair as
+# "<type>/<protocol>/<addr>/<port>" (host/udp/… or host/tcp/…), so the protocol a publisher is really on is
+# right there — that is how the Live tab tells a Go Live on TCP (the lossy-uplink switch) from one on UDP.
+# A session history is kept in MARK/webrtc-history.json so "when did he switch" survives the page being closed:
+# it is updated on every Live-tab poll AND every metrics scrape (Metricbeat, every few seconds), and a session
+# that is no longer listed is stamped ended. 24 h, 50 sessions per path.
+HISTORY_FILE = MARK / "webrtc-history.json"
+_history_lock = threading.Lock()
+
+def webrtc_sessions():
+    with urllib.request.urlopen(f"{MTX_API}/v3/webrtcsessions/list?itemsPerPage=1000", timeout=3) as response:
+        return json.load(response).get("items") or []
+
+def session_transport(item):
+    label = item.get("localCandidate") or item.get("remoteCandidate") or ""
+    parts = label.split("/")
+    return parts[1].lower() if len(parts) >= 2 and parts[1] else ""
+
+def track_sessions(items):
+    """Fold the current publish sessions into the history file; returns the history (path -> sessions)."""
+    now = datetime.now(TZ)
+    with _history_lock:
+        try:
+            hist = json.loads(HISTORY_FILE.read_text())
+        except (OSError, ValueError):
+            hist = {}
+        live_ids = set()
+        for it in items:
+            if str(it.get("state", "")).lower() != "publish":
+                continue
+            sid = it.get("id", ""); live_ids.add(sid)
+            rows = hist.setdefault(it.get("path", "?"), [])
+            row = next((r for r in rows if r["id"] == sid), None)
+            if row is None:
+                row = {"id": sid, "created": it.get("created"), "remote": it.get("remoteAddr"), "transport": "", "ended": None}
+                rows.append(row)
+            if session_transport(it): row["transport"] = session_transport(it)
+            row["received"] = it.get("rtpPacketsReceived", 0); row["lost"] = it.get("rtpPacketsLost", 0)
+            row["bytes"] = it.get("bytesReceived", 0); row["last_seen"] = now.isoformat(timespec="seconds")
+        cutoff = now - timedelta(hours=24)
+        for path in list(hist):
+            kept = []
+            for r in hist[path]:
+                if r["id"] not in live_ids and not r.get("ended"):
+                    r["ended"] = r.get("last_seen") or now.isoformat(timespec="seconds")
+                try:
+                    when = datetime.fromisoformat((r.get("ended") or r.get("last_seen") or "").replace("Z", "+00:00"))
+                    if when.tzinfo is None: when = when.replace(tzinfo=TZ)
+                except ValueError:
+                    when = now
+                if when >= cutoff: kept.append(r)
+            kept.sort(key=lambda r: r.get("created") or "")
+            hist[path] = kept[-50:]
+            if not hist[path]: del hist[path]
+        try:
+            MARK.mkdir(exist_ok=True)
+            tmp = HISTORY_FILE.with_suffix(".tmp"); tmp.write_text(json.dumps(hist)); os.replace(tmp, HISTORY_FILE)
+        except OSError as error:
+            log("history_write_failed", level="warn", error=str(error))
+        return hist
+
+def session_history():
+    """History as the page wants it: one list, newest first, with switch markers."""
+    try:
+        hist = json.loads(HISTORY_FILE.read_text())
+    except (OSError, ValueError):
+        return []
+    out = []
+    for path, rows in hist.items():
+        prev = None
+        for r in rows:
+            entry = {"path": path, **{k: r.get(k) for k in ("id", "created", "ended", "remote", "transport", "received", "lost", "bytes")}}
+            entry["id"] = (entry["id"] or "")[:8]
+            if prev is not None and prev.get("transport") and r.get("transport") and prev["transport"] != r["transport"]:
+                entry["switched_from"] = prev["transport"]
+            prev = r
+            out.append(entry)
+    out.sort(key=lambda e: e.get("created") or "", reverse=True)
+    return out
+
+# Turning a stream off from the Live tab. Two parts, because a kick alone is a hiccup: OBS reconnects on its
+# own within seconds and Go Live can be restarted with one click. So "stop" = kick the publish session through
+# MediaMTX's API AND put the streamer on a block list that /mtx-auth consults for every publish, for a chosen
+# number of minutes (0 = kick only). Blocks live in MARK/publish-blocks.json, survive a dumpd restart, expire on
+# their own, and the Live tab lists them with an unblock button.
+BLOCKS_FILE = MARK / "publish-blocks.json"
+_blocks_lock = threading.Lock()
+KICK_ENDPOINT = {"webrtc": "webrtcsessions", "rtmp": "rtmpconns", "srt": "srtconns", "rtsp": "rtspsessions"}
+
+def read_blocks():
+    try:
+        blocks = json.loads(BLOCKS_FILE.read_text())
+    except (OSError, ValueError):
+        blocks = {}
+    now = _time.time()
+    live = {u: b for u, b in blocks.items() if b.get("until", 0) > now}
+    if len(live) != len(blocks):
+        write_blocks(live)
+    return live
+
+def write_blocks(blocks):
+    try:
+        MARK.mkdir(exist_ok=True)
+        tmp = BLOCKS_FILE.with_suffix(".tmp"); tmp.write_text(json.dumps(blocks)); os.replace(tmp, BLOCKS_FILE)
+    except OSError as error:
+        log("blocks_write_failed", level="error", error=str(error))
+
+def block_user(user, minutes, by, reason=""):
+    with _blocks_lock:
+        blocks = read_blocks()
+        blocks[user] = {"until": _time.time() + minutes * 60, "by": by, "reason": reason,
+                        "since": datetime.now(TZ).isoformat(timespec="seconds")}
+        write_blocks(blocks)
+    log("publish_block", user=user, minutes=minutes, by=by)
+
+def unblock_user(user, by):
+    with _blocks_lock:
+        blocks = read_blocks()
+        if blocks.pop(user, None) is not None:
+            write_blocks(blocks); log("publish_unblock", user=user, by=by); return True
+    return False
+
+def blocked(user):
+    return read_blocks().get(user)
+
+def kick_publisher(kind, session_id):
+    endpoint = KICK_ENDPOINT.get(kind)
+    if not endpoint:
+        raise ValueError(f"cannot kick a {kind} publisher")
+    request = urllib.request.Request(f"{MTX_API}/v3/{endpoint}/kick/{urllib.parse.quote(session_id, safe='')}", method="POST")
+    with urllib.request.urlopen(request, timeout=5) as response:
+        return response.status
+
+def stop_stream(path, kind, session_id, minutes, who):
+    user = path.split("/", 1)[0].lower()
+    out = {"path": path, "user": user, "kicked": False, "blocked_minutes": minutes}
+    if minutes > 0:
+        block_user(user, minutes, who, reason=f"stopped from the Live tab ({path})")   # block first: no reconnect window
+    try:
+        kick_publisher(kind, session_id); out["kicked"] = True
+    except Exception as error:
+        out["error"] = f"kick failed: {error}"
+    # the Opus mirror of an OBS stream dies with its source (runOnReady exits), so nothing else to stop
+    _live_cache["data"] = None
+    log("stream_stop", level="info" if out["kicked"] else "error", stream=path, kind=kind, by=who, minutes=minutes,
+        result=out.get("error", "kicked"))
+    return out
+
+def live_status():
+    now = _time.time()
+    if _live_cache["data"] is not None and now - _live_cache["at"] < 5:
+        return _live_cache["data"]
+    out = {"paths": [], "publishers": [], "counts": {}, "log": [], "errors": [], "history": []}
+    sessions = {}
+    try:
+        items = webrtc_sessions()
+        track_sessions(items)
+        for it in items:
+            sessions[it.get("id", "")] = {"transport": session_transport(it), "since": it.get("created"),
+                                          "established": it.get("peerConnectionEstablished")}
+        out["history"] = session_history()
+    except Exception as error:
+        out["errors"].append(f"MediaMTX WebRTC sessions: {error}")
+    try:
+        with urllib.request.urlopen(f"{MTX_API}/v3/paths/list?itemsPerPage=1000", timeout=3) as response:
+            for item in json.load(response).get("items") or []:
+                if item.get("ready"):
+                    row = {"name": item["name"], "tracks": item.get("tracks") or [],
+                           "readers": len(item.get("readers") or []),
+                           "since": item.get("readyTime"), "bytes": item.get("bytesReceived", 0),
+                           "mirror": str(item["name"]).endswith(".webrtc")}
+                    if not row["mirror"]:
+                        try:
+                            row["health"] = stream_health(item["name"])
+                        except Exception as error:
+                            row["health_error"] = str(error)
+                    out["paths"].append(row)
+    except Exception as error:
+        out["errors"].append(f"MediaMTX API: {error}")
+    try:
+        with urllib.request.urlopen(MTX_METRICS, timeout=3) as response:
+            text = response.read().decode(errors="replace")
+        rows = {}
+        for match in re.finditer(r"^(webrtc_sessions|rtmp_conns|srt_conns)(_rtp_packets_received|_rtp_packets_lost|_bytes_received)?\{([^}]*)\} (\d+)$", text, re.M):
+            kind, field, labels_text, value = match.groups()
+            labels = dict(re.findall(r'(\w+)="([^"]*)"', labels_text))
+            if labels.get("state") != "publish":
+                continue
+            key = (labels.get("path", "?"), labels.get("id", ""), labels.get("remoteAddr", "?"))
+            row = rows.setdefault(key, {"kind": kind.split("_")[0]})
+            if field == "_rtp_packets_received": row["received"] = int(value)
+            elif field == "_rtp_packets_lost": row["lost"] = int(value)
+            elif field == "_bytes_received": row["bytes"] = int(value)
+        for (path, sid, addr), row in rows.items():
+            out["publishers"].append({"path": path, "id": sid[:8], "session": sid, "from": addr, **row, **sessions.get(sid, {})})
+        out["blocks"] = [{"user": u, **b, "until_iso": datetime.fromtimestamp(b["until"], TZ).isoformat(timespec="seconds")}
+                         for u, b in sorted(read_blocks().items())]
+    except Exception as error:
+        out["errors"].append(f"MediaMTX metrics: {error}")
+    try:
+        done = subprocess.run(["docker", "logs", "mediamtx", "--since", "10m"],
+                              capture_output=True, text=True, timeout=15)
+        log_text = done.stdout + done.stderr
+        out["counts"] = {
+            "publish_sessions": log_text.count("is publishing"),
+            "recorder_restarts": len(re.findall(r"recording \d+ tracks", log_text)),
+            "processing_errors": log_text.count("processing errors"),
+            "access_unit_errors": log_text.count("recorder] access unit"),
+            "window": "10m",
+        }
+        out["log"] = [l for l in log_text.splitlines() if re.search(r"recorder|is publishing|closed|ERR", l)][-14:]
+    except Exception as error:
+        out["errors"].append(f"MediaMTX log: {error}")
+    _live_cache.update(at=now, data=out)
+    return out
+
+# ---- disk report ------------------------------------------------------------------------------------
+# GET /streaming/api/disk: where the bytes are. Streamers (streams/<user>, month from the recording's own
+# "__YYYY-MM-DD" stamp, else mtime), clippers (clips/<user>, mtime), trash (its night folders), system (appdata
+# per app, /srv/live, /srv/bot, Docker's own accounting) and logs (Elasticsearch indices by the date in their
+# name, the ES data dir as the on-disk truth, journald, Docker's json logs). Walking a few TB of files takes
+# seconds, so the report is cached for ten minutes and ?refresh=1 forces it. Sizes are allocated blocks, i.e.
+# what the disk actually loses, not apparent length.
+_disk_cache = {"at": 0.0, "data": None}
+_disk_lock = threading.Lock()
+DISK_CACHE_S = 600
+APPDATA = Path("/srv/appdata")
+ES_NETRC = Path(os.environ.get("ES_NETRC_FILE", "/etc/vanguard/es-netrc"))
+ES_URL = os.environ.get("ES_URL", "http://127.0.0.1:9200")
+MONTH_IN_NAME = re.compile(r"__(\d{4})-(\d{2})-\d{2}_")
+MONTH_IN_INDEX = re.compile(r"(\d{4})\.(\d{2})\.\d{2}")
+
+def _blocks(st):
+    return st.st_blocks * 512 if hasattr(st, "st_blocks") else st.st_size
+
+def walk_sizes(root, month_of):
+    """{month: bytes} and total for everything under root; month_of(path, stat) -> 'YYYY-MM'."""
+    months, total = {}, 0
+    stack = [root]
+    while stack:
+        current = stack.pop()
+        try:
+            with os.scandir(current) as entries:
+                for entry in entries:
+                    try:
+                        if entry.is_symlink(): continue
+                        if entry.is_dir(follow_symlinks=False): stack.append(entry.path); continue
+                        st = entry.stat(follow_symlinks=False)
+                    except OSError:
+                        continue
+                    size = _blocks(st); total += size
+                    month = month_of(entry.path, st)
+                    months[month] = months.get(month, 0) + size
+        except OSError:
+            continue
+    return {"total": total, "months": months}
+
+def month_from_mtime(_path, st):
+    return datetime.fromtimestamp(st.st_mtime, TZ).strftime("%Y-%m")
+
+def month_from_name(path, st):
+    m = MONTH_IN_NAME.search(os.path.basename(path))
+    return f"{m.group(1)}-{m.group(2)}" if m else month_from_mtime(path, st)
+
+def dir_total(path):
+    return walk_sizes(path, lambda p, s: "all")["total"] if Path(path).is_dir() else 0
+
+def es_indices():
+    """[{index, bytes, month}] from _cat/indices, with the netrc the deadman also uses."""
+    import netrc as _netrc
+    request = urllib.request.Request(f"{ES_URL}/_cat/indices?bytes=b&format=json&h=index,store.size,creation.date.string&expand_wildcards=all")
+    try:
+        creds = _netrc.netrc(str(ES_NETRC)).authenticators(urllib.parse.urlparse(ES_URL).hostname or "127.0.0.1")
+    except (OSError, _netrc.NetrcParseError):
+        creds = None
+    if creds:
+        request.add_header("Authorization", "Basic " + base64.b64encode(f"{creds[0]}:{creds[2]}".encode()).decode())
+    with urllib.request.urlopen(request, timeout=10) as response:
+        rows = json.load(response)
+    out = []
+    for row in rows:
+        name = row.get("index", "")
+        m = MONTH_IN_INDEX.search(name)
+        month = f"{m.group(1)}-{m.group(2)}" if m else (row.get("creation.date.string") or "")[:7] or "other"
+        out.append({"index": name, "bytes": int(row.get("store.size") or 0), "month": month})
+    return out
+
+def docker_df():
+    done = subprocess.run(["docker", "system", "df", "--format", "{{json .}}"], capture_output=True, text=True, timeout=60)
+    rows = []
+    for line in done.stdout.splitlines():
+        try: rows.append(json.loads(line))
+        except ValueError: pass
+    return rows
+
+def journal_bytes():
+    done = subprocess.run(["journalctl", "--disk-usage"], capture_output=True, text=True, timeout=20)
+    m = re.search(r"([\d.]+)\s*([KMGT]?)B?", done.stdout.replace(",", ""))
+    if not m: return 0
+    return int(float(m.group(1)) * {"": 1, "K": 1e3, "M": 1e6, "G": 1e9, "T": 1e12}[m.group(2)])
+
+def disk_report(force=False):
+    now = _time.time()
+    with _disk_lock:
+        if not force and _disk_cache["data"] is not None and now - _disk_cache["at"] < DISK_CACHE_S:
+            return _disk_cache["data"]
+        t0 = _time.time()
+        out = {"generated": datetime.now(TZ).isoformat(timespec="seconds"), "fs": {}, "streamers": {}, "clippers": {},
+               "trash": {}, "system": {}, "logs": {}, "errors": []}
+        for label, path in (("media", ROOT), ("root", Path("/"))):
+            try:
+                u = shutil.disk_usage(path); out["fs"][label] = {"path": str(path), "total": u.total, "used": u.used, "free": u.free}
+            except OSError as error:
+                out["errors"].append(f"{path}: {error}")
+        same_fs = False
+        try: same_fs = os.stat(ROOT).st_dev == os.stat("/").st_dev
+        except OSError: pass
+        out["fs"]["same_filesystem"] = same_fs
+        for user_dir in sorted(p for p in (ROOT / "streams").glob("*") if p.is_dir()):
+            r = walk_sizes(user_dir, month_from_name)
+            r["raw"] = dir_total(user_dir / ".src"); r["unpublished"] = sum(dir_total(d) for d in user_dir.glob("*/.live"))
+            out["streamers"][user_dir.name] = r
+        for user_dir in sorted(p for p in (ROOT / "clips").glob("*") if p.is_dir()):
+            out["clippers"][user_dir.name] = walk_sizes(user_dir, month_from_mtime)
+        out["trash"] = walk_sizes(ROOT / "trash", lambda p, s: (re.search(r"/trash/(\d{4}-\d{2})-\d{2}/", p) or [None, month_from_mtime(p, s)])[1]) if (ROOT / "trash").is_dir() else {"total": 0, "months": {}}
+        other_media = 0
+        for entry in ROOT.iterdir():
+            if entry.name not in ("streams", "clips", "trash") and entry.is_dir() and not entry.is_symlink():
+                other_media += dir_total(entry)
+        out["system"]["media_other"] = other_media                       # .markers, .syncthing, .stversions, …
+        for app_dir in sorted(p for p in APPDATA.glob("*") if p.is_dir()) if APPDATA.is_dir() else []:
+            size = dir_total(app_dir)
+            if app_dir.name == "elasticsearch": out["logs"]["es_data_dir"] = size
+            else: out["system"][f"appdata/{app_dir.name}"] = size
+        for label, path in (("live portal", "/srv/live"), ("bot", "/srv/bot"), ("etc/vanguard", "/etc/vanguard")):
+            out["system"][label] = dir_total(path)
+        try:
+            for row in docker_df():
+                out["system"][f"docker {row.get('Type', '?').lower()}"] = row.get("Size", "?")   # docker's own figures (human strings)
+        except Exception as error:
+            out["errors"].append(f"docker system df: {error}")
+        try:
+            out["logs"]["indices"] = es_indices()
+            out["logs"]["es_total"] = sum(i["bytes"] for i in out["logs"]["indices"])
+            by_month = {}
+            for i in out["logs"]["indices"]: by_month[i["month"]] = by_month.get(i["month"], 0) + i["bytes"]
+            out["logs"]["months"] = by_month
+        except Exception as error:
+            out["errors"].append(f"Elasticsearch: {error}")
+        try: out["logs"]["journald"] = journal_bytes()
+        except Exception as error: out["errors"].append(f"journalctl: {error}")
+        out["logs"]["docker_json_logs"] = sum(_blocks(p.stat()) for p in Path("/var/lib/docker/containers").glob("*/*-json.log*")) if Path("/var/lib/docker/containers").is_dir() else 0
+        out["seconds"] = round(_time.time() - t0, 1)
+        _disk_cache.update(at=_time.time(), data=out)
+        return out
+
 # ---- stuck recordings -----------------------------------------------------------------------------
-# GET  /stuck/api   every unpublished recording and why, for anyone the gate let through
-# POST /stuck/api   {"action":"retry"|"drop","src":"<path>"} — admins only
+# GET  /streaming/api/stuck   every unpublished recording and why, for anyone the gate let through
+# POST /streaming/api/stuck   {"action":"retry"|"drop","src":"<path>"} — admins only
 # The classification and the file moves both live in bin/vgstuck.sh, so the web page and the CLI can never
 # disagree about why something is stuck, and "delete" means the same three-day trash in both.
 STUCK_TOOL = os.environ.get("VGSTUCK_TOOL", "/usr/local/bin/vgstuck.sh")
 
+_stuck_cache = {"at": 0.0, "items": None}
+STUCK_CACHE_S = 20          # the page refreshes every 30 s and several admins may have it open
+
 def stuck_list():
-    done = subprocess.run([STUCK_TOOL, "json"], capture_output=True, text=True, timeout=180)
+    """The survey, cached briefly. It runs ffprobe on every stuck file, so with many of them it is slow, and
+    without a cache every open tab re-ran it every 30 s. The cache is also invalidated by any action."""
+    now = _time.time()
+    if _stuck_cache["items"] is not None and now - _stuck_cache["at"] < STUCK_CACHE_S:
+        return _stuck_cache["items"]
     try:
-        return json.loads(done.stdout or "[]")
-    except ValueError:
-        log("stuck_list_failed", level="error", rc=done.returncode, err=(done.stderr or "")[:200])
-        return []
+        done = subprocess.run([STUCK_TOOL, "json"], capture_output=True, text=True, timeout=120)
+    except subprocess.TimeoutExpired:
+        log("stuck_list_timeout", level="error", seconds=120)
+        raise RuntimeError("the survey took over 120 s; too many files, or ffprobe is stuck") from None
+    if done.returncode != 0:
+        log("stuck_list_failed", level="error", rc=done.returncode, err=(done.stderr or "")[:300])
+        raise RuntimeError(f"vgstuck.sh exited {done.returncode}: {(done.stderr or '').strip()[-200:]}")
+    try:
+        items = json.loads(done.stdout or "[]")
+    except ValueError as error:
+        log("stuck_list_badjson", level="error", err=str(error), head=(done.stdout or "")[:120])
+        raise RuntimeError(f"vgstuck.sh did not return JSON: {error}") from None
+    _stuck_cache.update(at=now, items=items)
+    return items
 
 def stuck_target(src):
     """Resolve what the browser asked us to act on, or raise. This is the only thing between a request body
@@ -461,6 +895,7 @@ def stuck_target(src):
 def stuck_action(action, src, who):
     path = stuck_target(src)
     done = subprocess.run([STUCK_TOOL, action, str(path)], capture_output=True, text=True, timeout=600)
+    _stuck_cache["items"] = None                    # the list just changed; the next read must re-survey
     lines = [l for l in (done.stdout + done.stderr).splitlines() if l.strip()]
     log(f"stuck_{action}", level="info" if done.returncode == 0 else "error",
         file=str(path), by=who, rc=done.returncode)
@@ -579,6 +1014,10 @@ def metrics_text():
     # 4. MediaMTX. The Prometheus scrape of :9998 already has per-path detail; this is the count the portal
     #    and the dock actually mean by "live", with the derived .webrtc audio mirrors separated out.
     paths = mtx_paths()
+    try:
+        track_sessions(webrtc_sessions())          # keeps the transport history current between page views
+    except Exception:
+        pass
     add("vanguard_mediamtx_up", "gauge", "did the MediaMTX API answer this scrape", {}, 0 if paths is None else 1)
     if paths is not None:
         ready = [p for p in paths if p.get("ready")]
@@ -621,6 +1060,11 @@ class H(BaseHTTPRequestHandler):
     def _parse(self):
         n = int(self.headers.get("Content-Length") or 0)
         body = json.loads(self.rfile.read(n) or b"{}") if n else {}
+        # ?user=all&night=YYYY-MM-DD also work (the Dump tab's dry run). GET never moves anything: a link
+        # unfurled by Discord must not be able to dump a night.
+        query = urllib.parse.parse_qs(urllib.parse.urlparse(self.path).query)
+        for key in ("night", "user"):
+            if key not in body and query.get(key): body[key] = query[key][0]
         night = body.get("night")
         if night:
             datetime.strptime(night, "%Y-%m-%d")  # validate
@@ -637,8 +1081,15 @@ class H(BaseHTTPRequestHandler):
             if pf:
                 return prev, pf
         return n, files
+    def _is_dump(self):
+        path = self.path.split("?", 1)[0]
+        return path in ("/dump", "/streaming/api/dump")       # live. dock button / diag. Dump tab
+    def _admin_door(self):
+        return self.path.split("?", 1)[0] == "/streaming/api/dump"
     def _target(self, auth_user, is_admin, req_user):
-        if is_admin:
+        # Dumping someone else's night is triage, so it is only offered where triage lives: the diag host's
+        # streaming page. The dock's Dump tonight (live. /dump) is personal for everyone, admins included.
+        if is_admin and self._admin_door():
             return req_user or auth_user          # admins default to their own night; pass "all" or a name for others
         return auth_user
     def do_GET(self):
@@ -660,10 +1111,51 @@ class H(BaseHTTPRequestHandler):
             return self._json(200, {"user": who, "groups": groups, "admin": is_admin,
                                     "name": self.headers.get("X-Authentik-Name") or who, "email": self.headers.get("X-Authentik-Email") or "",
                                     "discord_id": proxy_claim(self.headers, "discord_id")})
-        if self.path == "/stuck/api":
+        if self.path == "/me/stream-health":
+            # The dock polls this after Go Live. Only the caller's own paths, and only what is live now.
+            who, _ = identity(self.headers)
+            if not who: return self._json(401, {"error": "not authenticated"})
+            streams = []
+            try:
+                with urllib.request.urlopen(f"{MTX_API}/v3/paths/list?itemsPerPage=1000", timeout=3) as response:
+                    for item in json.load(response).get("items") or []:
+                        name = str(item.get("name", ""))
+                        if item.get("ready") and name.lower().startswith(who.lower() + "/") and not name.endswith(".webrtc"):
+                            health = stream_health(name)
+                            streams.append({"path": name, "tracks": item.get("tracks") or [], **(health or {}),
+                                            "webrtc_ok": bool(health) and health["bframes"] == 0})
+            except Exception as error:
+                return self._json(502, {"error": f"MediaMTX: {error}"})
+            # Go Live polls this while live: an empty list means the server no longer has the stream (kicked, or
+            # the session died), and "blocked" says why it must not simply reconnect.
+            hold = blocked(who.lower())
+            return self._json(200, {"user": who, "streams": streams,
+                                    "blocked": ({"until": datetime.fromtimestamp(hold["until"], TZ).isoformat(timespec="seconds"),
+                                                 "by": hold.get("by")} if hold else None)})
+        if self.path.split("?", 1)[0] == "/streaming/api/disk":
             who, is_admin = identity(self.headers)
             if not who: return self._json(401, {"error": "not authenticated"})
-            return self._json(200, {"admin": is_admin, "user": who, "items": stuck_list()})
+            force = "refresh=1" in (self.path.split("?", 1) + [""])[1]
+            try:
+                return self._json(200, {"admin": is_admin, "user": who, **disk_report(force)})
+            except Exception as error:
+                return self._json(500, {"error": str(error)})
+        if self.path == "/streaming/api/live":
+            who, is_admin = identity(self.headers)
+            if not who: return self._json(401, {"error": "not authenticated"})
+            try:
+                return self._json(200, {"admin": is_admin, "user": who, **live_status()})
+            except Exception as error:
+                return self._json(500, {"error": str(error)})
+        if self.path == "/streaming/api/stuck":
+            who, is_admin = identity(self.headers)
+            if not who: return self._json(401, {"error": "not authenticated"})
+            try:
+                return self._json(200, {"admin": is_admin, "user": who, "items": stuck_list()})
+            except Exception as error:
+                # Always answer. A handler that raises sends nothing, Caddy turns that into an empty 502,
+                # and the page can only say "unexpected end of JSON" — which is exactly what happened.
+                return self._json(500, {"error": str(error)})
         if self.path.startswith("/me/clips"):
             who, _ = identity(self.headers)
             if not who: return self._json(401, {"error": "not authenticated"})
@@ -682,17 +1174,24 @@ class H(BaseHTTPRequestHandler):
                 return self._json(500, {"error": str(e)})
             # "pass" kept for the dock/portal: the token IS the password for MediaMTX
             return self._json(200, {"user": who, "pass": tok, "token": tok, "expires": exp, "ttl_hours": TOKEN_TTL_H})
-        if not self.path.startswith("/dump"):
+        if not self._is_dump():
             return self._json(404, {"error": "not found"})
         u, is_admin = identity(self.headers)
         if not u:
             return self._deny()
-        night, req_user = self._parse()
+        try:
+            night, req_user = self._parse()
+        except Exception as e:
+            return self._json(400, {"error": str(e)})
         tgt = self._target(u, is_admin, req_user)
         night, files = self._resolve(night, tgt)
-        self._json(200, {"night": night, "user": tgt, "dry_run": True,
-                         "files": [str(f.relative_to(ROOT)) for f in files],
-                         "gb": round(sum(f.stat().st_size for f in files) / 1e9, 2)})
+        start, end = night_window(night)
+        entries = [{"path": str(f.relative_to(ROOT)), "bytes": f.stat().st_size} for f in files]
+        self._json(200, {"night": night, "user": tgt, "by": u, "admin": is_admin, "dry_run": True,
+                         "window": [start.isoformat(timespec="minutes"), end.isoformat(timespec="minutes")],
+                         "default_night": default_night(),
+                         "files": [e["path"] for e in entries], "entries": entries,
+                         "gb": round(sum(e["bytes"] for e in entries) / 1e9, 2)})
     def do_POST(self):
         if self.server.server_port == AUTH_LISTEN[1] and self.path.startswith("/heartbeat"):
             # Alertmanager posts here every minute because the Watchdog alert always fires. Nothing is read from
@@ -776,7 +1275,7 @@ class H(BaseHTTPRequestHandler):
             path = (req.get("path") or "").lower()
             if path.endswith(".webrtc"):
                 # audio mirrors (<user>/<Title>.webrtc) are published by the hook inside the mediamtx container only
-                if req.get("ip") in ("127.0.0.1", "::1") and re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+\.webrtc", path):
+                if req.get("ip") in ("127.0.0.1", "::1") and re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_-]+\.webrtc", path):
                     return self._json(200, {"ok": True})
                 log("publish_deny", level="warn", stream=path, ip=req.get("ip"), reason="mirror paths are internal")
                 return self._json(401, {"error": ".webrtc paths are reserved"})
@@ -787,12 +1286,43 @@ class H(BaseHTTPRequestHandler):
             if not user:
                 log("publish_deny", level="warn", stream=path, ip=req.get("ip"), reason="bad or expired token")
                 return self._json(401, {"error": "invalid or expired stream token"})
-            if not re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+", path) or not path.startswith(user + "/"):
+            # title segment is dot-free (mirrors are <title>.webrtc and must never collide with a title)
+            if not re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_-]+", path) or not path.startswith(user + "/"):
                 log("publish_deny", level="warn", user=user, stream=path, reason="not their prefix")
                 return self._json(401, {"error": "path must be <your-username>/<Title>"})
+            hold = blocked(user.lower())
+            if hold:                                             # stopped from the Live tab: no reconnecting around it
+                log("publish_deny", level="warn", user=user, stream=path, ip=req.get("ip"), reason="blocked", by=hold.get("by"))
+                return self._json(401, {"error": "publishing is paused for this account by an admin"})
             log("publish_allow", user=user, stream=path, ip=req.get("ip"))
             return self._json(200, {"ok": True})
-        if self.path == "/stuck/api":
+        if self.path == "/streaming/api/live":
+            # {"action":"stop","path":..,"kind":..,"session":..,"minutes":N} or {"action":"unblock","user":..} — admins only
+            who, is_admin = identity(self.headers)
+            if not who: return self._deny()
+            if not is_admin:
+                return self._json(403, {"error": f"stopping streams is for {', '.join(sorted(ADMIN_GROUPS))}"})
+            try:
+                body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}")
+            except ValueError as error:
+                return self._json(400, {"error": str(error)})
+            action = body.get("action")
+            if action == "unblock":
+                user = str(body.get("user", "")).lower()
+                if not re.fullmatch(r"[a-z0-9_.-]+", user): return self._json(400, {"error": "bad user"})
+                _live_cache["data"] = None
+                return self._json(200, {"ok": unblock_user(user, who), "user": user})
+            if action == "stop":
+                path, kind, session = str(body.get("path", "")), str(body.get("kind", "")), str(body.get("session", ""))
+                if not re.fullmatch(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", path) or kind not in KICK_ENDPOINT or not re.fullmatch(r"[A-Za-z0-9-]{8,64}", session):
+                    return self._json(400, {"error": "need path, kind and session as the live list reports them"})
+                try:
+                    minutes = max(0, min(int(body.get("minutes", 30)), 24 * 60))
+                except (TypeError, ValueError):
+                    minutes = 30
+                return self._json(200, stop_stream(path, kind, session, minutes, who))
+            return self._json(400, {"error": "action must be stop or unblock"})
+        if self.path == "/streaming/api/stuck":
             who, is_admin = identity(self.headers)
             if not who: return self._deny()
             if not is_admin:
@@ -802,8 +1332,33 @@ class H(BaseHTTPRequestHandler):
             except ValueError as error:
                 return self._json(400, {"error": str(error)})
             action = body.get("action")
+            if action == "wipe":
+                # Everything unprocessed, deliberately (the Dump tab's button): stops the sweep, then every .live/
+                # file except one MediaMTX still has open, plus stale failure logs and .part leftovers. vgstuck.sh
+                # does the moving, into trash.
+                done = subprocess.run([STUCK_TOOL, "wipe"], capture_output=True, text=True, timeout=900)
+                _stuck_cache["items"] = None
+                lines = [l for l in (done.stdout + done.stderr).splitlines() if l.strip()]
+                log("stuck_wipe", level="info" if done.returncode == 0 else "error", by=who, rc=done.returncode,
+                    result=(lines[-1] if lines else ""))
+                return self._json(200, {"ok": done.returncode == 0, "rc": done.returncode, "output": lines[-6:]})
+            if action == "drop_under":
+                # Bulk: every unpublished STREAM recording smaller than N MB — reconnect stubs. Bounded so a
+                # typo cannot bin real recordings; clips never appear here because they never pass through
+                # .live/. Still-recording files are skipped by the tool itself.
+                try:
+                    mb = int(body.get("mb", 0))
+                except (TypeError, ValueError):
+                    mb = 0
+                if not 1 <= mb <= 50:
+                    return self._json(400, {"error": "mb must be between 1 and 50"})
+                done = subprocess.run([STUCK_TOOL, "drop", "--under", str(mb)], capture_output=True, text=True, timeout=900)
+                _stuck_cache["items"] = None
+                lines = [l for l in (done.stdout + done.stderr).splitlines() if l.strip()]
+                log("stuck_drop_under", level="info" if done.returncode == 0 else "error", mb=mb, by=who, rc=done.returncode)
+                return self._json(200, {"ok": done.returncode == 0, "rc": done.returncode, "output": lines[-6:]})
             if action not in ("retry", "drop"):
-                return self._json(400, {"error": "action must be retry or drop"})
+                return self._json(400, {"error": "action must be retry, drop, drop_under or wipe"})
             try:
                 return self._json(200, stuck_action(action, str(body.get("src", "")), who))
             except ValueError as error:
@@ -821,7 +1376,7 @@ class H(BaseHTTPRequestHandler):
                 return self._json(400, {"error": str(e)})
             except Exception as e:
                 return self._json(502, {"error": f"syncthing: {e}"})
-        if not self.path.startswith("/dump"):
+        if not self._is_dump():
             return self._json(404, {"error": "not found"})
         u, is_admin = identity(self.headers)
         if not u: