Soft per-user clip quotas: dumpd pauses/resumes Syncthing folders by folder size (CLIPS_QUOTA_GB, per-user overrides file); clips page shows usage

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

README.md

@@ -123,6 +123,12 @@ asks for theirs; on submit, dumpd creates `clips/<username>` with the video-only
 to our Syncthing (REST API, key read from `/srv/media/.syncthing/config.xml`) and creates the receive-only
 folder `clips-<username>` shared with them (min free 300 GB). They accept the device + folder on their PC,
 point it at their clip directory, Send Only. The page shows pairing/connection/sync state. Idempotent.
+**Per-user quota (soft)**: dumpd measures every `clips/<user>` every 5 min and pauses that user's Syncthing
+folder when it exceeds the quota (default `CLIPS_QUOTA_GB=100` in `dumpd.service`), resuming automatically
+under 90 %. The clips page shows usage and the paused state. Overrides per user in `/etc/vanguard/clip-quotas.json`,
+e.g. `{"techy2493": 250, "newguy": 50}` (GB) — read live, no restart. A gamer frees space by deleting clips in
+their own folder (Syncthing mirrors the deletion). Hard byte-exact quotas would be XFS project quotas on `/srv`
+(needs a remount with `prjquota`); not set up.
 Manual fallback (admin, same result for the server folder):
 
     ./addClipUser.sh <username>        # creates /srv/media/clips/<username> with the video-only .stignore

STATUS.md

@@ -15,6 +15,7 @@ Source of truth: this repo, `techy/Media-Server-Setup` on the stack's own Gitea
 | Every 5 min | Discord scheduled events snapshotted into a history so recordings can be matched to their session | Jellyfin plugin |
 | Every 30 min + startup | Every non-admin Jellyfin user set to direct-play-only (no hardware encoder on this box) | Jellyfin plugin |
 | Daily 04:10 ET | Recordings roll between weekday → week → month → year seasons as the calendar moves; no files change | Jellyfin plugin |
+| Every 5 min | Each user's clips folder is measured against their quota (100 GB default, per-user overrides); over quota → their Syncthing folder is paused, under 90 % → resumed; the clips page shows usage | dumpd quota loop — **pending deploy** |
 | Daily 05:00 ET | Dumped nights older than 3 days deleted for good | trash purge (systemd timer) |
 | On boot | Everything restarts itself after the media mount is up; interrupted recordings stay playable | systemd + Docker |
 | Continuous | Active streams are watchable live (WebRTC, HLS fallback) by Discord-server members; optional "is live" Discord post when a stream starts | MediaMTX + live portal (page deployed; MediaMTX port fix pending apply) |

bin/dumpd.py

@@ -100,6 +100,62 @@ ST_CONFIG = Path(os.environ.get("SYNCTHING_CONFIG", "/srv/media/.syncthing/confi
 ST_CLIPS_PREFIX = os.environ.get("SYNCTHING_CLIPS_PREFIX", "/var/syncthing/clips")      # path INSIDE the syncthing container
 STIGNORE_SRC = Path(os.environ.get("STIGNORE_SRC", "/etc/vanguard/stignore"))
 MIN_FREE_GB = int(os.environ.get("CLIPS_MIN_FREE_GB", "300"))
+CLIPS_QUOTA_GB = float(os.environ.get("CLIPS_QUOTA_GB", "100"))          # default per-user quota for clips/<user>
+CLIPS_QUOTA_FILE = Path(os.environ.get("CLIPS_QUOTA_FILE", "/etc/vanguard/clip-quotas.json"))   # {"user": GB, ...} overrides
+QUOTA_CHECK_S = int(os.environ.get("CLIPS_QUOTA_CHECK_S", "300"))
+QUOTA_STATE = Path(os.environ.get("CLIPS_QUOTA_STATE", "/srv/media/.markers/clip-quota-paused.json"))
+
+def quota_for(user):
+    try:
+        q = json.loads(CLIPS_QUOTA_FILE.read_text()).get(user)
+        if q is not None: return float(q)
+    except (OSError, json.JSONDecodeError, ValueError):
+        pass
+    return CLIPS_QUOTA_GB
+
+def folder_bytes(path):
+    total = 0
+    for root, dirs, files in os.walk(path):
+        for f in files:
+            try: total += os.lstat(os.path.join(root, f)).st_size
+            except OSError: pass
+    return total
+
+def quota_state():
+    try: return json.loads(QUOTA_STATE.read_text())
+    except (OSError, json.JSONDecodeError): return {}
+
+def quota_enforce_once():
+    """Pause a user's Syncthing folder when clips/<user> exceeds their quota; resume when back under 90 %."""
+    paused = quota_state(); changed = False
+    try:
+        folders = st("GET", "/rest/config/folders")
+    except Exception as e:
+        print(f"{datetime.now(TZ):%F %T} quota: syncthing unreachable ({e})", flush=True); return
+    for f in folders:
+        if not f["id"].startswith("clips-"): continue
+        user = f["id"][len("clips-"):]
+        path = Path(ROOT) / "clips" / user
+        if not path.is_dir(): continue
+        used = folder_bytes(path); q = quota_for(user) * 1e9
+        over = used > q; under = used < q * 0.9
+        if over and not f.get("paused"):
+            try: st("POST", f"/rest/db/pause?folder={f['id']}"); paused[user] = {"at": datetime.now(TZ).isoformat(), "used_gb": round(used / 1e9, 1), "quota_gb": q / 1e9}; changed = True
+            except Exception as e: print(f"{datetime.now(TZ):%F %T} quota: pause {user} failed: {e}", flush=True); continue
+            print(f"{datetime.now(TZ):%F %T} quota: PAUSED clips-{user} ({used/1e9:.1f} GB > {q/1e9:.0f} GB)", flush=True)
+        elif under and f.get("paused") and user in paused:
+            try: st("POST", f"/rest/db/resume?folder={f['id']}"); paused.pop(user, None); changed = True
+            except Exception as e: print(f"{datetime.now(TZ):%F %T} quota: resume {user} failed: {e}", flush=True); continue
+            print(f"{datetime.now(TZ):%F %T} quota: resumed clips-{user} ({used/1e9:.1f} GB)", flush=True)
+    if changed:
+        try: QUOTA_STATE.parent.mkdir(exist_ok=True); QUOTA_STATE.write_text(json.dumps(paused, indent=2))
+        except OSError: pass
+
+def quota_loop():
+    while True:
+        try: quota_enforce_once()
+        except Exception as e: print(f"{datetime.now(TZ):%F %T} quota loop error: {e}", flush=True)
+        _time.sleep(QUOTA_CHECK_S)
 
 def st_apikey():
     try:
@@ -146,6 +202,11 @@ def clips_status(user):
     except Exception as e:
         out["error"] = f"syncthing: {e}"
     out["folder_exists"] = Path(out["server_path"]).is_dir()
+    out["quota_gb"] = quota_for(user)
+    out["used_gb"] = round(folder_bytes(out["server_path"]) / 1e9, 2) if out["folder_exists"] else 0.0
+    ps = quota_state().get(user)
+    out["paused_for_quota"] = bool(ps)
+    if ps: out["paused_since"] = ps.get("at")
     return out
 
 def clips_setup(user, device_id, device_name=None):
@@ -357,4 +418,5 @@ if __name__ == "__main__":
     print(f"dumpd listening on {LISTEN[0]}:{LISTEN[1]} (dump, me) and {AUTH_LISTEN[0]}:{AUTH_LISTEN[1]} (mtx-auth)  tz={TZ.key}  night={NIGHT_START:%H:%M}-{NIGHT_END:%H:%M}  token-secret={'ok' if token_secret() else 'MISSING'}", flush=True)
     auth_srv = ThreadingHTTPServer(AUTH_LISTEN, H)
     threading.Thread(target=auth_srv.serve_forever, daemon=True).start()
+    threading.Thread(target=quota_loop, daemon=True).start()        # per-user clip quotas (pause/resume Syncthing folders)
     ThreadingHTTPServer(LISTEN, H).serve_forever()

clips/index.html

@@ -74,9 +74,11 @@ async function refresh(){
     pills.push(`<span class="pill ${d.folder_configured?'on':'off'}">share ${d.folder_configured?'configured':'not yet'}</span>`);
     if (d.device_id) pills.push(`<span class="pill ${d.device_connected?'on':'off'}">your device ${d.device_connected?'connected':'not connected'}</span>`);
     if (d.files_here != null) pills.push(`<span class="pill on">${d.files_here} file(s) here</span>`);
+    if (d.quota_gb) pills.push(`<span class="pill ${d.paused_for_quota?'off':(d.used_gb > d.quota_gb*0.9?'':'on')}">${d.used_gb} / ${d.quota_gb} GB${d.paused_for_quota?' — PAUSED (over quota)':''}</span>`);
     if (d.remote_completion != null && d.remote_completion < 100) pills.push(`<span class="pill">syncing ${Math.round(d.remote_completion)}%</span>`);
     $('pills').innerHTML = pills.join('');
     $('status').textContent = d.error ? d.error : (d.device_id ? `paired with ${d.device_name || 'your device'} (${d.device_id.slice(0,7)}…)` : 'not paired yet — do step 2');
+    if (d.paused_for_quota) $('status').textContent += `\nYour folder is over its ${d.quota_gb} GB quota, so syncing is paused. Delete older clips from your clip folder (they are removed here too) or ask an admin to raise the quota; syncing resumes automatically under 90 %.`;
     if (d.device_id) $('devid').value = d.device_id;
   } catch (e) { $('pills').innerHTML = `<span class="pill off">error</span>`; $('status').textContent = e.message; $('status').className = 'status bad'; }
 }

systemd/dumpd.service

@@ -9,6 +9,8 @@ Environment=DUMP_ADMIN_GROUPS=admins,media-admins
 Environment=PUBLISH_GROUPS=streamers,admins,media-admins
 Environment=STREAM_TOKEN_TTL_HOURS=12
 Environment=STREAM_TOKEN_SECRET_FILE=/etc/vanguard/stream-token.secret
+Environment=CLIPS_QUOTA_GB=100
+Environment=CLIPS_QUOTA_FILE=/etc/vanguard/clip-quotas.json
 Environment=DUMP_TZ=America/New_York
 Restart=always
 RestartSec=3