diff --git a/README.md b/README.md index fe55a24..85dfbc7 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,12 @@ asks for theirs; on submit, dumpd creates `clips/` with the video-only to our Syncthing (REST API, key read from `/srv/media/.syncthing/config.xml`) and creates the receive-only folder `clips-` shared with them (min free 300 GB). They accept the device + folder on their PC, point it at their clip directory, Send Only. The page shows pairing/connection/sync state. Idempotent. +**Per-user quota (soft)**: dumpd measures every `clips/` every 5 min and pauses that user's Syncthing +folder when it exceeds the quota (default `CLIPS_QUOTA_GB=100` in `dumpd.service`), resuming automatically +under 90 %. The clips page shows usage and the paused state. Overrides per user in `/etc/vanguard/clip-quotas.json`, +e.g. `{"techy2493": 250, "newguy": 50}` (GB) — read live, no restart. A gamer frees space by deleting clips in +their own folder (Syncthing mirrors the deletion). Hard byte-exact quotas would be XFS project quotas on `/srv` +(needs a remount with `prjquota`); not set up. Manual fallback (admin, same result for the server folder): ./addClipUser.sh # creates /srv/media/clips/ with the video-only .stignore diff --git a/STATUS.md b/STATUS.md index a6a32f3..149b6b7 100644 --- a/STATUS.md +++ b/STATUS.md @@ -15,6 +15,7 @@ Source of truth: this repo, `techy/Media-Server-Setup` on the stack's own Gitea | Every 5 min | Discord scheduled events snapshotted into a history so recordings can be matched to their session | Jellyfin plugin | | Every 30 min + startup | Every non-admin Jellyfin user set to direct-play-only (no hardware encoder on this box) | Jellyfin plugin | | Daily 04:10 ET | Recordings roll between weekday → week → month → year seasons as the calendar moves; no files change | Jellyfin plugin | +| Every 5 min | Each user's clips folder is measured against their quota (100 GB default, per-user overrides); over quota → their Syncthing folder is paused, under 90 % → resumed; the clips page shows usage | dumpd quota loop — **pending deploy** | | Daily 05:00 ET | Dumped nights older than 3 days deleted for good | trash purge (systemd timer) | | On boot | Everything restarts itself after the media mount is up; interrupted recordings stay playable | systemd + Docker | | Continuous | Active streams are watchable live (WebRTC, HLS fallback) by Discord-server members; optional "is live" Discord post when a stream starts | MediaMTX + live portal (page deployed; MediaMTX port fix pending apply) | diff --git a/bin/dumpd.py b/bin/dumpd.py index 86d4d9b..6f19a3d 100755 --- a/bin/dumpd.py +++ b/bin/dumpd.py @@ -100,6 +100,62 @@ ST_CONFIG = Path(os.environ.get("SYNCTHING_CONFIG", "/srv/media/.syncthing/confi ST_CLIPS_PREFIX = os.environ.get("SYNCTHING_CLIPS_PREFIX", "/var/syncthing/clips") # path INSIDE the syncthing container STIGNORE_SRC = Path(os.environ.get("STIGNORE_SRC", "/etc/vanguard/stignore")) MIN_FREE_GB = int(os.environ.get("CLIPS_MIN_FREE_GB", "300")) +CLIPS_QUOTA_GB = float(os.environ.get("CLIPS_QUOTA_GB", "100")) # default per-user quota for clips/ +CLIPS_QUOTA_FILE = Path(os.environ.get("CLIPS_QUOTA_FILE", "/etc/vanguard/clip-quotas.json")) # {"user": GB, ...} overrides +QUOTA_CHECK_S = int(os.environ.get("CLIPS_QUOTA_CHECK_S", "300")) +QUOTA_STATE = Path(os.environ.get("CLIPS_QUOTA_STATE", "/srv/media/.markers/clip-quota-paused.json")) + +def quota_for(user): + try: + q = json.loads(CLIPS_QUOTA_FILE.read_text()).get(user) + if q is not None: return float(q) + except (OSError, json.JSONDecodeError, ValueError): + pass + return CLIPS_QUOTA_GB + +def folder_bytes(path): + total = 0 + for root, dirs, files in os.walk(path): + for f in files: + try: total += os.lstat(os.path.join(root, f)).st_size + except OSError: pass + return total + +def quota_state(): + try: return json.loads(QUOTA_STATE.read_text()) + except (OSError, json.JSONDecodeError): return {} + +def quota_enforce_once(): + """Pause a user's Syncthing folder when clips/ exceeds their quota; resume when back under 90 %.""" + paused = quota_state(); changed = False + try: + folders = st("GET", "/rest/config/folders") + except Exception as e: + print(f"{datetime.now(TZ):%F %T} quota: syncthing unreachable ({e})", flush=True); return + for f in folders: + if not f["id"].startswith("clips-"): continue + user = f["id"][len("clips-"):] + path = Path(ROOT) / "clips" / user + if not path.is_dir(): continue + used = folder_bytes(path); q = quota_for(user) * 1e9 + over = used > q; under = used < q * 0.9 + if over and not f.get("paused"): + try: st("POST", f"/rest/db/pause?folder={f['id']}"); paused[user] = {"at": datetime.now(TZ).isoformat(), "used_gb": round(used / 1e9, 1), "quota_gb": q / 1e9}; changed = True + except Exception as e: print(f"{datetime.now(TZ):%F %T} quota: pause {user} failed: {e}", flush=True); continue + print(f"{datetime.now(TZ):%F %T} quota: PAUSED clips-{user} ({used/1e9:.1f} GB > {q/1e9:.0f} GB)", flush=True) + elif under and f.get("paused") and user in paused: + try: st("POST", f"/rest/db/resume?folder={f['id']}"); paused.pop(user, None); changed = True + except Exception as e: print(f"{datetime.now(TZ):%F %T} quota: resume {user} failed: {e}", flush=True); continue + print(f"{datetime.now(TZ):%F %T} quota: resumed clips-{user} ({used/1e9:.1f} GB)", flush=True) + if changed: + try: QUOTA_STATE.parent.mkdir(exist_ok=True); QUOTA_STATE.write_text(json.dumps(paused, indent=2)) + except OSError: pass + +def quota_loop(): + while True: + try: quota_enforce_once() + except Exception as e: print(f"{datetime.now(TZ):%F %T} quota loop error: {e}", flush=True) + _time.sleep(QUOTA_CHECK_S) def st_apikey(): try: @@ -146,6 +202,11 @@ def clips_status(user): except Exception as e: out["error"] = f"syncthing: {e}" out["folder_exists"] = Path(out["server_path"]).is_dir() + out["quota_gb"] = quota_for(user) + out["used_gb"] = round(folder_bytes(out["server_path"]) / 1e9, 2) if out["folder_exists"] else 0.0 + ps = quota_state().get(user) + out["paused_for_quota"] = bool(ps) + if ps: out["paused_since"] = ps.get("at") return out def clips_setup(user, device_id, device_name=None): @@ -357,4 +418,5 @@ if __name__ == "__main__": print(f"dumpd listening on {LISTEN[0]}:{LISTEN[1]} (dump, me) and {AUTH_LISTEN[0]}:{AUTH_LISTEN[1]} (mtx-auth) tz={TZ.key} night={NIGHT_START:%H:%M}-{NIGHT_END:%H:%M} token-secret={'ok' if token_secret() else 'MISSING'}", flush=True) auth_srv = ThreadingHTTPServer(AUTH_LISTEN, H) threading.Thread(target=auth_srv.serve_forever, daemon=True).start() + threading.Thread(target=quota_loop, daemon=True).start() # per-user clip quotas (pause/resume Syncthing folders) ThreadingHTTPServer(LISTEN, H).serve_forever() diff --git a/clips/index.html b/clips/index.html index ab7956c..dc2aaf1 100644 --- a/clips/index.html +++ b/clips/index.html @@ -74,9 +74,11 @@ async function refresh(){ pills.push(`share ${d.folder_configured?'configured':'not yet'}`); if (d.device_id) pills.push(`your device ${d.device_connected?'connected':'not connected'}`); if (d.files_here != null) pills.push(`${d.files_here} file(s) here`); + if (d.quota_gb) pills.push(`${d.used_gb} / ${d.quota_gb} GB${d.paused_for_quota?' — PAUSED (over quota)':''}`); if (d.remote_completion != null && d.remote_completion < 100) pills.push(`syncing ${Math.round(d.remote_completion)}%`); $('pills').innerHTML = pills.join(''); $('status').textContent = d.error ? d.error : (d.device_id ? `paired with ${d.device_name || 'your device'} (${d.device_id.slice(0,7)}…)` : 'not paired yet — do step 2'); + if (d.paused_for_quota) $('status').textContent += `\nYour folder is over its ${d.quota_gb} GB quota, so syncing is paused. Delete older clips from your clip folder (they are removed here too) or ask an admin to raise the quota; syncing resumes automatically under 90 %.`; if (d.device_id) $('devid').value = d.device_id; } catch (e) { $('pills').innerHTML = `error`; $('status').textContent = e.message; $('status').className = 'status bad'; } } diff --git a/systemd/dumpd.service b/systemd/dumpd.service index 04ba279..f8bace8 100644 --- a/systemd/dumpd.service +++ b/systemd/dumpd.service @@ -9,6 +9,8 @@ Environment=DUMP_ADMIN_GROUPS=admins,media-admins Environment=PUBLISH_GROUPS=streamers,admins,media-admins Environment=STREAM_TOKEN_TTL_HOURS=12 Environment=STREAM_TOKEN_SECRET_FILE=/etc/vanguard/stream-token.secret +Environment=CLIPS_QUOTA_GB=100 +Environment=CLIPS_QUOTA_FILE=/etc/vanguard/clip-quotas.json Environment=DUMP_TZ=America/New_York Restart=always RestartSec=3