commit c602a6ff52963d7ea207b25e6caaab0f98e35650 Author: Brandyn / Techy Date: Fri Oct 30 15:43:30 2026 -0500 Vanguard media stack: ingest, copyparty, Syncthing, normalize sweep, Authentik, Jellyfin + Vanguard Metadata plugin, MariaDB, Gitea, bot runtime Co-Authored-By: Claude Fable 5.1 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d9914ce --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# copy to .env (configure.py --full generates it) — never commit the real one +AUTHENTIK_SECRET_KEY= +AUTHENTIK_PG_PASS= +AUTHENTIK_TAG=2025.8 +WATCH_HOST=watch.example.com +MARIADB_ROOT_PASSWORD= +BOT_DB_NAME=vanguardbot +BOT_DB_USER=bot +BOT_DB_PASSWORD= +DISCORD_BOT_TOKEN= +DISCORD_GUILD_ID= +GIT_HOST=git.example.com +GITEA_DB_PASSWORD= +GITEA_RUNNER_TOKEN= +BOT_HOST=bot.example.com diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4d32d07 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# never commit real secrets; copy the templates and edit the copies on the server +*.local.* +__pycache__/ +.env +bot/app/__pycache__/ +*.pyc diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..f88af66 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,29 @@ +media.example.com { + encode zstd gzip + + # dump endpoint (python service on the host, does its own auth) + handle /dump* { + reverse_proxy 127.0.0.1:8080 + } + + # Syncthing admin UI (its own login; strip the prefix, present a local Host to pass its host check) + redir /sync /sync/ 308 + handle_path /sync/* { + reverse_proxy 127.0.0.1:8384 { + header_up Host {upstream_hostport} + } + } + + # OBS browser dock (static) + handle_path /dock* { + root * /srv/dock + file_server + } + + # everything else is copyparty (web UI + WebDAV) + handle { + reverse_proxy 127.0.0.1:3923 { + flush_interval -1 + } + } +} diff --git a/CommunityDiscordMedia.md b/CommunityDiscordMedia.md new file mode 100644 index 0000000..ac2f5fb --- /dev/null +++ b/CommunityDiscordMedia.md @@ -0,0 +1,317 @@ +# Community Discord Media — plan + +> Current state lives in **STATUS.md** (what runs, who does what, what's unfinished). This file is the design and the phase checklists. + +One login for everything (Discord via Authentik), a watch page (Jellyfin), and the existing +ingest/browse/download stack (MediaMTX, copyparty, Syncthing) moved behind that login. + +Host: OVH SYS-1 (Xeon E-2136, 32 GB, 2×4 TB mirror), Debian 13, Docker Compose, Caddy TLS. +Bundle: `vanguard-stream/` in this repo; live copy at `~/vanguard-stream` on the server. + +## Where we are + +| piece | state | +|-------|-------| +| MediaMTX ingest (RTMP 1935 / SRT 8890), per-streamer keys | live | +| copyparty browse/scrub/download/WebDAV at `https://ovh1.dumpstervigilantes.rip/` | live, local accounts | +| OBS dock (`/dock/`), dump-tonight (`/dump`) | live, dump uses its own password file | +| Syncthing clips intake (22000), admin UI at `/sync/` | live, Syncthing's own GUI login | +| normalize-sweep (one editing standard, raw files in hidden `.src/`) | live | +| nftables host firewall | live | +| Authentik (Discord login, groups) at `auth.` | live | +| Jellyfin at `watch.`, SSO via Authentik, direct play only | live | +| Vanguard Metadata plugin v0.1 | installed, libraries being switched to Shows | +| MariaDB (bot database) + `discord-bot` Python container (`bot/app`) | **in bundle, not yet applied** | +| Gitea (`git.`, MariaDB, ssh 2222) + Actions runner | live (local admin account; Authentik OIDC → Phase 10) | + +## Decisions already made + +- **Discord is the identity.** Authentik is the only self-hosted IdP that does Discord social login, + groups, OIDC *and* forward-auth for apps with no login of their own. Keycloak/Authelia/Pocket-ID don't fit. +- **Groups:** `streamers`, `editors`, `admins`. Access everywhere is by group, not by person. +- **Jellyfin is direct-play only** until a SYS-3 (Xeon E-2288G, Quick Sync). The E-2136 has no + hardware encoder; software transcoding tops out at ~2 concurrent viewers. Transcoding is off at + the server *and* per user. +- **MediaMTX stream keys stay machine credentials.** OBS can't do a Discord login. Keys are managed by + `configure.py`; the dock will fetch the logged-in streamer's key after Authentik login (phase 6). +- **Raw clipper files live in hidden `.src/` folders** so Jellyfin and copyparty show one item per clip. +- **Hostnames:** `auth.` and `watch.` are CNAMEs to the main host. Caddy routes by SNI, Let's Encrypt + follows CNAMEs. One A record to change if the IP ever moves. +- **WebDAV through Authentik** uses Authentik app passwords as HTTP basic auth (a mounted drive + can't do a browser login). + +## Target layout + +``` +auth.dumpstervigilantes.rip Authentik — Discord login, groups, OIDC provider, forward-auth gate +watch.dumpstervigilantes.rip Jellyfin — libraries: streams / clips / finals, SSO plugin → Authentik +ovh1.dumpstervigilantes.rip Caddy + / copyparty (forward-auth → Authentik; user+groups via headers; perms by @group) + /dock/ OBS dock (requires Authentik session; shows the caller's own stream key) + /dump dumpd (trusts X-Authentik-Username; admins may dump anyone) + /sync/ Syncthing (forward-auth, admins only) +rtmp://ovh1:1935 srt://ovh1:8890 MediaMTX (unchanged; per-streamer user/pass in stream key) +``` + +## Phases + +### Phase 1 — DNS + Authentik/Jellyfin containers ✅ done → `addCommunityMediaServer.sh` +- [ ] CNAME `auth` → `ovh1.dumpstervigilantes.rip` +- [ ] CNAME `watch` → `ovh1.dumpstervigilantes.rip` +- [ ] from your machine: `scp -r bin docker-compose.yml configure.py install.sh addCommunityMediaServer.sh README.md CommunityDiscordMedia.md .env.example debian@ovh1…:~/vanguard-stream/` +- [ ] on the server: `cd ~/vanguard-stream && sudo install -m 0755 bin/normalize-sweep.sh /usr/local/bin/ && ./addCommunityMediaServer.sh` + — asks for the two hostnames (defaults derived from the main host), checks they resolve to this + box, writes Caddy blocks + `.env` (secrets) via `configure.py --hosts`, starts the containers, + waits for HTTPS on both names, then prints the Phase 2/3 steps with the real hostnames. + Re-runnable; remembers the hostnames. +- [ ] `docker compose ps` → 9 containers up; `free -h` sane + +### Phase 2 — Authentik + Discord ✅ done (loose ends → Phase 8) +- [ ] `https://auth.HOST/if/flow/initial-setup/` → set `akadmin` password (store it somewhere safe) +- [ ] Discord developer portal → application → OAuth2 redirect `https://auth.HOST/source/oauth/callback//` + (slug = the Authentik source slug, e.g. `discord`; they must match exactly) +- [ ] Authentik → Directory → Federation & Social login → Create → Discord OAuth Source + (slug `discord`; consumer key/secret = Discord client id/secret; user matching "link on unique + identifier" is fine; scopes empty; flows default-source-authentication / default-source-enrollment) +- [ ] rotate the Discord client secret after setup if it was ever pasted into a chat/screenshot +- [ ] Show the source on the login page: Flows & Stages → Stages → `default-authentication-identification` + → Sources: select Discord, tick "Show sources' labels" (new sources are hidden until this is done) +- [ ] Make enrolled users Internal: Flows & Stages → Stages → `default-source-enrollment-write` → + User type: Internal (External users get "Interface can only be accessed by internal users") +- [ ] Directory → Groups: `streamers`, `editors`, `admins` +- [ ] Log in with Discord as a normal user; confirm the account is created and Internal; add yourself to `admins` +- [x] only members of our Discord server may log in — `authentik/discord-guild-policy.py`: + 1. Discord Developer Mode → right-click server → Copy Server ID + 2. Discord source → Scopes: `guilds` + 3. Customization → Policies → Create Expression Policy `discord-guild-member`, paste the file, + set `ACCEPTED_GUILD_ID` / `GUILD_NAME` + 4. bind it to `default-source-enrollment` **and** `default-source-authentication` + (flow → Policy / Group / User Bindings → Bind existing policy), then edit both flows → + Policy engine mode **all** (with `any`, the existing if-sso binding passes on its own) + 5. test with a member and a non-member; existing users re-consent once at Discord + Later: a per-role variant uses `guilds.members.read` and `/users/@me/guilds//member` + to map Discord roles → Authentik groups automatically. +- [ ] Pin `AUTHENTIK_TAG` in `.env` to the version that was installed (don't float on upgrades) + +### Phase 3 — Jellyfin ✅ done (loose ends → Phase 8) +- [ ] Wizard at `https://watch.HOST/`: admin user; libraries `/media/streams`, `/media/clips`, + `/media/finals` as "Home Videos", all metadata fetchers off +- [ ] Dashboard → Playback → Transcoding: hardware acceleration none; per user untick + "Allow video playback that requires transcoding" +- [ ] Plugins → Repositories → add `https://raw.githubusercontent.com/9p4/jellyfin-plugin-sso/manifest-release/manifest.json` → Catalog → "SSO Authentication" → install → restart +- [ ] Authentik → Applications → Create with provider (OAuth2/OIDC) for Jellyfin, redirect + `https://watch.HOST/sso/OID/redirect/authentik` +- [ ] SSO-Auth → add provider `authentik`: OID endpoint `https://auth.HOST/application/o//`, + client id/secret, roles claim `groups`, admin roles `admins`, enabled folders = 3 libraries, + default: transcoding disabled; Scheme Override `https`, Port Override EMPTY; Save the page +- [ ] direct test `https://watch.HOST/sso/OID/start/authentik` logs a Discord user in + (gotchas hit: unsaved provider → "Provider does not exist"; Port Override set → redirect gets `:443` + → Authentik mismatch; opening the /redirect/ URL by hand → "missing state") +- [ ] login page: Discord button first, local form behind "Admin login" — Dashboard → General → Branding: + disclaimer = `jellyfin/branding-disclaimer.html` (set the hostname), CSS = `jellyfin/branding.css` +- [ ] button works for a Discord user in `editors` + +### Phase 4 — "Vanguard Metadata" Jellyfin plugin (NEXT — usability for everyone) + +Decision: this lives **inside Jellyfin as a metadata provider plugin**, not as server scripts and +timers. Files stay exactly where they land (Syncthing mirrors the gamer's folder; streams stay in +`streams///`). No sidecar files, no symlink view, no moves. Jellyfin asks the plugin +for metadata and the plugin answers from the filename, the folder, Discord and Authentik. + +**What the plugin provides** (C#, Jellyfin plugin SDK, targets the installed server's ABI) +- `IRemoteMetadataProvider<Series>` — show = the `<user>` folder: display name from Discord, + overview from the member's roles/bio +- `IRemoteImageProvider` (Series) — poster = the Discord avatar +- `IRemoteMetadataProvider<Season>` — season **name** for a season number (policy / override / event) +- `IRemoteMetadataProvider<Episode>` — **season + episode numbers from the timestamp** (rolling + policy below), title from the Discord event overlapping the recording (fallback: the dock title = + folder name), overview = event description + duration + streamer, premiere date +- `IScheduledTask` "Discord snapshot" — members + scheduled events into the plugin's data store + every 5 min (Discord only exposes upcoming/active events; history must be kept) +- `IScheduledTask` "Season roll" — nightly after 04:00 Eastern: refresh metadata on items whose + bucket changed (weekday → week → month → year). Same path, same item ID → watched state kept. +- configuration page: Discord bot token + guild ID, Authentik URL + API token (streamer → Discord ID + via the OAuth source connection), broadcast-day hour, rolling thresholds, season name overrides, + provider on/off per library + +**Timestamp precedence** (plugin-side): bot `clip.json` sidecar if present (Medal/Twitch/YouTube +creation time) → container `creation_time` (Jellyfin's MediaInfo exposes it) → MediaMTX filename → +file date. Normalised to America/New_York. + +**Rolling seasons by tag** (ParentIndexNumber/IndexNumber set by the provider; pure function of +timestamp + policy): + +| age of item | season # | season name | +|---|---|---| +| current week | `YYYYMMDD` | "Thursday 30 Oct" | +| current month, past week | `YYYYWW` (ISO) | "Week 44" | +| current year, past month | `YYYYMM` | "October 2026" | +| past year | `YYYY` | "2026" | + +episode # derived from the timestamp (`DDHHMM`-style; stable; same-minute collisions get a digit). +Broadcast day rolls at 04:00. Season names: generated → override table → Discord event series name +when every item in the bucket belongs to one. + +**Libraries**: Shows type for `streams` and `clips` with this plugin as the only Series/Season/Episode +provider (TheTVDB/TMDB off), Movies type for `finals` (plugin optional). Jellyfin's scanner treats +`<user>` as the series and any file below it as an episode; the plugin supplies the numbers. + +**Build & ship**: .NET 8 SDK, `Jellyfin.Controller` matching the server's major version; CI builds +the DLL + zip; served from our Caddy as a plugin repository (`/plugins/manifest.json`) so installs +and updates go through Jellyfin's own Catalog. Cost: C#, and a rebuild on each Jellyfin major. + +Status: **v0.1.1.0** — v0.1.0 installed and **verified**: Jellyfin honours the provider's season numbers +(day seasons appeared on a Shows library). v0.1.1 adds the playback-policy task. Providers, Discord/Authentik +clients, event history, season-roll + snapshot tasks, Discord feed with frame-grab attachment, config page; +10 unit tests. + +Checklist: +- [x] skeleton + core season policy (tests) + providers + tasks + feed → `./package.sh` → `dist/VanguardMetadata_0.1.0.0.zip` +- [ ] Discord dev portal → the existing Authentik application → **Bot** → Reset Token; enable **Server Members** + intent; OAuth2 URL generator: scope `bot`, perms View Channels + Send Messages + Embed Links + Attach Files; + invite to the server +- [ ] Authentik: read-only service account + API token for the plugin +- [ ] install the zip (README in the plugin folder), restart Jellyfin, configure the plugin page +- [x] libraries → Shows type; providers: only Vanguard; scan — day seasons appeared +- [ ] install v0.1.1 (`./installJellyfinPlugin.sh`), run "enforce playback policy" once, check a user's policy +- [ ] verify: seasons appear as "Thursday, Oct 29" / "Week 44" / "October 2026"; poster = avatar; + a new file posts to the feed channel with the frame grab +- [ ] Medal local filename pattern → add to `ItemTimestamp` (need a sample filename) +- [ ] later: serve `dist/` at `/plugins/` via Caddy as a plugin repo; theme CSS + +### Phase 5 — copyparty behind Authentik (backend, few users; the one that can lock editors out) +- [ ] Authentik → Providers → Proxy provider, *forward auth (single application)*, external host + `https://ovh1.HOST`; Application bound to it; add to the embedded outpost +- [ ] Caddy: `forward_auth 127.0.0.1:9000 { uri /outpost.goauthentik.io/auth/caddy; copy_headers X-Authentik-Username X-Authentik-Groups … }` + on `/`, `/dock*`, `/dump*`, `/sync*`; `reverse_proxy /outpost.goauthentik.io/* 127.0.0.1:9000` +- [ ] copyparty: `idp-h-usr: X-Authentik-Username`, `idp-h-grp: X-Authentik-Groups`, `xff-src` = Caddy only; + volume accs become `r: @editors`, `rw: @editors` (finals), `rwmd: @admins` +- [ ] keep the local accounts in `copyparty.conf` until an `editors` Discord user has browsed and + downloaded through the new path; then remove them (`configure.py` drops the editors section) +- [ ] WebDAV: editor creates an app password in Authentik → mounts with Discord username + app password +- [ ] policy on the Application: only `editors` + `admins` + `streamers` may pass + +### Phase 6 — Syncthing UI + dump behind Authentik +- [ ] `/sync/`: forward-auth, policy `admins` only; Syncthing GUI auth can stay as a second layer +- [ ] `dumpd.py`: read `X-Authentik-Username` / `X-Authentik-Groups` from Caddy instead of basic auth; + admin = member of `admins`; drop `dump-users.txt` +- [ ] dock: `Dump tonight` sends no credentials; session cookie carries identity + +### Phase 7 — dock polish + cleanup +- [ ] `dumpd.py` gains `GET /me` → `{user, groups, stream_key}` for the logged-in streamer; + dock fills OBS settings from it — nobody types a stream password again +- [ ] `configure.py` shrinks: hostnames + streamers only (editors and admin live in Authentik) +- [ ] README: replace the per-app login notes with "log in with Discord" + +### Phase 8 — hardening (loose ends from phases 2–3, plus backups) +- [ ] Jellyfin login branding: Discord button first, "Admin login" reveals the local form + (`jellyfin/branding-disclaimer.html` + `jellyfin/branding.css`) +- [ ] group check: a Discord user in `editors` but not `admins` sees the three libraries and is NOT a + Jellyfin administrator; one in `admins` is +- [x] per-user transcoding OFF for every user, automatically: the Vanguard plugin's "enforce playback + policy" task (startup + every 30 min) sets direct-play-only on all non-admin users, SSO-created + ones included. Config toggle `EnforceDirectPlayOnly`; turn off on the SYS-3 +- [ ] pin Authentik: `.env` → `AUTHENTIK_TAG=<tag now running>` (`docker inspect authentik-server --format '{{.Config.Image}}'`) +- [x] guild restriction so only members of our Discord server can log in +- [ ] rotate the Discord client secret that appeared in a screenshot; paste the new one into the source +- [ ] nightly `pg_dump` of the Authentik database into `/srv/appdata/authentik/backup/` (timer), keep 14 days; + Jellyfin config and Authentik media are plain directories, covered by the archive/backup of `/srv/appdata` +- [ ] Jellyfin: library "real time monitoring" on, metadata fetchers confirmed off, scheduled scan daily + +### Discord bot (separate dev) — runtime provided by the stack +- `mariadb` container, db `vanguardbot` / user `bot`, loopback only (ssh tunnel for dev access) +- `discord-bot` container: python 3.12 + ffmpeg + yt-dlp baseline; code bind-mounted from `bot/app/`; + env gives token, guild, DB creds, rw `/media/clips`, `/data` scratch; restarts on exit +- web UI at `bot.` behind Caddy + Authentik forward-auth (first use of the forward-auth pattern that + Phase 5 applies to copyparty); the bot trusts `X-Authentik-*` headers and implements no login +- `bot/AGENT.md` is the build brief for the agent writing the bot; `bot/README.md` the ops cheat-sheet. The clip-submissions feature (Option below) is a natural home + for this bot: write `clips/<user>/<name>.mp4` + `<name>.json` sidecar and the Jellyfin plugin picks it up. +- [ ] CNAME `bot` → main host; `configure.py --full` (asks the hostname, bot token + guild) → `docker compose up -d --build` +- [ ] Authentik: Proxy Provider (forward auth, single app, external host `https://bot.HOST`) → embedded outpost → group policy +- [ ] dev drops `main.py` into `bot/app/` → `docker compose restart discord-bot` + +### Phase 9 — Gitea + Actions runner +- `gitea` on MariaDB (db/user from `mariadb/init/10-gitea.sh` on first init, else `./createGiteaDb.sh`), + `git.` via Caddy, ssh 2222 (firewall opened), registration closed, Actions enabled +- `act_runner` with the Docker socket; labels ubuntu-latest / ubuntu-22.04 / dotnet / python +- [ ] CNAME `git` → main host; `./configure.py --full` (asks the hostname) or `--hosts AUTH WATCH GIT` +- [ ] https://git.HOST/ install page → admin account +- [ ] `./registerGiteaRunner.sh`; confirm the runner is idle under Site Administration → Actions → Runners +- [ ] create users (or OIDC via Authentik, same pattern as Jellyfin: Gitea → Authentication Sources → OpenID Connect) +- [ ] first use: push `vanguard-stream` itself; a workflow that runs `package.sh` on the `dotnet` label and + publishes the plugin zip is the obvious first pipeline +- risk: the runner's Docker socket is root on the host for anyone who can push a workflow + +### Phase 10 — Gitea login via Authentik (OIDC) (later) +Same pattern as Jellyfin: Authentik is the identity, Gitea trusts it. +- [ ] Authentik → Applications → Create with provider → OAuth2/OpenID: name `gitea`, redirect + `https://git.HOST/user/oauth2/authentik/callback`, authorization flow implicit-consent, scopes + openid email profile (+ `groups` claim via the default profile mapping) +- [ ] Gitea → Site Administration → Identity & Access → Authentication Sources → Add → OAuth2, + provider OpenID Connect, name `authentik` (must match the callback path above), + client ID/secret, discovery URL `https://auth.HOST/application/o/<slug>/.well-known/openid-configuration`, + "Skip local 2FA", claim mapping: group claim `groups` → Gitea admin group `admins` +- [ ] Gitea app.ini / env: `GITEA__oauth2_client__ENABLE_AUTO_REGISTRATION=true`, + `GITEA__oauth2_client__ACCOUNT_LINKING=auto`, `GITEA__oauth2_client__USERNAME=nickname` so first + Discord login creates the Gitea user with no form (registration stays closed for local accounts) +- [ ] optional: `GITEA__service__ALLOW_ONLY_EXTERNAL_REGISTRATION=true`, hide the password form on the + login page (`GITEA__oauth2_client__OPENID_CONNECT_SCOPES`, custom template) so Discord is the only way in +- [ ] policy on the Authentik application: who may reach Gitea at all (e.g. `admins` + a `devs` group) +- [ ] test with a non-admin Discord user; confirm admin mapping via `admins` group + +### Option — more clip sources (Medal, Twitch/YouTube/Kick clips, Discord uploads) + +Today the only clip intake is a local clipper folder synced by Syncthing. Members without that +setup have no way in. Two additions cover everyone, and neither depends on Authentik. + +**A. `#clip-submissions` Discord bot** (one container, outbound-only, no new ports) +- member posts a Medal.tv / Twitch clip / YouTube / Kick / Streamable link, or drops a file +- bot maps the poster's Discord username → `/srv/media/clips/<username>/` (same identity Authentik uses) +- downloads at best quality with **yt-dlp** (has extractors for all of the above incl. Medal) +- writes `clip.json` beside it: source URL, title, submitter, timestamp — context for editors +- reacts ✅ / ❌ on the message; replies with the reason on failure +- normalize-sweep handles whatever arrives (Medal's HEVC/VFR etc.) +- config: bot token + channel id in `.env`; yt-dlp pinned, self-updates weekly (extractors rot) +- quality note: Medal/Twitch/YouTube deliver their re-encode (~1080p). Fine for clips; originals + still come via Syncthing or upload below. +- skipped on purpose: polling Medal profiles via its API — needs per-member IDs, pulls clips nobody + submitted, and only returns embed URLs; the bot covers it with explicit intent. + +**B. Web upload for members (after Phase 5)** +- copyparty per-user volume `clips/${u}` writable by its owner, readable by `editors`/`admins` +- anyone who can log in with Discord drags an original into the browser or WebDAV — full quality, + no bot, no install + +| source | path in | quality | +|--------|---------|---------| +| local clipper + Syncthing | automatic | original | +| Medal / Twitch / YouTube / Kick link | bot (A) | platform re-encode | +| Discord attachment | bot (A) | original, Discord size cap | +| file in hand | web upload (B) | original | + +Checklist if adopted: +- [ ] Discord developer portal → bot user, Message Content intent, invite to the server with + read/react/reply in `#clip-submissions` +- [ ] `bot/` in the bundle: python + discord.py + yt-dlp, runs as uid 1000, mounts `/srv/media/clips` +- [ ] `.env`: `DISCORD_BOT_TOKEN`, `CLIPS_CHANNEL_ID` +- [ ] per-user clips volume in `copyparty.conf` once forward-auth is live (Phase 5) + +### Later — when it hurts +- [ ] **SYS-3 (E-2288G)**: Quick Sync → Jellyfin transcoding on, sweep re-encodes in minutes. + Swap, not redesign: same compose, same `/srv` layout; restore `/srv/appdata` + `/srv/media`. +- [ ] home archive puller (`home/rclone-archive.sh`) on the home box, nightly cron + +## Risks / gotchas + +- **Phase 5 lockout.** Keep one SSH session open and a copyparty local admin account until the + Authentik path is proven. Caddy config errors stop *all* sites — `docker logs caddy` first. +- **Authentik outpost header names** differ by version (`X-Authentik-Username`, `X-Authentik-Groups`, + `X-Authentik-Email`). Verify with a test route that echoes headers before wiring copyparty. +- **Jellyfin SSO plugin** creates users with transcoding allowed by default unless the provider's + defaults are set — set them, or the per-user toggle has to be fixed after every first login. +- **RAM**: Authentik ~1.5 GB, Jellyfin ~0.5 GB idle. Fine on 32 GB; check `free -h` after phase 1. +- **nftables**: never `systemctl stop|restart nftables` (flushes Docker's NAT). Reload only. +- **Firewall unchanged**: Authentik (9000) and Jellyfin (8096) bind to 127.0.0.1; only Caddy is public. + +## Monthly cost +Unchanged: the SYS-1 (~$30 + disks). No new services cost money. Discord OAuth is free. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2f2889a --- /dev/null +++ b/README.md @@ -0,0 +1,255 @@ +# vanguard-stream — SYS-1 (Xeon E-2136, 32 GB ECC, 2×4 TB SATA soft RAID) + +> **STATUS.md** = what runs, who does what, what's unfinished (kept current). **CommunityDiscordMedia.md** = design + phases. + +Ingest from OBS, browse/scrub/download for editors, automatic normalization to one editing +standard, dump button, 15-day archive pull to a home HDD. Everything is plain files under /srv/media. + +## 1. Reinstall from the OVH manager + +Server → Reinstall → Debian 13 (Trixie). +Tick **custom partitioning** and **use distribution kernel**. Layout (RAID 1 across both HDDs): + +| mount | size | fs | +|-------|------|----| +| /boot | 1 GB | ext4 | +| / | 250 GB | ext4 | +| swap | 8 GB per disk | swap | +| /srv | rest (~3.4 TiB) | xfs | + +Add your SSH key. OVH's template creates a `debian` user with sudo; root SSH is disabled. Wait for the mail. + +## 2. Install + + scp -r vanguard-stream debian@SERVER:~/ + ssh debian@SERVER + cd ~/vanguard-stream && sudo bash install.sh + +install.sh installs docker, ffmpeg, rclone; creates /srv/media/{streams,clips,finals,trash,.markers} +owned by uid 1000 (the `debian` user); copies the systemd units; adds `debian` to the docker group; +starts the compose stack. It prints what you still have to edit. + +The folder stays where it is: compose bind-mounts the configs and the dock page from it. +After install, log out and back in once so the docker group applies, then plain `docker compose` +works from ~/vanguard-stream without sudo. + +## 3. Configure (domain, admin, streamers, editors) + + cd ~/vanguard-stream + ./configure.py --full # first time: asks everything, then applies + ./configure.py # later: add/remove/reset streamers and editors, then apply + ./configure.py --show # print current users and the OBS settings to hand out + +Stateless: it reads the current values out of `Caddyfile`, `mediamtx.yml` and `copyparty.conf`, +regenerates those three plus `dump-users.txt`, installs the dump users to `/etc/vanguard`, creates +`streams/<name>` and `clips/<name>` (with `.stignore`) for each streamer, restarts dumpd and the +containers. Removing a streamer or editor revokes access only; their files stay on disk. +Passwords may not contain `: & ? # / \ quotes or spaces` (they ride inside the stream key). + +## 4. Firewall (host nftables; OVH edge firewall is optional on top) + +Applied automatically at the end of `install.sh`. To re-apply by hand: + + cd ~/vanguard-stream && sudo bash firewall/firewall.sh + +Installs `firewall/nftables.conf` as `/etc/nftables.conf`, syntax-checks, applies, enables at boot. +It uses its own table (`inet vanguard`) so applying or reloading never disturbs Docker's rules, +and filters input only so container networking keeps working. **Keep your SSH session open and +test a new one before closing it.** + +| port | proto | who | +|------|-------|-----| +| 22 | tcp | anywhere, rate limited (edit `ssh_allow` in /etc/nftables.conf to restrict to your IPs) | +| 80, 443 | tcp | world (Caddy) | +| 443 | udp | world (Caddy HTTP/3) | +| 1935 | tcp | world (RTMP ingest) | +| 8890 | udp | world (SRT ingest) | +| 22000 | tcp+udp | world (Syncthing) | +| 3306 | tcp | world, rate limited (MariaDB for the bot dev; restrict via `db_allow` in /etc/nftables.conf) | +| 2222 | tcp | world (Gitea ssh) | + +Everything else is dropped and a trickle is logged (`journalctl -k | grep vg-drop`). Authentik (9000), +Jellyfin (8096), copyparty, the Syncthing UI and dumpd bind to 127.0.0.1 only and are reached via Caddy +or an ssh tunnel, so they are deliberately not opened. MariaDB runs on the host network (not a Docker +published port) precisely so the firewall can govern it. +Syncthing's admin UI is proxied at `https://HOST/sync/` behind Syncthing's own GUI login (set it in +Actions → Settings → GUI before relying on this). The ssh tunnel to 8384 still works as a fallback. + +To change rules: edit `/etc/nftables.conf`, then `sudo nft -f /etc/nftables.conf` or +`sudo systemctl reload nftables`. **Never `systemctl stop|restart nftables`** — Debian's unit flushes +the whole ruleset on stop, which also wipes Docker's NAT rules and takes the containers offline +until Docker is restarted. + +## 5. Streamer setup (once) + +OBS → Settings → Stream → Custom: Server `rtmp://media.example.com:1935/`, key anything (the dock sets it). +OBS → Tools → WebSocket Server Settings → enable, set a password. +OBS → Docks → Custom Browser Docks → `https://media.example.com/dock/`. +In the dock: fill host/user/pass/OBS-ws-password once (stored in the dock's localStorage). +Per stream: type a title, click **Go Live**. After a bad night: **Dump tonight**. + +Stream key format the dock produces: `techy/Title?user=techy&pass=...` +SRT equivalent if preferred: `srt://media.example.com:8890?streamid=publish:techy/Title:techy:PASS` + +## 5b. Syncthing hardening (do once, in the server's Syncthing UI) + +Syncthing receives straight into `clips/<user>` (Syncthing UI path `/var/syncthing/clips/<user>`), +receive-only, with a `.stignore` (from `syncthing/stignore`) that pulls video files only. The gamer's +folder structure is mirrored as-is. **One copy per clip on the server**: compliant files are left +alone; a non-compliant file (HEVC/MKV/VFR) is converted to `name.mp4` (`name.h264.mp4` if the raw was +already `.mp4`), the raw's name is appended to `.stlocal` (an `#include`d ignore list, so Syncthing +drops it without flagging or re-pulling), and the raw is deleted locally — it still exists on the +gamer's PC. `configure.py` creates the folder + ignore file for each streamer. + +In the UI (https://HOST/sync/): +- each clips folder → path `/var/syncthing/clips/<user>`, **Folder Type: Receive Only**, + Edit → Advanced → **Minimum Free Disk Space: 300 GB** (so clips can never fill the mirror and stall + the recordings). +- Actions → Settings → Connections → **Incoming Rate Limit: 12500 KiB/s** (~100 Mbps) so a gamer + sharing a year of old clips cannot starve the ingest during stream hours. + +## 5c. Authentik (Discord login) and Jellyfin + +DNS: `auth` and `watch` as CNAMEs to the main host. Then run `./addCommunityMediaServer.sh` — it asks +for both hostnames, checks DNS, generates `.env` (Authentik secret + DB password; never commit it), +adds the Caddy site blocks, starts the containers, waits for them and prints the steps below with your +hostnames filled in. (`./configure.py --full` asks for the same hostnames if you prefer.) Then: + +**Authentik first run** — https://auth.HOST/if/flow/initial-setup/ → set the `akadmin` password. +1. Discord developer portal → New Application → OAuth2: add redirect + `https://auth.HOST/source/oauth/callback/discord/` (last segment = the source slug below), copy Client ID + Secret. +2. Authentik → Directory → Federation & Social login → Create → Discord OAuth Source: slug `discord`, + consumer key/secret = Client ID/Secret, user matching "link on unique identifier", scopes empty. +3. Flows & Stages → Stages → `default-authentication-identification` → Sources: select Discord, + tick "Show sources' labels". Without this the Discord button never appears on the login page. +4. Flows & Stages → Stages → `default-source-enrollment-write` → User type: Internal. Otherwise + Discord users are External and get "Interface can only be accessed by internal users". +5. Directory → Groups: create `streamers`, `editors`, `admins`. Add people after their first login. +6. (optional) Restrict login to members of your Discord server: add scope `guilds` to the source; + Customization → Policies → Create Expression Policy from `authentik/discord-guild-policy.py` (fill in + your server ID); bind it to both `default-source-enrollment` and `default-source-authentication` + (flow → Policy / Group / User Bindings) and set each flow's Policy engine mode to `all`. Server ID: Discord Developer Mode → right-click server → Copy ID. + +**Jellyfin first run** — https://watch.HOST/ → wizard: admin user, add libraries: `/media/streams` and +`/media/clips` as **Shows** with only "Vanguard Metadata" as metadata downloader (Series/Season/Episode) +and Screen Grabber + Vanguard Metadata as image fetchers; `/media/finals` as Home Videos with fetchers off. +The Vanguard Metadata plugin (`jellyfin-plugin-vanguard/`, built with its `package.sh`) is installed by +`addCommunityMediaServer.sh` / `installJellyfinPlugin.sh` from `jellyfin-plugin-vanguard/dist/`. +- Dashboard → Playback → Transcoding: **disable** hardware acceleration. Per-user "allow transcoding" is + switched off automatically for everyone (SSO users included) by the Vanguard plugin's "enforce playback + policy" task; turn that toggle off in the plugin config once the server has Quick Sync (SYS-3). +- Dashboard → Plugins → Repositories → add `https://raw.githubusercontent.com/9p4/jellyfin-plugin-sso/manifest-release/manifest.json`, then Catalog → **SSO Authentication** → install → restart Jellyfin. +- Authentik → Applications → Create with provider → OAuth2/OpenID: redirect + `https://watch.HOST/sso/OID/redirect/authentik`, note client ID/secret. +- Jellyfin → Plugins → SSO-Auth → add OID provider "authentik": endpoint `https://auth.HOST/application/o/<slug>/`, + client ID/secret, enable, roles claim `groups`, admin roles `admins`, enabled folders = the three libraries, + Scheme Override `https`, Port Override empty (a value appends `:443` and Authentik rejects the redirect). Save. +- The plugin adds no button. Test with `https://watch.HOST/sso/OID/start/authentik`, then Dashboard → + General → Branding: Login disclaimer = `jellyfin/branding-disclaimer.html` (set the hostname), + Custom CSS = `jellyfin/branding.css`. The page then shows only "Sign in with Discord"; a small + "Admin login" link reveals the local username/password form. + Jellyfin creates the account on first SSO login. + +Phase 4 is the Vanguard Metadata Jellyfin plugin; Phase 5 puts copyparty, /sync and /dump behind Authentik forward-auth. + +## 5d. MariaDB + Discord bot (for the bot dev) + +`mariadb` (public on 3306, host network, root local-only, 32-char generated passwords) with a database/user +for the bot, and `discord-bot`, a Python 3.12 container that runs `bot/app/main.py` and idles until that file +exists. Credentials live in `.env` (generated by `configure.py`; bot token + guild ID are asked in `--full` +or the `[d]` menu item; `./configure.py --show` prints them). + +The bot also serves a web UI on 127.0.0.1:8000, published as `https://bot.HOST/` behind **Authentik +forward-auth**: Caddy asks Authentik, then injects `X-Authentik-Username/-Groups/-Email`. Create the matching +Proxy Provider in Authentik (forward auth, single application, external host `https://bot.HOST`), add it to +the embedded outpost, and bind a group policy. Until then the site returns an error, never an open page. + +The build brief for whoever writes the bot is `bot/AGENT.md`; `bot/README.md` is the ops cheat-sheet. + + docker compose logs -f discord-bot # watch it + docker compose restart discord-bot # after code changes + docker compose build discord-bot && docker compose up -d discord-bot # after requirements.txt changes + +## 5e. Gitea + Actions runner + +`gitea` (MariaDB-backed, `git.HOST` via Caddy, git-over-ssh on **2222**, registration closed) and +`act_runner` (jobs run as Docker containers; labels `ubuntu-latest`, `ubuntu-22.04`, `dotnet`, `python`). +The gitea db/user are created by `mariadb/init/10-gitea.sh` on MariaDB's first start; for a MariaDB that +already existed run `./createGiteaDb.sh` once. + +1. `./configure.py --full` asks the Gitea hostname (CNAME `git`); apply starts both containers. +2. https://git.HOST/ → install page (db pre-filled) → create the admin account → Install. +3. `./registerGiteaRunner.sh` → generates a registration token, stores it in `.env`, starts the runner. +4. Remotes: `ssh://git@git.HOST:2222/<org>/<repo>.git` or `https://git.HOST/<org>/<repo>.git`. + +Runner caveat: it holds the Docker socket, so anyone who can push a workflow can run as root on the box. +Keep repo write access to trusted people. + +## 6. Editors + +Browser: https://media.example.com/ → log in. Scrub any `.mp4` in the browser, download what you want. +WebDAV mount: same URL, same login. Upload finished work to `/finals`. + +Every `.mp4` an editor sees is the same kind of file (see "Editing standard"). Raw clipper output that +needed converting is kept in a hidden `.src/` folder next to it (admins can show dotfiles in copyparty). + +## 7. Home archive box + +Copy `home/rclone-archive.sh` to the home machine, set the vars, run from cron nightly. +It connects over SFTP as `debian` (all media files are owned by uid 1000 so it can delete them). +Pulls everything older than 15 days off the server, including the hidden `.src/` raw files. + +## Layout on the server + + /srv/media/streams/<user>/<Title>/2025-10-10_21-03-11.mp4 stream, remuxed flat (lossless) by the sweep + /srv/media/clips/<user>/clip.mp4 clip (synced as-is, or converted; raw removed) + /srv/media/clips/<user>/.stlocal Syncthing ignore list maintained by the sweep + /srv/media/clips/<user>/.src/clip.mkv raw source, only for clips that did NOT arrive via Syncthing + /srv/media/clips/<user>/clip.normalize-failed.log only if normalization failed; delete to retry + /srv/media/finals/ editors upload here + /srv/media/trash/<night>/... dumped, purged after 3 days + /srv/media/.markers/ dump markers + /srv/media/.syncthing/ Syncthing config + +## Editing standard (what normalize-sweep enforces) + +MP4 · H.264 High, 8-bit 4:2:0 · constant frame rate · AAC audio (all tracks kept) · index at the +front (instant seek in browser and NLE) · at most 2560x1440 and 60 fps (downscaled to fit, never +upscaled; aspect kept). + +| input | action | +|-------|--------| +| already compliant | nothing | +| MediaMTX recording (fragmented MP4, good codecs) | lossless remux to flat MP4, replaced in place | +| in a Syncthing folder (`.stfolder` marker) | compliant / index-only: untouched; otherwise convert → `name.mp4`, add raw to `.stlocal`, delete raw (one copy) | +| HEVC / 10-bit / VFR / MKV / >1440p / >60fps / non-AAC | full-res re-encode (x264 veryfast CRF 18) → `name.mp4`; source kept in `.src/` | +| ffmpeg fails or duration mismatch | `name.normalize-failed.log` written, file skipped until that log is deleted | + +Runs every 5 min on files idle ≥3 min and not open. Software x264 on the E-2136 does ~0.5x realtime +at 1440p60, so a 2 h clip batch takes ~4 h; streams only remux (minutes). All at idle priority. + +## Reboot behaviour + +Everything comes back on its own, in this order: + +1. mdadm assembles the mirror, systemd mounts /srv (local-fs). A degraded mirror still boots. +2. docker.service has `RequiresMountsFor=/srv/media` (drop-in installed by install.sh), so it never + starts with the media tree missing. All containers are `restart: unless-stopped`. +3. dumpd.service starts; normalize-sweep.timer fires 2 min after boot and every 5 min after; + trash-purge.timer is `Persistent=true` so a missed 05:00 run fires at boot. +4. A stream cut by a reboot is playable up to the last second (fMP4 parts are flushed every 1 s); + OBS reconnects and MediaMTX opens a new file in the same title folder. +5. Half-written `*.part` files are overwritten by the next sweep. Dumps are idempotent. + +Time: install.sh sets the server clock to America/New_York so journal, timers, file timestamps +and the dump window all read in Eastern. Dump "night" = 17:00 → 02:00 Eastern (pinned in dumpd via +DUMP_TZ, independent of the host zone). Between 02:00 and 17:00 the default night is the previous +evening; after 17:00 it is tonight. The 05:00 trash purge is 05:00 Eastern. + +## Timers + +| unit | when | does | +|------|------|------| +| normalize-sweep.timer | every 5 min | remux/re-encode finished files to the editing standard | +| trash-purge.timer | daily 05:00 Eastern | deletes trash older than 3 days | +| dumpd.service | always | the dump endpoint on 127.0.0.1:8080, behind Caddy at /dump | diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000..0caa0ee --- /dev/null +++ b/STATUS.md @@ -0,0 +1,78 @@ +# Vanguard stack — what runs, what people do, what's unfinished + +Living document. Updated whenever something is deployed or changes behaviour. Last update: 2026-10-30. +Host: `ovh1.dumpstervigilantes.rip` (OVH SYS-1, Debian 13). Bundle: `~/vanguard-stream` on the box, `vanguard-stream/` in this repo. + +## Runs on its own + +| Cadence | What happens | Done by | +|---|---|---| +| Continuous | OBS streams land as files the moment Go Live is pressed, one per stream under `streams/<user>/<Title>/` | MediaMTX | +| Continuous | Gamers' clip folders mirror onto the server (video files only, their structure untouched, receive-only) | Syncthing | +| Continuous | New recordings and clips are announced in a Discord channel with the streamer's name and a frame grab | Jellyfin plugin (needs bot token + channel in its config) | +| Every 5 min | Finished files are checked. Streams: lossless remux so they seek instantly. Non-compliant clips: converted to the editing standard, raw removed, one copy per clip | normalize sweep (systemd timer) | +| Every 5 min | Discord scheduled events snapshotted into a history so recordings can be matched to their session | Jellyfin plugin | +| Every 30 min + startup | Every non-admin Jellyfin user set to direct-play-only (no hardware encoder on this box) | Jellyfin plugin | +| Daily 04:10 ET | Recordings roll between weekday → week → month → year seasons as the calendar moves; no files change | Jellyfin plugin | +| Daily 05:00 ET | Dumped nights older than 3 days deleted for good | trash purge (systemd timer) | +| On boot | Everything restarts itself after the media mount is up; interrupted recordings stay playable | systemd + Docker | +| **Not yet running** | Pull of files older than 15 days to the home HDD | `home/rclone-archive.sh`, needs the home box | +| **Not yet running** | Nightly backup of the Authentik database (Postgres) | planned | + +## What people do + +| Who | Activity | Where | Login | +|---|---|---|---| +| Streamer | Type a title, Go Live, Stop. "Dump tonight" if it was junk (3-day undo) | OBS browser dock `https://ovh1…/dock/` | stream password in dock settings | +| Gamer | Nothing; clips save where they always did | their clipper + Syncthing, paired once | — | +| Editor | Browse by streamer/title, scrub in browser, download originals, upload finals | copyparty `https://ovh1…/` | **local account** (to move to Discord) | +| Editor | Mount the library as a drive | WebDAV, same URL | same | +| Anyone in the Discord server | Watch recordings and clips by streamer and season | Jellyfin `https://watch.…/` | Discord (Authentik) | +| Devs | Host code, review, CI | Gitea `https://git.…/`, ssh 2222 | **local account** (Discord later) | +| Bot dev | Push code, restart container, read logs; web UI at `https://bot.…/` | Gitea + compose; `bot/AGENT.md` | web UI behind Discord (Authentik), once its provider exists | +| Admin | Add/remove streamers & editors, hostnames, bot token | `./configure.py` | ssh | +| Admin | Pair a gamer's device, approve their folder | Syncthing UI `https://ovh1…/sync/` | Syncthing GUI password | +| Admin | Groups, who may log in, app access | Authentik `https://auth.…/` | Discord or akadmin | +| Admin | Install/upgrade the Jellyfin plugin | `./installJellyfinPlugin.sh` | ssh | +| Admin | Register the CI runner | `./registerGiteaRunner.sh` | ssh | + +## Services and their state + +| Service | Role | State | +|---|---|---| +| MediaMTX | RTMP/SRT ingest, records fMP4 | live | +| copyparty | browse/scrub/download/WebDAV/finals upload | live, local accounts | +| dumpd + dock | dump-tonight endpoint, OBS dock page | live, own password file | +| Syncthing | clip intake, UI at `/sync/` | live, Syncthing GUI login | +| normalize sweep | editing standard, one copy per clip | live | +| nftables | host firewall (22, 80/443, 1935, 2222, 3306, 8890/udp, 22000) | live | +| Caddy | TLS + routing for all hostnames | live | +| Authentik (+Postgres, Redis) | Discord login, groups, guild-membership policy | live | +| Jellyfin | watch page, SSO via Authentik, direct play only | live | +| Vanguard Metadata plugin | seasons by tag, Discord names/avatars/events, feed, playback policy | **0.1.0 live, 0.1.1 built** (policy task) — install pending | +| MariaDB | databases for the bot and Gitea; public :3306, root local-only | live | +| Gitea | git hosting on MariaDB, Actions enabled | live, local admin | +| act_runner | CI jobs in Docker | **not registered** | +| discord-bot | Python runtime, idles until `bot/app/main.py` exists | live, no code yet | +| `bot.` host | bot web UI behind Authentik forward-auth | Caddy block ready; **CNAME + Authentik proxy provider pending** | + +## Unfinished, in rough priority + +1. **Install plugin 0.1.1** (`./installJellyfinPlugin.sh`), run "enforce playback policy", check a user. +2. **Register the runner** (`./registerGiteaRunner.sh`). +3. **`bot` CNAME + Authentik proxy provider** — first use of forward-auth; proves the pattern for copyparty. +4. **copyparty behind Authentik** (group permissions, WebDAV via app passwords, then drop local accounts). The one change that can lock editors out; keep a local admin until a Discord editor has used the new path. +5. **Syncthing UI + dump behind Authentik**; dump reads the identity header; dock fetches the caller's own stream key. +6. **Gitea login via Authentik** (OIDC source, auto-register, `admins` → Gitea admin). +7. **Housekeeping**: pin `AUTHENTIK_TAG` in `.env`, rotate the Discord client secret that was in a screenshot, nightly Postgres dump, Jellyfin login branding (Discord button first), check a non-admin Discord user's Jellyfin view. +8. **Medal filename pattern** in the plugin's timestamp parser — need one sample filename. +9. **Home archive puller** on the home box. +10. **Plugin repository** served by Caddy so Jellyfin updates the plugin from its catalog. +11. **Clip submissions** (Medal/Twitch/YouTube links, Discord uploads) — belongs to the bot. +12. **SYS-3** when transcoding or sweep speed hurts: Quick Sync, flip the plugin's policy toggle off. + +## Verified behaviours worth remembering +- Jellyfin honours provider-supplied season numbers on a Shows library (day seasons appeared). Next check: Monday 04:10 ET should fold them into a week season. +- Sweep leaves compliant Syncthing files untouched and keeps exactly one copy of converted ones (tested). +- Firewall: Docker-published ports bypass the input chain; MariaDB uses host networking precisely so the rules apply. +- Never `systemctl stop|restart nftables` (flushes Docker's NAT). Reload only. diff --git a/addCommunityMediaServer.sh b/addCommunityMediaServer.sh new file mode 100755 index 0000000..a9d8b62 --- /dev/null +++ b/addCommunityMediaServer.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Phase 1 of CommunityDiscordMedia.md: add Authentik (Discord login) + Jellyfin to a running +# vanguard-stream box. Asks for the two hostnames, checks DNS, writes configs/.env via configure.py, +# starts the containers, waits for them, then prints the Phase 2/3 setup steps with real hostnames. +# Run as the debian user from ~/vanguard-stream. Re-runnable. +set -euo pipefail +cd "$(dirname "$0")" +DRY="${VG_DRY:-0}" + +say() { printf '\n\033[1m== %s\033[0m\n' "$*"; } +ask() { local q="$1" d="${2:-}" a; read -r -p "$q${d:+ [$d]}: " a; echo "${a:-$d}"; } +yes() { local a; read -r -p "$1 [Y/n]: " a; [[ -z $a || $a =~ ^[Yy] ]]; } +ip_of(){ getent ahostsv4 "$1" 2>/dev/null | awk 'NR==1{print $1}' || true; } + +# ---- preconditions ------------------------------------------------------------------------------ +[[ -f configure.py && -f docker-compose.yml ]] || { echo "run from the vanguard-stream folder"; exit 1; } +[[ $EUID -ne 0 ]] || { echo "run as the debian user, not root (sudo is used where needed)"; exit 1; } +MAIN=$(python3 -c 'import configure as c; print(c.read_domain())') +[[ -n $MAIN ]] || { echo "main hostname not set — run ./configure.py --full first"; exit 1; } +if [[ $DRY != 1 ]]; then + docker info >/dev/null 2>&1 || { echo "docker not usable as this user — log out/in (docker group) or run install.sh"; exit 1; } +fi +read -r AUTH_CUR WATCH_CUR < <(python3 -c 'import configure as c; h=c.read_hosts(); print(h.get("auth",""), h.get("watch",""))') +BASE="${MAIN#*.}"; [[ $MAIN == *.*.* ]] || BASE="$MAIN" + +say "Community media server — Phase 1 (Authentik + Jellyfin) on $MAIN" +echo "Needs two DNS names as CNAMEs to $MAIN. Create them now if you haven't." +AUTH=$(ask "Authentik hostname" "${AUTH_CUR:-auth.$BASE}") +WATCH=$(ask "Jellyfin hostname" "${WATCH_CUR:-watch.$BASE}") +GIT_CUR=$(python3 -c 'import configure as c; print(c.read_hosts().get("git",""))') +GIT=$(ask "Gitea hostname" "${GIT_CUR:-git.$BASE}") +BOT_CUR=$(python3 -c 'import configure as c; print(c.read_hosts().get("bot",""))') +BOT=$(ask "Bot web UI hostname" "${BOT_CUR:-bot.$BASE}") + +# ---- DNS sanity ---------------------------------------------------------------------------------- +say "DNS check" +MAIN_IP=$(ip_of "$MAIN"); ok=1 +for h in "$AUTH" "$WATCH" "$GIT" "$BOT"; do + ip=$(ip_of "$h") + if [[ -z $ip ]]; then echo " $h: does not resolve yet"; ok=0 + elif [[ $ip != "$MAIN_IP" ]]; then echo " $h -> $ip (main is $MAIN_IP) — points somewhere else"; ok=0 + else echo " $h -> $ip ok"; fi +done +if [[ $ok == 0 ]]; then + echo " Let's Encrypt will fail for names that don't point here. Caddy retries on its own once DNS is right." + yes "continue anyway?" || exit 1 +fi + +# ---- write configs, .env, start containers ----------------------------------------------------- +say "Applying (configure.py --hosts)" +VG_DRY=$DRY python3 configure.py --hosts "$AUTH" "$WATCH" "$GIT" "$BOT" | sed -n '/== writing/,$p' | grep -vE '^ (techy|diceofire|[a-z0-9_-]+) password' || true +[[ $DRY == 1 ]] && { echo "(dry run: skipping container wait)"; exit 0; } + +# ---- wait --------------------------------------------------------------------------------------- +say "Waiting for containers" +for c in authentik-db authentik-redis authentik-server authentik-worker jellyfin mariadb discord-bot gitea caddy; do + for _ in $(seq 1 60); do + st=$(docker inspect -f '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' "$c" 2>/dev/null || echo missing) + case "$st" in running\ healthy|running\ ) echo " $c: $st"; break;; esac + sleep 3 + done + [[ $st == running* ]] || echo " $c: $st <- check: docker logs $c" +done + +say "Waiting for HTTPS (certificates + Authentik migrations; up to ~3 min)" +for h in "$AUTH" "$WATCH" "$GIT"; do + for _ in $(seq 1 60); do + code=$(curl -s -o /dev/null -m 5 -w '%{http_code}' "https://$h/" || true) + [[ $code =~ ^(200|302|301)$ ]] && { echo " https://$h/ $code"; break; } + sleep 3 + done + [[ $code =~ ^(200|302|301)$ ]] || echo " https://$h/ not answering yet (got '${code:-none}') — docker logs caddy / authentik-server" +done +echo; free -h | awk 'NR<=2{print " "$0}' + +# ---- Vanguard Metadata plugin (optional: only if a built zip was uploaded with the bundle) ------------ +if ls jellyfin-plugin-vanguard/dist/VanguardMetadata_*.zip >/dev/null 2>&1; then + say "Installing the Vanguard Metadata Jellyfin plugin" + bash installJellyfinPlugin.sh || echo " plugin install failed — see above; Jellyfin works without it" +fi + +# ---- instructions ------------------------------------------------------------------------------- +cat <<MSG + +$(printf '\033[1m== Phase 1 done. Next, by hand:\033[0m') + +PHASE 2 — Authentik + Discord + 1. https://$AUTH/if/flow/initial-setup/ -> set the akadmin password (keep it somewhere safe) + 2. Discord developer portal -> New Application -> OAuth2 -> Redirects: add + https://$AUTH/source/oauth/callback/discord/ + (the last path segment is the source SLUG you choose in step 3 — keep them identical) + copy Client ID and Client Secret + 3. Authentik -> Directory -> Federation & Social login -> Create -> Discord OAuth Source + Name: Discord Slug: discord (must match the redirect above) + User matching: "Link users on unique identifier" (default; creates the user on first login) + Consumer key = Client ID, Consumer secret = Client Secret; Scopes: leave empty + Flow settings (if shown): authentication default-source-authentication, + enrollment default-source-enrollment + 4. Put the source on the login page (a new source is NOT shown until you do this): + Flows & Stages -> Stages -> edit "default-authentication-identification" + -> Sources: select the Discord source; tick "Show sources' labels" -> Update + 5. Make Discord users "Internal" (External users can't open the portal or admin UI): + Flows & Stages -> Stages -> edit "default-source-enrollment-write" -> User type: Internal -> Update + 6. Directory -> Groups -> create: streamers, editors, admins + 7. Open a private window -> https://$AUTH/ -> click "Discord" -> log in; confirm the user appears + under Directory -> Users (if it says External, edit it -> User type Internal); add yourself to admins + 8. (optional) only members of YOUR Discord server may log in: + a. Discord -> User Settings -> Advanced -> Developer Mode; right-click your server -> Copy Server ID + b. edit the Discord source -> Protocol settings -> Scopes: guilds -> Update + c. Customization -> Policies -> Create -> Expression Policy, name discord-guild-member, + paste authentik/discord-guild-policy.py (set ACCEPTED_GUILD_ID and GUILD_NAME) -> Create + d. Flows & Stages -> Flows -> default-source-enrollment -> "Policy / Group / User Bindings" + -> Bind existing policy -> discord-guild-member. Repeat for default-source-authentication. + Then edit each of those two flows -> Policy engine mode: all (default "any" would let the + existing if-sso binding alone pass the login; keep that binding, just switch the mode). + e. test: private window, Discord login with a member (works) and a non-member (denied with message) + Users who logged in before (b) get a one-time Discord re-consent for the new scope. + 9. pin the version: edit .env -> AUTHENTIK_TAG=<the tag now running> (docker inspect authentik-server) + +PHASE 3 — Jellyfin (direct play only on this CPU) + 1. https://$WATCH/ -> wizard: admin user; libraries: + /media/streams and /media/clips -> type "Shows": metadata downloaders = ONLY "Vanguard Metadata" + (Series, Season, Episode); image fetchers = Vanguard Metadata (Series) + Screen Grabber (Episode); + real-time monitoring on + /media/finals -> type "Home Videos", all metadata OFF + (the Vanguard plugin is installed by this script when its zip is present; it needs no config to + do seasons — Discord/Authentik fields are optional and add names, avatars, event titles, the feed) + then Dashboard -> Scheduled Tasks -> run "Vanguard: roll seasons" once after the first scan + 2. Dashboard -> Playback -> Transcoding: hardware acceleration = none + Users -> each user -> Playback: untick "Allow video playback that requires transcoding" + 3. Dashboard -> Plugins -> Repositories -> "+" -> name SSO-Auth, URL: + https://raw.githubusercontent.com/9p4/jellyfin-plugin-sso/manifest-release/manifest.json + -> Save; Catalog tab -> "SSO Authentication" -> Install -> docker compose restart jellyfin + 4. Authentik -> Applications -> Create with provider -> OAuth2/OpenID + name: jellyfin; redirect URI: https://$WATCH/sso/OID/redirect/authentik + note the client ID / secret and the provider's slug + 5. Jellyfin -> Plugins -> SSO-Auth -> OID providers -> add "authentik": + OID endpoint: https://$AUTH/application/o/<slug>/ + client ID / secret; Enabled; Roles claim: groups; Admin roles: admins; + Enabled folders: the three libraries; default "disable transcoding" on + Scheme Override: https Port Override: EMPTY (a value adds ":443" to the redirect -> mismatch) + -> Save at the bottom of the page (unsaved providers give "Provider does not exist") + 6. test the plugin: open https://$WATCH/sso/OID/start/authentik -> Authentik -> back in, logged in. + Always start from the /start/ URL; opening the /redirect/ URL by hand gives "missing state". + 7. login page = Discord button first, local form behind an "Admin login" link: + Dashboard -> General -> Branding + Login disclaimer: paste jellyfin/branding-disclaimer.html (replace watch.HOST with $WATCH) + Custom CSS: paste jellyfin/branding.css + -> Save; reload https://$WATCH/ + +BOT WEB UI (https://$BOT/) — Authentik forward-auth gate, create once: + Authentik -> Applications -> Create with provider -> Proxy Provider + mode "Forward auth (single application)", External host https://$BOT + -> Applications -> Outposts -> edit "authentik Embedded Outpost" -> add the new application -> Update + -> policy on the application: groups admins / editors / streamers as you see fit + Until this exists the site returns an error (never open). The bot receives X-Authentik-Username/-Groups/-Email. + +GITEA (git.$GIT) + 1. https://$GIT/ -> the first visit shows the install page: database is pre-filled from env; set the + admin account at the bottom (registration is closed afterwards) -> Install Gitea + 2. ./registerGiteaRunner.sh -> registers "vanguard-runner"; Site Administration -> Actions -> Runners + 3. users: Site Administration -> Identity & Access -> User Accounts -> Create (or Authentik OIDC later) + 4. git remotes: ssh://git@$GIT:2222/<org>/<repo>.git or https://$GIT/<org>/<repo>.git + +Then continue with Phase 4 in CommunityDiscordMedia.md (the Vanguard Metadata Jellyfin plugin). +MSG diff --git a/authentik/discord-guild-policy.py b/authentik/discord-guild-policy.py new file mode 100644 index 0000000..9bb1375 --- /dev/null +++ b/authentik/discord-guild-policy.py @@ -0,0 +1,32 @@ +# Authentik expression policy: allow Discord logins only for members of one Discord server (guild). +# Paste into: Customization -> Policies -> Create -> Expression Policy (name: discord-guild-member) +# Bind to BOTH flows: default-source-enrollment and default-source-authentication +# (Flows & Stages -> Flows -> <flow> -> Policy / Group / User Bindings -> Bind existing policy) +# Requires the "guilds" scope on the Discord source (source -> Protocol settings -> Scopes: guilds). +# +# Server ID: Discord -> User Settings -> Advanced -> Developer Mode on; right-click the server -> Copy Server ID. + +ACCEPTED_GUILD_ID = "1387203044084551690" # <- your Discord server ID +GUILD_NAME = "Vanguard" # <- shown in the denial message + +GUILD_API_URL = "https://discord.com/api/users/@me/guilds" + +# Only act on logins that came through the Discord source; let everything else (akadmin password login) pass. +source = context.get("source") +if not source or getattr(source, "provider_type", "") != "discord": + return True + +connection = context.get("goauthentik.io/sources/connection") +if not connection: + ak_message("No Discord connection in context.") + return False + +resp = requests.get(GUILD_API_URL, headers={"Authorization": f"Bearer {connection.access_token}"}, timeout=10) +if resp.status_code != 200: + ak_message(f"Could not read Discord server list ({resp.status_code}). Try again.") + return False + +is_member = any(g.get("id") == ACCEPTED_GUILD_ID for g in resp.json()) +if not is_member: + ak_message(f"You must be a member of the {GUILD_NAME} Discord server to log in.") +return is_member diff --git a/bin/dumpd.py b/bin/dumpd.py new file mode 100755 index 0000000..f18a80a --- /dev/null +++ b/bin/dumpd.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Dump-tonight endpoint. Moves one streamer's night into trash (soft delete). + +POST /dump Authorization: Basic user:pass body (optional JSON): {"night":"YYYY-MM-DD","user":"name"} +GET /dump -> lists what a dump would move (dry run) + +"Night" = everything the user recorded from 17:00 Eastern on <night> until 02:00 Eastern the next day. +Default night: today if it is 17:00 or later, else yesterday (so 00:00-02:00 still counts as last night). +Users may dump only their own prefix; 'admin' may pass "user" to dump anyone, or "all". +""" +import base64, hashlib, json, os, shutil, sys +from datetime import datetime, timedelta, time +from zoneinfo import ZoneInfo +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +ROOT = Path(os.environ.get("MEDIA_ROOT", "/srv/media")) +SRC_DIRS = [ROOT / "streams", ROOT / "clips"] +TRASH = ROOT / "trash" +MARK = ROOT / ".markers" +USERS_FILE = Path(os.environ.get("DUMP_USERS", "/etc/vanguard/dump-users.txt")) +LISTEN = ("127.0.0.1", 8080) +TZ = ZoneInfo(os.environ.get("DUMP_TZ", "America/New_York")) +NIGHT_START = time(17, 0) # 5 pm +NIGHT_END = time(2, 0) # 2 am next day + +def load_users(): + users = {} + for line in USERS_FILE.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or ":" not in line: + continue + u, h = line.split(":", 1) + users[u.strip()] = h.strip().lower() + return users + +def check_auth(header): + if not header or not header.startswith("Basic "): + return None + try: + raw = base64.b64decode(header[6:]).decode() + u, p = raw.split(":", 1) + except Exception: + return None + users = load_users() + if users.get(u) == hashlib.sha256(p.encode()).hexdigest(): + return u + return None + +def default_night(now=None): + now = now or datetime.now(TZ) + d = now.date() if now.time() >= NIGHT_START else (now - timedelta(days=1)).date() + return d.isoformat() + +def night_window(night): + d = datetime.strptime(night, "%Y-%m-%d").date() + return (datetime.combine(d, NIGHT_START, tzinfo=TZ), + datetime.combine(d + timedelta(days=1), NIGHT_END, tzinfo=TZ)) + +def candidates(user, night): + start, end = night_window(night) + out = [] + for base in SRC_DIRS: + udirs = [p for p in base.iterdir() if p.is_dir()] if user == "all" else [base / user] + for ud in udirs: + if not ud.is_dir(): + continue + for f in ud.rglob("*"): + if not f.is_file(): + continue + mt = datetime.fromtimestamp(f.stat().st_mtime, TZ) + if start <= mt <= end: + out.append(f) + return out + +def do_move(files, night): + moved, size = 0, 0 + for f in files: + rel = f.relative_to(ROOT) + dst = TRASH / night / rel + dst.parent.mkdir(parents=True, exist_ok=True) + try: + size += f.stat().st_size + shutil.move(str(f), str(dst)) + moved += 1 + except FileNotFoundError: + pass + # drop now-empty title folders + for base in SRC_DIRS: + for d in sorted((p for p in base.rglob("*") if p.is_dir()), key=lambda p: -len(p.parts)): + if d.parent != base and not any(d.iterdir()): + d.rmdir() + return moved, size + +class H(BaseHTTPRequestHandler): + server_version = "dumpd/1" + def _json(self, code, obj): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def _deny(self): + self.send_response(401) + self.send_header("WWW-Authenticate", 'Basic realm="dump"') + self.end_headers() + def _parse(self): + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(n) or b"{}") if n else {} + night = body.get("night") + if night: + datetime.strptime(night, "%Y-%m-%d") # validate + return night, body.get("user") + def _resolve(self, night, tgt): + """Explicit night wins. Otherwise: tonight, falling back to last night if tonight is empty.""" + if night: + return night, candidates(tgt, night) + n = default_night() + files = candidates(tgt, n) + if not files: + prev = (datetime.strptime(n, "%Y-%m-%d") - timedelta(days=1)).strftime("%Y-%m-%d") + pf = candidates(tgt, prev) + if pf: + return prev, pf + return n, files + def _target(self, auth_user, req_user): + if auth_user == "admin": + return req_user or "all" + return auth_user + def do_GET(self): + if not self.path.startswith("/dump"): + return self._json(404, {"error": "not found"}) + u = check_auth(self.headers.get("Authorization")) + if not u: + return self._deny() + night, req_user = self._parse() + tgt = self._target(u, req_user) + night, files = self._resolve(night, tgt) + self._json(200, {"night": night, "user": tgt, "dry_run": True, + "files": [str(f.relative_to(ROOT)) for f in files], + "gb": round(sum(f.stat().st_size for f in files) / 1e9, 2)}) + def do_POST(self): + if not self.path.startswith("/dump"): + return self._json(404, {"error": "not found"}) + u = check_auth(self.headers.get("Authorization")) + if not u: + return self._deny() + try: + night, req_user = self._parse() + except Exception as e: + return self._json(400, {"error": str(e)}) + tgt = self._target(u, req_user) + night, files = self._resolve(night, tgt) + moved, size = do_move(files, night) + MARK.mkdir(exist_ok=True) + (MARK / f"{night}-{tgt}.dumped").write_text(f"{datetime.now(TZ).isoformat()} by {u}\n") + print(f"{datetime.now(TZ):%F %T} dump night={night} user={tgt} by={u} files={moved} gb={size/1e9:.2f}", flush=True) + self._json(200, {"night": night, "user": tgt, "moved": moved, "gb": round(size / 1e9, 2), + "undo_until": (datetime.now(TZ) + timedelta(days=3)).strftime("%F")}) + def log_message(self, *a): # quiet + pass + +if __name__ == "__main__": + TRASH.mkdir(exist_ok=True); MARK.mkdir(exist_ok=True) + print(f"dumpd listening on {LISTEN[0]}:{LISTEN[1]} tz={TZ.key} night={NIGHT_START:%H:%M}-{NIGHT_END:%H:%M}", flush=True) + ThreadingHTTPServer(LISTEN, H).serve_forever() diff --git a/bin/normalize-sweep.sh b/bin/normalize-sweep.sh new file mode 100755 index 0000000..a6f011b --- /dev/null +++ b/bin/normalize-sweep.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# Normalize every finished original so editors always get the same kind of file: +# MP4 · H.264 8-bit 4:2:0 · constant frame rate · AAC audio · index up front · ≤2560x1440 · ≤60 fps +# - compliant files: untouched +# - fragmented/unindexed MP4 (MediaMTX output) with good codecs: lossless remux, replaced in place +# - anything else (HEVC, 10-bit, VFR, MKV, >1440p, >60fps): full-res re-encode -> name.mp4, +# raw source kept in a hidden .src/ subfolder next to it (Jellyfin/copyparty hide dot-dirs) +# - Syncthing folders (marker dir .stfolder in the folder root, receive-only): compliant files are left +# alone (zero copies); a non-compliant file is converted to name.mp4 (name.h264.mp4 if the raw is +# already .mp4), the raw's name is appended to .stlocal (an #include'd ignore list) so Syncthing +# drops it without flagging or re-pulling, then the raw is deleted. One copy on the server. +# - failure: name.normalize-failed.log is written and the file is skipped until that log is deleted +# Idempotent; safe every few minutes. +set -uo pipefail + +ROOT="${MEDIA_ROOT:-/srv/media}" +MINS_IDLE=3 +MAX_W=2560; MAX_H=1440; MAX_FPS=60 +exec 9>"${LOCK_FILE:-/run/lock/normalize-sweep.lock}" +flock -n 9 || exit 0 + +log() { echo "$(date '+%F %T') $*"; } +dur() { ffprobe -v error -show_entries format=duration -of csv=p=0 "$1" 2>/dev/null | cut -d. -f1; } + +# --- probe: prints "verdict reason" where verdict ∈ ok|remux|encode|skip ---------------------- +probe() { + local f="$1" ext="${1##*.}" + local json; json=$(ffprobe -v error -show_entries stream=codec_type,codec_name,pix_fmt,width,height,r_frame_rate,avg_frame_rate -of json "$f" 2>/dev/null) || { echo "skip unreadable"; return; } + local vcodec pix w h rfr afr acodecs + vcodec=$(jq -r '[.streams[]|select(.codec_type=="video")][0].codec_name // "none"' <<<"$json") + pix=$(jq -r '[.streams[]|select(.codec_type=="video")][0].pix_fmt // ""' <<<"$json") + w=$(jq -r '[.streams[]|select(.codec_type=="video")][0].width // 0' <<<"$json") + h=$(jq -r '[.streams[]|select(.codec_type=="video")][0].height // 0' <<<"$json") + rfr=$(jq -r '[.streams[]|select(.codec_type=="video")][0].r_frame_rate // "0/1"' <<<"$json") + afr=$(jq -r '[.streams[]|select(.codec_type=="video")][0].avg_frame_rate // "0/1"' <<<"$json") + acodecs=$(jq -r '[.streams[]|select(.codec_type=="audio")|.codec_name]|unique|join(",")' <<<"$json") + [[ $vcodec == none ]] && { echo "skip no-video"; return; } + local fps; fps=$(awk -F/ '{ if ($2>0) printf "%.3f", $1/$2; else print 0 }' <<<"$afr") + local reasons=() + [[ $vcodec == h264 ]] || reasons+=("codec=$vcodec") + [[ $pix == yuv420p ]] || reasons+=("pix=$pix") + (( w <= MAX_W && h <= MAX_H )) || reasons+=("size=${w}x${h}") + awk "BEGIN{exit !($fps > $MAX_FPS + 0.01)}" && reasons+=("fps=$fps") + [[ -z $acodecs || $acodecs == aac ]] || reasons+=("audio=$acodecs") + [[ ${ext,,} == mp4 ]] || reasons+=("container=$ext") + if (( ${#reasons[@]} )); then echo "encode ${reasons[*]}"; return; fi + # codecs fine. Fragmented MP4 (MediaMTX output) -> lossless remux. ffprobe's frame-rate figures are + # meaningless on fragmented files, so the VFR check only runs on flat files (i.e. after the remux). + local head; head=$(head -c 2097152 "$f" | grep -aob -m3 -E 'moov|mdat|moof' | tr '\n' ' ') + if grep -q moof <<<"$head"; then echo "remux fragmented"; return; fi + # real VFR test: do the gaps between the first ~400 frames vary? (dts is monotonic even with B-frames) + local vfr; vfr=$(ffprobe -v error -select_streams v:0 -show_entries packet=dts_time -of csv=p=0 -read_intervals '%+#400' "$f" 2>/dev/null | + awk 'NF && $1!="N/A" { if (p!="") d[n++]=$1-p; p=$1 } + END { if (n<20) { print "cfr"; exit } + for(i=0;i<n;i++) a[i]=d[i]; asort(a); med=a[int(n/2)]; if (med<=0) { print "cfr"; exit } + bad=0; for(i=0;i<n;i++){ x=d[i]-med; if(x<0)x=-x; if (x > med*0.25) bad++ } + print (bad > n*0.02) ? "vfr" : "cfr" }') + [[ $vfr == vfr ]] && { echo "encode vfr"; return; } + local moov mdat; moov=$(grep -o '[0-9]*:moov' <<<"$head" | cut -d: -f1 | head -1); mdat=$(grep -o '[0-9]*:mdat' <<<"$head" | cut -d: -f1 | head -1) + if [[ -z $moov || ( -n $mdat && $mdat -lt $moov ) ]]; then echo "remux index-last"; return; fi + echo "ok" +} + +find "$ROOT/streams" "$ROOT/clips" -type f \ + \( -iname '*.mp4' -o -iname '*.mkv' -o -iname '*.mov' -o -iname '*.ts' -o -iname '*.m4v' -o -iname '*.webm' \) \ + ! -path '*/.src/*' ! -path '*/.stfolder/*' ! -path '*/.stversions/*' ! -name '*.part' -mmin +"$MINS_IDLE" -print0 2>/dev/null | +while IFS= read -r -d '' src; do + base="${src%.*}"; ext="${src##*.}" + synced=0; stroot="" + d="$(dirname "$src")" + while [[ $d == "$ROOT"/* ]]; do # find the Syncthing folder root, if any + if [[ -d "$d/.stfolder" ]]; then synced=1; stroot="$d"; break; fi + d="$(dirname "$d")" + done + [[ -e "$base.normalize-failed.log" ]] && continue + fuser -s "$src" 2>/dev/null && continue # still being written + [[ -e "$src" ]] || continue # dumped meanwhile + + read -r verdict reason <<<"$(probe "$src")" + case "$verdict" in ok|skip) continue ;; esac + out="$base.mp4" + if (( synced )); then + [[ $verdict == remux ]] && continue # synced + only an index issue: not worth touching + [[ ${ext,,} == mp4 ]] && out="$base.h264.mp4" # raw is already name.mp4: don't collide + [[ -e "$out" ]] && continue + fi + tmp="$out.part" + log "$verdict $src ($reason)" + if [[ $verdict == remux ]]; then + nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \ + -map 0 -c copy -movflags +faststart -f mp4 "$tmp" + else + # scale down into the cap (never up), keep aspect, even dimensions; cap fps; CFR; all audio -> AAC + vf="scale='min(iw,$MAX_W)':'min(ih,$MAX_H)':force_original_aspect_ratio=decrease:force_divisible_by=2" + fpsargs=(); fpsnow=$(awk -F/ '{ if ($2>0) printf "%.3f", $1/$2; else print 0 }' <<<"$(ffprobe -v error -select_streams v:0 -show_entries stream=avg_frame_rate -of csv=p=0 "$src")") + awk "BEGIN{exit !($fpsnow > $MAX_FPS + 0.01)}" && fpsargs=(-r "$MAX_FPS") + nice -n 15 ionice -c 3 ffmpeg -nostdin -hide_banner -loglevel error -y -i "$src" \ + -map 0:v:0 -map '0:a?' \ + -c:v libx264 -preset veryfast -crf 18 -maxrate 30M -bufsize 60M -pix_fmt yuv420p -profile:v high \ + -vf "$vf" -fps_mode cfr "${fpsargs[@]}" \ + -c:a aac -b:a 160k \ + -movflags +faststart -f mp4 "$tmp" + fi + rc=$? + d1=$(dur "$src"); d2=$(dur "$tmp") + if (( rc == 0 )) && [[ -n $d1 && -n $d2 ]] && (( d2 >= d1 - 2 && d2 <= d1 + 2 )); then + if (( synced )); then + rel_src="${src#"$stroot"/}"; rel_out="${out#"$stroot"/}" + { echo "/$rel_out"; echo "/$rel_src"; } >> "$stroot/.stlocal" # ignore our output and the raw + chown 1000:1000 "$stroot/.stlocal" 2>/dev/null + sleep 2 # let Syncthing re-read ignores before the delete + mv -f "$tmp" "$out" + rm -f "$src" # raw stays on the gamer's PC; one copy here + elif [[ $verdict == remux ]]; then + mv -f "$tmp" "$src" # lossless: replace in place + else + srcdir="$(dirname "$src")/.src"; mkdir -p "$srcdir"; chown 1000:1000 "$srcdir" 2>/dev/null + mv -f "$src" "$srcdir/$(basename "$src")" # keep raw source, hidden from editors/Jellyfin + mv -f "$tmp" "$out" + fi + chown 1000:1000 "$out" 2>/dev/null + log "done $out" + else + rm -f "$tmp" + { echo "$(date '+%F %T') $verdict failed rc=$rc src_dur=${d1:-?} out_dur=${d2:-?} ($reason)"; } > "$base.normalize-failed.log" + log "FAILED $src (see $base.normalize-failed.log)" >&2 + fi +done diff --git a/bin/trash-purge.sh b/bin/trash-purge.sh new file mode 100755 index 0000000..371576e --- /dev/null +++ b/bin/trash-purge.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Delete dumped nights older than 3 days; drop empty dirs; clean stale markers. +set -uo pipefail +TRASH=/srv/media/trash +MARK=/srv/media/.markers +[[ -d $TRASH ]] || exit 0 +find "$TRASH" -mindepth 1 -type f -mtime +3 -delete +find "$TRASH" -mindepth 1 -type d -empty -delete +find "$MARK" -type f -mtime +30 -delete 2>/dev/null +echo "$(date '+%F %T') trash purge done" diff --git a/bot/.dockerignore b/bot/.dockerignore new file mode 100644 index 0000000..5812ae6 --- /dev/null +++ b/bot/.dockerignore @@ -0,0 +1,2 @@ +app/__pycache__/ +app/*.pyc diff --git a/bot/AGENT.md b/bot/AGENT.md new file mode 100644 index 0000000..1f8823d --- /dev/null +++ b/bot/AGENT.md @@ -0,0 +1,108 @@ +# Vanguard Discord bot — brief for the agent building it + +You are building the Discord bot for Vanguard. What the bot *does* is up to the community and you. +This file only covers where it runs, what it is given, and the few rules that keep it from breaking +the box it shares. Nothing here obliges you to integrate with the other services on that box. + +## 1. Hosting + +One Debian server runs a set of Docker Compose services from `~/vanguard-stream/`. You own exactly +one of them: the `discord-bot` container. You do not own the compose file, the reverse proxy (Caddy), +the firewall, the identity provider (Authentik), or the database server config. If you need any of +those changed, ask the admin with the exact change (see §6). + +What you are given: + +| | | +|---|---| +| **Runtime** | `python:3.12-slim` + ffmpeg + curl. Dependencies from `app/requirements.txt`, installed at image build (`docker compose build discord-bot`). | +| **Your code** | bind-mounted at `/app` from `bot/app/`. Not baked into the image. Edit, then `docker compose restart discord-bot`. | +| **Entry point** | the container runs `python -u /app/main.py` and restarts it if it exits. With no `main.py` it idles. | +| **User** | uid 1000 | +| **Database** | MariaDB, database `vanguardbot`, user `bot`, reachable at `DB_HOST:DB_PORT`. Yours alone. Other databases on the same server belong to other services; never touch them. | +| **Persistent disk** | `/data`, yours | +| **Web UI** | bind `WEB_HOST:WEB_PORT` (0.0.0.0:8000). The proxy publishes it as `https://bot.<domain>/` behind login (§3). | +| **Discord** | `DISCORD_BOT_TOKEN`, `DISCORD_GUILD_ID`. The bot user belongs to the community's existing Discord application. | +| **Clock** | `TZ=America/New_York` | + +### Environment variables +`DISCORD_BOT_TOKEN`, `DISCORD_GUILD_ID`, `DB_HOST`, `DB_PORT`, `DB_NAME`, `DB_USER`, `DB_PASSWORD`, +`WEB_HOST`, `WEB_PORT`, `WEB_PUBLIC_URL`, `AUTH_HEADER_USER`, `AUTH_HEADER_GROUPS`, `AUTH_HEADER_EMAIL`, +`MEDIA_CLIPS`, `TZ`. Secrets arrive only through env. Never write them to the repo, `/data`, or logs. + +SQLAlchemy URL: `mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}` (async: `mysql+aiomysql`). + +## 2. Process model + +- One process, one asyncio loop: the Discord gateway client and the web server run together + (`discord.py` plus `aiohttp` or FastAPI via uvicorn's programmatic API). Do not spawn a second container. +- Log to stdout. The admin reads `docker compose logs -f discord-bot`. +- On fatal misconfiguration (no token, DB unreachable after retries) log the reason and exit non-zero; + the restart policy retries. +- Handle SIGTERM: close the gateway and the web server cleanly. + +## 3. Web UI: login is handled outside your code + +Every request to `https://bot.<domain>/` passes through Caddy, which sends the user to Authentik +(Discord login) and, once authenticated, forwards the request to you with identity headers: + +| header | content | +|---|---| +| `X-Authentik-Username` | the user's account name | +| `X-Authentik-Groups` | group names, separated by `\|` or `,` — the community uses `admins`, `editors`, `streamers`; others may be added | +| `X-Authentik-Email`, `X-Authentik-Name`, `X-Authentik-Uid` | extras | + +Rules: +1. **Implement no login page, no sessions, no passwords.** Identity is the headers, per request. +2. Port 8000 is loopback-only on the host, so the headers can be trusted. Still fail closed: a request + without the username header gets 401. +3. Authorize by group, read from the header each time. Don't cache roles in your DB. +4. Serve under `/`; the proxy does no path rewriting for this host. +5. Build absolute links from `WEB_PUBLIC_URL`. +6. The Authentik username may differ from the Discord username. Use the header value as the user key; + if you need the Discord snowflake, resolve it through your gateway client and cache it in your DB. + +Until the admin creates the matching Authentik application, the hostname returns an error, never an +open page. Test against `http://127.0.0.1:8000/` on the server with the headers set by hand. + +## 4. Repository layout (recommended, not enforced) + +``` +bot/ + AGENT.md, README.md this brief; the ops cheat-sheet (keep README current) + Dockerfile, entrypoint.sh admin-owned; ask before changing + app/ + main.py config → migrations → gateway + web in one loop → clean shutdown + requirements.txt pin versions once past the prototype + config.py the only module that reads os.environ + bot/ discord.py client, cogs/ (one feature per cog), permission checks + web/ routes/, templates/, static/, auth.py (header → user/groups) + db/ SQLAlchemy models, session, alembic/ migrations (run at startup) + services/ the actual logic, shared by cogs and web; no Discord or HTTP types + tests/ pytest; must pass offline with no token +``` +Slash commands scoped to the guild so they appear instantly. Prefer slash commands over reading +message content; Message Content is a privileged intent, enable it only if you need it. + +## 5. Shared ground on the box + +- `/media/clips` is mounted read-write. It is the community's clip storage, organized as one folder per + user. You may **add** files there; never modify, move or delete files you did not create (parts of it + are mirrored from members' machines and will fight back). If you write a video, write + `<name>.part` then rename to `<name>.mp4`, and put a `<name>.json` beside it with at least + `{"timestamp": "<UTC ISO-8601 creation time>", "source": "...", "uploader": "<discord username>"}`. + Other services on the box index that folder and read the sidecar. If you never touch this mount, + none of this applies. +- The same bot user is used by another service for REST calls (posting notices, reading scheduled + events). Sharing a token is fine. Don't rename the bot or change its avatar; pick your own channels. +- Don't bind other ports, don't start other containers, don't write outside `/app`, `/data`, `/media/clips`. + +## 6. Working with the admin + +- **Source**: push to the community Gitea, `ssh://git@git.<domain>:2222/vanguard/discord-bot.git`. + The admin deploys by pulling into `bot/app` and restarting. CI on the `python` runner label runs your tests. +- **Need infra?** Open a Gitea issue titled `infra: …` stating the exact change: a new env var and why, + a mount, a Caddy path that should skip login (e.g. a public webhook endpoint), a new database. + Don't work around the proxy or the firewall. +- **Dependencies changed?** Say so in the PR; the image must be rebuilt. +- Keep `README.md` updated with every command, env var, and table you add. diff --git a/bot/Dockerfile b/bot/Dockerfile new file mode 100644 index 0000000..155b7a8 --- /dev/null +++ b/bot/Dockerfile @@ -0,0 +1,13 @@ +# Python runtime for the Discord bot. Source is bind-mounted at /app from ./bot/app (not baked in), +# so the dev edits code and restarts the container. Dependencies come from app/requirements.txt and +# are installed at BUILD time: after changing it run docker compose build discord-bot && docker compose up -d discord-bot +FROM python:3.12-slim +ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 +RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* +COPY app/requirements.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +WORKDIR /app +ENTRYPOINT ["/entrypoint.sh"] diff --git a/bot/README.md b/bot/README.md new file mode 100644 index 0000000..8971a4d --- /dev/null +++ b/bot/README.md @@ -0,0 +1,43 @@ +# discord-bot — for the bot dev + +Start with **AGENT.md** in this folder: the stack, the rules, the layout, the web-UI auth contract. + +Your code goes in this folder (`bot/app/`). The container runs `python -u /app/main.py` and restarts +it if it exits. No `main.py` yet → the container idles and logs a hint. + +## Day to day (on the server, in ~/vanguard-stream) + docker compose logs -f discord-bot # live log + docker compose restart discord-bot # after editing code + docker compose build discord-bot && docker compose up -d discord-bot # after editing requirements.txt + docker compose exec discord-bot python # a REPL inside the container + +Edit files directly in `~/vanguard-stream/bot/app/` (they are bind-mounted) or scp them up. + +## Environment the container gives you +| var | value | +|-----|-------| +| `DISCORD_BOT_TOKEN` | from `.env` (bundle root) — the Bot added to the existing Authentik Discord application | +| `DISCORD_GUILD_ID` | the server | +| `DB_HOST` / `DB_PORT` | `host.docker.internal` / `3306` (MariaDB runs on the host network) | +| `DB_NAME` / `DB_USER` / `DB_PASSWORD` | your own database + user (`vanguardbot` / `bot` / generated) | +| `MEDIA_CLIPS` | `/media/clips` — rw mount of `/srv/media/clips` (folder per user, for clip submissions) | +| `/data` | persistent scratch dir owned by the bot | +| `WEB_HOST` / `WEB_PORT` | `0.0.0.0` / `8000` — your web UI; public as `https://bot.<domain>/` via Caddy + Authentik | +| `WEB_PUBLIC_URL` | `https://bot.<domain>` for absolute links | +| `AUTH_HEADER_USER` / `_GROUPS` / `_EMAIL` | `X-Authentik-Username` / `X-Authentik-Groups` / `X-Authentik-Email` — identity injected by Caddy, trust these, implement no login | +| `TZ` | America/New_York | + +SQLAlchemy URL: `mysql+pymysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}` +async: `mysql+aiomysql://...` + +## Database from your own machine +MariaDB is reachable directly: `ovh1.dumpstervigilantes.rip:3306`, user `bot`, database `vanguardbot`, +password from the admin (`./configure.py --show` on the server). Root is local-only. New connections are +rate limited at the firewall; if yours get dropped, ask the admin to add your IP to `db_allow`. + +## Conventions +- Run as uid 1000 (files you write under `/media/clips` are owned like everything else there). +- If you write clips: `/media/clips/<discord-username>/<name>.mp4` plus `<name>.json` with + `{"timestamp": "<UTC ISO-8601>", "source": "medal", "source_url": "...", "uploader": "...", "title": "..."}` + — the Jellyfin plugin reads that sidecar for the exact recording time. +- Log to stdout; the container captures it. diff --git a/bot/app/requirements.txt b/bot/app/requirements.txt new file mode 100644 index 0000000..d14ef59 --- /dev/null +++ b/bot/app/requirements.txt @@ -0,0 +1,8 @@ +# Baseline for a discord.py bot with MariaDB. Add what you need; then: +# docker compose build discord-bot && docker compose up -d discord-bot +discord.py>=2.4 +aiomysql>=0.2 +PyMySQL>=1.1 +SQLAlchemy>=2.0 +python-dotenv>=1.0 +yt-dlp diff --git a/bot/entrypoint.sh b/bot/entrypoint.sh new file mode 100644 index 0000000..7becf9a --- /dev/null +++ b/bot/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Run the bot if there is one; otherwise idle so the container stays up until code arrives. +if [ -f /app/main.py ]; then + exec python -u /app/main.py +fi +echo "discord-bot: no /app/main.py yet — drop the bot code into ./bot/app and 'docker compose restart discord-bot'" +exec sleep infinity diff --git a/configure.py b/configure.py new file mode 100755 index 0000000..f5fbb1c --- /dev/null +++ b/configure.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +"""vanguard-stream post-install configuration. Stateless: the config files ARE the state. + + ./configure.py --full first-time setup: asks every question (current values as defaults) + ./configure.py manage: skips what is already set; add/remove/reset streamers & editors + ./configure.py --show print current streamers, editors, domain and the OBS settings + ./configure.py --hosts AUTH WATCH [GIT [BOT]] set the Authentik/Jellyfin/Gitea/bot hostnames and apply (non-interactive; + used by addCommunityMediaServer.sh) + +Reads : Caddyfile (hostnames), mediamtx.yml (streamers), copyparty.conf (editors + admin), .env +Writes : Caddyfile, mediamtx.yml, copyparty.conf, dump-users.txt -> /etc/vanguard, .env (generated secrets + + Discord bot token/guild), folders, restarts +Set VG_DRY=1 to write files only (no sudo / docker / folders). +""" +import getpass, hashlib, os, re, secrets, shutil, subprocess, sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +os.chdir(HERE) +DRY = os.environ.get("VG_DRY") == "1" +NAME_RE = re.compile(r"^[a-z0-9_-]{2,32}$") +BAD_PW = set(':&?#/ "\'\\\t') # would break the RTMP query string / SRT streamid / YAML + +# ------------------------------------------------------------------ read current +def read_hosts(): + """{'main': host, 'auth': host, 'watch': host} from '# vanguard:<role>' markers; first block = main.""" + hosts, role = {}, None + try: + for line in Path("Caddyfile").read_text().splitlines(): + if line[:1].isspace(): continue # site blocks start at column 0 + t = line.strip() + m = re.match(r"#\s*vanguard:(\w+)", t) + if m: role = m.group(1); continue + if t and not t.startswith(("#", "}", "{")) and t.endswith("{") and " " in t: + host = t.split()[0] + if "example.com" in host: role = None; continue + hosts[role or ("main" if "main" not in hosts else host)] = host + role = None + except FileNotFoundError: + pass + return hosts + +def read_domain(): return read_hosts().get("main", "") + +def read_env(): + env = {} + try: + for line in Path(".env").read_text().splitlines(): + if "=" in line and not line.startswith("#"): + k, v = line.split("=", 1); env[k.strip()] = v.strip() + except FileNotFoundError: + pass + return env + +def default_sub(main, sub): + labels = main.split(".") + base = ".".join(labels[1:]) if len(labels) >= 3 else main + return f"{sub}.{base}" if main else "" + +def read_streamers(): + out, cur = {}, None + try: + in_users = False + for line in Path("mediamtx.yml").read_text().splitlines(): + if line.startswith("authInternalUsers:"): + in_users = True; continue + if in_users and line and not line.startswith(" "): + break + if in_users: + m = re.match(r"\s*-\s*user:\s*(\S+)", line) + if m: cur = m.group(1); continue + m = re.match(r"\s*pass:\s*(\S+)", line) + if m and cur: + if not m.group(1).startswith("CHANGEME"): out[cur] = m.group(1) + cur = None + except FileNotFoundError: + pass + return out + +def read_accounts(): + out = {} + try: + in_acc = False + for line in Path("copyparty.conf").read_text().splitlines(): + if line.strip() == "[accounts]": in_acc = True; continue + if in_acc and line.startswith("["): break + if in_acc: + m = re.match(r"\s*([A-Za-z0-9_-]+):\s*(\S+)", line) + if m and not m.group(2).startswith("CHANGEME"): out[m.group(1)] = m.group(2) + except FileNotFoundError: + pass + return out + +# ------------------------------------------------------------------ prompts +def ask(q, default=""): + s = input(f"{q}{f' [{default}]' if default else ''}: ").strip() + return s or default + +def yes(q, default=True): + d = "Y/n" if default else "y/N" + s = input(f"{q} [{d}]: ").strip().lower() + return default if not s else s.startswith("y") + +def gen_pw(): return "".join(c for c in secrets.token_urlsafe(24) if c.isalnum())[:20] + +def ask_pw(label, current=None): + """keep / generate / type. Returns the password.""" + opts = "(enter = generate" + (", k = keep current" if current else "") + ", or type one)" + while True: + prompt = f" password for {label} {opts}: " + s = (getpass.getpass(prompt) if sys.stdin.isatty() else input(prompt)).strip() + if not s: p = gen_pw(); print(f" generated: {p}"); return p + if s == "k" and current: return current + if BAD_PW & set(s): print(" avoid : & ? # / \\ quotes and spaces (they break stream keys)"); continue + if len(s) < 8: print(" at least 8 characters"); continue + return s + +def ask_name(kind, taken): + while True: + n = ask(f" {kind} name (blank to stop)").lower() + if not n: return None + if not NAME_RE.match(n): print(" lowercase letters, digits, _ - (2-32 chars)"); continue + if n in taken: print(" already exists"); continue + return n + +# ------------------------------------------------------------------ writers +AUTHENTIK_FORWARD_AUTH = """ # Authentik forward-auth (embedded outpost). Needs a Proxy Provider (forward auth, single application) + # for this external host in Authentik, assigned to the embedded outpost. Until then: 4xx, never open. + reverse_proxy /outpost.goauthentik.io/* 127.0.0.1:9000 + forward_auth 127.0.0.1:9000 { + uri /outpost.goauthentik.io/auth/caddy + copy_headers X-Authentik-Username X-Authentik-Groups X-Authentik-Entitlements X-Authentik-Email X-Authentik-Name X-Authentik-Uid X-Authentik-Jwt X-Authentik-Meta-Jwks X-Authentik-Meta-Outpost X-Authentik-Meta-Provider X-Authentik-Meta-App X-Authentik-Meta-Version + trusted_proxies private_ranges + } +""" + +def write_caddy(domain, auth_host="", watch_host="", git_host="", bot_host=""): + extra = "" + if bot_host: + extra += f""" +# vanguard:bot +{bot_host} {{ + encode zstd gzip +{AUTHENTIK_FORWARD_AUTH} reverse_proxy 127.0.0.1:8000 +}} +""" + if git_host: + extra += f""" +# vanguard:git +{git_host} {{ + encode zstd gzip + reverse_proxy 127.0.0.1:3000 +}} +""" + if auth_host: + extra += f""" +# vanguard:auth +{auth_host} {{ + encode zstd gzip + reverse_proxy 127.0.0.1:9000 +}} +""" + if watch_host: + extra += f""" +# vanguard:watch +{watch_host} {{ + encode zstd gzip + reverse_proxy 127.0.0.1:8096 +}} +""" + Path("Caddyfile").write_text(f"""# vanguard:main +{domain} {{ + encode zstd gzip + + # dump endpoint (python service on the host, does its own auth) + handle /dump* {{ + reverse_proxy 127.0.0.1:8080 + }} + + # Syncthing admin UI (its own login; strip the prefix, present a local Host to pass its host check) + redir /sync /sync/ 308 + handle_path /sync/* {{ + reverse_proxy 127.0.0.1:8384 {{ + header_up Host {{upstream_hostport}} + }} + }} + + # OBS browser dock (static) + handle_path /dock* {{ + root * /srv/dock + file_server + }} + + # everything else is copyparty (web UI + WebDAV) + handle {{ + reverse_proxy 127.0.0.1:3923 {{ + flush_interval -1 + }} + }} +}} +""" + extra) + +def write_env(watch_host, discord_token=None, discord_guild=None, git_host="", bot_host=""): + env = read_env() + env["GIT_HOST"] = git_host or env.get("GIT_HOST", "git.example.com") + env["BOT_HOST"] = bot_host or env.get("BOT_HOST", "bot.example.com") + env.setdefault("GITEA_DB_PASSWORD", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32]) + env.setdefault("GITEA_RUNNER_TOKEN", "") # filled by ./registerGiteaRunner.sh + if discord_token is not None: env["DISCORD_BOT_TOKEN"] = discord_token + if discord_guild is not None: env["DISCORD_GUILD_ID"] = discord_guild + env.setdefault("AUTHENTIK_SECRET_KEY", secrets.token_urlsafe(48)) + env.setdefault("AUTHENTIK_PG_PASS", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32]) + env.setdefault("AUTHENTIK_TAG", "2025.8") + env.setdefault("MARIADB_ROOT_PASSWORD", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32]) + env.setdefault("BOT_DB_NAME", "vanguardbot") + env.setdefault("BOT_DB_USER", "bot") + env.setdefault("BOT_DB_PASSWORD", "".join(c for c in secrets.token_urlsafe(40) if c.isalnum())[:32]) + env.setdefault("DISCORD_BOT_TOKEN", "") # fill in: Discord dev portal -> the Authentik app -> Bot + env.setdefault("DISCORD_GUILD_ID", "") + env["WATCH_HOST"] = watch_host or "watch.example.com" + Path(".env").write_text("".join(f"{k}={v}\n" for k, v in env.items())) + os.chmod(".env", 0o600) + +def write_mediamtx(streamers): + users = "".join(f""" - user: {u} + pass: {p} + permissions: + - action: publish + path: ~^{u}/[A-Za-z0-9_.-]+$ +""" for u, p in streamers.items()) or " []\n" + alt = "|".join(streamers) or "nobody" + Path("mediamtx.yml").write_text(f"""# MediaMTX — ingest only. No playback endpoints exposed. (generated by configure.py) +logLevel: info + +rtsp: no +webrtc: no +hls: no +api: no +metrics: no +pprof: no + +rtmp: yes +rtmpAddress: :1935 + +srt: yes +srtAddress: :8890 + +# ---- auth ------------------------------------------------------------- +# Each streamer may publish only under their own prefix. +# RTMP key: <user>/Title?user=<user>&pass=<pass> +# SRT: srt://HOST:8890?streamid=publish:<user>/Title:<user>:<pass> +authMethod: internal +authInternalUsers: +{users} +# ---- recording -------------------------------------------------------- +pathDefaults: + record: yes + recordPath: /recordings/%path/%Y-%m-%d_%H-%M-%S + recordFormat: fmp4 + recordPartDuration: 1s + recordSegmentDuration: 12h # one file per stream unless it runs >12h + recordDeleteAfter: 0s # never auto-delete; the archive job handles it + +paths: + # any name under a streamer's prefix; inherits pathDefaults + ~^({alt})/[A-Za-z0-9_.-]+$: {{}} +""") + +def write_copyparty(admin_pw, editors): + acc = f" admin: {admin_pw}\n" + "".join(f" {u}: {p}\n" for u, p in editors.items()) + eds = ", ".join(editors) + r_line = f" r: {eds}\n" if eds else "" + rw_line = f" rw: {eds}\n" if eds else "" + Path("copyparty.conf").write_text(f"""# copyparty — editors browse/preview/download; finals are writable. (generated by configure.py) +# Docs: https://github.com/9001/copyparty#server-config + +[global] + e2dsa # index files for search + e2ts # read media tags + th-ff-jpg # video thumbnails via ffmpeg (cached under /cfg/hist) + hist: /cfg/hist + rproxy: 1 # trust first X-Forwarded-For hop (Caddy) + xff-src: 172.16.0.0/12, 10.0.0.0/8, 192.168.0.0/16, 127.0.0.0/8 + no-robots + name: vanguard + usernames # login needs username + password (default is password-only) + # dark skin for the login page (served by Caddy from ./dock) + html-head: <link rel="stylesheet" href="/dock/vanguard.css"> + +[accounts] +{acc} +# streams: read for editors, full for admin +[/streams] + /w/streams + accs: +{r_line} rwmd: admin + +# gamer clips (Syncthing drops them here) +[/clips] + /w/clips + accs: +{r_line} rwmd: admin + +# editors upload finished work here +[/finals] + /w/finals + accs: +{rw_line} rwmd: admin + +# dumped nights, purged after 3 days — admin only +[/trash] + /w/trash + accs: + rwmd: admin +""") + +def write_dump_users(admin_pw, streamers): + h = lambda p: hashlib.sha256(p.encode()).hexdigest() + body = "# user:sha256hex(password) (generated by configure.py)\n" + body += "".join(f"{u}:{h(p)}\n" for u, p in streamers.items()) + body += f"admin:{h(admin_pw)}\n" + Path("dump-users.txt").write_text(body) + +def run(cmd, **kw): + print(" $", " ".join(cmd)) + if DRY: return 0 + return subprocess.call(cmd, **kw) + +def apply(domain, admin_pw, streamers, editors, auth_host="", watch_host="", discord_token=None, discord_guild=None, git_host="", bot_host=""): + print("\n== writing configs") + write_caddy(domain, auth_host, watch_host, git_host, bot_host); write_mediamtx(streamers); write_copyparty(admin_pw, editors); write_dump_users(admin_pw, streamers) + write_env(watch_host, discord_token, discord_guild, git_host, bot_host) + print("== app data folders") + run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/jellyfin/config", "/srv/appdata/jellyfin/cache", + "/srv/appdata/authentik/media", "/srv/appdata/authentik/templates", "/srv/appdata/authentik/certs"]) + run(["sudo", "install", "-d", "/srv/appdata/authentik/db", "/srv/appdata/authentik/redis", "/srv/appdata/mariadb"]) + run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", "/srv/appdata/discord-bot", "/srv/appdata/gitea", "/srv/appdata/act_runner"]) + print("== folders for streamers") + for u in streamers: + run(["sudo", "install", "-d", "-o", "1000", "-g", "1000", f"/srv/media/streams/{u}", f"/srv/media/clips/{u}"]) + if not DRY and not Path(f"/srv/media/clips/{u}/.stignore").exists(): + run(["sudo", "install", "-m", "0644", "-o", "1000", "-g", "1000", "syncthing/stignore", f"/srv/media/clips/{u}/.stignore"]) + print("== dump users + restart dumpd") + run(["sudo", "install", "-d", "-m", "0750", "/etc/vanguard"]) + run(["sudo", "install", "-m", "0640", "dump-users.txt", "/etc/vanguard/dump-users.txt"]) + run(["sudo", "systemctl", "restart", "dumpd"]) + print("== containers") + dc = ["docker", "compose"] if (DRY or subprocess.call(["docker", "info"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) == 0) else ["sudo", "docker", "compose"] + run(dc + ["up", "-d", "--build"]) + run(dc + ["restart", "mediamtx", "copyparty", "caddy"]) + +def show(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host=""): + print(f"\n== domain : {domain or '(not set)'}") + print(f"== bot web : https://{bot_host}/ (behind Authentik forward-auth; create the Proxy Provider for it)" if bot_host else "== bot web : (not set — run --full)") + print(f"== gitea : https://{git_host}/ ssh port 2222 (runner: ./registerGiteaRunner.sh)" if git_host else "== gitea : (not set — run --full)") + print(f"== authentik: https://{auth_host}/ (first run: https://{auth_host}/if/flow/initial-setup/)" if auth_host else "== authentik: (not set — run --full)") + print(f"== jellyfin : https://{watch_host}/" if watch_host else "== jellyfin : (not set — run --full)") + env = read_env() + tok = env.get("DISCORD_BOT_TOKEN", "") + print(f"== discord : guild {env.get('DISCORD_GUILD_ID') or '(not set)'}; bot token {'set (' + tok[:6] + '…)' if tok else '(not set)'}") + print(f"== mariadb : {domain or 'HOST'}:3306 (public, root local-only) root pass {env.get('MARIADB_ROOT_PASSWORD','(run apply)')}") + print(f" bot : db {env.get('BOT_DB_NAME','vanguardbot')} user {env.get('BOT_DB_USER','bot')} pass {env.get('BOT_DB_PASSWORD','(run apply)')}") + print(f" gitea : db gitea user gitea pass {env.get('GITEA_DB_PASSWORD','(run apply)')} (created on MariaDB first init, else ./createGiteaDb.sh)") + print(f"== admin : admin / {admin_pw or '(not set)'} (copyparty + dump)") + print("== editors :", ", ".join(f"{u} / {p}" for u, p in editors.items()) or "(none)") + print("== streamers:") + for u, p in streamers.items(): + print(f"\n {u} password {p}") + print(f" OBS server : rtmp://{domain or 'HOST'}:1935/") + print(f" OBS key : {u}/TITLE?user={u}&pass={p} (the dock sets this for them)") + print(f" SRT : srt://{domain or 'HOST'}:8890?streamid=publish:{u}/TITLE:{u}:{p}") + print(f" Dock URL : https://{domain or 'HOST'}/dock/ (settings: host={domain or 'HOST'}, user={u}, pass={p})") + if not streamers: print(" (none)") + print(f"\n editors log in at https://{domain or 'HOST'}/ — WebDAV at the same URL") + print(f" verify: curl -sI https://{domain or 'HOST'}/dock/ | head -1 (expect 200)") + print(f" curl -s -o /dev/null -w '%{{http_code}}\\n' https://{domain or 'HOST'}/dump (expect 401)\n") + +# ------------------------------------------------------------------ modes +def manage_users(kind, users): + """add / remove / reset loop for one kind of user; mutates dict.""" + while True: + print(f"\n {kind}s: {', '.join(users) or '(none)'}") + c = ask(" [a]dd [r]emove [p]assword reset [d]one", "d").lower()[:1] + if c == "d": return + if c == "a": + n = ask_name(kind, users) + if n: users[n] = ask_pw(n) + elif c in "rp": + n = ask(f" which {kind}").lower() + if n not in users: print(" no such user"); continue + if c == "r": + if yes(f" remove {n}? (their files stay on disk)", False): del users[n] + else: + users[n] = ask_pw(n, users[n]) + +def full(domain, admin_pw, streamers, editors, auth_host="", watch_host="", git_host="", bot_host=""): + print("== FULL SETUP (enter keeps the shown default)\n") + while True: + domain = ask("public hostname (DNS A record must point here)", domain) + if domain and "." in domain and " " not in domain: break + print(" need a hostname like media.example.com") + auth_host = ask("Authentik hostname (CNAME to the above)", auth_host or default_sub(domain, "auth")) + watch_host = ask("Jellyfin hostname (CNAME to the above)", watch_host or default_sub(domain, "watch")) + git_host = ask("Gitea hostname (CNAME to the above)", git_host or default_sub(domain, "git")) + bot_host = ask("Bot web UI hostname (CNAME to the above)", bot_host or default_sub(domain, "bot")) + print("\n-- Discord bot (optional; the Bot added to the Authentik Discord application). Enter keeps the current value.") + discord_token, discord_guild = ask_discord() + print("\n-- admin (copyparty admin + may dump anyone's night)") + admin_pw = ask_pw("admin", admin_pw or None) + print("\n-- streamers (OBS publishers; each gets streams/<name> and clips/<name>)") + for u in list(streamers): + k = ask(f" {u}: [k]eep [r]emove [p]assword reset", "k").lower()[:1] + if k == "r": del streamers[u] + elif k == "p": streamers[u] = ask_pw(u, streamers[u]) + print(" add streamers:") + while True: + n = ask_name("streamer", streamers) + if not n: break + streamers[n] = ask_pw(n) + print("\n-- editors (copyparty logins: read streams/clips, write finals)") + for u in list(editors): + k = ask(f" {u}: [k]eep [r]emove [p]assword reset", "k").lower()[:1] + if k == "r": del editors[u] + elif k == "p": editors[u] = ask_pw(u, editors[u]) + print(" add editors:") + while True: + n = ask_name("editor", editors) + if not n: break + editors[n] = ask_pw(n) + return domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host + +def ask_discord(): + env = read_env() + cur_tok, cur_guild = env.get("DISCORD_BOT_TOKEN", ""), env.get("DISCORD_GUILD_ID", "") + prompt = f" bot token{' [keep ' + cur_tok[:6] + '…]' if cur_tok else ' [none]'}: " + tok = (getpass.getpass(prompt) if sys.stdin.isatty() else input(prompt)).strip() or cur_tok + guild = ask(" server (guild) ID", cur_guild) + return tok, guild + +def normal(domain, admin_pw, streamers, editors): + missing = [x for x, v in (("domain", domain), ("admin password", admin_pw)) if not v] + if missing: + print(f"not set yet: {', '.join(missing)} — run ./configure.py --full"); sys.exit(1) + print(f"== MANAGE (domain {domain}; run --full to change hostnames/admin)") + discord = (None, None) + while True: + c = ask("\n[s]treamers [e]ditors [d]iscord bot token/guild [v]iew [a]pply & quit [q]uit without applying", "a").lower()[:1] + if c == "s": manage_users("streamer", streamers) + elif c == "e": manage_users("editor", editors) + elif c == "d": discord = ask_discord() + elif c == "v": show(domain, admin_pw, streamers, editors) + elif c == "a": return discord + elif c == "q": return False + +def main(): + mode = sys.argv[1] if len(sys.argv) > 1 else "" + hosts = read_hosts() + domain, auth_host, watch_host, git_host, bot_host = hosts.get("main", ""), hosts.get("auth", ""), hosts.get("watch", ""), hosts.get("git", ""), hosts.get("bot", "") + streamers, accounts = read_streamers(), read_accounts() + admin_pw = accounts.pop("admin", "") + editors = accounts + if mode == "--show": + show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host); return + if mode == "--hosts": + if len(sys.argv) not in (4, 5, 6): print("usage: configure.py --hosts AUTH_HOST WATCH_HOST [GIT_HOST [BOT_HOST]]"); sys.exit(2) + if not domain or not admin_pw: print("domain/admin not set yet — run ./configure.py --full first"); sys.exit(1) + auth_host, watch_host = sys.argv[2].strip(), sys.argv[3].strip() + if len(sys.argv) >= 5: git_host = sys.argv[4].strip() + if len(sys.argv) == 6: bot_host = sys.argv[5].strip() + apply(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host=git_host, bot_host=bot_host) + show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host); return + discord_token = discord_guild = None + if mode == "--full": + domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host = full(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host) + if not yes("\napply now (write configs, restart services)?"): print("nothing written"); return + elif mode == "": + r = normal(domain, admin_pw, streamers, editors) + if r is False: print("nothing written"); return + discord_token, discord_guild = r + else: + print(__doc__); sys.exit(2) + apply(domain, admin_pw, streamers, editors, auth_host, watch_host, discord_token, discord_guild, git_host, bot_host) + show(domain, admin_pw, streamers, editors, auth_host, watch_host, git_host, bot_host) + +if __name__ == "__main__": + try: main() + except (KeyboardInterrupt, EOFError): print("\naborted, nothing written") diff --git a/copyparty.conf b/copyparty.conf new file mode 100644 index 0000000..3225745 --- /dev/null +++ b/copyparty.conf @@ -0,0 +1,47 @@ +# copyparty — editors browse/preview/download; finals are writable. +# Docs: https://github.com/9001/copyparty#server-config + +[global] + e2dsa # index files for search + e2ts # read media tags + th-ff-jpg # video thumbnails via ffmpeg (cached under /cfg/hist) + hist: /cfg/hist + rproxy: 1 # trust first X-Forwarded-For hop (Caddy) + xff-src: 172.16.0.0/12, 10.0.0.0/8, 192.168.0.0/16 + no-robots + name: vanguard + usernames # login needs username + password (default is password-only) + # dark skin for the login page (served by Caddy from ./dock) + html-head: <link rel="stylesheet" href="/dock/vanguard.css"> + +[accounts] + admin: CHANGEME_ADMIN + editor1: CHANGEME_E1 + editor2: CHANGEME_E2 + +# streams: read for editors, full for admin +[/streams] + /w/streams + accs: + r: editor1, editor2 + rwmd: admin + +# gamer clips (Syncthing drops them here) +[/clips] + /w/clips + accs: + r: editor1, editor2 + rwmd: admin + +# editors upload finished work here +[/finals] + /w/finals + accs: + rw: editor1, editor2 + rwmd: admin + +# dumped nights, purged after 3 days — admin only +[/trash] + /w/trash + accs: + rwmd: admin diff --git a/createGiteaDb.sh b/createGiteaDb.sh new file mode 100755 index 0000000..019403d --- /dev/null +++ b/createGiteaDb.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Fallback for a MariaDB that was initialised before Gitea was added: create the gitea db/user now. +set -euo pipefail +cd "$(dirname "$0")" +ROOT=$(grep -E '^MARIADB_ROOT_PASSWORD=' .env | cut -d= -f2-) +PW=$(grep -E '^GITEA_DB_PASSWORD=' .env | cut -d= -f2-) +[[ -n $ROOT && -n $PW ]] || { echo "MARIADB_ROOT_PASSWORD / GITEA_DB_PASSWORD missing in .env — run ./configure.py and apply first"; exit 1; } +docker exec -i mariadb mariadb -uroot -p"$ROOT" <<SQL +CREATE DATABASE IF NOT EXISTS gitea CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +CREATE USER IF NOT EXISTS 'gitea'@'%' IDENTIFIED BY '${PW}'; +ALTER USER 'gitea'@'%' IDENTIFIED BY '${PW}'; +GRANT ALL PRIVILEGES ON gitea.* TO 'gitea'@'%'; +FLUSH PRIVILEGES; +SQL +echo "gitea db/user ready; now: docker compose up -d gitea" diff --git a/dock/index.html b/dock/index.html new file mode 100644 index 0000000..a4c89ce --- /dev/null +++ b/dock/index.html @@ -0,0 +1,151 @@ +<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<title>Vanguard Stream + + + + +
+

Vanguard Stream

+ + + + +
+ + +
+
idle
+ +
+ Settings (saved in this dock) + + + + + + +
+
+ + + + diff --git a/dock/vanguard.css b/dock/vanguard.css new file mode 100644 index 0000000..818c638 --- /dev/null +++ b/dock/vanguard.css @@ -0,0 +1,55 @@ +/* Vanguard skin for copyparty's control-panel / login page only. + Scoped to #wrap (splash) so the file browser keeps copyparty's own theme. */ + +html:has(#wrap > #lf), html:has(#wrap > div > #lf) { background:#1e1e1e; } +body:has(#wrap > div > #lf), body:has(#wrap > #b) { + margin:0; background:#1e1e1e; color:#e6e6e6; + font:15px/1.5 system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; + min-height:100vh; display:flex; align-items:center; justify-content:center; +} + +#wrap { + width:min(420px, calc(100vw - 32px)); margin:24px auto; padding:28px 28px 22px; + background:#2a2a2a; border:1px solid #3a3a3a; border-radius:12px; box-shadow:0 12px 40px rgba(0,0,0,.45); +} +#wrap::before { + content:"Vanguard Stream"; display:block; font-size:22px; font-weight:700; letter-spacing:.3px; + color:#fff; margin:0 0 4px; +} + +/* greeting */ +#wrap > #b { margin:0 0 20px; color:#9a9a9a; font-size:13px; } +#wrap > #b small { color:#7a7a7a; } + +/* clutter: refresh/connect links, "other stuff", recent uploads, reset, footer, repl */ +#wrap > #a, #wrap > #v, #wrap > #cc, #wrap .agr, #repl, #pb { display:none !important; } + +/* headings */ +#wrap h1 { font-size:12px; text-transform:uppercase; letter-spacing:.08em; color:#9a9a9a; margin:18px 0 8px; } +#wrap > #l { margin-top:0; } + +/* login form */ +#lf { display:grid; gap:10px; } +#lf input[type=text], #lf input[type=password] { + width:100%; box-sizing:border-box; padding:11px 12px; font-size:15px; + background:#1e1e1e; color:#e6e6e6; border:1px solid #444; border-radius:8px; outline:none; +} +#lf input:focus { border-color:#3d8bfd; box-shadow:0 0 0 3px rgba(61,139,253,.25); } +#lf input[type=submit] { + width:100%; padding:12px; margin-top:4px; font-size:15px; font-weight:600; cursor:pointer; + background:#3d8bfd; color:#fff; border:0; border-radius:8px; +} +#lf input[type=submit]:hover { background:#2f7af0; } +#lm { color:#e74c3c; font-size:13px; min-height:1em; } + +/* logged in: volume list becomes tiles */ +#wrap ul { list-style:none; padding:0; margin:6px 0 0; display:grid; gap:8px; } +#wrap ul li a { + display:block; padding:12px 14px; border-radius:8px; background:#1e1e1e; border:1px solid #3a3a3a; + color:#e6e6e6; text-decoration:none; font-weight:600; +} +#wrap ul li a:hover { border-color:#3d8bfd; background:#232833; } +#wrap ul li a::before { content:"▸ "; color:#3d8bfd; } + +/* logout / misc links that remain */ +#wrap a { color:#3d8bfd; } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ab7ddf7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,248 @@ +services: + mediamtx: + image: bluenviron/mediamtx:latest + container_name: mediamtx + user: "1000:1000" # recordings owned by the debian user, so the archive pull can delete them + ports: + - "1935:1935" # RTMP ingest + - "8890:8890/udp" # SRT ingest + volumes: + - ./mediamtx.yml:/mediamtx.yml:ro + - /srv/media/streams:/recordings + restart: unless-stopped + + copyparty: + image: copyparty/ac:latest + container_name: copyparty + ports: + - "127.0.0.1:3923:3923" + volumes: + - /srv/media:/w + - ./copyparty.conf:/cfg/copyparty.conf:ro + - copyparty_hist:/cfg/hist + restart: unless-stopped + + syncthing: + image: syncthing/syncthing:latest + container_name: syncthing + environment: + - PUID=1000 + - PGID=1000 + ports: + - "22000:22000/tcp" + - "22000:22000/udp" + - "127.0.0.1:8384:8384" # admin UI, reach via ssh tunnel + volumes: + - /srv/media/.syncthing:/var/syncthing/config # created by install.sh, owned by uid 1000 + - /srv/media/clips:/var/syncthing/clips + restart: unless-stopped + + caddy: + image: caddy:latest + container_name: caddy + depends_on: [copyparty] + network_mode: host # talks to copyparty (127.0.0.1:3923) and dumpd (127.0.0.1:8080) on loopback + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ./dock:/srv/dock:ro + - caddy_data:/data + - caddy_config:/config + restart: unless-stopped + + # ---------------------------------------------------------------- identity (Authentik) + authentik-db: + image: docker.io/library/postgres:16-alpine + container_name: authentik-db + environment: + POSTGRES_DB: authentik + POSTGRES_USER: authentik + POSTGRES_PASSWORD: ${AUTHENTIK_PG_PASS} + volumes: + - /srv/appdata/authentik/db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -d authentik -U authentik"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 20s + restart: unless-stopped + + authentik-redis: + image: docker.io/library/redis:alpine + container_name: authentik-redis + command: --save 60 1 --loglevel warning + volumes: + - /srv/appdata/authentik/redis:/data + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + interval: 30s + timeout: 3s + retries: 5 + restart: unless-stopped + + authentik-server: + image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-2025.8} + container_name: authentik-server + command: server + environment: &authentik_env + AUTHENTIK_SECRET_KEY: ${AUTHENTIK_SECRET_KEY} + AUTHENTIK_REDIS__HOST: authentik-redis + AUTHENTIK_POSTGRESQL__HOST: authentik-db + AUTHENTIK_POSTGRESQL__USER: authentik + AUTHENTIK_POSTGRESQL__NAME: authentik + AUTHENTIK_POSTGRESQL__PASSWORD: ${AUTHENTIK_PG_PASS} + AUTHENTIK_ERROR_REPORTING__ENABLED: "false" + ports: + - "127.0.0.1:9000:9000" + volumes: + - /srv/appdata/authentik/media:/media + - /srv/appdata/authentik/templates:/templates + depends_on: + authentik-db: { condition: service_healthy } + authentik-redis: { condition: service_healthy } + restart: unless-stopped + + authentik-worker: + image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-2025.8} + container_name: authentik-worker + command: worker + environment: *authentik_env + volumes: + - /srv/appdata/authentik/media:/media + - /srv/appdata/authentik/templates:/templates + - /srv/appdata/authentik/certs:/certs + depends_on: + authentik-db: { condition: service_healthy } + authentik-redis: { condition: service_healthy } + restart: unless-stopped + + # ---------------------------------------------------------------- watching (Jellyfin, direct play only) + jellyfin: + image: jellyfin/jellyfin:latest + container_name: jellyfin + user: "1000:1000" + environment: + JELLYFIN_PublishedServerUrl: https://${WATCH_HOST:-watch.example.com} + ports: + - "127.0.0.1:8096:8096" + volumes: + - /srv/appdata/jellyfin/config:/config + - /srv/appdata/jellyfin/cache:/cache + - /srv/media/streams:/media/streams:ro + - /srv/media/clips:/media/clips:ro + - /srv/media/finals:/media/finals:ro + restart: unless-stopped + + # ---------------------------------------------------------------- database (for the Discord bot and friends) + mariadb: + image: mariadb:11 + container_name: mariadb + network_mode: host # PUBLIC on 3306: host networking so the nftables input rules (db_allow / rate + # limit) actually apply — Docker-published ports bypass the host firewall. + environment: + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD} + MARIADB_ROOT_HOST: localhost # root only from the box itself; the bot user may connect from anywhere + MARIADB_DATABASE: ${BOT_DB_NAME:-vanguardbot} + MARIADB_USER: ${BOT_DB_USER:-bot} + MARIADB_PASSWORD: ${BOT_DB_PASSWORD} + GITEA_DB_PASSWORD: ${GITEA_DB_PASSWORD} # used by mariadb/init/10-gitea.sh on first init + volumes: + - /srv/appdata/mariadb:/var/lib/mysql + - ./mariadb/init:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 20s + timeout: 5s + retries: 6 + start_period: 30s + restart: unless-stopped + + # ---------------------------------------------------------------- Discord bot (python; code lives in ./bot/app) + discord-bot: + build: ./bot + container_name: discord-bot + user: "1000:1000" + environment: + DISCORD_BOT_TOKEN: ${DISCORD_BOT_TOKEN:-} + DISCORD_GUILD_ID: ${DISCORD_GUILD_ID:-} + DB_HOST: host.docker.internal # mariadb runs on the host network + DB_PORT: "3306" + DB_NAME: ${BOT_DB_NAME:-vanguardbot} + DB_USER: ${BOT_DB_USER:-bot} + DB_PASSWORD: ${BOT_DB_PASSWORD} + MEDIA_CLIPS: /media/clips + WEB_HOST: 0.0.0.0 + WEB_PORT: "8000" + WEB_PUBLIC_URL: https://${BOT_HOST:-bot.example.com} + AUTH_HEADER_USER: X-Authentik-Username + AUTH_HEADER_GROUPS: X-Authentik-Groups + AUTH_HEADER_EMAIL: X-Authentik-Email + TZ: America/New_York + PYTHONUNBUFFERED: "1" + ports: + - "127.0.0.1:8000:8000" # web UI; public only via Caddy + Authentik forward-auth at https://BOT_HOST/ + volumes: + - ./bot/app:/app # the bot's source; edit and `docker compose restart discord-bot` + - /srv/media/clips:/media/clips # rw: for a clip-submissions feature (writes into clips//) + - /srv/appdata/discord-bot:/data # persistent scratch (sqlite, caches) if the bot wants files + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + mariadb: { condition: service_healthy } + restart: unless-stopped + + # ---------------------------------------------------------------- Gitea + Actions runner + gitea: + image: docker.gitea.com/gitea:1.24 + container_name: gitea + environment: + USER_UID: "1000" + USER_GID: "1000" + GITEA__database__DB_TYPE: mysql + GITEA__database__HOST: host.docker.internal:3306 # mariadb runs on the host network + GITEA__database__NAME: gitea + GITEA__database__USER: gitea + GITEA__database__PASSWD: ${GITEA_DB_PASSWORD} + GITEA__server__DOMAIN: ${GIT_HOST:-git.example.com} + GITEA__server__ROOT_URL: https://${GIT_HOST:-git.example.com}/ + GITEA__server__HTTP_PORT: "3000" + GITEA__server__SSH_DOMAIN: ${GIT_HOST:-git.example.com} + GITEA__server__SSH_PORT: "2222" # what users put in their remotes + GITEA__server__SSH_LISTEN_PORT: "22" # inside the container + GITEA__service__DISABLE_REGISTRATION: "true" # accounts are created by an admin (or via Authentik OIDC later) + GITEA__actions__ENABLED: "true" + GITEA__actions__DEFAULT_ACTIONS_URL: github + GITEA__webhook__ALLOWED_HOST_LIST: private,loopback,external + GITEA__time__DEFAULT_UI_LOCATION: America/New_York + ports: + - "127.0.0.1:3000:3000" + - "2222:22" + volumes: + - /srv/appdata/gitea:/data + - /etc/timezone:/etc/timezone:ro + - /etc/localtime:/etc/localtime:ro + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + mariadb: { condition: service_healthy } + restart: unless-stopped + + act_runner: + image: docker.gitea.com/act_runner:latest + container_name: act_runner + environment: + GITEA_INSTANCE_URL: https://${GIT_HOST:-git.example.com}/ + GITEA_RUNNER_REGISTRATION_TOKEN: ${GITEA_RUNNER_TOKEN:-} # set by ./registerGiteaRunner.sh + GITEA_RUNNER_NAME: vanguard-runner + GITEA_RUNNER_LABELS: "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest,ubuntu-22.04:docker://docker.gitea.com/runner-images:ubuntu-22.04,dotnet:docker://mcr.microsoft.com/dotnet/sdk:10.0,python:docker://python:3.12" + volumes: + - /srv/appdata/act_runner:/data + - /var/run/docker.sock:/var/run/docker.sock # jobs run as sibling containers; root-equivalent on the host + depends_on: + - gitea + restart: unless-stopped + +volumes: + copyparty_hist: + caddy_data: + caddy_config: diff --git a/dump-users.txt b/dump-users.txt new file mode 100644 index 0000000..8627a29 --- /dev/null +++ b/dump-users.txt @@ -0,0 +1,6 @@ +# user:sha256hex(password) — same user names as mediamtx.yml +# generate: printf '%s' 'password' | sha256sum +techy:REPLACE_WITH_SHA256 +diceofire:REPLACE_WITH_SHA256 +# admin may dump everyone's night +admin:REPLACE_WITH_SHA256 diff --git a/firewall/firewall.sh b/firewall/firewall.sh new file mode 100755 index 0000000..81e7ef1 --- /dev/null +++ b/firewall/firewall.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Install + enable the host firewall. Run with sudo from the bundle dir. +# Keep your current SSH session open and test a NEW connection before closing it. +set -euo pipefail +cd "$(dirname "$0")" +[[ $EUID -eq 0 ]] || { echo "run with sudo"; exit 1; } + +apt-get install -y -q nftables >/dev/null +install -m 0644 nftables.conf /etc/nftables.conf + +echo "== syntax check"; nft -c -f /etc/nftables.conf +echo "== apply"; nft -f /etc/nftables.conf +systemctl enable nftables >/dev/null 2>&1 || true # loads /etc/nftables.conf at boot, before docker + +echo "== active ruleset (vanguard table)" +nft list table inet vanguard | sed 's/^/ /' +echo +echo "== listening on public interfaces (should be 22 80 443 1935 2222 3306 8890 22000 only)" +ss -tulpnH | awk '$5 !~ /^(127\.|\[::1\])/ {print " "$1, $5}' | sort -u +echo +echo "Now open a SECOND terminal and confirm: ssh debian@$(hostname -f 2>/dev/null || hostname)" +echo "To restrict SSH to your IPs, edit the ssh_allow set in /etc/nftables.conf and re-run: sudo nft -f /etc/nftables.conf" diff --git a/firewall/nftables.conf b/firewall/nftables.conf new file mode 100644 index 0000000..85b47fd --- /dev/null +++ b/firewall/nftables.conf @@ -0,0 +1,63 @@ +#!/usr/sbin/nft -f +# vanguard-stream host firewall. Own table so (re)applying never disturbs Docker's rules. +# Input only: Docker-published ports (1935, 8890, 22000) arrive via Docker's forward chain and are +# listed here for the host-network case (Caddy) and clarity. Nothing else should listen publicly. + +destroy table inet vanguard +table inet vanguard { + set ssh_allow { + type ipv4_addr + flags interval + # leave empty to allow SSH from anywhere (key auth, rate limited); or list CIDRs: + # elements = { 203.0.113.0/24, 198.51.100.7 } + } + + set db_allow { + type ipv4_addr + flags interval + # MariaDB clients. Leave empty = anywhere (rate limited); or list the bot dev's IPs and then + # delete the "tcp dport 3306 ct state new accept" line below to restrict to this set. + # elements = { 203.0.113.7 } + } + + chain input { + type filter hook input priority 0; policy drop; + + iif lo accept + ct state established,related accept + ct state invalid drop + + # icmp (ping, pmtu) and all icmpv6 (neighbor discovery is mandatory) + ip protocol icmp icmp type { echo-request, destination-unreachable, time-exceeded, parameter-problem } accept + ip6 nexthdr icmpv6 accept + + # dhcp client replies (OVH may hand out the address by DHCP) + udp sport 67 udp dport 68 accept + + # ssh: restricted to ssh_allow if it has entries, otherwise anywhere, rate limited + tcp dport 22 ct state new ip saddr @ssh_allow accept + tcp dport 22 ct state new limit rate over 15/minute burst 10 packets drop + tcp dport 22 ct state new accept + + # public services + tcp dport { 80, 443 } accept # Caddy: copyparty, dock, dump + udp dport 443 accept # Caddy HTTP/3 (QUIC) + tcp dport 1935 accept # RTMP ingest + udp dport 8890 accept # SRT ingest + tcp dport 22000 accept # Syncthing + tcp dport 2222 accept # Gitea ssh (git clone/push) + udp dport 22000 accept # Syncthing QUIC + + # mariadb (host network): allowlist first, then anywhere with a brute-force limit + tcp dport 3306 ct state new ip saddr @db_allow accept + tcp dport 3306 ct state new limit rate over 30/minute burst 20 packets drop + tcp dport 3306 ct state new accept + + # NOT opened on purpose — these bind to 127.0.0.1 only and are reached via Caddy or an ssh tunnel: + # 9000 authentik · 8096 jellyfin · 3923 copyparty · 8384 syncthing UI · 8080 dumpd + # Docker publishes them on loopback, so its NAT never forwards outside traffic to them either. + + # everything else: drop (policy), but log a trickle for diagnosis + limit rate 5/minute log prefix "vg-drop " level info + } +} diff --git a/home/rclone-archive.sh b/home/rclone-archive.sh new file mode 100755 index 0000000..7d35132 --- /dev/null +++ b/home/rclone-archive.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Run nightly on the HOME box. Pulls originals older than 15 days off the server, +# moves originals AND their .src raw files; nothing is left behind. +# Setup once: rclone config -> new remote "vanguard", type sftp, host media.example.com, user debian, key file. +set -euo pipefail +REMOTE=vanguard +ARCHIVE=/mnt/archive # local HDD +MIN_AGE=15d +LOG=/var/log/vanguard-archive.log + +for sub in streams clips; do + rclone move "$REMOTE:/srv/media/$sub" "$ARCHIVE/$sub" \ + --min-age "$MIN_AGE" \ + --exclude '*.part' --exclude '*.normalize-failed.log' \ + --transfers 4 --checkers 8 \ + --delete-empty-src-dirs=false \ + --log-file "$LOG" --log-level INFO +done +echo "$(date '+%F %T') archive pull done" >> "$LOG" diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..87463db --- /dev/null +++ b/install.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Debian 13 host installer for vanguard-stream. Run with sudo from this directory. +set -euo pipefail +cd "$(dirname "$0")" +[[ $EUID -eq 0 ]] || { echo "run with sudo"; exit 1; } +OWNER="${SUDO_USER:-debian}" # the login user; owns /srv/media and may run docker + +echo "== packages" +apt-get update -q +apt-get install -y -q ca-certificates curl gnupg ffmpeg jq psmisc rclone python3 xfsprogs + +if ! command -v docker >/dev/null; then + echo "== docker" + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -q + apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-compose-plugin +fi + +echo "== timezone: server clock, logs, timers and the dump window all in Eastern" +timedatectl set-timezone "${VG_TZ:-America/New_York}" +timedatectl set-ntp true +echo " timezone is: $(timedatectl show -p Timezone --value) (override: VG_TZ=... bash install.sh)" + +echo "== docker waits for /srv/media (never record onto the OS disk)" +install -d /etc/systemd/system/docker.service.d +cat > /etc/systemd/system/docker.service.d/10-srv-media.conf <<'DROP' +[Unit] +RequiresMountsFor=/srv/media +DROP +systemctl daemon-reload +systemctl restart docker 2>/dev/null || true + +echo "== media user + folders" +# media files are owned by uid 1000 — on OVH's template that is the 'debian' user +getent passwd 1000 >/dev/null || useradd -r -u 1000 -d /srv/media -s /usr/sbin/nologin media +usermod -aG docker "$OWNER" 2>/dev/null || true +install -d -o 1000 -g 1000 /srv/media/{streams,clips,finals,trash,.markers,.syncthing} +install -d -o 1000 -g 1000 /srv/appdata/jellyfin/{config,cache} /srv/appdata/authentik/{media,templates,certs} +install -d /srv/appdata/authentik/{db,redis} /srv/appdata/mariadb +install -d -o 1000 -g 1000 /srv/appdata/discord-bot /srv/appdata/gitea /srv/appdata/act_runner +install -d -o 1000 -g 1000 /srv/media/streams/{techy,diceofire} /srv/media/clips/{techy,diceofire} +for d in /srv/media/clips/*/; do + [[ -e "$d/.stignore" ]] || install -m 0644 -o 1000 -g 1000 syncthing/stignore "$d/.stignore" +done + +echo "== host scripts + units" +install -m 0755 bin/normalize-sweep.sh bin/trash-purge.sh bin/dumpd.py /usr/local/bin/ +install -d -m 0750 /etc/vanguard +install -m 0640 dump-users.txt /etc/vanguard/dump-users.txt +install -m 0644 systemd/*.service systemd/*.timer /etc/systemd/system/ +systemctl daemon-reload +systemctl enable --now normalize-sweep.timer trash-purge.timer dumpd.service + +echo "== sysctl" +cat > /etc/sysctl.d/90-vanguard.conf <<'SYS' +vm.swappiness = 10 +net.core.rmem_max = 16777216 +net.core.wmem_max = 16777216 +SYS +sysctl -q --system + +echo "== compose images (the stack is started by ./configure.py --full, which generates .env first)" +chown -R "$OWNER:" . +docker compose pull -q --ignore-buildable 2>/dev/null || docker compose pull -q || true + +echo "== host firewall (nftables; own table, input only; keeps this SSH session alive)" +if [[ -f firewall/firewall.sh ]]; then + bash firewall/firewall.sh +else + echo " firewall/firewall.sh missing — skipped" +fi + +cat <<'MSG' + +== done. Next: ./configure.py --full (hostnames, passwords, .env secrets; starts the containers) +== firewall is active: open a SECOND terminal and confirm ssh still works before closing this one. + +== log out and back in once so the docker group applies; then 'docker compose' works without sudo. +== syncthing UI: ssh -L 8384:127.0.0.1:8384 debian@SERVER -> http://localhost:8384 +MSG diff --git a/installJellyfinPlugin.sh b/installJellyfinPlugin.sh new file mode 100755 index 0000000..e8693db --- /dev/null +++ b/installJellyfinPlugin.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Install/upgrade the Vanguard Metadata Jellyfin plugin from jellyfin-plugin-vanguard/dist/*.zip +# (built on a dev box with ./package.sh and scp'd up with the bundle). Re-runnable. +# Removes older versions of the same plugin so Jellyfin loads exactly one. +set -euo pipefail +cd "$(dirname "$0")" +PLUGDIR=/srv/appdata/jellyfin/config/plugins +NAME="Vanguard Metadata" + +mkdir -p jellyfin-plugin-vanguard/dist +ZIP=$(ls -1 jellyfin-plugin-vanguard/dist/VanguardMetadata_*.zip 2>/dev/null | sort -V | tail -1 || true) +[[ -n $ZIP ]] || { echo "no plugin zip in jellyfin-plugin-vanguard/dist/ — build with ./package.sh on the dev box, then:"; echo " scp jellyfin-plugin-vanguard/dist/VanguardMetadata_*.zip debian@HOST:~/vanguard-stream/jellyfin-plugin-vanguard/dist/"; exit 1; } +VER=$(basename "$ZIP" .zip | sed 's/^VanguardMetadata_//') +DEST="$PLUGDIR/${NAME}_${VER}" + +if [[ -f "$DEST/meta.json" ]] && grep -q "\"version\": \"$VER\"" "$DEST/meta.json"; then + echo "plugin $VER already installed at $DEST"; exit 0 +fi +sudo install -d -o 1000 -g 1000 "$PLUGDIR" +for old in "$PLUGDIR/${NAME}_"*/; do [[ -d $old && $old != "$DEST/" ]] && { echo "removing old $old"; sudo rm -rf "$old"; }; done +sudo -u "#1000" mkdir -p "$DEST" +sudo -u "#1000" python3 -m zipfile -e "$ZIP" "$DEST/" +echo "installed $NAME $VER -> $DEST" +ls -la "$DEST" +if docker ps --format '{{.Names}}' | grep -qx jellyfin; then + docker compose restart jellyfin + sleep 8 + docker logs jellyfin --tail 60 2>&1 | grep -iE "vanguard|Loaded plugin" | tail -3 || true +fi diff --git a/jellyfin-plugin-vanguard/.gitignore b/jellyfin-plugin-vanguard/.gitignore new file mode 100644 index 0000000..bafb0f8 --- /dev/null +++ b/jellyfin-plugin-vanguard/.gitignore @@ -0,0 +1,4 @@ +bin/ +obj/ +*.user +dist/ diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Jellyfin.Plugin.Vanguard.Core.csproj b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Jellyfin.Plugin.Vanguard.Core.csproj new file mode 100644 index 0000000..c6660de --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Jellyfin.Plugin.Vanguard.Core.csproj @@ -0,0 +1,11 @@ + + + + net10.0 + Jellyfin.Plugin.Vanguard + enable + enable + 0.1.0.0 + + diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Seasons/ItemTimestamp.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Seasons/ItemTimestamp.cs new file mode 100644 index 0000000..a0c4065 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Seasons/ItemTimestamp.cs @@ -0,0 +1,79 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace Jellyfin.Plugin.Vanguard.Seasons; + +/// Finds the best "when was this recorded" for a media file. +public static partial class ItemTimestamp +{ + public enum Source { Sidecar, Filename, FileTime } + + public sealed record Result(DateTimeOffset Utc, Source Source); + + /// + /// Precedence: clip.json / <name>.json sidecar "timestamp" (UTC ISO-8601, written by the clip bot) + /// -> MediaMTX filename yyyy-MM-dd_HH-mm-ss (local time of the server, i.e. ) + /// -> file last-write time. + /// + public static Result Resolve(string path, TimeZoneInfo tz) + { + var sidecar = TryReadSidecar(path); + if (sidecar is not null) return new Result(sidecar.Value, Source.Sidecar); + + var fromName = TryParseFilename(Path.GetFileNameWithoutExtension(path), tz); + if (fromName is not null) return new Result(fromName.Value, Source.Filename); + + var mtime = File.Exists(path) ? File.GetLastWriteTimeUtc(path) : DateTime.UtcNow; + return new Result(new DateTimeOffset(mtime, TimeSpan.Zero), Source.FileTime); + } + + // MediaMTX 2026-10-29_21-03-11 Xbox Game Bar Game 2026-10-29 21-03-11 + [GeneratedRegex(@"(?\d{4})-(?\d{2})-(?\d{2})[_ T](?\d{2})-(?\d{2})-(?\d{2})")] + private static partial Regex DashedName(); + // ShadowPlay / NVIDIA App Game 2026.10.29 - 21.03.11.05.DVR + [GeneratedRegex(@"(?\d{4})\.(?\d{2})\.(?\d{2}) - (?\d{2})\.(?\d{2})\.(?\d{2})")] + private static partial Regex ShadowPlayName(); + // Steam Game Recording clip_730_20261029_210311 (also plain 20261029_210311 / 20261029-210311) + [GeneratedRegex(@"(?20\d{2})(?\d{2})(?\d{2})[_-](?\d{2})(?\d{2})(?\d{2})(?![\d])")] + private static partial Regex CompactName(); + + public static DateTimeOffset? TryParseFilename(string stem, TimeZoneInfo tz) + { + var m = DashedName().Match(stem); + if (!m.Success) m = ShadowPlayName().Match(stem); + if (!m.Success) m = CompactName().Match(stem); + if (!m.Success) return null; + try + { + var local = new DateTime( + int.Parse(m.Groups["y"].Value, CultureInfo.InvariantCulture), int.Parse(m.Groups["mo"].Value, CultureInfo.InvariantCulture), int.Parse(m.Groups["d"].Value, CultureInfo.InvariantCulture), + int.Parse(m.Groups["h"].Value, CultureInfo.InvariantCulture), int.Parse(m.Groups["mi"].Value, CultureInfo.InvariantCulture), int.Parse(m.Groups["s"].Value, CultureInfo.InvariantCulture), + DateTimeKind.Unspecified); + var offset = tz.GetUtcOffset(local); + return new DateTimeOffset(local, offset); + } + catch (ArgumentOutOfRangeException) { return null; } + } + + private static DateTimeOffset? TryReadSidecar(string path) + { + var dir = Path.GetDirectoryName(path) ?? string.Empty; + var stem = Path.GetFileNameWithoutExtension(path); + foreach (var candidate in new[] { Path.Combine(dir, stem + ".json"), Path.Combine(dir, stem + ".clip.json") }) + { + if (!File.Exists(candidate)) continue; + try + { + using var doc = JsonDocument.Parse(File.ReadAllText(candidate)); + if (doc.RootElement.TryGetProperty("timestamp", out var ts) && ts.ValueKind == JsonValueKind.String && + DateTimeOffset.TryParse(ts.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed; + } + } + catch (JsonException) { /* ignore broken sidecar */ } + } + return null; + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Seasons/SeasonPolicy.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Seasons/SeasonPolicy.cs new file mode 100644 index 0000000..20b7467 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Core/Seasons/SeasonPolicy.cs @@ -0,0 +1,119 @@ +using System.Globalization; + +namespace Jellyfin.Plugin.Vanguard.Seasons; + +/// Where an item sits in the rolling season scheme. +public sealed record SeasonSlot(int SeasonNumber, string SeasonName, int EpisodeNumber, DateOnly BroadcastDay, SeasonKind Kind); + +public enum SeasonKind { Day, Week, Month, Year } + +/// +/// Pure function: (item timestamp, now, settings) -> season/episode numbers and season name. +/// Rolling consolidation: items in the current ISO week are bucketed by broadcast day; in the +/// current month by ISO week; in the current year by month; older by year. Episode numbers are +/// MMDDHHMMSS of the local timestamp, so they never change when the season rolls. +/// +public static class SeasonPolicy +{ + /// Week seasons are 1YYYYWW so they never collide with YYYYMM month seasons. Lengths: year 4, month 6, week 7, day 8. + public const int WeekBase = 1_000_000; + + /// Recover kind and display name from a season number alone (used by the Season provider). + public static (SeasonKind Kind, string Name)? Describe(int seasonNumber, IReadOnlyDictionary? nameOverrides = null) + { + (SeasonKind, string)? r = seasonNumber switch + { + >= 10_000_000 and <= 99_999_999 when TryDay(seasonNumber, out var d) => (SeasonKind.Day, d.ToString("dddd, MMM d", CultureInfo.InvariantCulture)), + >= WeekBase and < 10_000_000 => (SeasonKind.Week, $"Week {seasonNumber % 100}"), + >= 100_000 and <= 999_999 when seasonNumber % 100 is >= 1 and <= 12 => (SeasonKind.Month, new DateOnly(seasonNumber / 100, seasonNumber % 100, 1).ToString("MMMM yyyy", CultureInfo.InvariantCulture)), + >= 1000 and <= 9999 => (SeasonKind.Year, seasonNumber.ToString(CultureInfo.InvariantCulture)), + _ => null, + }; + if (r is null) return null; + if (nameOverrides is not null && nameOverrides.TryGetValue(seasonNumber, out var over) && !string.IsNullOrWhiteSpace(over)) + { + return (r.Value.Item1, over.Trim()); + } + return r; + } + + private static bool TryDay(int n, out DateOnly d) + { + try { d = new DateOnly(n / 10000, n / 100 % 100, n % 100); return true; } + catch (ArgumentOutOfRangeException) { d = default; return false; } + } + + public static SeasonSlot Resolve(DateTimeOffset itemUtc, DateTimeOffset nowUtc, TimeZoneInfo tz, int rolloverHour, IReadOnlyDictionary? nameOverrides = null) + { + var item = TimeZoneInfo.ConvertTime(itemUtc, tz); + var now = TimeZoneInfo.ConvertTime(nowUtc, tz); + var itemDay = BroadcastDay(item, rolloverHour); + var today = BroadcastDay(now, rolloverHour); + + var (itemIsoYear, itemIsoWeek) = IsoWeek(itemDay); + var (todayIsoYear, todayIsoWeek) = IsoWeek(today); + + int season; string name; SeasonKind kind; + if (itemIsoYear == todayIsoYear && itemIsoWeek == todayIsoWeek) + { + kind = SeasonKind.Day; + season = itemDay.Year * 10000 + itemDay.Month * 100 + itemDay.Day; // 20261030 + name = itemDay.ToString("dddd, MMM d", CultureInfo.InvariantCulture); // Thursday, Oct 30 + } + else if (itemDay.Year == today.Year && itemDay.Month == today.Month) + { + kind = SeasonKind.Week; + season = WeekBase + itemIsoYear * 100 + itemIsoWeek; // 1202644 (leading 1 keeps weeks distinct from months) + name = $"Week {itemIsoWeek}"; + } + else if (itemDay.Year == today.Year) + { + kind = SeasonKind.Month; + season = itemDay.Year * 100 + itemDay.Month; // 202610 + name = itemDay.ToString("MMMM yyyy", CultureInfo.InvariantCulture); // October 2026 + } + else + { + kind = SeasonKind.Year; + season = itemDay.Year; // 2026 + name = itemDay.Year.ToString(CultureInfo.InvariantCulture); + } + + if (nameOverrides is not null && nameOverrides.TryGetValue(season, out var over) && !string.IsNullOrWhiteSpace(over)) + { + name = over.Trim(); + } + + // MMDDHHMMSS of the local timestamp: 1030210311 -> fits int32 (max 1231235959 < 2147483647) + var episode = item.Month * 100000000 + item.Day * 1000000 + item.Hour * 10000 + item.Minute * 100 + item.Second; + return new SeasonSlot(season, name, episode, itemDay, kind); + } + + /// Calendar day the item belongs to after shifting the day boundary to . + public static DateOnly BroadcastDay(DateTimeOffset local, int rolloverHour) => + DateOnly.FromDateTime(local.DateTime.AddHours(-rolloverHour)); + + public static (int IsoYear, int IsoWeek) IsoWeek(DateOnly d) + { + var dt = d.ToDateTime(TimeOnly.MinValue); + return (ISOWeek.GetYear(dt), ISOWeek.GetWeekOfYear(dt)); + } + + /// Parse "202611=Winter Scrims" lines into a map. + public static IReadOnlyDictionary ParseOverrides(string? text) + { + var map = new Dictionary(); + foreach (var raw in (text ?? string.Empty).Split('\n')) + { + var line = raw.Trim(); + if (line.Length == 0 || line.StartsWith('#')) continue; + var eq = line.IndexOf('='); + if (eq <= 0) continue; + if (int.TryParse(line[..eq].Trim(), NumberStyles.None, CultureInfo.InvariantCulture, out var key)) + { + map[key] = line[(eq + 1)..].Trim(); + } + } + return map; + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Tests/Jellyfin.Plugin.Vanguard.Tests.csproj b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Tests/Jellyfin.Plugin.Vanguard.Tests.csproj new file mode 100644 index 0000000..4b46a4f --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Tests/Jellyfin.Plugin.Vanguard.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Tests/SeasonPolicyTests.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Tests/SeasonPolicyTests.cs new file mode 100644 index 0000000..3e9feed --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard.Tests/SeasonPolicyTests.cs @@ -0,0 +1,115 @@ +using Jellyfin.Plugin.Vanguard.Seasons; +using Xunit; + +namespace Jellyfin.Plugin.Vanguard.Tests; + +public class SeasonPolicyTests +{ + private static readonly TimeZoneInfo NY = TimeZoneInfo.FindSystemTimeZoneById("America/New_York"); + private static DateTimeOffset Ny(int y, int mo, int d, int h, int mi, int s = 0) + { + var local = new DateTime(y, mo, d, h, mi, s, DateTimeKind.Unspecified); + return new DateTimeOffset(local, NY.GetUtcOffset(local)); + } + + // "now" = Thursday 2026-10-30 12:00 Eastern (ISO week 44) + private static readonly DateTimeOffset Now = Ny(2026, 10, 30, 12, 0); + + [Fact] + public void SameWeek_IsDaySeason() + { + var slot = SeasonPolicy.Resolve(Ny(2026, 10, 29, 21, 3, 11), Now, NY, 4); + Assert.Equal(SeasonKind.Day, slot.Kind); + Assert.Equal(20261029, slot.SeasonNumber); + Assert.Equal("Thursday, Oct 29", slot.SeasonName); + Assert.Equal(1029210311, slot.EpisodeNumber); + } + + [Fact] + public void AfterMidnight_BelongsToPreviousBroadcastDay() + { + var slot = SeasonPolicy.Resolve(Ny(2026, 10, 30, 1, 15), Now, NY, 4); + Assert.Equal(new DateOnly(2026, 10, 29), slot.BroadcastDay); + Assert.Equal(20261029, slot.SeasonNumber); + Assert.Equal(1030011500, slot.EpisodeNumber); // episode number keeps the real clock time + } + + [Fact] + public void EarlierInMonth_IsWeekSeason() + { + var slot = SeasonPolicy.Resolve(Ny(2026, 10, 14, 20, 0), Now, NY, 4); // ISO week 42 + Assert.Equal(SeasonKind.Week, slot.Kind); + Assert.Equal(1202642, slot.SeasonNumber); + Assert.Equal("Week 42", slot.SeasonName); + } + + [Fact] + public void EarlierInYear_IsMonthSeason() + { + var slot = SeasonPolicy.Resolve(Ny(2026, 7, 4, 20, 0), Now, NY, 4); + Assert.Equal(SeasonKind.Month, slot.Kind); + Assert.Equal(202607, slot.SeasonNumber); + Assert.Equal("July 2026", slot.SeasonName); + } + + [Fact] + public void PreviousYear_IsYearSeason() + { + var slot = SeasonPolicy.Resolve(Ny(2025, 12, 31, 23, 0), Now, NY, 4); + Assert.Equal(SeasonKind.Year, slot.Kind); + Assert.Equal(2025, slot.SeasonNumber); + Assert.Equal("2025", slot.SeasonName); + } + + [Fact] + public void EpisodeNumber_IsStableAcrossRolls() + { + var ts = Ny(2026, 10, 29, 21, 3, 11); + var e1 = SeasonPolicy.Resolve(ts, Now, NY, 4).EpisodeNumber; + var e2 = SeasonPolicy.Resolve(ts, Ny(2026, 11, 20, 12, 0), NY, 4).EpisodeNumber; // weeks later: week bucket? no -> month bucket + var e3 = SeasonPolicy.Resolve(ts, Ny(2027, 3, 1, 12, 0), NY, 4).EpisodeNumber; // next year -> year bucket + Assert.Equal(e1, e2); + Assert.Equal(e1, e3); + Assert.Equal(202610, SeasonPolicy.Resolve(ts, Ny(2026, 11, 20, 12, 0), NY, 4).SeasonNumber); + Assert.Equal(2026, SeasonPolicy.Resolve(ts, Ny(2027, 3, 1, 12, 0), NY, 4).SeasonNumber); + } + + [Fact] + public void Overrides_ReplaceGeneratedName() + { + var over = SeasonPolicy.ParseOverrides("# comment\n202610 = Fall Training Block\nbad line\n"); + var slot = SeasonPolicy.Resolve(Ny(2026, 10, 14, 20, 0), Ny(2026, 12, 1, 12, 0), NY, 4, over); + Assert.Equal("Fall Training Block", slot.SeasonName); + } + + [Fact] + public void DstTransition_UsesCorrectOffset() + { + // 2026-11-01 02:30 local does not exist twice in a way that breaks us; 01:30 is ambiguous: must not throw + var slot = SeasonPolicy.Resolve(Ny(2026, 11, 1, 1, 30), Ny(2026, 11, 2, 12, 0), NY, 4); + Assert.Equal(new DateOnly(2026, 10, 31), slot.BroadcastDay); + } + + [Fact] + public void Describe_RecoversKindAndName() + { + Assert.Equal((SeasonKind.Day, "Thursday, Oct 29"), SeasonPolicy.Describe(20261029)); + Assert.Equal((SeasonKind.Week, "Week 42"), SeasonPolicy.Describe(1202642)); + Assert.Equal((SeasonKind.Month, "October 2026"), SeasonPolicy.Describe(202610)); + Assert.Equal((SeasonKind.Year, "2025"), SeasonPolicy.Describe(2025)); + Assert.Null(SeasonPolicy.Describe(7)); + Assert.Equal("Fall Block", SeasonPolicy.Describe(202610, new Dictionary { [202610] = "Fall Block" })!.Value.Name); + } + + [Fact] + public void Filename_ParsesMediaMtxPattern() + { + var ts = ItemTimestamp.TryParseFilename("2026-10-29_21-03-11", NY); + Assert.NotNull(ts); + Assert.Equal(Ny(2026, 10, 29, 21, 3, 11), ts!.Value); + Assert.Null(ItemTimestamp.TryParseFilename("clip-final-v2", NY)); + Assert.Equal(Ny(2026, 10, 29, 21, 3, 11), ItemTimestamp.TryParseFilename("Overwatch 2 2026.10.29 - 21.03.11.05.DVR", NY)); + Assert.Equal(Ny(2026, 10, 29, 21, 3, 11), ItemTimestamp.TryParseFilename("Halo Infinite 2026-10-29 21-03-11", NY)); + Assert.Equal(Ny(2026, 10, 29, 21, 3, 11), ItemTimestamp.TryParseFilename("clip_730_20261029_210311", NY)); + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Configuration/PluginConfiguration.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Configuration/PluginConfiguration.cs new file mode 100644 index 0000000..5e8a13c --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Configuration/PluginConfiguration.cs @@ -0,0 +1,51 @@ +using MediaBrowser.Model.Plugins; + +namespace Jellyfin.Plugin.Vanguard.Configuration; + +/// Plugin settings (edited on the plugin's config page). +public class PluginConfiguration : BasePluginConfiguration +{ + // --- Discord ------------------------------------------------------------------------- + public string DiscordBotToken { get; set; } = string.Empty; + public string DiscordGuildId { get; set; } = string.Empty; + + // --- Discord feed (new stream / new clip notices) --------------------------------------- + /// Channel to post notices in (same guild). Empty = feed off. + public string NotifyChannelId { get; set; } = string.Empty; + public bool NotifyNewStreams { get; set; } = true; + public bool NotifyNewClips { get; set; } = true; + /// Public Jellyfin URL used in links, e.g. https://watch.example.com + public string PublicUrl { get; set; } = string.Empty; + /// Library names that count as streams / clips (case-insensitive substring match on the library name or path). + public string StreamsLibraryHint { get; set; } = "stream"; + public string ClipsLibraryHint { get; set; } = "clip"; + + // --- Playback policy (no hardware encoder on this box) -------------------------------------- + /// Set direct-play-only on all users automatically (new SSO users included). Turn off when the server can transcode. + public bool EnforceDirectPlayOnly { get; set; } = true; + public bool EnforceOnAdminsToo { get; set; } = false; + + // --- Authentik (streamer username -> Discord user id via the OAuth source connection) -- + public string AuthentikUrl { get; set; } = "https://auth.example.com"; + public string AuthentikApiToken { get; set; } = string.Empty; + + // --- Time / seasons --------------------------------------------------------------------- + public string TimeZoneId { get; set; } = "America/New_York"; + /// Hour (0-23) at which the "broadcast day" rolls over. 4 = a 01:00 clip belongs to the previous evening. + public int BroadcastDayRolloverHour { get; set; } = 4; + /// Season name overrides: "202611=Winter Scrims Block", one per line. + public string SeasonNameOverrides { get; set; } = string.Empty; + + /// Prefer the overlapping Discord scheduled event's name as the episode title; fallback is the dock title (folder name). + public bool EpisodeTitleFromEvent { get; set; } = true; + + /// + /// Folder name -> identity, one per line: "techy=123456789012345678" (Discord user id) or + /// "techy=authentik:someuser" (Authentik username). Folders not listed are looked up in Authentik + /// by username = folder name. + /// + public string FolderIdentityMap { get; set; } = string.Empty; + + /// Hours of slack when matching a recording to a Discord scheduled event (before start / after end). + public double EventMatchSlackHours { get; set; } = 1.0; +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Configuration/configPage.html b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Configuration/configPage.html new file mode 100644 index 0000000..f61553f --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Configuration/configPage.html @@ -0,0 +1,59 @@ + + +Vanguard Metadata + +
+
+
+

Discord

+
+
+

Discord feed

+
+
+
+
+
+
+

Playback policy

+
+
+

Authentik

+
+
+
+

Seasons

+
+
+
+
+ +
+
+ +
+ + diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Jellyfin.Plugin.Vanguard.csproj b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Jellyfin.Plugin.Vanguard.csproj new file mode 100644 index 0000000..0a2330e --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Jellyfin.Plugin.Vanguard.csproj @@ -0,0 +1,23 @@ + + + net10.0 + Jellyfin.Plugin.Vanguard + 0.1.1.0 + 0.1.1.0 + enable + enable + false + false + + + + + + + + + + + + + diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Plugin.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Plugin.cs new file mode 100644 index 0000000..f9dc395 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Plugin.cs @@ -0,0 +1,33 @@ +using System.Globalization; +using Jellyfin.Plugin.Vanguard.Configuration; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Plugins; +using MediaBrowser.Model.Plugins; +using MediaBrowser.Model.Serialization; + +namespace Jellyfin.Plugin.Vanguard; + +/// Vanguard Metadata plugin entry point. +public class Plugin : BasePlugin, IHasWebPages +{ + public Plugin(IApplicationPaths applicationPaths, IXmlSerializer xmlSerializer) + : base(applicationPaths, xmlSerializer) + { + Instance = this; + } + + public override string Name => "Vanguard Metadata"; + + public override Guid Id => Guid.Parse("d00ebd97-f987-4678-8089-c8f0f8a6ab3e"); + + public static Plugin? Instance { get; private set; } + + public IEnumerable GetPages() => + [ + new PluginPageInfo + { + Name = Name, + EmbeddedResourcePath = string.Format(CultureInfo.InvariantCulture, "{0}.Configuration.configPage.html", GetType().Namespace) + } + ]; +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardEpisodeProvider.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardEpisodeProvider.cs new file mode 100644 index 0000000..6749924 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardEpisodeProvider.cs @@ -0,0 +1,75 @@ +using System.Globalization; +using Jellyfin.Plugin.Vanguard.Configuration; +using Jellyfin.Plugin.Vanguard.Seasons; +using Jellyfin.Plugin.Vanguard.Services; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Providers; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Providers; + +/// Season/episode numbers from the timestamp; title from the Discord event or the dock title. +public sealed class VanguardEpisodeProvider : IRemoteMetadataProvider, IHasOrder +{ + private readonly LibraryPaths _paths; + private readonly EventStore _store; + private readonly IdentityResolver _identity; + private readonly IHttpClientFactory _http; + private readonly ILogger _log; + + public VanguardEpisodeProvider(LibraryPaths paths, EventStore store, IdentityResolver identity, IHttpClientFactory http, ILogger log) + { _paths = paths; _store = store; _identity = identity; _http = http; _log = log; } + + public string Name => "Vanguard Metadata"; + public int Order => 0; + + private static PluginConfiguration Cfg => Plugin.Instance?.Configuration ?? new PluginConfiguration(); + + public static TimeZoneInfo Tz(PluginConfiguration cfg) + { + try { return TimeZoneInfo.FindSystemTimeZoneById(cfg.TimeZoneId); } + catch (TimeZoneNotFoundException) { return TimeZoneInfo.Utc; } + } + + public async Task> GetMetadata(EpisodeInfo info, CancellationToken ct) + { + var result = new MetadataResult(); + if (string.IsNullOrEmpty(info.Path)) return result; + var cfg = Cfg; + var tz = Tz(cfg); + var when = ItemTimestamp.Resolve(info.Path, tz); + var slot = SeasonPolicy.Resolve(when.Utc, DateTimeOffset.UtcNow, tz, cfg.BroadcastDayRolloverHour, SeasonPolicy.ParseOverrides(cfg.SeasonNameOverrides)); + var parts = _paths.Split(info.Path); + + var ev = _store.FindEventAt(when.Utc, TimeSpan.FromHours(cfg.EventMatchSlackHours), TimeSpan.FromHours(4)); + var local = TimeZoneInfo.ConvertTime(when.Utc, tz); + var dockTitle = parts?.Title ?? parts?.Stem ?? Path.GetFileNameWithoutExtension(info.Path); + var title = cfg.EpisodeTitleFromEvent && ev is not null ? ev.Name : dockTitle; + + var who = parts is null ? null : await _identity.ResolveFolderAsync(parts.SeriesFolder, ct).ConfigureAwait(false); + var lines = new List(); + if (ev is not null && !string.IsNullOrWhiteSpace(ev.Description)) lines.Add(ev.Description!.Trim()); + lines.Add($"Recorded {local:dddd, MMMM d yyyy 'at' h:mm tt} by {who?.DisplayName ?? parts?.SeriesFolder ?? "unknown"}."); + if (ev is not null && title != dockTitle && !string.IsNullOrWhiteSpace(dockTitle)) lines.Add($"Stream title: {dockTitle}"); + + result.Item = new Episode + { + Name = $"{title} — {local:MMM d, h:mm tt}", + IndexNumber = slot.EpisodeNumber, + ParentIndexNumber = slot.SeasonNumber, + PremiereDate = when.Utc.UtcDateTime, + ProductionYear = local.Year, + Overview = string.Join("\n\n", lines), + }; + result.HasMetadata = true; + _log.LogDebug("Episode {Path}: season {Season} ({Name}) episode {Ep} from {Src}", info.Path, slot.SeasonNumber, slot.SeasonName, slot.EpisodeNumber, when.Source); + return result; + } + + public Task> GetSearchResults(EpisodeInfo searchInfo, CancellationToken ct) => + Task.FromResult>([]); + + public Task GetImageResponse(string url, CancellationToken ct) => + _http.CreateClient(MediaBrowser.Common.Net.NamedClient.Default).GetAsync(url, ct); +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardSeasonProvider.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardSeasonProvider.cs new file mode 100644 index 0000000..6a44bd5 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardSeasonProvider.cs @@ -0,0 +1,35 @@ +using Jellyfin.Plugin.Vanguard.Configuration; +using Jellyfin.Plugin.Vanguard.Seasons; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Providers; + +namespace Jellyfin.Plugin.Vanguard.Providers; + +/// Names a season from its number: "Thursday, Oct 29", "Week 44", "October 2026", "2026" or an override. +public sealed class VanguardSeasonProvider : IRemoteMetadataProvider, IHasOrder +{ + private readonly IHttpClientFactory _http; + public VanguardSeasonProvider(IHttpClientFactory http) { _http = http; } + + public string Name => "Vanguard Metadata"; + public int Order => 0; + + public Task> GetMetadata(SeasonInfo info, CancellationToken ct) + { + var result = new MetadataResult(); + if (!info.IndexNumber.HasValue) return Task.FromResult(result); + var cfg = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + var d = SeasonPolicy.Describe(info.IndexNumber.Value, SeasonPolicy.ParseOverrides(cfg.SeasonNameOverrides)); + if (d is null) return Task.FromResult(result); + result.Item = new Season { Name = d.Value.Name, IndexNumber = info.IndexNumber }; + result.HasMetadata = true; + return Task.FromResult(result); + } + + public Task> GetSearchResults(SeasonInfo searchInfo, CancellationToken ct) => + Task.FromResult>([]); + + public Task GetImageResponse(string url, CancellationToken ct) => + _http.CreateClient(MediaBrowser.Common.Net.NamedClient.Default).GetAsync(url, ct); +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardSeriesProvider.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardSeriesProvider.cs new file mode 100644 index 0000000..e394da1 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Providers/VanguardSeriesProvider.cs @@ -0,0 +1,71 @@ +using Jellyfin.Plugin.Vanguard.Services; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.Entities; +using MediaBrowser.Model.Providers; + +namespace Jellyfin.Plugin.Vanguard.Providers; + +/// A show is a streamer/submitter folder: display name and overview from Discord. +public sealed class VanguardSeriesProvider : IRemoteMetadataProvider, IHasOrder +{ + private readonly IdentityResolver _identity; + private readonly IHttpClientFactory _http; + public VanguardSeriesProvider(IdentityResolver identity, IHttpClientFactory http) { _identity = identity; _http = http; } + + public string Name => "Vanguard Metadata"; + public int Order => 0; + + public async Task> GetMetadata(SeriesInfo info, CancellationToken ct) + { + var result = new MetadataResult(); + var folder = string.IsNullOrEmpty(info.Path) ? info.Name : Path.GetFileName(info.Path.TrimEnd(Path.DirectorySeparatorChar)); + if (string.IsNullOrEmpty(folder)) return result; + var m = await _identity.ResolveFolderAsync(folder, ct).ConfigureAwait(false); + result.Item = new Series + { + Name = m?.DisplayName ?? folder, + Overview = m is null ? $"Recordings from {folder}." : $"Recordings from {m.DisplayName} (@{m.Username} on Discord).", + Status = SeriesStatus.Continuing, + }; + result.HasMetadata = true; + return result; + } + + public Task> GetSearchResults(SeriesInfo searchInfo, CancellationToken ct) => + Task.FromResult>([]); + + public Task GetImageResponse(string url, CancellationToken ct) => + _http.CreateClient(MediaBrowser.Common.Net.NamedClient.Default).GetAsync(url, ct); +} + +/// Series poster = the Discord avatar (guild-specific if set). +public sealed class VanguardSeriesImageProvider : IRemoteImageProvider, IHasOrder +{ + private readonly IdentityResolver _identity; + private readonly IHttpClientFactory _http; + public VanguardSeriesImageProvider(IdentityResolver identity, IHttpClientFactory http) { _identity = identity; _http = http; } + + public string Name => "Vanguard Metadata"; + public int Order => 0; + + public bool Supports(BaseItem item) => item is Series; + public IEnumerable GetSupportedImages(BaseItem item) => [ImageType.Primary, ImageType.Thumb]; + + public async Task> GetImages(BaseItem item, CancellationToken ct) + { + var folder = Path.GetFileName(item.Path?.TrimEnd(Path.DirectorySeparatorChar) ?? string.Empty); + if (string.IsNullOrEmpty(folder)) return []; + var m = await _identity.ResolveFolderAsync(folder, ct).ConfigureAwait(false); + if (m?.AvatarUrl is null) return []; + return + [ + new RemoteImageInfo { ProviderName = Name, Type = ImageType.Primary, Url = m.AvatarUrl, Width = 512, Height = 512 }, + new RemoteImageInfo { ProviderName = Name, Type = ImageType.Thumb, Url = m.AvatarUrl, Width = 512, Height = 512 }, + ]; + } + + public Task GetImageResponse(string url, CancellationToken ct) => + _http.CreateClient(MediaBrowser.Common.Net.NamedClient.Default).GetAsync(url, ct); +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/ServiceRegistrator.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/ServiceRegistrator.cs new file mode 100644 index 0000000..30277e7 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/ServiceRegistrator.cs @@ -0,0 +1,20 @@ +using Jellyfin.Plugin.Vanguard.Services; +using MediaBrowser.Controller; +using MediaBrowser.Controller.Plugins; +using Microsoft.Extensions.DependencyInjection; + +namespace Jellyfin.Plugin.Vanguard; + +/// Registers the plugin's services with Jellyfin's DI container. +public sealed class ServiceRegistrator : IPluginServiceRegistrator +{ + public void RegisterServices(IServiceCollection services, IServerApplicationHost applicationHost) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.AddSingleton(sp => new EventStore(Plugin.Instance?.DataFolderPath ?? Path.Combine(Path.GetTempPath(), "vanguard"))); + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/AuthentikClient.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/AuthentikClient.cs new file mode 100644 index 0000000..c4846af --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/AuthentikClient.cs @@ -0,0 +1,50 @@ +using System.Net.Http.Headers; +using System.Text.Json; +using MediaBrowser.Common.Net; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Services; + +/// Resolves an Authentik username to the Discord user id stored on their OAuth source connection. +public sealed class AuthentikClient +{ + private readonly IHttpClientFactory _http; + private readonly ILogger _log; + + public AuthentikClient(IHttpClientFactory http, ILogger log) { _http = http; _log = log; } + + public async Task GetDiscordIdForUsernameAsync(string baseUrl, string token, string username, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(baseUrl) || string.IsNullOrWhiteSpace(token)) return null; + var c = _http.CreateClient(NamedClient.Default); + c.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + var b = baseUrl.TrimEnd('/'); + try + { + using var u = await c.GetAsync($"{b}/api/v3/core/users/?username={Uri.EscapeDataString(username)}", ct).ConfigureAwait(false); + if (!u.IsSuccessStatusCode) { _log.LogWarning("Authentik users lookup {User}: {Status}", username, u.StatusCode); return null; } + using var ud = JsonDocument.Parse(await u.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + var results = ud.RootElement.GetProperty("results"); + if (results.GetArrayLength() == 0) return null; + var pk = results[0].GetProperty("pk").GetInt32(); + + using var cres = await c.GetAsync($"{b}/api/v3/sources/user_connections/oauth/?user={pk}", ct).ConfigureAwait(false); + if (!cres.IsSuccessStatusCode) { _log.LogWarning("Authentik connections lookup {User}: {Status}", username, cres.StatusCode); return null; } + using var cd = JsonDocument.Parse(await cres.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + foreach (var conn in cd.RootElement.GetProperty("results").EnumerateArray()) + { + var slug = conn.TryGetProperty("source", out var src) && src.ValueKind == JsonValueKind.Object && src.TryGetProperty("slug", out var sl) ? sl.GetString() : null; + var ident = conn.TryGetProperty("identifier", out var idp) ? idp.GetString() : null; + if (!string.IsNullOrEmpty(ident) && (slug is null || slug.Contains("discord", StringComparison.OrdinalIgnoreCase))) + { + return ident; + } + } + } + catch (Exception ex) when (ex is HttpRequestException or JsonException or TaskCanceledException) + { + _log.LogWarning(ex, "Authentik lookup failed for {User}", username); + } + return null; + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/DiscordClient.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/DiscordClient.cs new file mode 100644 index 0000000..f10731b --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/DiscordClient.cs @@ -0,0 +1,111 @@ +using System.Net.Http.Headers; +using System.Text.Json; +using MediaBrowser.Common.Net; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Services; + +public sealed record DiscordMember(string UserId, string Username, string DisplayName, string? AvatarUrl); +public sealed record DiscordEvent(string Id, string Name, string? Description, DateTimeOffset Start, DateTimeOffset? End, string? Status, DateTimeOffset SeenAt); + +/// Thin REST client for the Discord bot API (no gateway). Uses the same application Authentik uses; add a bot user to it. +public sealed class DiscordClient +{ + private const string Api = "https://discord.com/api/v10"; + private readonly IHttpClientFactory _http; + private readonly ILogger _log; + + public DiscordClient(IHttpClientFactory http, ILogger log) { _http = http; _log = log; } + + private HttpClient Client(string token) + { + var c = _http.CreateClient(NamedClient.Default); + c.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bot", token); + return c; + } + + public async Task GetGuildMemberAsync(string token, string guildId, string userId, CancellationToken ct) + { + using var resp = await Client(token).GetAsync($"{Api}/guilds/{guildId}/members/{userId}", ct).ConfigureAwait(false); + if (!resp.IsSuccessStatusCode) + { + _log.LogWarning("Discord member lookup {User} failed: {Status}", userId, resp.StatusCode); + return null; + } + using var doc = JsonDocument.Parse(await resp.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + var root = doc.RootElement; + var user = root.GetProperty("user"); + var username = user.GetProperty("username").GetString() ?? userId; + var global = user.TryGetProperty("global_name", out var g) && g.ValueKind == JsonValueKind.String ? g.GetString() : null; + var nick = root.TryGetProperty("nick", out var n) && n.ValueKind == JsonValueKind.String ? n.GetString() : null; + var guildAvatar = root.TryGetProperty("avatar", out var ga) && ga.ValueKind == JsonValueKind.String ? ga.GetString() : null; + var userAvatar = user.TryGetProperty("avatar", out var ua) && ua.ValueKind == JsonValueKind.String ? ua.GetString() : null; + string? avatarUrl = guildAvatar is not null + ? $"https://cdn.discordapp.com/guilds/{guildId}/users/{userId}/avatars/{guildAvatar}.png?size=512" + : userAvatar is not null ? $"https://cdn.discordapp.com/avatars/{userId}/{userAvatar}.png?size=512" + : $"https://cdn.discordapp.com/embed/avatars/{(ulong.TryParse(userId, out var id) ? (id >> 22) % 6 : 0)}.png"; + return new DiscordMember(userId, username, nick ?? global ?? username, avatarUrl); + } + + /// Post an embed to a channel. are (name, value) pairs. + public async Task PostEmbedAsync(string token, string channelId, string title, string? description, string? url, string? thumbnailUrl, IEnumerable<(string Name, string Value)> fields, int color, CancellationToken ct, byte[]? imageBytes = null, string imageName = "thumb.jpg") + { + var embed = new Dictionary + { + ["title"] = title.Length > 256 ? title[..256] : title, + ["description"] = description is { Length: > 0 } ? (description.Length > 2000 ? description[..2000] : description) : null, + ["url"] = url, + ["color"] = color, + ["timestamp"] = DateTimeOffset.UtcNow.ToString("o"), + ["fields"] = fields.Select(f => new { name = f.Name, value = f.Value, inline = true }).ToArray(), + }; + if (thumbnailUrl is not null) embed["thumbnail"] = new { url = thumbnailUrl }; + if (imageBytes is { Length: > 0 }) embed["image"] = new { url = $"attachment://{imageName}" }; // big picture = the frame grab + var payload = JsonSerializer.Serialize(new { embeds = new[] { embed } }, new JsonSerializerOptions { DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull }); + + HttpContent content; + if (imageBytes is { Length: > 0 }) + { + var mp = new MultipartFormDataContent(); + mp.Add(new StringContent(payload, System.Text.Encoding.UTF8, "application/json"), "payload_json"); + var file = new ByteArrayContent(imageBytes); + file.Headers.ContentType = new MediaTypeHeaderValue(imageName.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ? "image/png" : "image/jpeg"); + mp.Add(file, "files[0]", imageName); + content = mp; + } + else + { + content = new StringContent(payload, System.Text.Encoding.UTF8, "application/json"); + } + using var _ = content; + using var resp = await Client(token).PostAsync($"{Api}/channels/{channelId}/messages", content, ct).ConfigureAwait(false); + if (!resp.IsSuccessStatusCode) _log.LogWarning("Discord post to {Channel} failed: {Status} {Body}", channelId, resp.StatusCode, await resp.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + return resp.IsSuccessStatusCode; + } + + public async Task> GetScheduledEventsAsync(string token, string guildId, CancellationToken ct) + { + using var resp = await Client(token).GetAsync($"{Api}/guilds/{guildId}/scheduled-events", ct).ConfigureAwait(false); + if (!resp.IsSuccessStatusCode) + { + _log.LogWarning("Discord scheduled-events failed: {Status}", resp.StatusCode); + return []; + } + using var doc = JsonDocument.Parse(await resp.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + var list = new List(); + var now = DateTimeOffset.UtcNow; + foreach (var e in doc.RootElement.EnumerateArray()) + { + var start = e.GetProperty("scheduled_start_time").GetDateTimeOffset(); + DateTimeOffset? end = e.TryGetProperty("scheduled_end_time", out var et) && et.ValueKind == JsonValueKind.String ? et.GetDateTimeOffset() : null; + list.Add(new DiscordEvent( + e.GetProperty("id").GetString()!, + e.GetProperty("name").GetString() ?? "(event)", + e.TryGetProperty("description", out var d) && d.ValueKind == JsonValueKind.String ? d.GetString() : null, + start, end, + e.TryGetProperty("status", out var st) ? st.ToString() : null, + now)); + } + return list; + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/EventStore.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/EventStore.cs new file mode 100644 index 0000000..93e7f36 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/EventStore.cs @@ -0,0 +1,77 @@ +using System.Text.Json; + +namespace Jellyfin.Plugin.Vanguard.Services; + +/// +/// Persistent history of Discord scheduled events (Discord only returns upcoming/active ones, so we keep +/// everything we have ever seen) plus an in-memory member cache. +/// +public sealed class EventStore +{ + private readonly object _lock = new(); + private readonly string _path; + private Dictionary _events = new(); + private readonly Dictionary _members = new(); + private readonly Dictionary _identities = new(StringComparer.OrdinalIgnoreCase); + + public EventStore(string dataFolder) + { + Directory.CreateDirectory(dataFolder); + _path = Path.Combine(dataFolder, "events.json"); + Load(); + } + + private void Load() + { + try + { + if (File.Exists(_path)) + { + var list = JsonSerializer.Deserialize>(File.ReadAllText(_path)) ?? []; + _events = list.ToDictionary(e => e.Id); + } + } + catch (Exception ex) when (ex is JsonException or IOException) { _events = new(); } + } + + public void Merge(IEnumerable fresh) + { + lock (_lock) + { + foreach (var e in fresh) _events[e.Id] = e; + var tmp = _path + ".tmp"; + File.WriteAllText(tmp, JsonSerializer.Serialize(_events.Values.OrderBy(e => e.Start).ToList(), new JsonSerializerOptions { WriteIndented = true })); + File.Move(tmp, _path, overwrite: true); + } + } + + /// Event whose [start - slack, end + slack] window contains ; closest start wins. + public DiscordEvent? FindEventAt(DateTimeOffset at, TimeSpan slack, TimeSpan defaultDuration) + { + lock (_lock) + { + return _events.Values + .Where(e => at >= e.Start - slack && at <= (e.End ?? e.Start + defaultDuration) + slack) + .OrderBy(e => Math.Abs((e.Start - at).Ticks)) + .FirstOrDefault(); + } + } + + public int Count { get { lock (_lock) return _events.Count; } } + + public DiscordMember? GetCachedMember(string discordId, TimeSpan maxAge) + { + lock (_lock) return _members.TryGetValue(discordId, out var m) && DateTimeOffset.UtcNow - m.At < maxAge ? m.Member : null; + } + public void CacheMember(DiscordMember m) { lock (_lock) _members[m.UserId] = (m, DateTimeOffset.UtcNow); } + + public bool TryGetCachedIdentity(string folder, TimeSpan maxAge, out string? discordId) + { + lock (_lock) + { + if (_identities.TryGetValue(folder, out var v) && DateTimeOffset.UtcNow - v.At < maxAge) { discordId = v.DiscordId; return true; } + discordId = null; return false; + } + } + public void CacheIdentity(string folder, string? discordId) { lock (_lock) _identities[folder] = (discordId, DateTimeOffset.UtcNow); } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/FeedService.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/FeedService.cs new file mode 100644 index 0000000..6346b39 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/FeedService.cs @@ -0,0 +1,104 @@ +using System.Globalization; +using Jellyfin.Plugin.Vanguard.Configuration; +using Jellyfin.Plugin.Vanguard.Providers; +using Jellyfin.Plugin.Vanguard.Seasons; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Entities; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Services; + +/// Posts "new stream recording" / "new clip" embeds to a Discord channel when items land in the library. +public sealed class FeedService : IHostedService +{ + private readonly ILibraryManager _library; + private readonly DiscordClient _discord; + private readonly IdentityResolver _identity; + private readonly LibraryPaths _paths; + private readonly ILogger _log; + private readonly HashSet _announced = new(); + + public FeedService(ILibraryManager library, DiscordClient discord, IdentityResolver identity, LibraryPaths paths, ILogger log) + { _library = library; _discord = discord; _identity = identity; _paths = paths; _log = log; } + + public Task StartAsync(CancellationToken ct) { _library.ItemAdded += OnItemAdded; return Task.CompletedTask; } + public Task StopAsync(CancellationToken ct) { _library.ItemAdded -= OnItemAdded; return Task.CompletedTask; } + + private void OnItemAdded(object? sender, ItemChangeEventArgs e) + { + if (e.Item is not Episode ep || string.IsNullOrEmpty(ep.Path)) return; + _ = Task.Run(() => AnnounceAsync(ep)); + } + + private async Task AnnounceAsync(Episode ep) + { + try + { + var cfg = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + if (string.IsNullOrWhiteSpace(cfg.NotifyChannelId) || string.IsNullOrWhiteSpace(cfg.DiscordBotToken)) return; + lock (_announced) { if (!_announced.Add(ep.Id)) return; } + + var kind = Classify(ep.Path, cfg); + if (kind == Kind.Other) return; + if (kind == Kind.Stream && !cfg.NotifyNewStreams) return; + if (kind == Kind.Clip && !cfg.NotifyNewClips) return; + + var parts = _paths.Split(ep.Path); + var who = parts is null ? null : await _identity.ResolveFolderAsync(parts.SeriesFolder, CancellationToken.None).ConfigureAwait(false); + var tz = VanguardEpisodeProvider.Tz(cfg); + var when = TimeZoneInfo.ConvertTime(ItemTimestamp.Resolve(ep.Path, tz).Utc, tz); + var title = parts?.Title ?? parts?.Stem ?? Path.GetFileNameWithoutExtension(ep.Path); + var link = string.IsNullOrWhiteSpace(cfg.PublicUrl) ? null : $"{cfg.PublicUrl.TrimEnd('/')}/web/#/details?id={ep.Id:N}"; + var name = who?.DisplayName ?? parts?.SeriesFolder ?? "someone"; + + var heading = kind == Kind.Stream ? $"🔴 {name} is recording: {title}" : $"🎬 New clip from {name}: {title}"; + var fields = new List<(string, string)> { ("When", when.ToString("ddd MMM d, h:mm tt", CultureInfo.InvariantCulture)) }; + if (parts?.Title is not null && kind == Kind.Clip) fields.Add(("Folder", parts.Title)); + + // Jellyfin's Screen Grabber extracts the episode's primary image during the refresh that follows + // ItemAdded; wait a little for it so the embed can carry the frame. Falls back to avatar only. + var (bytes, imgName) = await WaitForPrimaryImageAsync(ep.Id, TimeSpan.FromSeconds(90)).ConfigureAwait(false); + await _discord.PostEmbedAsync(cfg.DiscordBotToken, cfg.NotifyChannelId, heading, null, link, who?.AvatarUrl, fields, + kind == Kind.Stream ? 0xE74C3C : 0x3D8BFD, CancellationToken.None, bytes, imgName).ConfigureAwait(false); + } + catch (Exception ex) { _log.LogWarning(ex, "Feed announce failed for {Path}", ep.Path); } + } + + private async Task<(byte[]? Bytes, string Name)> WaitForPrimaryImageAsync(Guid id, TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + if (_library.GetItemById(id) is BaseItem fresh && fresh.HasImage(ImageType.Primary)) + { + var path = fresh.GetImagePath(ImageType.Primary, 0); + if (!string.IsNullOrEmpty(path) && File.Exists(path)) + { + try { return (await File.ReadAllBytesAsync(path).ConfigureAwait(false), "thumb" + Path.GetExtension(path)); } + catch (IOException) { /* still being written; retry */ } + } + } + await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); + } + return (null, "thumb.jpg"); + } + + private enum Kind { Stream, Clip, Other } + + private Kind Classify(string path, PluginConfiguration cfg) + { + var full = Path.GetFullPath(path); + foreach (var vf in _library.GetVirtualFolders()) + { + var hit = (vf.Locations ?? []).Any(l => full.StartsWith(Path.GetFullPath(l).TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar, StringComparison.Ordinal)); + if (!hit) continue; + var hay = (vf.Name ?? string.Empty) + " " + string.Join(' ', vf.Locations ?? []); + if (!string.IsNullOrWhiteSpace(cfg.StreamsLibraryHint) && hay.Contains(cfg.StreamsLibraryHint, StringComparison.OrdinalIgnoreCase)) return Kind.Stream; + if (!string.IsNullOrWhiteSpace(cfg.ClipsLibraryHint) && hay.Contains(cfg.ClipsLibraryHint, StringComparison.OrdinalIgnoreCase)) return Kind.Clip; + } + return Kind.Other; + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/IdentityResolver.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/IdentityResolver.cs new file mode 100644 index 0000000..82770f8 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/IdentityResolver.cs @@ -0,0 +1,57 @@ +using Jellyfin.Plugin.Vanguard.Configuration; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Services; + +/// Folder name (streamer / submitter) -> Discord member, via the config map or Authentik. +public sealed class IdentityResolver +{ + private static readonly TimeSpan MemberTtl = TimeSpan.FromHours(6); + private static readonly TimeSpan IdentityTtl = TimeSpan.FromHours(24); + + private readonly DiscordClient _discord; + private readonly AuthentikClient _authentik; + private readonly EventStore _store; + private readonly ILogger _log; + + public IdentityResolver(DiscordClient discord, AuthentikClient authentik, EventStore store, ILogger log) + { _discord = discord; _authentik = authentik; _store = store; _log = log; } + + private static PluginConfiguration Cfg => Plugin.Instance?.Configuration ?? new PluginConfiguration(); + + public async Task ResolveFolderAsync(string folderName, CancellationToken ct) + { + var cfg = Cfg; + if (string.IsNullOrWhiteSpace(cfg.DiscordBotToken) || string.IsNullOrWhiteSpace(cfg.DiscordGuildId)) return null; + + if (!_store.TryGetCachedIdentity(folderName, IdentityTtl, out var discordId)) + { + discordId = await LookupDiscordIdAsync(cfg, folderName, ct).ConfigureAwait(false); + _store.CacheIdentity(folderName, discordId); + } + if (discordId is null) return null; + + var cached = _store.GetCachedMember(discordId, MemberTtl); + if (cached is not null) return cached; + var m = await _discord.GetGuildMemberAsync(cfg.DiscordBotToken, cfg.DiscordGuildId, discordId, ct).ConfigureAwait(false); + if (m is not null) _store.CacheMember(m); + return m; + } + + private async Task LookupDiscordIdAsync(PluginConfiguration cfg, string folder, CancellationToken ct) + { + string authentikUser = folder; + foreach (var raw in (cfg.FolderIdentityMap ?? string.Empty).Split('\n')) + { + var line = raw.Trim(); + var eq = line.IndexOf('='); + if (eq <= 0 || !line[..eq].Trim().Equals(folder, StringComparison.OrdinalIgnoreCase)) continue; + var val = line[(eq + 1)..].Trim(); + if (val.StartsWith("authentik:", StringComparison.OrdinalIgnoreCase)) { authentikUser = val["authentik:".Length..].Trim(); break; } + if (val.All(char.IsDigit) && val.Length >= 15) return val; // a Discord snowflake + } + var id = await _authentik.GetDiscordIdForUsernameAsync(cfg.AuthentikUrl, cfg.AuthentikApiToken, authentikUser, ct).ConfigureAwait(false); + if (id is null) _log.LogInformation("No Discord identity for folder {Folder} (authentik user {User})", folder, authentikUser); + return id; + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/LibraryPaths.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/LibraryPaths.cs new file mode 100644 index 0000000..322aa36 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Services/LibraryPaths.cs @@ -0,0 +1,31 @@ +using MediaBrowser.Controller.Library; + +namespace Jellyfin.Plugin.Vanguard.Services; + +/// Splits a media path into (series folder, dock title or null, file stem) relative to its library root. +public sealed class LibraryPaths +{ + private readonly ILibraryManager _library; + public LibraryPaths(ILibraryManager library) { _library = library; } + + public sealed record Parts(string SeriesFolder, string? Title, string Stem); + + public Parts? Split(string path) + { + var full = Path.GetFullPath(path); + string? root = null; + foreach (var vf in _library.GetVirtualFolders()) + { + foreach (var loc in vf.Locations ?? []) + { + var l = Path.GetFullPath(loc).TrimEnd(Path.DirectorySeparatorChar); + if (full.StartsWith(l + Path.DirectorySeparatorChar, StringComparison.Ordinal) && (root is null || l.Length > root.Length)) root = l; + } + } + if (root is null) return null; + var rel = full[(root.Length + 1)..].Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries); + if (rel.Length < 2) return null; + var stem = Path.GetFileNameWithoutExtension(rel[^1]); + return rel.Length >= 3 ? new Parts(rel[0], rel[^2], stem) : new Parts(rel[0], null, stem); + } +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/DiscordSnapshotTask.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/DiscordSnapshotTask.cs new file mode 100644 index 0000000..0cf2ed9 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/DiscordSnapshotTask.cs @@ -0,0 +1,36 @@ +using Jellyfin.Plugin.Vanguard.Configuration; +using Jellyfin.Plugin.Vanguard.Services; +using MediaBrowser.Model.Tasks; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Tasks; + +/// Every 5 minutes: pull the guild's scheduled events into the persistent history. +public sealed class DiscordSnapshotTask : IScheduledTask +{ + private readonly DiscordClient _discord; + private readonly EventStore _store; + private readonly ILogger _log; + public DiscordSnapshotTask(DiscordClient discord, EventStore store, ILogger log) { _discord = discord; _store = store; _log = log; } + + public string Name => "Vanguard: snapshot Discord events"; + public string Key => "VanguardDiscordSnapshot"; + public string Description => "Keeps a history of Discord scheduled events so recordings can be matched to them later."; + public string Category => "Vanguard"; + + public async Task ExecuteAsync(IProgress progress, CancellationToken ct) + { + var cfg = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + if (string.IsNullOrWhiteSpace(cfg.DiscordBotToken) || string.IsNullOrWhiteSpace(cfg.DiscordGuildId)) { _log.LogInformation("Discord not configured; skipping"); return; } + var events = await _discord.GetScheduledEventsAsync(cfg.DiscordBotToken, cfg.DiscordGuildId, ct).ConfigureAwait(false); + _store.Merge(events); + _log.LogInformation("Discord snapshot: {Fresh} current events, {Total} in history", events.Count, _store.Count); + progress.Report(100); + } + + public IEnumerable GetDefaultTriggers() => + [ + new TaskTriggerInfo { Type = TaskTriggerInfoType.IntervalTrigger, IntervalTicks = TimeSpan.FromMinutes(5).Ticks }, + new TaskTriggerInfo { Type = TaskTriggerInfoType.StartupTrigger }, + ]; +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/PlaybackPolicyTask.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/PlaybackPolicyTask.cs new file mode 100644 index 0000000..e6537b7 --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/PlaybackPolicyTask.cs @@ -0,0 +1,59 @@ +using Jellyfin.Data; +using Jellyfin.Database.Implementations.Enums; +using Jellyfin.Plugin.Vanguard.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Tasks; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Tasks; + +/// +/// Enforces a direct-play-only policy on every non-administrator user (this box has no hardware encoder). +/// Runs at startup and every 30 minutes so users created by the SSO plugin are covered automatically. +/// Turn off in the plugin config when the server gets Quick Sync. +/// +public sealed class PlaybackPolicyTask : IScheduledTask +{ + private readonly IUserManager _users; + private readonly ILogger _log; + public PlaybackPolicyTask(IUserManager users, ILogger log) { _users = users; _log = log; } + + public string Name => "Vanguard: enforce playback policy"; + public string Key => "VanguardPlaybackPolicy"; + public string Description => "Sets direct-play-only (no video transcoding) on all non-admin users, including new SSO users."; + public string Category => "Vanguard"; + + public async Task ExecuteAsync(IProgress progress, CancellationToken ct) + { + var cfg = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + if (!cfg.EnforceDirectPlayOnly) { _log.LogDebug("playback policy enforcement is off"); return; } + + int changed = 0; + var all = _users.GetUsers().ToList(); + foreach (var u in all) + { + ct.ThrowIfCancellationRequested(); + if (u.HasPermission(PermissionKind.IsAdministrator) && !cfg.EnforceOnAdminsToo) continue; + bool dirty = false; + dirty |= Set(u, PermissionKind.EnableVideoPlaybackTranscoding, false); + dirty |= Set(u, PermissionKind.EnableAudioPlaybackTranscoding, false); + dirty |= Set(u, PermissionKind.EnablePlaybackRemuxing, true); // container remux is cheap and keeps odd clients working + if (dirty) { await _users.UpdateUserAsync(u).ConfigureAwait(false); changed++; } + } + _log.LogInformation("playback policy: {Changed} of {Total} users updated", changed, all.Count); + progress.Report(100); + } + + private static bool Set(Jellyfin.Database.Implementations.Entities.User u, PermissionKind kind, bool value) + { + if (u.HasPermission(kind) == value) return false; + u.SetPermission(kind, value); + return true; + } + + public IEnumerable GetDefaultTriggers() => + [ + new TaskTriggerInfo { Type = TaskTriggerInfoType.StartupTrigger }, + new TaskTriggerInfo { Type = TaskTriggerInfoType.IntervalTrigger, IntervalTicks = TimeSpan.FromMinutes(30).Ticks }, + ]; +} diff --git a/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/SeasonRollTask.cs b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/SeasonRollTask.cs new file mode 100644 index 0000000..8157b9b --- /dev/null +++ b/jellyfin-plugin-vanguard/Jellyfin.Plugin.Vanguard/Tasks/SeasonRollTask.cs @@ -0,0 +1,87 @@ +using Jellyfin.Data.Enums; +using Jellyfin.Plugin.Vanguard.Configuration; +using Jellyfin.Plugin.Vanguard.Providers; +using Jellyfin.Plugin.Vanguard.Seasons; +using MediaBrowser.Controller.Entities; +using MediaBrowser.Controller.Entities.TV; +using MediaBrowser.Controller.Library; +using MediaBrowser.Controller.Providers; +using MediaBrowser.Model.IO; +using MediaBrowser.Model.Tasks; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Plugin.Vanguard.Tasks; + +/// +/// Nightly after the broadcast-day rollover: re-bucket every episode (day -> week -> month -> year), +/// rename seasons, and refresh series so virtual seasons appear/disappear. Paths never change, so +/// item ids and watched state are kept. +/// +public sealed class SeasonRollTask : IScheduledTask +{ + private readonly ILibraryManager _library; + private readonly IFileSystem _fs; + private readonly ILogger _log; + public SeasonRollTask(ILibraryManager library, IFileSystem fs, ILogger log) { _library = library; _fs = fs; _log = log; } + + public string Name => "Vanguard: roll seasons"; + public string Key => "VanguardSeasonRoll"; + public string Description => "Moves recordings between day/week/month/year seasons as time passes and fixes season names."; + public string Category => "Vanguard"; + + public async Task ExecuteAsync(IProgress progress, CancellationToken ct) + { + var cfg = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + var tz = VanguardEpisodeProvider.Tz(cfg); + var overrides = SeasonPolicy.ParseOverrides(cfg.SeasonNameOverrides); + var now = DateTimeOffset.UtcNow; + + var episodes = _library.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.Episode], Recursive = true, IsVirtualItem = false }); + var touchedSeries = new HashSet(); + int moved = 0, i = 0; + foreach (var item in episodes.OfType()) + { + ct.ThrowIfCancellationRequested(); + progress.Report(80.0 * ++i / Math.Max(1, episodes.Count)); + if (string.IsNullOrEmpty(item.Path)) continue; + var when = ItemTimestamp.Resolve(item.Path, tz); + var slot = SeasonPolicy.Resolve(when.Utc, now, tz, cfg.BroadcastDayRolloverHour, overrides); + if (item.ParentIndexNumber == slot.SeasonNumber && item.IndexNumber == slot.EpisodeNumber) continue; + item.ParentIndexNumber = slot.SeasonNumber; + item.IndexNumber = slot.EpisodeNumber; + await item.UpdateToRepositoryAsync(ItemUpdateType.MetadataEdit, ct).ConfigureAwait(false); + if (item.SeriesId != Guid.Empty) touchedSeries.Add(item.SeriesId); + moved++; + } + + // let each touched series re-evaluate its (virtual) seasons, then name every season from its number + var options = new MetadataRefreshOptions(new DirectoryService(_fs)) { MetadataRefreshMode = MetadataRefreshMode.Default, ImageRefreshMode = MetadataRefreshMode.None }; + foreach (var sid in touchedSeries) + { + if (_library.GetItemById(sid) is Series s) await s.RefreshMetadata(options, ct).ConfigureAwait(false); + } + var seasons = _library.GetItemList(new InternalItemsQuery { IncludeItemTypes = [BaseItemKind.Season], Recursive = true }); + int renamed = 0; + foreach (var s in seasons.OfType()) + { + if (!s.IndexNumber.HasValue) continue; + var d = SeasonPolicy.Describe(s.IndexNumber.Value, overrides); + if (d is null || string.Equals(s.Name, d.Value.Name, StringComparison.Ordinal)) continue; + s.Name = d.Value.Name; + await s.UpdateToRepositoryAsync(ItemUpdateType.MetadataEdit, ct).ConfigureAwait(false); + renamed++; + } + _log.LogInformation("Season roll: {Moved} episodes re-bucketed, {Renamed} seasons renamed, {Series} series refreshed", moved, renamed, touchedSeries.Count); + progress.Report(100); + } + + public IEnumerable GetDefaultTriggers() + { + var cfg = Plugin.Instance?.Configuration ?? new PluginConfiguration(); + var hour = Math.Clamp(cfg.BroadcastDayRolloverHour, 0, 23); + return + [ + new TaskTriggerInfo { Type = TaskTriggerInfoType.DailyTrigger, TimeOfDayTicks = TimeSpan.FromHours(hour).Add(TimeSpan.FromMinutes(10)).Ticks }, + ]; + } +} diff --git a/jellyfin-plugin-vanguard/README.md b/jellyfin-plugin-vanguard/README.md new file mode 100644 index 0000000..f596baf --- /dev/null +++ b/jellyfin-plugin-vanguard/README.md @@ -0,0 +1,60 @@ +# Vanguard Metadata — Jellyfin plugin + +Metadata provider for gameplay recordings. Each streamer/submitter folder is a show; season and +episode numbers come from the recording timestamp under a rolling policy; names, avatars and episode +titles come from Discord via Authentik's identity link; new recordings and clips are announced in a +Discord channel. + +## Layout +- `Jellyfin.Plugin.Vanguard.Core/` — pure logic: `SeasonPolicy` (rolling buckets, naming, overrides), + `ItemTimestamp` (sidecar → filename patterns → mtime). No Jellyfin dependency. Unit-tested. +- `Jellyfin.Plugin.Vanguard/` — the plugin: providers (Series, Series images, Season, Episode), + services (Discord REST, Authentik REST, event store, identity resolver, feed), scheduled tasks + (Discord snapshot every 5 min; season roll nightly after the rollover hour), config page. +- `Jellyfin.Plugin.Vanguard.Tests/` — xunit. + +## Season scheme +| age of item | season # | season name | +|---|---|---| +| current ISO week | `YYYYMMDD` | "Thursday, Oct 29" | +| current month | `1YYYYWW` | "Week 44" | +| current year | `YYYYMM` | "October 2026" | +| older | `YYYY` | "2026" | + +Episode # = `MMDDHHMMSS` of the local timestamp — never changes when the season rolls. +Broadcast day rolls at the configured hour (default 04:00). Overrides: `seasonNumber=Name`, one per line. + +## Timestamp sources, in order +1. `.json` / `.clip.json` sidecar with `"timestamp"` (UTC ISO-8601) — written by the clip bot +2. filename: MediaMTX `2026-10-29_21-03-11`, Game Bar `2026-10-29 21-03-11`, + ShadowPlay `2026.10.29 - 21.03.11`, Steam `20261029_210311` +3. file modification time (Syncthing preserves the gamer's mtime) + +## Build / package + ./package.sh # tests + Release build → dist/VanguardMetadata_.zip + manifest.json +Requires .NET 10 SDK. Bump `version` in `build.yaml` for a release. + +## Install on the server (manual, v0.1) + scp dist/VanguardMetadata_0.1.0.0.zip debian@HOST:/tmp/ + ssh debian@HOST 'sudo -u "#1000" mkdir -p "/srv/appdata/jellyfin/config/plugins/Vanguard Metadata_0.1.0.0" && sudo -u "#1000" python3 -m zipfile -e /tmp/VanguardMetadata_0.1.0.0.zip "/srv/appdata/jellyfin/config/plugins/Vanguard Metadata_0.1.0.0/" && cd ~/vanguard-stream && docker compose restart jellyfin' +Later: serve `dist/` at `https://HOST/plugins/` (Caddy) and add it as a repository in Jellyfin so updates +come through the Catalog. + +## Runs with an empty config +Seasons, episode numbers, timestamps and the nightly roll need nothing external. Without a Discord bot +token the show is named after its folder, has no poster, and episode titles are the dock title; without +Authentik no folder→Discord mapping is attempted; without a feed channel nothing is posted. Add the +credentials later and re-run "roll seasons" / refresh the libraries to pick up names and avatars. + +## Configure (Dashboard → Plugins → Vanguard Metadata) +- Discord: bot token (add a **Bot** to the existing Authentik Discord application; enable the Server + Members intent; invite it to the server), guild ID, feed channel ID, public Jellyfin URL. +- Authentik: URL + API token of a read-only service account (used to map folder → Discord user). +- Folder → identity overrides if a streamer's folder name isn't their Authentik username. +- Libraries (Dashboard → Libraries → each of streams/clips): content type Shows; metadata downloaders: + **only** "Vanguard Metadata" for Series, Season and Episode; image fetchers: Vanguard Metadata (Series) + and **Screen Grabber** (Episode) — the grab is what the Discord embed attaches. + +## Tasks (Dashboard → Scheduled Tasks → Vanguard) +- "snapshot Discord events": every 5 min + startup. Run it once after configuring. +- "roll seasons": daily at rollover+10 min. Run it once after the first library scan. diff --git a/jellyfin-plugin-vanguard/build.yaml b/jellyfin-plugin-vanguard/build.yaml new file mode 100644 index 0000000..5f6ea1e --- /dev/null +++ b/jellyfin-plugin-vanguard/build.yaml @@ -0,0 +1,18 @@ +name: "Vanguard Metadata" +guid: "d00ebd97-f987-4678-8089-c8f0f8a6ab3e" +version: "0.1.1.0" +targetAbi: "12.4.0.0" +framework: "net10.0" +overview: "Season/episode metadata for gameplay recordings from timestamps, Discord and Authentik." +description: > + Treats each streamer/submitter folder as a show. Season and episode numbers come from the + recording timestamp under a rolling policy (weekday -> week -> month -> year). Names, avatars and + episode titles come from Discord (members, scheduled events) via Authentik's identity link. +category: "Metadata" +owner: "techy" +artifacts: +- "Jellyfin.Plugin.Vanguard.dll" +- "Jellyfin.Plugin.Vanguard.Core.dll" +changelog: > + 0.1.1.0 playback-policy enforcement task (direct-play-only for all users) + 0.1.0.0 initial skeleton diff --git a/jellyfin-plugin-vanguard/package.sh b/jellyfin-plugin-vanguard/package.sh new file mode 100755 index 0000000..4fa82c5 --- /dev/null +++ b/jellyfin-plugin-vanguard/package.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Build Release and produce dist/VanguardMetadata_.zip (DLLs + meta.json) ready to drop into +# Jellyfin's plugins folder, plus dist/manifest.json for serving as a plugin repository. +set -euo pipefail +cd "$(dirname "$0")" +command -v yq >/dev/null 2>&1 || true +get() { grep -E "^$1:" build.yaml | head -1 | sed -E 's/^[a-zA-Z]+: *"?([^"]*)"?$/\1/'; } +NAME=$(get name); GUID=$(get guid); VERSION=$(get version); ABI=$(get targetAbi); OVERVIEW=$(get overview); OWNER=$(get owner); CATEGORY=$(get category) +DESC=$(python3 -c "import yaml,sys; print(yaml.safe_load(open('build.yaml'))['description'].strip())") +CHANGELOG=$(python3 -c "import yaml,sys; print(yaml.safe_load(open('build.yaml'))['changelog'].strip())") + +dotnet test Jellyfin.Plugin.Vanguard.Tests --nologo -v q +dotnet build Jellyfin.Plugin.Vanguard -c Release --nologo -v q + +OUT=Jellyfin.Plugin.Vanguard/bin/Release/net10.0 +STAGE=dist/stage; rm -rf dist; mkdir -p "$STAGE" +cp "$OUT/Jellyfin.Plugin.Vanguard.dll" "$OUT/Jellyfin.Plugin.Vanguard.Core.dll" "$STAGE/" +TS=$(date -u +%Y-%m-%dT%H:%M:%SZ) +cat > "$STAGE/meta.json" < dist/manifest.json < + + + + diff --git a/jellyfin/branding.css b/jellyfin/branding.css new file mode 100644 index 0000000..849fa16 --- /dev/null +++ b/jellyfin/branding.css @@ -0,0 +1,14 @@ +/* Jellyfin -> Dashboard -> General -> Branding -> Custom CSS + Login page shows only the Discord (Authentik SSO) button; "Admin login" reveals the local form. */ +.disclaimerContainer { display: block; } +a.raised.emby-button { padding: .9em 1em; color: inherit !important; } + +#loginPage .manualLoginForm, +#loginPage .visualLoginForm, +#loginPage .btnForgotPassword { display: none !important; } + +#showAdmin { display: none; } +.showAdminLabel { display: block; text-align: center; margin-top: 1.2em; color: #999; cursor: pointer; text-decoration: underline; font-size: .9em; } + +#loginPage:has(#showAdmin:checked) .manualLoginForm { display: block !important; } +#loginPage:has(#showAdmin:checked) .showAdminLabel { display: none; } diff --git a/mariadb/init/10-gitea.sh b/mariadb/init/10-gitea.sh new file mode 100755 index 0000000..8f44d59 --- /dev/null +++ b/mariadb/init/10-gitea.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Runs once, on MariaDB's first initialisation (docker-entrypoint-initdb.d). Creates Gitea's database and user. +# Already-initialised server? run ./createGiteaDb.sh from the bundle root instead. +set -e +: "${GITEA_DB_PASSWORD:?GITEA_DB_PASSWORD not set}" +mariadb -uroot -p"$MARIADB_ROOT_PASSWORD" <12h + recordDeleteAfter: 0s # never auto-delete; the archive job handles it + +paths: + # any name under a streamer's prefix; inherits pathDefaults + ~^(techy|diceofire)/[A-Za-z0-9_.-]+$: {} diff --git a/registerGiteaRunner.sh b/registerGiteaRunner.sh new file mode 100755 index 0000000..9e90391 --- /dev/null +++ b/registerGiteaRunner.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Generate a runner registration token from Gitea, store it in .env, (re)start the runner. +# Re-run any time the runner shows as offline/unregistered. +set -euo pipefail +cd "$(dirname "$0")" +docker ps --format '{{.Names}}' | grep -qx gitea || { echo "gitea container not running"; exit 1; } +TOKEN=$(docker exec -u 1000 gitea gitea --config /data/gitea/conf/app.ini actions generate-runner-token | tail -1 | tr -d '\r') +[[ ${#TOKEN} -ge 20 ]] || { echo "could not get a token: $TOKEN"; exit 1; } +if grep -q '^GITEA_RUNNER_TOKEN=' .env; then sed -i "s|^GITEA_RUNNER_TOKEN=.*|GITEA_RUNNER_TOKEN=$TOKEN|" .env; else echo "GITEA_RUNNER_TOKEN=$TOKEN" >> .env; fi +sudo rm -rf /srv/appdata/act_runner/.runner 2>/dev/null || true # forget any previous registration +docker compose up -d --force-recreate act_runner +sleep 6; docker logs act_runner --tail 8 2>&1 | sed 's/^/ /' +echo "check: Gitea -> Site Administration -> Actions -> Runners (should list 'vanguard-runner' as idle)" diff --git a/syncthing/stignore b/syncthing/stignore new file mode 100644 index 0000000..33777d4 --- /dev/null +++ b/syncthing/stignore @@ -0,0 +1,14 @@ +// Server-side ignore list for a clips receive folder (clips/): pull video only. +// .stlocal is appended to by the sweep: raws it converted (then deleted locally) and the standard +// files it produced. Ignored = not tracked, not flagged, never re-pulled, never sent back. +#include .stlocal +// Ignored files are never downloaded from the sender. Subfolders are still traversed +// because include (!) patterns are present. (?i) = case-insensitive. +(?i)!*.mp4 +(?i)!*.mkv +(?i)!*.mov +(?i)!*.m4v +(?i)!*.webm +(?i)!*.ts +// everything else is ignored +* diff --git a/systemd/dumpd.service b/systemd/dumpd.service new file mode 100644 index 0000000..58f55d1 --- /dev/null +++ b/systemd/dumpd.service @@ -0,0 +1,15 @@ +[Unit] +RequiresMountsFor=/srv/media +Description=Dump-tonight endpoint (127.0.0.1:8080) +After=network.target + +[Service] +ExecStart=/usr/bin/python3 /usr/local/bin/dumpd.py +Environment=DUMP_USERS=/etc/vanguard/dump-users.txt +Environment=DUMP_TZ=America/New_York +Restart=always +RestartSec=3 +User=root + +[Install] +WantedBy=multi-user.target diff --git a/systemd/normalize-sweep.service b/systemd/normalize-sweep.service new file mode 100644 index 0000000..58d4c9e --- /dev/null +++ b/systemd/normalize-sweep.service @@ -0,0 +1,10 @@ +[Unit] +Description=Normalize finished recordings/clips to the editing standard +RequiresMountsFor=/srv/media +After=docker.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/normalize-sweep.sh +Nice=15 +IOSchedulingClass=idle diff --git a/systemd/normalize-sweep.timer b/systemd/normalize-sweep.timer new file mode 100644 index 0000000..3625619 --- /dev/null +++ b/systemd/normalize-sweep.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run normalize sweep every 5 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +AccuracySec=30s + +[Install] +WantedBy=timers.target diff --git a/systemd/trash-purge.service b/systemd/trash-purge.service new file mode 100644 index 0000000..2cb4f47 --- /dev/null +++ b/systemd/trash-purge.service @@ -0,0 +1,7 @@ +[Unit] +RequiresMountsFor=/srv/media +Description=Purge dumped nights older than 3 days + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/trash-purge.sh diff --git a/systemd/trash-purge.timer b/systemd/trash-purge.timer new file mode 100644 index 0000000..930fed5 --- /dev/null +++ b/systemd/trash-purge.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily trash purge + +[Timer] +OnCalendar=*-*-* 05:00:00 +Persistent=true + +[Install] +WantedBy=timers.target