From 9bdf9f1947f77c92798341574d0d178672232122 Mon Sep 17 00:00:00 2001 From: Brandyn / Techy Date: Sun, 1 Nov 2026 13:58:15 -0600 Subject: [PATCH] WebRTC audio mirror: AAC cannot cross WebRTC, so transcode audio to Opus into .webrtc bin/mtx-on-ready.sh (runOnReady inside the mediamtx container) posts the is-live notice and, for sources whose audio is not Opus, runs ffmpeg: video copy, audio -> Opus, published over MediaMTX's loopback RTSP to /.webrtc (record: no). dumpd accepts .webrtc publishes only from 127.0.0.1. The portal hides mirror paths, plays the mirror over WHEP (HLS stays on the source), holds an AAC source for 15 s while its mirror comes up, and corrects the viewer count. Verified headless with a mocked path list. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --- README.md | 11 +++++++++-- STATUS.md | 3 ++- bin/dumpd.py | 8 +++++++- bin/mtx-on-ready.sh | 36 ++++++++++++++++++++++++++++++++++++ configure.py | 20 +++++++++++++------- docker-compose.yml | 1 + live/src/Player.jsx | 5 +++-- live/src/api.js | 41 ++++++++++++++++++++++++++++------------- 8 files changed, 99 insertions(+), 26 deletions(-) create mode 100755 bin/mtx-on-ready.sh diff --git a/README.md b/README.md index 817af72..00fc8a9 100644 --- a/README.md +++ b/README.md @@ -299,8 +299,15 @@ Optional later: hide the password form (custom template) so Discord is the only React + dockview page (`live/`) behind Authentik forward-auth: every active stream is a dockable window, audio follows the focused window, per-window mute + volume. WebRTC (WHEP) with HLS fallback, served by MediaMTX (HLS 8888, WebRTC 8889, API 9997 on loopback; WebRTC media on **8189** udp/tcp, firewall opened). -MediaMTX runs the `-ffmpeg` image so the optional "is live" Discord hook (`LIVE_WEBHOOK_URL` in `.env`, -asked by `configure.py --full`) can post when a stream starts. +MediaMTX runs the `-ffmpeg` image for the on-ready hook `bin/mtx-on-ready.sh` (bind-mounted into the container): +the optional "is live" Discord post (`LIVE_WEBHOOK_URL` in `.env`, asked by `configure.py --full`) and the +**WebRTC audio mirror**. Browsers cannot receive AAC over WebRTC (MediaMTX forwards Opus/G711/G722/LPCM only) and +OBS sends AAC, so for every stream whose audio is not already Opus the hook runs one ffmpeg that copies the video +and transcodes the audio to Opus into the sibling path `<user>/<Title>.webrtc` (via MediaMTX's RTSP on the +container loopback). The portal plays the mirror over WebRTC and hides it as a panel; recordings, HLS and the +Discord post stay on the original. Mirrors are never recorded and can only be published from inside the container +(dumpd checks the source IP). Browser Go Live already sends Opus and gets no mirror. Cost: a few % of a core per stream. +Viewer requirement on the streamer side: **B-frames 0** (section 5), or WebRTC is refused and the portal falls back to HLS. 1. CNAME `live` → main host; `./configure.py --full` asks the hostname (also `--hosts … LIVE`). 2. Authentik: Proxy Provider (forward auth, single application, external host `https://live.HOST`) → diff --git a/STATUS.md b/STATUS.md index 1c84b99..afe7d52 100644 --- a/STATUS.md +++ b/STATUS.md @@ -50,7 +50,7 @@ Source of truth: this repo, `techy/Media-Server-Setup` on the stack's own Gitea | Service | Role | State | |---|---|---| -| MediaMTX | RTMP/SRT ingest, records fMP4; publish auth delegated to dumpd (tokens via `/mtx-auth`) | live, token auth verified | +| MediaMTX | RTMP/SRT ingest, records fMP4; publish auth delegated to dumpd (tokens via `/mtx-auth`); on-ready hook posts "is live" and runs the WebRTC audio mirror (`<path>.webrtc`, audio → Opus, never recorded) | live, token auth verified | | copyparty | browse/scrub/download/finals upload | behind Authentik, group permissions (`editors`, `media-admins`), Discord-only, no WebDAV | live | | dumpd + dock | dump, stream tokens, MediaMTX auth hook, `/time`, `/me` identity (on `live.`), `/me/clips` Syncthing self-service (on `clips.`); OBS dock with timecode-band setup | identity from Authentik headers | live | | `media.` gateway, `clips.` self-service | static pages behind the gate | — | live (first real pairing pending) | @@ -71,6 +71,7 @@ Source of truth: this repo, `techy/Media-Server-Setup` on the stack's own Gitea ## Unfinished, in rough priority +0a. **WebRTC audio** (found 2026-11-01 with the 5-stream test): AAC cannot cross WebRTC, so the portal was silent unless it fell back to HLS. Fix committed: audio mirror in `bin/mtx-on-ready.sh` + portal mapping. **Deploy pending** (MediaMTX recreate between streams; dumpd restart; portal push). Streamers must set B-frames 0 for WebRTC playback. 0. **Band removal**: deployed 2026-11-01; `vgdeband.sh` catch-up ran on the Vanguard-Training recordings (check `journalctl -u vgdeband` for the done lines). Live portal: viewer now clips the band with a measured game-aspect frame (refits on every dockview resize/move) — committed 2026-11-01, deploy with `./deployLive.sh`. 1. Roles→groups policy: bound and updated (revokes only its own grants); will be proven by the next natural Discord logins (`/whoami` should show `bot-users`). Nested groups **verified 2026-11-01**: `git-admins` now reaches the header via the `Vanguard profile (nested groups)` scope mapping (README 5e). `akadmin` remains the recovery login if a policy misfires. diff --git a/bin/dumpd.py b/bin/dumpd.py index c1cac8d..479e0a3 100755 --- a/bin/dumpd.py +++ b/bin/dumpd.py @@ -418,11 +418,17 @@ class H(BaseHTTPRequestHandler): return self._json(200, {"ok": True}) # viewers are gated by Caddy + Authentik if action != "publish": return self._json(401, {"error": f"action {action} not allowed"}) + path = (req.get("path") or "").lower() + if path.endswith(".webrtc"): + # audio mirrors (<user>/<Title>.webrtc) are published by the hook inside the mediamtx container only + if req.get("ip") in ("127.0.0.1", "::1") and re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+\.webrtc", path): + return self._json(200, {"ok": True}) + print(f"{datetime.now(TZ):%F %T} mtx-auth DENY publish path={path} ip={req.get('ip')} (mirror paths are internal)", flush=True) + return self._json(401, {"error": ".webrtc paths are reserved"}) cred = req.get("password") or req.get("token") or "" if cred.lower().startswith("bearer "): cred = cred[7:] if ":" in cred and "." in cred.split(":", 1)[1]: cred = cred.split(":", 1)[1] # "user:token" form user = verify_token(cred) - path = (req.get("path") or "").lower() if not user: print(f"{datetime.now(TZ):%F %T} mtx-auth DENY publish path={path} ip={req.get('ip')} (bad/expired token)", flush=True) return self._json(401, {"error": "invalid or expired stream token"}) diff --git a/bin/mtx-on-ready.sh b/bin/mtx-on-ready.sh new file mode 100755 index 0000000..5fc0803 --- /dev/null +++ b/bin/mtx-on-ready.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# MediaMTX runOnReady hook (runs INSIDE the mediamtx container; busybox sh + wget + ffmpeg from the -ffmpeg image). +# Env from MediaMTX: MTX_PATH (user/Title). From compose: LIVE_WEBHOOK_URL, LIVE_HOST. +# 1. optional Discord "is live" notice +# 2. WebRTC audio mirror: browsers cannot receive AAC over WebRTC (MediaMTX forwards Opus/G711/G722/LPCM only), and +# OBS sends AAC. For every stream whose audio is not already Opus, copy the video and transcode the audio to Opus +# into the sibling path "<user>/<Title>.webrtc". The live portal plays that path over WebRTC; recordings, HLS and +# the Discord notice stay on the original. Mirrors are published from 127.0.0.1 inside this container only, which +# is the only publisher dumpd accepts for ".webrtc" paths. Audio-only transcode: a few % of one core per stream. +# MediaMTX kills this process when the source path goes away. +case "$MTX_PATH" in *.webrtc) exit 0 ;; esac +u="${MTX_PATH%%/*}"; t="${MTX_PATH#*/}" +if [ -n "$LIVE_WEBHOOK_URL" ]; then + wget -q -O /dev/null --header="Content-Type: application/json" \ + --post-data="{\"content\":\"🔴 **$u** is live: $t — https://$LIVE_HOST/\"}" "$LIVE_WEBHOOK_URL" 2>/dev/null & +fi +# which audio does the source carry? (API on loopback; "Opus" needs no mirror, no audio needs no mirror) +tracks="" +for i in 1 2 3 4 5; do + tracks=$(wget -q -O - "http://127.0.0.1:9997/v3/paths/get/$MTX_PATH" 2>/dev/null) && [ -n "$tracks" ] && break + sleep 1 +done +case "$tracks" in + *Opus*) exit 0 ;; + *"MPEG-4 Audio"*|*"MPEG-1 Audio"*|*G711*|*G722*|*LPCM*|*AC-3*) ;; + *) exit 0 ;; # no (recognised) audio: nothing to mirror +esac +echo "mirror: $MTX_PATH -> $MTX_PATH.webrtc (audio -> Opus)" +n=0 +while [ $n -lt 10 ]; do # retry: the source may not be readable for the first second + ffmpeg -nostdin -hide_banner -loglevel warning -fflags nobuffer -flags low_delay \ + -rtsp_transport tcp -i "rtsp://127.0.0.1:8554/$MTX_PATH" \ + -map 0:v:0 -map 0:a:0 -c:v copy -c:a libopus -b:a 128k -ar 48000 -ac 2 -application lowdelay -frame_duration 20 \ + -f rtsp -rtsp_transport tcp "rtsp://127.0.0.1:8554/$MTX_PATH.webrtc" + n=$((n+1)); sleep 2 +done diff --git a/configure.py b/configure.py index 53c6c33..1549d5a 100755 --- a/configure.py +++ b/configure.py @@ -372,7 +372,12 @@ def write_mediamtx(public_host=""): Path("mediamtx.yml").write_text(f"""# MediaMTX — ingest only. No playback endpoints exposed. (generated by configure.py) logLevel: info -rtsp: no +# RTSP only on the container loopback: the audio-mirror hook (bin/mtx-on-ready.sh) reads the source and publishes +# "<path>.webrtc" through it. Not published by compose, unreachable from outside. +rtsp: yes +rtspAddress: 127.0.0.1:8554 +rtspTransports: [tcp] +rtspEncryption: "no" moq: no metrics: no pprof: no @@ -417,15 +422,16 @@ pathDefaults: recordPartDuration: 1s recordSegmentDuration: 12h # one file per stream unless it runs >12h recordDeleteAfter: 0s # never auto-delete; the archive job handles it - # optional Discord "is live" notice (LIVE_WEBHOOK_URL set in .env). busybox wget in the -ffmpeg image. - runOnReady: > - sh -c '[ -n "$LIVE_WEBHOOK_URL" ] || exit 0; - u="${{MTX_PATH%%/*}}"; t="${{MTX_PATH#*/}}"; - wget -q -O /dev/null --header="Content-Type: application/json" - --post-data="{{\\"content\\":\\"🔴 **$u** is live: $t — https://$LIVE_HOST/\\"}}" "$LIVE_WEBHOOK_URL"' + # on ready: optional Discord "is live" notice (LIVE_WEBHOOK_URL in .env) + the WebRTC audio mirror + # (bin/mtx-on-ready.sh, bind-mounted by compose; AAC cannot cross WebRTC, so audio is transcoded to Opus into <path>.webrtc) + runOnReady: /vanguard/on-ready.sh runOnReadyRestart: no paths: + # audio mirrors (first rule wins): never recorded, no hook; only publishable from inside the container (dumpd) + ~^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\\.webrtc$: + record: no + runOnReady: "" # <username>/<Title>; dumpd decides who may publish where (token user == prefix) ~^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$: {{}} """) diff --git a/docker-compose.yml b/docker-compose.yml index 7a6bf7d..ca3b5e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: - "127.0.0.1:9997:9997" # API -> Caddy live.HOST/api/* volumes: - ./mediamtx.yml:/mediamtx.yml:ro + - ./bin/mtx-on-ready.sh:/vanguard/on-ready.sh:ro # is-live notice + WebRTC audio mirror (runOnReady) - /srv/media/streams:/recordings restart: unless-stopped diff --git a/live/src/Player.jsx b/live/src/Player.jsx index 98c3464..1d86d2b 100644 --- a/live/src/Player.jsx +++ b/live/src/Player.jsx @@ -6,6 +6,7 @@ import { audio, effectiveMuted } from './useAudio.js' // WebRTC (WHEP) first, HLS fallback. Mute/volume come from the shared audio store. export default function Player({ params, api }) { const { id, user, title } = params + const whep = params.whep || id // Opus audio mirror path when the source audio is AAC const videoRef = useRef(null) const [status, setStatus] = useState('connecting…') const [viewers, setViewers] = useState(params.viewers ?? 0) @@ -78,7 +79,7 @@ export default function Player({ params, api }) { const offer = await pc.createOffer() await pc.setLocalDescription(offer) await new Promise(res => { if (pc.iceGatheringState === 'complete') return res(); const t = setTimeout(res, 1500); pc.onicegatheringstatechange = () => { if (pc.iceGatheringState === 'complete') { clearTimeout(t); res() } } }) - const r = await fetch(whepUrl(id), { method: 'POST', headers: { 'Content-Type': 'application/sdp' }, body: pc.localDescription.sdp, credentials: 'same-origin' }) + const r = await fetch(whepUrl(whep), { method: 'POST', headers: { 'Content-Type': 'application/sdp' }, body: pc.localDescription.sdp, credentials: 'same-origin' }) if (!r.ok) throw new Error('whep ' + r.status) await pc.setRemoteDescription({ type: 'answer', sdp: await r.text() }) setStatus('webrtc') @@ -90,7 +91,7 @@ export default function Player({ params, api }) { } startWhep() return () => { cancelled = true; try { pc?.close() } catch {}; try { hls?.destroy() } catch {}; v.srcObject = null; v.removeAttribute('src') } - }, [id]) + }, [id, whep]) const icon = muted ? '🔇' : volume === 0 ? '🔈' : volume < 0.5 ? '🔉' : '🔊' return ( diff --git a/live/src/api.js b/live/src/api.js index 278aae7..074958d 100644 --- a/live/src/api.js +++ b/live/src/api.js @@ -1,22 +1,37 @@ // MediaMTX control API, proxied by Caddy at /api (-> 127.0.0.1:9997). Returns active streams. +const VIDEO = new Set(['H264', 'H265', 'VP8', 'VP9', 'AV1', 'MJPEG', 'M-JPEG', 'MPEG-4 Video', 'MPEG-1 Video']) +const firstSeen = new Map() // path -> first time we saw it ready (mirror grace period) +const MIRROR_WAIT_MS = 15000 + export async function fetchStreams() { const r = await fetch('/api/v3/paths/list?itemsPerPage=200', { credentials: 'same-origin' }) if (!r.ok) throw new Error(`api ${r.status}`) const data = await r.json() - return (data.items || []) - .filter(p => p.ready && (p.tracks || []).length > 0) - .map(p => { - const [user, ...rest] = p.name.split('/') - return { - id: p.name, // e.g. techy/VanguardTraining - user, - title: rest.join('/') || p.name, - viewers: (p.readers || []).length, - since: p.readyTime, - tracks: p.tracks || [], - } + const ready = (data.items || []).filter(p => p.ready && (p.tracks || []).length > 0) + const byName = new Map(ready.map(p => [p.name, p])) + const now = Date.now() + for (const p of ready) if (!firstSeen.has(p.name)) firstSeen.set(p.name, now) + for (const k of [...firstSeen.keys()]) if (!byName.has(k)) firstSeen.delete(k) + const out = [] + for (const p of ready) { + if (p.name.endsWith('.webrtc')) continue // audio mirror: played through its source's panel, never its own + const tracks = p.tracks || [] + const audio = tracks.filter(t => !VIDEO.has(t)) + const needsMirror = audio.length > 0 && !audio.includes('Opus') // AAC etc. cannot cross WebRTC + const mirror = byName.get(p.name + '.webrtc') + if (needsMirror && !mirror && now - firstSeen.get(p.name) < MIRROR_WAIT_MS) continue // give the mirror a moment to come up + const [user, ...rest] = p.name.split('/') + out.push({ + id: p.name, // e.g. techy/VanguardTraining + user, + title: rest.join('/') || p.name, + whep: mirror ? mirror.name : p.name, // WebRTC plays the Opus mirror when there is one; HLS always the source + viewers: (p.readers || []).length + (mirror ? (mirror.readers || []).length - 1 : 0), // the mirror's ffmpeg is one reader + since: p.readyTime, + tracks, }) - .sort((a, b) => (a.since || '').localeCompare(b.since || '')) + } + return out.sort((a, b) => (a.since || '').localeCompare(b.since || '')) } export const whepUrl = id => `/webrtc/${id}/whep`