Files
Media-Server-Setup/stuck/index.html
Brandyn / Techy ca01516d42 Stuck-recording triage, a diag landing page, and the fixes behind Tuesday's symptoms
Triage
- bin/vgstuck.sh classifies every unpublished recording (parked by a failure log, published
  collision, waiting, still writing), with ffmpeg's own reason for parked ones. retry clears the
  blocker; drop moves to trash/<today>/ with the same 3-day undo as a dumped night; drop --all is
  refused. A leftover .mp4.part blocks nothing (ffmpeg overwrites with -y) and is only cleaned up.
- The same tool backs a page at diag.HOST/stuck. dumpd serves /stuck/api, lets anyone past the gate
  read, and allows retry/bin only for the admin groups; the path validation is the single thing
  between a request body and a file move, so it accepts only an existing .mp4 under .live/ inside
  the streams tree.
- The sweep logs event=parked once per run so Kibana can name stuck files rather than only count
  them.

Landing
- diag.HOST opens on a "Vanguard ops" dashboard via Kibana's defaultRoute: a markdown panel of links
  to every host and the triage page, plus Discover, Dashboards, Stack Rules and Index Management.
  Kibana stays at the root; moving it under /kibana would have meant the prefix in both Beats'
  setup.kibana, two scripts and the deadman, with Metricbeat crash-looping on whichever was missed.
- /api/kibana/settings is an internal API behind server.restrictInternalApis, which Kibana 9 turns
  on by default, so the script sends x-elastic-internal-origin. kibana-setup.py takes a step name and
  verifies each step by reading it back.

Installers
- bin/install.list is the one list of what lands in /usr/local/bin, read by install.sh,
  configure.py and deploy.sh. deploy.sh's own copy had drifted to five of ten files. check.sh fails
  on drift. deploy.sh also recreates a Beat or Kibana when its bind-mounted config changes, which
  nothing did before.
- check.sh distinguishes absent from broken: the server checkout is partial, and a yaml file or node
  binary that is not there is a skip, not a failure.

Fixes
- Go Live pixelation: x-google-min-bitrate=6000 forbade Chrome from encoding below 6 Mbps, so on a
  weak uplink the excess was dropped and MediaMTX logged invalid FU-A packets. Loss was on the
  publish sessions only. Floor is 1200 and scaleResolutionDownBy is unpinned.
- HLS manifestParsingError: handle_path strips /hls, so MediaMTX's cookieCheck redirect came back
  root-relative and landed on the SPA catch-all. header_down rewrites Location.
- Sweep audio: the band-removal branch overwrote $reason and lost the probe's audio=opus marker, so a
  Go Live recording with the band shipped Opus inside an MP4. probe_reason is kept separately.
- Deadman false alarm: alerts.state was only written when something changed, so quiet runs let it go
  stale. Every run stamps last_run now, and the deadman also checks is-failed on the unit.
- Both timer scripts load /etc/vanguard/alerts.env themselves, so a run by hand matches the timer.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-04 21:25:43 -06:00

114 lines
5.6 KiB
HTML

<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Stuck recordings — Vanguard</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
:root{--bg:#141414;--fg:#e6e6e6;--mut:#9a9a9a;--acc:#3d8bfd;--ok:#2ecc71;--warn:#e3a008;--bad:#e05260;--box:#1f1f1f;--line:#2e2e2e}
html,body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-serif}
main{max-width:1040px;margin:0 auto;padding:28px 16px 48px}
header{display:flex;align-items:baseline;gap:14px;flex-wrap:wrap;margin-bottom:6px}
h1{font-size:26px;margin:0}.who{color:var(--mut)}.who b{color:var(--fg)}
p.lead{color:var(--mut);margin:4px 0 20px}
section{background:var(--box);border:1px solid var(--line);border-radius:14px;padding:16px 18px;margin-bottom:14px}
.row{display:grid;grid-template-columns:96px 1fr auto;gap:12px;align-items:start}
.state{font-size:12px;border-radius:999px;padding:2px 10px;text-align:center;align-self:start;white-space:nowrap}
.parked{background:#43242a;color:var(--bad)}.published{background:#3a3220;color:var(--warn)}
.waiting{background:#1e3147;color:var(--acc)}.writing{background:#1f3b2a;color:var(--ok)}
.name{font-family:ui-monospace,monospace;font-size:13px;word-break:break-all}
.meta{color:var(--mut);font-size:13px;margin-top:2px}
.why{color:var(--mut);font-size:13px;margin-top:6px;border-left:2px solid var(--line);padding-left:10px;white-space:pre-wrap}
.acts{display:flex;gap:8px}
button{font:inherit;font-size:13px;border-radius:8px;border:1px solid #3a3a3a;background:#262626;color:var(--fg);padding:6px 12px;cursor:pointer}
button:hover:not(:disabled){border-color:var(--acc)}button:disabled{opacity:.45;cursor:default}
button.danger:hover:not(:disabled){border-color:var(--bad);color:var(--bad)}
.note{color:var(--mut);font-size:13px}.err{color:var(--bad)}
pre{background:#121212;border-radius:8px;padding:10px;overflow:auto;font-size:12px;margin:8px 0 0}
footer{color:var(--mut);font-size:13px;margin-top:20px}
</style>
</head>
<body>
<main>
<header><h1>Stuck recordings</h1><div class="who" id="who"></div></header>
<p class="lead">Recordings that finished but never reached Jellyfin or copyparty. Retry puts one back in the
queue; bin moves it to the trash, recoverable for three days just like a dumped night.</p>
<div id="list"><section class="note">Loading…</section></div>
<footer>Refreshes every 30&nbsp;seconds. Trends and history live on the diagnostics dashboard.</footer>
</main>
<script>
const API = '/stuck/api';
let admin = false;
const hms = s => { s = Math.max(0, s|0); const h = s/3600|0, m = (s%3600)/60|0; return h ? `${h}h ${m}m` : `${m}m`; };
const dur = s => { s = Math.max(0, s|0); return `${s/3600|0}:${String((s%3600)/60|0).padStart(2,'0')}:${String(s%60).padStart(2,'0')}`; };
const short = p => p.replace(/^.*\/streams\//, '').replace('/.live/', ' · ');
async function load() {
const list = document.getElementById('list');
let data;
try {
const r = await fetch(API, { credentials: 'same-origin' });
if (r.status === 401) { list.innerHTML = '<section class="note err">Not signed in. Reload the page.</section>'; return; }
data = await r.json();
} catch (e) {
list.innerHTML = `<section class="note err">Could not reach the server: ${e.message}</section>`; return;
}
admin = !!data.admin;
document.getElementById('who').innerHTML = `signed in as <b>${data.user || '?'}</b>${admin ? '' : ' · read only'}`;
const items = (data.items || []).sort((a, b) =>
(a.state === b.state ? b.age_s - a.age_s : a.state.localeCompare(b.state)));
if (!items.length) { list.innerHTML = '<section class="note">Nothing is stuck. Everything recorded has published.</section>'; return; }
list.innerHTML = '';
for (const it of items) {
const el = document.createElement('section');
// "writing" means it is still being recorded: acting on it would cut a live recording, so no buttons.
const actionable = admin && it.state !== 'writing';
el.innerHTML = `
<div class="row">
<div class="state ${it.state}">${it.state}</div>
<div>
<div class="name">${short(it.src)}</div>
<div class="meta">waiting ${hms(it.age_s)} · ${dur(it.duration_s)} long · ${(it.bytes/1e6).toFixed(0)} MB</div>
<div class="why">${it.reason || ''}</div>
</div>
<div class="acts">
<button data-act="retry" ${actionable ? '' : 'disabled'}>Retry</button>
<button data-act="drop" class="danger" ${actionable ? '' : 'disabled'}>Bin</button>
</div>
</div>`;
for (const b of el.querySelectorAll('button')) {
b.onclick = () => act(it, b.dataset.act, el);
}
list.appendChild(el);
}
}
async function act(item, action, el) {
if (action === 'drop' && !confirm(`Move this recording to the trash?\n\n${short(item.src)}\n\nRecoverable for three days.`)) return;
for (const b of el.querySelectorAll('button')) b.disabled = true;
const out = document.createElement('pre');
out.textContent = action === 'retry' ? 'Clearing the blocker…' : 'Moving to trash…';
el.appendChild(out);
try {
const r = await fetch(API, {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ action, src: item.src }),
});
const body = await r.json();
out.textContent = body.error ? `Failed: ${body.error}` : (body.output || []).join('\n') || 'Done.';
if (!body.error) setTimeout(load, 1200);
} catch (e) {
out.textContent = `Failed: ${e.message}`;
}
}
load();
setInterval(load, 30000);
</script>
</body>
</html>