Stuck-recording triage, a diag landing page, and the fixes behind Tuesday's symptoms

Triage
- bin/vgstuck.sh classifies every unpublished recording (parked by a failure log, published
  collision, waiting, still writing), with ffmpeg's own reason for parked ones. retry clears the
  blocker; drop moves to trash/<today>/ with the same 3-day undo as a dumped night; drop --all is
  refused. A leftover .mp4.part blocks nothing (ffmpeg overwrites with -y) and is only cleaned up.
- The same tool backs a page at diag.HOST/stuck. dumpd serves /stuck/api, lets anyone past the gate
  read, and allows retry/bin only for the admin groups; the path validation is the single thing
  between a request body and a file move, so it accepts only an existing .mp4 under .live/ inside
  the streams tree.
- The sweep logs event=parked once per run so Kibana can name stuck files rather than only count
  them.

Landing
- diag.HOST opens on a "Vanguard ops" dashboard via Kibana's defaultRoute: a markdown panel of links
  to every host and the triage page, plus Discover, Dashboards, Stack Rules and Index Management.
  Kibana stays at the root; moving it under /kibana would have meant the prefix in both Beats'
  setup.kibana, two scripts and the deadman, with Metricbeat crash-looping on whichever was missed.
- /api/kibana/settings is an internal API behind server.restrictInternalApis, which Kibana 9 turns
  on by default, so the script sends x-elastic-internal-origin. kibana-setup.py takes a step name and
  verifies each step by reading it back.

Installers
- bin/install.list is the one list of what lands in /usr/local/bin, read by install.sh,
  configure.py and deploy.sh. deploy.sh's own copy had drifted to five of ten files. check.sh fails
  on drift. deploy.sh also recreates a Beat or Kibana when its bind-mounted config changes, which
  nothing did before.
- check.sh distinguishes absent from broken: the server checkout is partial, and a yaml file or node
  binary that is not there is a skip, not a failure.

Fixes
- Go Live pixelation: x-google-min-bitrate=6000 forbade Chrome from encoding below 6 Mbps, so on a
  weak uplink the excess was dropped and MediaMTX logged invalid FU-A packets. Loss was on the
  publish sessions only. Floor is 1200 and scaleResolutionDownBy is unpinned.
- HLS manifestParsingError: handle_path strips /hls, so MediaMTX's cookieCheck redirect came back
  root-relative and landed on the SPA catch-all. header_down rewrites Location.
- Sweep audio: the band-removal branch overwrote $reason and lost the probe's audio=opus marker, so a
  Go Live recording with the band shipped Opus inside an MP4. probe_reason is kept separately.
- Deadman false alarm: alerts.state was only written when something changed, so quiet runs let it go
  stale. Every run stamps last_run now, and the deadman also checks is-failed on the unit.
- Both timer scripts load /etc/vanguard/alerts.env themselves, so a run by hand matches the timer.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

stuck/index.html

@@ -0,0 +1,113 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>Stuck recordings — Vanguard</title>
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<style>
+  :root{--bg:#141414;--fg:#e6e6e6;--mut:#9a9a9a;--acc:#3d8bfd;--ok:#2ecc71;--warn:#e3a008;--bad:#e05260;--box:#1f1f1f;--line:#2e2e2e}
+  html,body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-serif}
+  main{max-width:1040px;margin:0 auto;padding:28px 16px 48px}
+  header{display:flex;align-items:baseline;gap:14px;flex-wrap:wrap;margin-bottom:6px}
+  h1{font-size:26px;margin:0}.who{color:var(--mut)}.who b{color:var(--fg)}
+  p.lead{color:var(--mut);margin:4px 0 20px}
+  section{background:var(--box);border:1px solid var(--line);border-radius:14px;padding:16px 18px;margin-bottom:14px}
+  .row{display:grid;grid-template-columns:96px 1fr auto;gap:12px;align-items:start}
+  .state{font-size:12px;border-radius:999px;padding:2px 10px;text-align:center;align-self:start;white-space:nowrap}
+  .parked{background:#43242a;color:var(--bad)}.published{background:#3a3220;color:var(--warn)}
+  .waiting{background:#1e3147;color:var(--acc)}.writing{background:#1f3b2a;color:var(--ok)}
+  .name{font-family:ui-monospace,monospace;font-size:13px;word-break:break-all}
+  .meta{color:var(--mut);font-size:13px;margin-top:2px}
+  .why{color:var(--mut);font-size:13px;margin-top:6px;border-left:2px solid var(--line);padding-left:10px;white-space:pre-wrap}
+  .acts{display:flex;gap:8px}
+  button{font:inherit;font-size:13px;border-radius:8px;border:1px solid #3a3a3a;background:#262626;color:var(--fg);padding:6px 12px;cursor:pointer}
+  button:hover:not(:disabled){border-color:var(--acc)}button:disabled{opacity:.45;cursor:default}
+  button.danger:hover:not(:disabled){border-color:var(--bad);color:var(--bad)}
+  .note{color:var(--mut);font-size:13px}.err{color:var(--bad)}
+  pre{background:#121212;border-radius:8px;padding:10px;overflow:auto;font-size:12px;margin:8px 0 0}
+  footer{color:var(--mut);font-size:13px;margin-top:20px}
+</style>
+</head>
+<body>
+<main>
+  <header><h1>Stuck recordings</h1><div class="who" id="who"></div></header>
+  <p class="lead">Recordings that finished but never reached Jellyfin or copyparty. Retry puts one back in the
+  queue; bin moves it to the trash, recoverable for three days just like a dumped night.</p>
+  <div id="list"><section class="note">Loading…</section></div>
+  <footer>Refreshes every 30&nbsp;seconds. Trends and history live on the diagnostics dashboard.</footer>
+</main>
+<script>
+const API = '/stuck/api';
+let admin = false;
+
+const hms = s => { s = Math.max(0, s|0); const h = s/3600|0, m = (s%3600)/60|0; return h ? `${h}h ${m}m` : `${m}m`; };
+const dur = s => { s = Math.max(0, s|0); return `${s/3600|0}:${String((s%3600)/60|0).padStart(2,'0')}:${String(s%60).padStart(2,'0')}`; };
+const short = p => p.replace(/^.*\/streams\//, '').replace('/.live/', ' · ');
+
+async function load() {
+  const list = document.getElementById('list');
+  let data;
+  try {
+    const r = await fetch(API, { credentials: 'same-origin' });
+    if (r.status === 401) { list.innerHTML = '<section class="note err">Not signed in. Reload the page.</section>'; return; }
+    data = await r.json();
+  } catch (e) {
+    list.innerHTML = `<section class="note err">Could not reach the server: ${e.message}</section>`; return;
+  }
+  admin = !!data.admin;
+  document.getElementById('who').innerHTML = `signed in as <b>${data.user || '?'}</b>${admin ? '' : ' · read only'}`;
+
+  const items = (data.items || []).sort((a, b) =>
+    (a.state === b.state ? b.age_s - a.age_s : a.state.localeCompare(b.state)));
+  if (!items.length) { list.innerHTML = '<section class="note">Nothing is stuck. Everything recorded has published.</section>'; return; }
+
+  list.innerHTML = '';
+  for (const it of items) {
+    const el = document.createElement('section');
+    // "writing" means it is still being recorded: acting on it would cut a live recording, so no buttons.
+    const actionable = admin && it.state !== 'writing';
+    el.innerHTML = `
+      <div class="row">
+        <div class="state ${it.state}">${it.state}</div>
+        <div>
+          <div class="name">${short(it.src)}</div>
+          <div class="meta">waiting ${hms(it.age_s)} · ${dur(it.duration_s)} long · ${(it.bytes/1e6).toFixed(0)} MB</div>
+          <div class="why">${it.reason || ''}</div>
+        </div>
+        <div class="acts">
+          <button data-act="retry" ${actionable ? '' : 'disabled'}>Retry</button>
+          <button data-act="drop" class="danger" ${actionable ? '' : 'disabled'}>Bin</button>
+        </div>
+      </div>`;
+    for (const b of el.querySelectorAll('button')) {
+      b.onclick = () => act(it, b.dataset.act, el);
+    }
+    list.appendChild(el);
+  }
+}
+
+async function act(item, action, el) {
+  if (action === 'drop' && !confirm(`Move this recording to the trash?\n\n${short(item.src)}\n\nRecoverable for three days.`)) return;
+  for (const b of el.querySelectorAll('button')) b.disabled = true;
+  const out = document.createElement('pre');
+  out.textContent = action === 'retry' ? 'Clearing the blocker…' : 'Moving to trash…';
+  el.appendChild(out);
+  try {
+    const r = await fetch(API, {
+      method: 'POST', credentials: 'same-origin',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ action, src: item.src }),
+    });
+    const body = await r.json();
+    out.textContent = body.error ? `Failed: ${body.error}` : (body.output || []).join('\n') || 'Done.';
+    if (!body.error) setTimeout(load, 1200);
+  } catch (e) {
+    out.textContent = `Failed: ${e.message}`;
+  }
+}
+
+load();
+setInterval(load, 30000);
+</script>
+</body>
+</html>