Stuck-recording triage, a diag landing page, and the fixes behind Tuesday's symptoms
Triage - bin/vgstuck.sh classifies every unpublished recording (parked by a failure log, published collision, waiting, still writing), with ffmpeg's own reason for parked ones. retry clears the blocker; drop moves to trash/<today>/ with the same 3-day undo as a dumped night; drop --all is refused. A leftover .mp4.part blocks nothing (ffmpeg overwrites with -y) and is only cleaned up. - The same tool backs a page at diag.HOST/stuck. dumpd serves /stuck/api, lets anyone past the gate read, and allows retry/bin only for the admin groups; the path validation is the single thing between a request body and a file move, so it accepts only an existing .mp4 under .live/ inside the streams tree. - The sweep logs event=parked once per run so Kibana can name stuck files rather than only count them. Landing - diag.HOST opens on a "Vanguard ops" dashboard via Kibana's defaultRoute: a markdown panel of links to every host and the triage page, plus Discover, Dashboards, Stack Rules and Index Management. Kibana stays at the root; moving it under /kibana would have meant the prefix in both Beats' setup.kibana, two scripts and the deadman, with Metricbeat crash-looping on whichever was missed. - /api/kibana/settings is an internal API behind server.restrictInternalApis, which Kibana 9 turns on by default, so the script sends x-elastic-internal-origin. kibana-setup.py takes a step name and verifies each step by reading it back. Installers - bin/install.list is the one list of what lands in /usr/local/bin, read by install.sh, configure.py and deploy.sh. deploy.sh's own copy had drifted to five of ten files. check.sh fails on drift. deploy.sh also recreates a Beat or Kibana when its bind-mounted config changes, which nothing did before. - check.sh distinguishes absent from broken: the server checkout is partial, and a yaml file or node binary that is not there is a skip, not a failure. Fixes - Go Live pixelation: x-google-min-bitrate=6000 forbade Chrome from encoding below 6 Mbps, so on a weak uplink the excess was dropped and MediaMTX logged invalid FU-A packets. Loss was on the publish sessions only. Floor is 1200 and scaleResolutionDownBy is unpinned. - HLS manifestParsingError: handle_path strips /hls, so MediaMTX's cookieCheck redirect came back root-relative and landed on the SPA catch-all. header_down rewrites Location. - Sweep audio: the band-removal branch overwrote $reason and lost the probe's audio=opus marker, so a Go Live recording with the band shipped Opus inside an MP4. probe_reason is kept separately. - Deadman false alarm: alerts.state was only written when something changed, so quiet runs let it go stale. Every run stamps last_run now, and the deadman also checks is-failed on the unit. - Both timer scripts load /etc/vanguard/alerts.env themselves, so a run by hand matches the timer. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
41
check.sh
41
check.sh
@@ -13,12 +13,43 @@ for f in configure.py bin/dumpd.py bin/vgstrip.py observability/kibana-setup.py
|
||||
python3 -m py_compile "$f" && say "$f" ok || { say "$f" "PYTHON SYNTAX"; fail=1; }
|
||||
done
|
||||
rm -rf __pycache__ bin/__pycache__
|
||||
python3 -c "import yaml,sys; [yaml.safe_load(open(f)) for f in sys.argv[1:]]" docker-compose.yml mediamtx.example.yml observability/filebeat.yml observability/metricbeat.yml observability/kibana.yml && say "yaml" ok || { say "yaml" "INVALID"; fail=1; }
|
||||
node -e "
|
||||
# bin/install.list is the single source for what lands in /usr/local/bin. Fail if it names something that
|
||||
# does not exist, or if a file in bin/ is neither listed nor explicitly excluded — that drift is exactly how
|
||||
# deploy.sh ended up installing five of ten scripts.
|
||||
if python3 - <<'PYMAN' >/dev/null 2>&1
|
||||
import re, sys
|
||||
from pathlib import Path
|
||||
text = Path("bin/install.list").read_text()
|
||||
listed = dict((m.group(2), m.group(1)) for m in re.finditer(r"^(0\d{3}) (\S+)$", text, re.M))
|
||||
excluded = set(re.findall(r"^#\s+exclude (\S+)", text, re.M))
|
||||
present = set("bin/" + p.name for p in Path("bin").iterdir() if p.is_file() and p.name != "install.list")
|
||||
missing = set(listed) - present
|
||||
unclaimed = present - set(listed) - excluded
|
||||
assert not missing, f"install.list names files that do not exist: {sorted(missing)}"
|
||||
assert not unclaimed, f"bin/ files neither listed nor excluded in install.list: {sorted(unclaimed)}"
|
||||
PYMAN
|
||||
then say "bin/install.list" ok; else say "bin/install.list" "DRIFTED — see the assertion"; fail=1; fi
|
||||
# Only files that are actually here. The server checkout is partial (scp, not a full clone), and a tool that
|
||||
# reports "broken" when it means "absent" gets ignored — the same reason the deadman reports on change only.
|
||||
yamls=(); for f in docker-compose.yml mediamtx.example.yml observability/filebeat.yml observability/metricbeat.yml observability/kibana.yml; do [[ -f $f ]] && yamls+=("$f"); done
|
||||
python3 -c "import yaml,sys; [yaml.safe_load(open(f)) for f in sys.argv[1:]]" "${yamls[@]}" \
|
||||
&& say "yaml (${#yamls[@]} files)" ok || { say "yaml" "INVALID"; fail=1; }
|
||||
if ! command -v node >/dev/null; then
|
||||
say "page JS" "(skipped: node not installed here)"
|
||||
else
|
||||
node -e "
|
||||
const fs=require('fs');
|
||||
for (const f of ['dock/index.html','dock/clock/index.html','clips/index.html','about/index.html']) { const html=fs.readFileSync(f,'utf8'); for (const m of html.matchAll(/<script(?:[^>]*)>([\s\S]*?)<\/script>/g)) if (m[1].trim()) new Function(m[1]); }
|
||||
new Function(fs.readFileSync('dock/clock/vgclock.js','utf8'));" && say "dock + clock JS" ok || { say "dock + clock JS" "SYNTAX ERROR"; fail=1; }
|
||||
diff <(sed '$d' live/src/vgclock.js | sed 's/})(typeof window.*/})(window);/') dock/clock/vgclock.js >/dev/null && say "vgclock.js in sync (dock/portal)" ok || say "vgclock.js in sync (dock/portal)" "DIFFERS — copy dock/clock/vgclock.js to live/src and re-add the export line"
|
||||
for (const f of ['dock/index.html','dock/clock/index.html','clips/index.html','about/index.html','stuck/index.html']) { if (!fs.existsSync(f)) continue; const html=fs.readFileSync(f,'utf8'); for (const m of html.matchAll(/<script(?:[^>]*)>([\s\S]*?)<\/script>/g)) if (m[1].trim()) new Function(m[1]); }
|
||||
if (fs.existsSync('dock/clock/vgclock.js')) new Function(fs.readFileSync('dock/clock/vgclock.js','utf8'));" \
|
||||
&& say "page JS" ok || { say "page JS" "SYNTAX ERROR"; fail=1; }
|
||||
fi
|
||||
if [[ -f live/src/vgclock.js && -f dock/clock/vgclock.js ]]; then
|
||||
diff <(sed '$d' live/src/vgclock.js | sed 's/})(typeof window.*/})(window);/') dock/clock/vgclock.js >/dev/null \
|
||||
&& say "vgclock.js in sync (dock/portal)" ok \
|
||||
|| say "vgclock.js in sync (dock/portal)" "DIFFERS — copy dock/clock/vgclock.js to live/src and re-add the export line"
|
||||
else
|
||||
say "vgclock.js in sync" "(skipped: only one copy in this checkout)"
|
||||
fi
|
||||
# generated Caddyfile / mediamtx / copyparty from a dry configure run must at least be produced + parse
|
||||
T=$(mktemp -d); cp configure.py "$T/"; ( cd "$T" && VG_DRY=1 python3 -c "
|
||||
import configure as c, yaml
|
||||
|
||||
Reference in New Issue
Block a user