Triage - bin/vgstuck.sh classifies every unpublished recording (parked by a failure log, published collision, waiting, still writing), with ffmpeg's own reason for parked ones. retry clears the blocker; drop moves to trash/<today>/ with the same 3-day undo as a dumped night; drop --all is refused. A leftover .mp4.part blocks nothing (ffmpeg overwrites with -y) and is only cleaned up. - The same tool backs a page at diag.HOST/stuck. dumpd serves /stuck/api, lets anyone past the gate read, and allows retry/bin only for the admin groups; the path validation is the single thing between a request body and a file move, so it accepts only an existing .mp4 under .live/ inside the streams tree. - The sweep logs event=parked once per run so Kibana can name stuck files rather than only count them. Landing - diag.HOST opens on a "Vanguard ops" dashboard via Kibana's defaultRoute: a markdown panel of links to every host and the triage page, plus Discover, Dashboards, Stack Rules and Index Management. Kibana stays at the root; moving it under /kibana would have meant the prefix in both Beats' setup.kibana, two scripts and the deadman, with Metricbeat crash-looping on whichever was missed. - /api/kibana/settings is an internal API behind server.restrictInternalApis, which Kibana 9 turns on by default, so the script sends x-elastic-internal-origin. kibana-setup.py takes a step name and verifies each step by reading it back. Installers - bin/install.list is the one list of what lands in /usr/local/bin, read by install.sh, configure.py and deploy.sh. deploy.sh's own copy had drifted to five of ten files. check.sh fails on drift. deploy.sh also recreates a Beat or Kibana when its bind-mounted config changes, which nothing did before. - check.sh distinguishes absent from broken: the server checkout is partial, and a yaml file or node binary that is not there is a skip, not a failure. Fixes - Go Live pixelation: x-google-min-bitrate=6000 forbade Chrome from encoding below 6 Mbps, so on a weak uplink the excess was dropped and MediaMTX logged invalid FU-A packets. Loss was on the publish sessions only. Floor is 1200 and scaleResolutionDownBy is unpinned. - HLS manifestParsingError: handle_path strips /hls, so MediaMTX's cookieCheck redirect came back root-relative and landed on the SPA catch-all. header_down rewrites Location. - Sweep audio: the band-removal branch overwrote $reason and lost the probe's audio=opus marker, so a Go Live recording with the band shipped Opus inside an MP4. probe_reason is kept separately. - Deadman false alarm: alerts.state was only written when something changed, so quiet runs let it go stale. Every run stamps last_run now, and the deadman also checks is-failed on the unit. - Both timer scripts load /etc/vanguard/alerts.env themselves, so a run by hand matches the timer. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
127 lines
8.2 KiB
Bash
Executable File
127 lines
8.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# Minimal-impact deployer for the server checkout. Run as root (sudo) — installed as /usr/local/bin/vanguard-deploy.
|
|
# vanguard-deploy pull fetch + fast-forward DEPLOY_BRANCH, then apply only what changed
|
|
# vanguard-deploy apply [OLDREV] apply changes between OLDREV (default: last deployed rev) and HEAD
|
|
# vanguard-deploy rollback check out the previously deployed rev and apply the reverse diff
|
|
# vanguard-deploy pending restart MediaMTX if a restart is pending and no stream is live (timer runs this)
|
|
# vanguard-deploy status
|
|
# Env (from /etc/vanguard/deploy.env): DEPLOY_REPO_DIR, DEPLOY_BRANCH, DEPLOY_NOTIFY_WEBHOOK_URL. VG_DRY=1 prints only.
|
|
set -uo pipefail
|
|
ENVF=/etc/vanguard/deploy.env; [[ -f $ENVF ]] && set -a && . "$ENVF" && set +a
|
|
REPO="${DEPLOY_REPO_DIR:-/home/debian/vanguard-stream}"; BRANCH="${DEPLOY_BRANCH:-main}"
|
|
STATE=/var/lib/vanguard; mkdir -p "$STATE" 2>/dev/null || true
|
|
LASTF="$STATE/deployed-rev"; PREVF="$STATE/previous-rev"; PENDING="$STATE/mediamtx-restart-pending"
|
|
DRY="${VG_DRY:-0}"; OWNER=$(stat -c %U "$REPO" 2>/dev/null || echo debian)
|
|
log() { printf 'ts=%s level=info component=deploy msg="%s"\n' "$(date -Is)" "${*//\"/\\\"}"; }
|
|
run() { log "\$ $*"; [[ $DRY == 1 ]] || "$@"; }
|
|
asowner() { log "\$ (as $OWNER) $*"; [[ $DRY == 1 ]] || sudo -u "$OWNER" -H "$@"; }
|
|
notify() {
|
|
[[ -n ${DEPLOY_NOTIFY_WEBHOOK_URL:-} && $DRY != 1 ]] || return 0
|
|
curl -s -m 10 -H 'Content-Type: application/json' -d "$(python3 -c 'import json,sys; print(json.dumps({"content": sys.argv[1][:1900]}))' "$1")" "$DEPLOY_NOTIFY_WEBHOOK_URL" >/dev/null || true
|
|
}
|
|
cd "$REPO" || { log "repo dir $REPO missing"; exit 1; }
|
|
# "pending" (the 5-min timer) needs no git; every other mode does
|
|
[[ ${1:-} == pending ]] || git rev-parse --is-inside-work-tree >/dev/null 2>&1 || { log "$REPO is not a git checkout (run switchToClone.sh once)"; exit 1; }
|
|
|
|
live_streams() { curl -s -m 3 'http://127.0.0.1:9997/v3/paths/list' 2>/dev/null | python3 -c 'import json,sys; d=json.load(sys.stdin); print(sum(1 for p in d.get("items",[]) if p.get("ready")))' 2>/dev/null || echo 0; }
|
|
|
|
restart_mediamtx_or_defer() {
|
|
local n; n=$(live_streams)
|
|
if [[ $n == 0 ]]; then
|
|
run docker compose restart mediamtx; rm -f "$PENDING"; log "mediamtx restarted"
|
|
else
|
|
[[ $DRY == 1 ]] || touch "$PENDING"; log "mediamtx restart DEFERRED: $n live stream(s); deploy-pending.timer retries every 5 min"
|
|
notify "⏳ deploy: MediaMTX restart deferred ($n live stream(s)); will restart when idle"
|
|
fi
|
|
}
|
|
|
|
apply_changes() {
|
|
local old="$1" new="$2"
|
|
local changed; changed=$(git diff --name-only "$old" "$new" 2>/dev/null || git ls-files)
|
|
[[ -n $changed ]] || { log "nothing changed between $old and $new"; return 0; }
|
|
log "changed files:"; echo "$changed" | sed 's/^/ /'
|
|
has() { echo "$changed" | grep -qE "$1"; }
|
|
|
|
# 0. static checks on the new tree
|
|
if [[ -x check.sh ]]; then asowner bash check.sh >/tmp/vg-check.log 2>&1 || { log "check.sh FAILED — aborting"; tail -20 /tmp/vg-check.log; notify "❌ deploy $new: check.sh failed"; return 1; }; fi
|
|
|
|
# 1. host scripts + units
|
|
# bin/install.list, not a second hand-kept list: this one had drifted to five of ten files, so a push
|
|
# never installed vgstuck.sh, vgmetrics.sh, vgnotify.sh or either timer script.
|
|
if has '^bin/'; then
|
|
while read -r mode path; do
|
|
[[ -f $path ]] && run install -m "$mode" "$path" /usr/local/bin/
|
|
done < <(grep -vE '^[[:space:]]*(#|$)' bin/install.list)
|
|
fi
|
|
if has '^systemd/'; then run install -m 0644 systemd/*.service systemd/*.timer /etc/systemd/system/; run systemctl daemon-reload; fi
|
|
if has '^deploy\.sh$'; then run install -m 0755 deploy.sh /usr/local/bin/vanguard-deploy; fi
|
|
if has '^deployBotBundle\.sh$'; then run install -m 0755 deployBotBundle.sh /usr/local/bin/vanguard-deploy-bot; fi
|
|
if has '^firewall/nftables\.conf$'; then run install -m 0644 firewall/nftables.conf /etc/nftables.conf; run nft -f /etc/nftables.conf; fi
|
|
|
|
# 2. regenerate configs (Caddy reload, copyparty restart) — configure.py decides; MediaMTX is handled below
|
|
if has '^configure\.py$|^syncthing/stignore$|^dock/|^clips/|^about/|^stuck/|^observability/kibana-setup\.py$'; then
|
|
local before_mtx; before_mtx=$(md5sum mediamtx.yml 2>/dev/null | cut -d' ' -f1)
|
|
if [[ $DRY == 1 ]]; then log "\$ (as $OWNER) VG_DRY=1 python3 configure.py --apply"; else sudo -u "$OWNER" -H python3 configure.py --apply >/tmp/vg-configure.log 2>&1 || { log "configure --apply FAILED"; tail -20 /tmp/vg-configure.log; return 1; }; fi
|
|
local after_mtx; after_mtx=$(md5sum mediamtx.yml 2>/dev/null | cut -d' ' -f1)
|
|
[[ $before_mtx == "$after_mtx" ]] || { log "mediamtx.yml changed"; restart_mediamtx_or_defer; }
|
|
fi
|
|
if has '^bin/dumpd\.py$|^systemd/dumpd\.service$'; then run systemctl restart dumpd; fi
|
|
# Beat and Kibana configs are bind-mounted, so a content change needs the container recreated. Nothing
|
|
# handled this before: a push that changed filebeat.yml silently did nothing.
|
|
if has '^observability/filebeat\.yml$'; then run docker compose up -d --force-recreate filebeat; fi
|
|
if has '^observability/metricbeat\.yml$'; then run docker compose up -d --force-recreate metricbeat; fi
|
|
if has '^observability/kibana\.yml$'; then run docker compose up -d --force-recreate kibana; fi
|
|
|
|
# 3. compose: recreate changed services, but never MediaMTX while live
|
|
if has '^docker-compose\.yml$|^bot/Dockerfile$|^bot/entrypoint\.sh$'; then
|
|
local mtx_changed=0
|
|
git diff "$old" "$new" -- docker-compose.yml | grep -qE '^[+-]' && git diff -U0 "$old" "$new" -- docker-compose.yml | grep -qiE '^[+-].*(mediamtx|8189|8890|1935)' && mtx_changed=1
|
|
if [[ $mtx_changed == 1 && $(live_streams) != 0 ]]; then
|
|
log "compose touches mediamtx and streams are live: recreating everything except mediamtx"
|
|
run docker compose up -d --build --no-deps $(docker compose config --services | grep -v '^mediamtx$' | tr '\n' ' ')
|
|
[[ $DRY == 1 ]] || touch "$PENDING"; notify "⏳ deploy: MediaMTX recreate deferred (live streams)"
|
|
else
|
|
run docker compose up -d --build
|
|
fi
|
|
fi
|
|
|
|
# 4. Jellyfin plugin (committed zip)
|
|
if has '^jellyfin-plugin-vanguard/dist/.*\.zip$'; then run bash installJellyfinPlugin.sh; fi
|
|
|
|
# 5. live portal
|
|
if has '^live/'; then
|
|
if command -v npm >/dev/null; then
|
|
asowner bash -c 'cd live && npm install --no-audit --no-fund --loglevel=error && npm run build' && { run install -d -o "$OWNER" /srv/live; run rsync -a --delete live/dist/ /srv/live/; }
|
|
else log "npm not installed on the server: live/ changed but not built (apt-get install nodejs npm)"; fi
|
|
fi
|
|
log "applied $old..$new"
|
|
}
|
|
|
|
case "${1:-}" in
|
|
pull)
|
|
old=$(git rev-parse HEAD)
|
|
asowner git fetch --quiet origin "$BRANCH" || { log "fetch failed"; exit 1; }
|
|
asowner git merge --ff-only "origin/$BRANCH" || { log "fast-forward failed (local changes?)"; exit 1; }
|
|
new=$(git rev-parse HEAD)
|
|
[[ $old == "$new" ]] && { log "already at $new"; exit 0; }
|
|
if apply_changes "$old" "$new"; then
|
|
[[ $DRY == 1 ]] || { cp -f "$LASTF" "$PREVF" 2>/dev/null; echo "$new" > "$LASTF"; }
|
|
notify "✅ deploy $(git log -1 --format='%h %s' "$new")"
|
|
else notify "❌ deploy $new failed — see journalctl -u dumpd"; exit 1; fi ;;
|
|
apply)
|
|
old="${2:-$(cat "$LASTF" 2>/dev/null || git rev-parse HEAD~1)}"; new=$(git rev-parse HEAD)
|
|
apply_changes "$old" "$new" && { [[ $DRY == 1 ]] || { cp -f "$LASTF" "$PREVF" 2>/dev/null; echo "$new" > "$LASTF"; }; } ;;
|
|
rollback)
|
|
prev=$(cat "$PREVF" 2>/dev/null) || { log "no previous rev recorded"; exit 1; }
|
|
cur=$(git rev-parse HEAD); log "rolling back $cur -> $prev"
|
|
asowner git checkout --quiet "$prev" && apply_changes "$cur" "$prev" && { [[ $DRY == 1 ]] || { echo "$cur" > "$PREVF"; echo "$prev" > "$LASTF"; }; notify "↩️ rolled back to $prev"; } ;;
|
|
pending)
|
|
[[ -f $PENDING ]] || exit 0
|
|
restart_mediamtx_or_defer ;;
|
|
status)
|
|
echo "repo: $REPO ($(git rev-parse --abbrev-ref HEAD) @ $(git rev-parse --short HEAD))"
|
|
echo "deployed: $(cat "$LASTF" 2>/dev/null || echo '-') previous: $(cat "$PREVF" 2>/dev/null || echo '-')"
|
|
echo "auto: DEPLOY_AUTO=${DEPLOY_AUTO:-0} branch: $BRANCH pending mediamtx restart: $([[ -f $PENDING ]] && echo yes || echo no) live streams: $(live_streams)" ;;
|
|
*) sed -n '2,9p' "$0"; exit 2 ;;
|
|
esac
|