Brandyn / Techy
88bb28989c
Eight observability containers become four. Elasticsearch stores logs and metrics, Kibana is the one UI for Discover, dashboards and alerting, Filebeat ships container logs plus our journal units, and Metricbeat covers host and Docker metrics — which retires node_exporter and cAdvisor too — while still scraping MediaMTX's and Caddy's Prometheus endpoints. Pinned rather than left to chance: a fixed 2 GB heap (a self-sizing JVM would claim a quarter of a 62 GB box whose spare capacity belongs to x264), single-node discovery, security off because it only listens on loopback behind the Authentik gate, ML off, and vm.max_map_count set and persisted since ES will not start without it. Kibana gets a fixed encryption key or it loses its saved objects on every restart. diag.HOST is now just Kibana at the root: no path prefixes to get wrong and no index page of links. The deadman is the only thing kept from the old stack, repointed at Elastic. It checks that ES answers and is not red, that no index has been flipped read-only by the disk watermark (which does not clear itself when space is freed, so ingestion stays silently dead), that Kibana is available, that each Beat has actually written a document recently rather than merely running, that the Kibana heartbeat still arrives, and the disks. Verified against a mock Elasticsearch for all five failure modes plus the healthy case staying silent. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
106 lines
5.5 KiB
Bash
Executable File
106 lines
5.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Debian 13 host installer for vanguard-stream. Run with sudo from this directory.
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
[[ $EUID -eq 0 ]] || { echo "run with sudo"; exit 1; }
|
|
OWNER="${SUDO_USER:-debian}" # the login user; owns /srv/media and may run docker
|
|
|
|
echo "== packages"
|
|
apt-get update -q
|
|
apt-get install -y -q ca-certificates curl gnupg ffmpeg jq psmisc rclone python3 python3-yaml xfsprogs git nodejs npm rsync
|
|
|
|
if ! command -v docker >/dev/null; then
|
|
echo "== docker"
|
|
install -m 0755 -d /etc/apt/keyrings
|
|
curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
|
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \
|
|
> /etc/apt/sources.list.d/docker.list
|
|
apt-get update -q
|
|
apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
|
fi
|
|
|
|
echo "== timezone: server clock, logs, timers and the dump window all in Eastern"
|
|
timedatectl set-timezone "${VG_TZ:-America/New_York}"
|
|
timedatectl set-ntp true
|
|
echo " timezone is: $(timedatectl show -p Timezone --value) (override: VG_TZ=... bash install.sh)"
|
|
|
|
echo "== docker waits for /srv/media (never record onto the OS disk)"
|
|
install -d /etc/systemd/system/docker.service.d
|
|
cat > /etc/systemd/system/docker.service.d/10-srv-media.conf <<'DROP'
|
|
[Unit]
|
|
RequiresMountsFor=/srv/media
|
|
DROP
|
|
systemctl daemon-reload
|
|
systemctl restart docker 2>/dev/null || true
|
|
|
|
echo "== media user + folders"
|
|
# media files are owned by uid 1000 — on OVH's template that is the 'debian' user
|
|
getent passwd 1000 >/dev/null || useradd -r -u 1000 -d /srv/media -s /usr/sbin/nologin media
|
|
usermod -aG docker "$OWNER" 2>/dev/null || true
|
|
install -d -o 1000 -g 1000 /srv/media/{streams,clips,finals,trash,.markers,.syncthing}
|
|
install -d -o 1000 -g 1000 /srv/appdata/jellyfin/{config,cache} /srv/appdata/authentik/{media,templates,certs}
|
|
install -d /srv/appdata/authentik/{db,redis} /srv/appdata/mariadb
|
|
install -d -o 1000 -g 1000 /srv/appdata/discord-bot /srv/appdata/gitea /srv/appdata/act_runner
|
|
install -d -o 1000 -g 1000 /srv/bot /srv/bot/app # the Discord bot's released CI bundle (bind-mounted at /app; see bot/README.md)
|
|
install -d -o 1000 -g 1000 /srv/appdata/elasticsearch # the ES image runs as uid 1000
|
|
install -d /var/lib/vanguard # deadman state
|
|
# Elasticsearch refuses to start below this. Host setting, so set it now and persist it.
|
|
sysctl -q -w vm.max_map_count=262144
|
|
printf 'vm.max_map_count=262144\n' > /etc/sysctl.d/99-vanguard-elasticsearch.conf
|
|
install -d -o "$OWNER" /srv/live # live portal static build (deployLive.sh)
|
|
install -d -o 1000 -g 1000 /srv/media/streams/{techy,diceofire} /srv/media/clips/{techy,diceofire}
|
|
for d in /srv/media/clips/*/; do
|
|
[[ -e "$d/.stignore" ]] || install -m 0644 -o 1000 -g 1000 syncthing/stignore "$d/.stignore"
|
|
[[ -e "$d/.stlocal" ]] || install -m 0644 -o 1000 -g 1000 /dev/null "$d/.stlocal" # #include'd by .stignore
|
|
done
|
|
|
|
echo "== host scripts + units"
|
|
install -m 0755 bin/normalize-sweep.sh bin/trash-purge.sh bin/dumpd.py bin/vgstrip.py bin/vgdeband.sh /usr/local/bin/
|
|
install -m 0755 deployBotBundle.sh /usr/local/bin/vanguard-deploy-bot # dumpd runs this for the bot CI's POST /deploy-bot
|
|
install -m 0755 bin/vanguard-deadman.sh /usr/local/bin/ # watches the observability stack from outside it
|
|
# Compose bind-mounts this as a FILE; without it Docker would create a directory and alertmanager would not
|
|
# start. configure.py rewrites it from ALERT_WEBHOOK_URL on every apply; an empty one means "log only".
|
|
if [[ ! -f /etc/vanguard/alert-webhook ]]; then
|
|
install -d -m 0750 /etc/vanguard
|
|
install -m 0640 -o root -g 65534 /dev/null /etc/vanguard/alert-webhook
|
|
fi
|
|
install -d -m 0750 /etc/vanguard
|
|
install -m 0640 dump-users.txt /etc/vanguard/dump-users.txt
|
|
install -m 0644 systemd/*.service systemd/*.timer /etc/systemd/system/
|
|
systemctl daemon-reload
|
|
systemctl enable --now normalize-sweep.timer trash-purge.timer dumpd.service vanguard-deadman.timer
|
|
|
|
echo "== sysctl"
|
|
cat > /etc/sysctl.d/90-vanguard.conf <<'SYS'
|
|
vm.swappiness = 10
|
|
net.core.rmem_max = 16777216
|
|
net.core.wmem_max = 16777216
|
|
SYS
|
|
sysctl -q --system
|
|
|
|
echo "== compose images (the stack is started by ./configure.py --full, which generates .env first)"
|
|
chown -R "$OWNER:" .
|
|
docker compose pull -q --ignore-buildable 2>/dev/null || docker compose pull -q || true
|
|
|
|
echo "== live portal: build into /srv/live (node)"
|
|
if [[ -d live && -f live/package.json ]]; then
|
|
( cd live && sudo -u "$OWNER" -H npm install --no-audit --no-fund --loglevel=error && sudo -u "$OWNER" -H npm run build ) \
|
|
&& install -d -o "$OWNER" /srv/live && rsync -a --delete live/dist/ /srv/live/ && echo " built" || echo " portal build failed (run ./deployLive.sh from a dev box later)"
|
|
fi
|
|
|
|
echo "== host firewall (nftables; own table, input only; keeps this SSH session alive)"
|
|
if [[ -f firewall/firewall.sh ]]; then
|
|
bash firewall/firewall.sh
|
|
else
|
|
echo " firewall/firewall.sh missing — skipped"
|
|
fi
|
|
|
|
cat <<'MSG'
|
|
|
|
== done. Next: ./configure.py --full (hostnames, passwords, .env secrets; starts the containers)
|
|
== firewall is active: open a SECOND terminal and confirm ssh still works before closing this one.
|
|
|
|
== log out and back in once so the docker group applies; then 'docker compose' works without sudo.
|
|
== syncthing UI: ssh -L 8384:127.0.0.1:8384 debian@SERVER -> http://localhost:8384
|
|
MSG
|