Logging framework: logfmt events in our scripts, container labels + rotated json-file logs, Caddy JSON access logs; LOGGING.md
Store-agnostic: journald + Docker logs are the only taps any collector (Alloy/Loki, VictoriaLogs, Netdata) needs. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
42
bin/dumpd.py
42
bin/dumpd.py
@@ -134,7 +134,7 @@ def quota_enforce_once():
|
||||
try:
|
||||
folders = st("GET", "/rest/config/folders")
|
||||
except Exception as e:
|
||||
print(f"{datetime.now(TZ):%F %T} quota: syncthing unreachable ({e})", flush=True); return
|
||||
log("quota_unreachable", level="warn", error=e); return
|
||||
for f in folders:
|
||||
if not f["id"].startswith("clips-"): continue
|
||||
user = f["id"][len("clips-"):]
|
||||
@@ -144,12 +144,12 @@ def quota_enforce_once():
|
||||
over = used > q; under = used < q * 0.9
|
||||
if over and not f.get("paused"):
|
||||
try: st("POST", f"/rest/db/pause?folder={f['id']}"); paused[user] = {"at": datetime.now(TZ).isoformat(), "used_gb": round(used / 1e9, 1), "quota_gb": q / 1e9}; changed = True
|
||||
except Exception as e: print(f"{datetime.now(TZ):%F %T} quota: pause {user} failed: {e}", flush=True); continue
|
||||
print(f"{datetime.now(TZ):%F %T} quota: PAUSED clips-{user} ({used/1e9:.1f} GB > {q/1e9:.0f} GB)", flush=True)
|
||||
except Exception as e: log("quota_pause_failed", level="error", user=user, error=e); continue
|
||||
log("quota_paused", level="warn", user=user, folder=f"clips-{user}", used_gb=f"{used/1e9:.1f}", quota_gb=f"{q/1e9:.0f}")
|
||||
elif under and f.get("paused") and user in paused:
|
||||
try: st("POST", f"/rest/db/resume?folder={f['id']}"); paused.pop(user, None); changed = True
|
||||
except Exception as e: print(f"{datetime.now(TZ):%F %T} quota: resume {user} failed: {e}", flush=True); continue
|
||||
print(f"{datetime.now(TZ):%F %T} quota: resumed clips-{user} ({used/1e9:.1f} GB)", flush=True)
|
||||
except Exception as e: log("quota_resume_failed", level="error", user=user, error=e); continue
|
||||
log("quota_resumed", user=user, folder=f"clips-{user}", used_gb=f"{used/1e9:.1f}")
|
||||
if changed:
|
||||
try: QUOTA_STATE.parent.mkdir(exist_ok=True); QUOTA_STATE.write_text(json.dumps(paused, indent=2))
|
||||
except OSError: pass
|
||||
@@ -157,7 +157,7 @@ def quota_enforce_once():
|
||||
def quota_loop():
|
||||
while True:
|
||||
try: quota_enforce_once()
|
||||
except Exception as e: print(f"{datetime.now(TZ):%F %T} quota loop error: {e}", flush=True)
|
||||
except Exception as e: log("quota_loop_error", level="error", error=e)
|
||||
_time.sleep(QUOTA_CHECK_S)
|
||||
|
||||
def st_apikey():
|
||||
@@ -247,7 +247,7 @@ def clips_setup(user, device_id, device_name=None):
|
||||
cur["devices"].append({"deviceID": device_id})
|
||||
cur["type"] = "receiveonly"; cur["minDiskFree"] = spec["minDiskFree"]
|
||||
st("PUT", f"/rest/config/folders/{fid}", cur)
|
||||
print(f"{datetime.now(TZ):%F %T} clips setup user={user} device={device_id[:7]}… folder={fid}", flush=True)
|
||||
log("clips_setup", user=user, device=device_id[:7], folder=fid)
|
||||
return clips_status(user)
|
||||
|
||||
DEPLOY_ENV = Path(os.environ.get("DEPLOY_ENV_FILE", "/etc/vanguard/deploy.env"))
|
||||
@@ -267,17 +267,25 @@ def run_deploy(reason):
|
||||
_deploy_lock["running"] = True
|
||||
def go():
|
||||
try:
|
||||
print(f"{datetime.now(TZ):%F %T} deploy: start ({reason})", flush=True)
|
||||
log("deploy_start", reason=reason)
|
||||
r = subprocess.run(["/usr/local/bin/vanguard-deploy", "pull"], capture_output=True, text=True, timeout=1800)
|
||||
for line in (r.stdout + r.stderr).splitlines()[-40:]: print(f"deploy| {line}", flush=True)
|
||||
print(f"{datetime.now(TZ):%F %T} deploy: exit {r.returncode}", flush=True)
|
||||
log("deploy_exit", level=("info" if r.returncode == 0 else "error"), rc=r.returncode)
|
||||
except Exception as e:
|
||||
print(f"{datetime.now(TZ):%F %T} deploy: error {e}", flush=True)
|
||||
log("deploy_error", level="error", error=e)
|
||||
finally:
|
||||
_deploy_lock["running"] = False
|
||||
threading.Thread(target=go, daemon=True).start()
|
||||
return True
|
||||
|
||||
def log(event, level="info", **kv):
|
||||
"""One logfmt line per event (see LOGGING.md): ts= level= component=dumpd event= key=value ..."""
|
||||
def q(v):
|
||||
v = str(v)
|
||||
return v if v and not re.search(r'[\s"=]', v) else '"' + v.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||
print(f"ts={datetime.now(TZ).isoformat(timespec='seconds')} level={level} component=dumpd event={event} "
|
||||
+ " ".join(f"{k}={q(v)}" for k, v in kv.items() if v is not None), flush=True)
|
||||
|
||||
def token_secret():
|
||||
try:
|
||||
sec = TOKEN_SECRET_FILE.read_bytes().strip()
|
||||
@@ -394,7 +402,7 @@ class H(BaseHTTPRequestHandler):
|
||||
env = deploy_env(); secret = env.get("DEPLOY_WEBHOOK_SECRET", "")
|
||||
sig = self.headers.get("X-Gitea-Signature") or self.headers.get("X-Hub-Signature-256", "").replace("sha256=", "")
|
||||
if not secret or not sig or not hmac.compare_digest(hmac.new(secret.encode(), raw, hashlib.sha256).hexdigest(), sig):
|
||||
print(f"{datetime.now(TZ):%F %T} deploy webhook: bad signature from {self.client_address[0]}", flush=True)
|
||||
log("deploy_webhook_bad_signature", level="warn", ip=self.client_address[0])
|
||||
return self._json(401, {"error": "bad signature"})
|
||||
try: payload = json.loads(raw or b"{}")
|
||||
except json.JSONDecodeError: return self._json(400, {"error": "bad json"})
|
||||
@@ -403,7 +411,7 @@ class H(BaseHTTPRequestHandler):
|
||||
if ref != f"refs/heads/{branch}":
|
||||
return self._json(200, {"ignored": f"ref {ref} is not {branch}"})
|
||||
if env.get("DEPLOY_AUTO", "0") != "1":
|
||||
print(f"{datetime.now(TZ):%F %T} deploy webhook: push {after} by {pusher} on {branch} — auto-deploy disabled (DEPLOY_AUTO=0)", flush=True)
|
||||
log("deploy_webhook_push", rev=after, pusher=pusher, branch=branch, auto="0")
|
||||
return self._json(202, {"queued": False, "note": "auto-deploy disabled (DEPLOY_AUTO=0); run vanguard-deploy pull by hand"})
|
||||
started = run_deploy(f"push {after} by {pusher}")
|
||||
return self._json(202, {"queued": started, "note": "deploy started" if started else "a deploy is already running"})
|
||||
@@ -426,19 +434,19 @@ class H(BaseHTTPRequestHandler):
|
||||
# audio mirrors (<user>/<Title>.webrtc) are published by the hook inside the mediamtx container only
|
||||
if req.get("ip") in ("127.0.0.1", "::1") and re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+\.webrtc", path):
|
||||
return self._json(200, {"ok": True})
|
||||
print(f"{datetime.now(TZ):%F %T} mtx-auth DENY publish path={path} ip={req.get('ip')} (mirror paths are internal)", flush=True)
|
||||
log("publish_deny", level="warn", stream=path, ip=req.get("ip"), reason="mirror paths are internal")
|
||||
return self._json(401, {"error": ".webrtc paths are reserved"})
|
||||
cred = req.get("password") or req.get("token") or ""
|
||||
if cred.lower().startswith("bearer "): cred = cred[7:]
|
||||
if ":" in cred and "." in cred.split(":", 1)[1]: cred = cred.split(":", 1)[1] # "user:token" form
|
||||
user = verify_token(cred)
|
||||
if not user:
|
||||
print(f"{datetime.now(TZ):%F %T} mtx-auth DENY publish path={path} ip={req.get('ip')} (bad/expired token)", flush=True)
|
||||
log("publish_deny", level="warn", stream=path, ip=req.get("ip"), reason="bad or expired token")
|
||||
return self._json(401, {"error": "invalid or expired stream token"})
|
||||
if not re.fullmatch(r"[a-z0-9_.-]+/[a-z0-9_.-]+", path) or not path.startswith(user + "/"):
|
||||
print(f"{datetime.now(TZ):%F %T} mtx-auth DENY publish user={user} path={path} (not their prefix)", flush=True)
|
||||
log("publish_deny", level="warn", user=user, stream=path, reason="not their prefix")
|
||||
return self._json(401, {"error": "path must be <your-username>/<Title>"})
|
||||
print(f"{datetime.now(TZ):%F %T} mtx-auth allow publish user={user} path={path} ip={req.get('ip')}", flush=True)
|
||||
log("publish_allow", user=user, stream=path, ip=req.get("ip"))
|
||||
return self._json(200, {"ok": True})
|
||||
if self.path.startswith("/me/clips"):
|
||||
who, _ = identity(self.headers)
|
||||
@@ -465,7 +473,7 @@ class H(BaseHTTPRequestHandler):
|
||||
moved, size = do_move(files, night)
|
||||
MARK.mkdir(exist_ok=True)
|
||||
(MARK / f"{night}-{tgt}.dumped").write_text(f"{datetime.now(TZ).isoformat()} by {u}\n")
|
||||
print(f"{datetime.now(TZ):%F %T} dump night={night} user={tgt} by={u} files={moved} gb={size/1e9:.2f}", flush=True)
|
||||
log("dump", night=night, user=tgt, by=u, files=moved, gb=f"{size/1e9:.2f}")
|
||||
self._json(200, {"night": night, "user": tgt, "moved": moved, "gb": round(size / 1e9, 2),
|
||||
"undo_until": (datetime.now(TZ) + timedelta(days=3)).strftime("%F")})
|
||||
def log_message(self, *a): # quiet
|
||||
|
||||
Reference in New Issue
Block a user