watchdog: off-thread event-loop stall detector

Incidents where the main thread wedged on synchronous work (the FUSE/SQLite
class of bug) left no diagnostic trail: in-process HTTP dies with the loop, and
main-thread timers can't fire to log anything until/unless it recovers — which
it sometimes never does. The run logs just went silent.

Add a worker thread with its OWN event loop that watches a heartbeat the main
thread stamps into shared memory. When the heartbeat goes stale it logs the
stall to stderr (→ the Space's run logs) in real time: how long, what the main
thread was doing (a breadcrumb set at each build phase), and process RSS (read
from the worker, so it's available even while main is frozen). A 30s baseline
"ok" line means a gap in the logs is itself the alarm.

Breadcrumbs (mark/tracked, save+restore so they nest) wrap the trace build, the
opencode/hermes SQLite reads, and the usage build — the known-heavy paths.
Fail-safe: worker is unref'd and error-swallowing, mark() is a no-op if the
worker never started, and AM_NO_WATCHDOG=1 disables it.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

server/src/index.js

@@ -19,6 +19,7 @@ import { buildUsage } from './usage.js';
 import { buildTraces, traceDigests, digestFor } from './traces.js';
 import { initPush, publicKey, deviceCount, addSubscription, removeSubscription, sendToAll } from './push.js';
 import { startVisibilityWatch, isPublic, visibility } from './visibility.js';
+import { startWatchdog } from './watchdog.js';
 
 ensureDirs();
 refreshVersions();
@@ -1106,6 +1107,10 @@ generateEnvSkill(loadSecretNotes()); // keep the environment skill current on bo
 // (skeletons + stale-while-revalidate), so nothing is lost but the boot risk.
 setTimeout(() => { traceDigests().catch(() => {}); }, 4000);
 
+// Off-thread stall detector — must start early so it's watching before the
+// first heavy build. Logs any event-loop wedge to the run logs (see watchdog.js).
+startWatchdog();
+
 server.listen(PORT, () => {
   console.log(`Agent Manager :${PORT}  tmux=${USE_TMUX}  data=${DATA_DIR}`);
   console.log('⚠  No authentication: this app trusts whoever can reach it.');

server/src/traces.js

@@ -3,6 +3,7 @@ import fsp from 'node:fs/promises';
 import path from 'node:path';
 import * as store from './sessions.js';
 import { WORKSPACES_DIR } from './config.js';
+import { mark, tracked, PHASE } from './watchdog.js';
 
 // Workspace-wide trace analytics: parse every Claude transcript and Codex
 // rollout on the Space into per-conversation stats (turns, tool calls, web
@@ -343,7 +344,10 @@ function readOpencode() {
   if (ocMemo.key && Date.now() - ck.hotMs < 8_000) return ocMemo.rows;
   const key = ck.key;
   let db;
-  try { db = new DatabaseSync(p, { readOnly: true }); } catch { return ocMemo.rows; }
+  // Synchronous DB read — the historically wedge-prone spot. Breadcrumb it so a
+  // stall while it runs is attributed (see watchdog.js).
+  const ocPhase = mark(PHASE.readOpencode);
+  try { db = new DatabaseSync(p, { readOnly: true }); } catch { mark(ocPhase); return ocMemo.rows; }
   const rows = [];
   try {
     const sessions = db.prepare('select * from session').all();
@@ -399,6 +403,7 @@ function readOpencode() {
     }
   } catch { /* torn read / schema drift: keep previous */ } finally {
     try { db.close(); } catch {}
+    mark(ocPhase);
   }
   ocMemo = { key, rows };
   return rows;
@@ -436,7 +441,8 @@ function readHermes() {
   if (hermesMemo.key === ck.key) return hermesMemo.rows;
   if (hermesMemo.key && Date.now() - ck.hotMs < 8_000) return hermesMemo.rows;
   let db;
-  try { db = new DatabaseSync(p, { readOnly: true }); } catch { return hermesMemo.rows; }
+  const hermesPhase = mark(PHASE.readHermes);
+  try { db = new DatabaseSync(p, { readOnly: true }); } catch { mark(hermesPhase); return hermesMemo.rows; }
   const rows = [];
   try {
     const sessions = db.prepare('select * from sessions').all();
@@ -492,6 +498,7 @@ function readHermes() {
     }
   } catch { /* torn read / schema drift: keep previous */ } finally {
     try { db.close(); } catch {}
+    mark(hermesPhase);
   }
   hermesMemo = { key: ck.key, rows };
   return rows;
@@ -599,7 +606,12 @@ async function codexFilesUncached() {
 
 const UUID_RE = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:\.jsonl)?$/;
 
-async function build() {
+// Breadcrumb the whole build so a wedge anywhere in it is attributed to
+// 'buildTraces' (the db reads narrow it further to their own phase). tracked()
+// restores the prior breadcrumb even if the build throws.
+function build() { return tracked(PHASE.buildTraces, buildImpl); }
+
+async function buildImpl() {
   const sessions = store.list();
   const byClaudeUuid = new Map(sessions.filter((s) => s.cli === 'claude' && s.sessionUuid).map((s) => [s.sessionUuid, s]));
   const byCodexId = new Map(sessions.filter((s) => s.codexSessionId).map((s) => [s.codexSessionId, s]));

server/src/usage.js

@@ -3,6 +3,7 @@ import fsp from 'node:fs/promises';
 import path from 'node:path';
 import { execFile as execFileCb } from 'node:child_process';
 import { promisify } from 'node:util';
+import { tracked, PHASE } from './watchdog.js';
 
 const execFile = promisify(execFileCb);
 
@@ -239,7 +240,11 @@ async function debugInfo() {
   };
 }
 
-export async function buildUsage(debug = false, only = null) {
+export function buildUsage(debug = false, only = null) {
+  return tracked(PHASE.buildUsage, () => buildUsageImpl(debug, only));
+}
+
+async function buildUsageImpl(debug = false, only = null) {
   // `only` narrows to one provider so the Usage page can fetch each in
   // parallel and render whichever answers first — one slow/hung provider
   // (ccusage has a 20s timeout) no longer blocks the rest.

server/src/watchdog-worker.js

@@ -0,0 +1,51 @@
+import { workerData } from 'node:worker_threads';
+
+// Runs on its own event loop, so it keeps ticking while the MAIN thread is
+// blocked. Reads the heartbeat + phase the main thread writes to shared memory
+// and logs to stderr — the one channel that survives a wedge (the Space's run
+// logs). See watchdog.js for the contract.
+
+const { sab, beatMs, stallMs, phases } = workerData;
+const hb = new BigInt64Array(sab, 0, 1);
+const meta = new Int32Array(sab, 8, 2);
+
+const mb = (b) => Math.round(b / 1048576);
+const phaseName = (c) => phases[c] || `#${c}`;
+const now = () => Date.now();
+
+let stalling = false;
+let stallStart = 0;
+let lastStallLog = 0;
+let maxLag = 0;
+let lastOk = now();
+
+setInterval(() => {
+  const t = now();
+  const last = Number(Atomics.load(hb, 0));
+  const age = t - last;
+  if (age > maxLag) maxLag = age;
+
+  if (age > stallMs) {
+    // Loop is stuck. Log on onset, then at most every 10s while it stays stuck,
+    // so a long wedge leaves a trail without flooding.
+    if (!stalling) { stalling = true; stallStart = last; lastStallLog = 0; }
+    if (lastStallLog === 0 || t - lastStallLog >= 10000) {
+      lastStallLog = t;
+      console.error(`[watchdog] main loop STALLED ${Math.round(age / 1000)}s`
+        + ` — activity=${phaseName(Atomics.load(meta, 0))} rss=${mb(process.memoryUsage().rss)}MB`);
+    }
+    return;
+  }
+
+  if (stalling) {
+    stalling = false;
+    console.error(`[watchdog] main loop recovered after ~${Math.round((t - stallStart) / 1000)}s`);
+  }
+  // Baseline heartbeat every 30s: a steady "ok" line means alive; a GAP in
+  // these lines is itself the alarm, and rss/maxLag give a cheap trend.
+  if (t - lastOk >= 30000) {
+    lastOk = t;
+    console.error(`[watchdog] ok rss=${mb(process.memoryUsage().rss)}MB maxLag=${maxLag}ms`);
+    maxLag = 0;
+  }
+}, beatMs);

server/src/watchdog.js

@@ -0,0 +1,68 @@
+import { Worker } from 'node:worker_threads';
+
+// Observability that survives a BLOCKED event loop. When the main thread wedges
+// on synchronous work (the FUSE/SQLite class of bug), in-process HTTP dies and
+// main-thread timers can't fire — so nothing gets logged until/unless the loop
+// recovers, and it may never. A worker thread has its OWN event loop and keeps
+// watching regardless: it reports the stall to stderr (→ the Space's run logs)
+// in real time, with the last activity breadcrumb and current RSS, so an
+// incident is a labelled event instead of a sudden silence.
+//
+// Mechanism: the main thread stamps the time into a SharedArrayBuffer every
+// beat and records a small integer "current activity" code; the worker reads
+// both and, when the stamp goes stale, reports how long the loop has been
+// unresponsive, what it was doing, and process memory (RSS is process-wide, so
+// the worker can read it even while the main thread is frozen).
+
+// Activity breadcrumbs. The worker maps code → name (order defines the code).
+export const PHASE = {
+  idle: 0,
+  buildTraces: 1,
+  readOpencode: 2,
+  readHermes: 3,
+  buildUsage: 4,
+};
+const PHASE_NAMES = Object.keys(PHASE);
+
+const BEAT_MS = 1000;
+const STALL_MS = 5000; // report once the loop has missed ~5 beats
+
+let shared = null; // { hb: BigInt64Array, meta: Int32Array }
+let worker = null;
+
+export function startWatchdog() {
+  if (worker || process.env.AM_NO_WATCHDOG) return;
+  // BigInt64 for the heartbeat (Atomics needs an integer view; ms-since-epoch
+  // overflows Int32), Int32 for the phase code — 8-byte aligned so both are
+  // valid Atomics targets.
+  const sab = new SharedArrayBuffer(16);
+  const hb = new BigInt64Array(sab, 0, 1);
+  const meta = new Int32Array(sab, 8, 2);
+  Atomics.store(hb, 0, BigInt(Date.now()));
+  shared = { hb, meta };
+  const beat = setInterval(() => Atomics.store(hb, 0, BigInt(Date.now())), BEAT_MS);
+  if (beat.unref) beat.unref();
+  try {
+    worker = new Worker(new URL('./watchdog-worker.js', import.meta.url), {
+      workerData: { sab, beatMs: BEAT_MS, stallMs: STALL_MS, phases: PHASE_NAMES },
+    });
+    worker.unref();
+    worker.on('error', () => {}); // the watchdog must never take down the app
+  } catch { worker = null; }
+}
+
+// Cheap breadcrumb: record what the main thread is about to do, so a stall
+// report can name the culprit. Returns the PREVIOUS code so callers can restore
+// it — breadcrumbs then nest correctly (a build that dips into a db read shows
+// the read while it runs, the build again after). A safe no-op (returns idle)
+// before startWatchdog() runs.
+export function mark(code) {
+  return shared ? Atomics.exchange(shared.meta, 0, code | 0) : PHASE.idle;
+}
+
+// Run fn under a breadcrumb, restoring the previous one even on throw. Works for
+// sync or async fn; the breadcrumb is what the worker reports if fn wedges.
+export async function tracked(code, fn) {
+  const prev = mark(code);
+  try { return await fn(); } finally { mark(prev); }
+}