fix(test): out-of-process watchdog for the httpapi exerciser

.github/workflows/ci-test.yml

@@ -143,6 +143,11 @@ jobs:
         # default 6h. The full gate baseline is ~6m in CI, so 15m is generous;
         # test:httpapi:ci adds --progress/--trace to effect mode so the log
         # names the exact scenario and phase if it ever hangs again.
+        # 2026-08-05: a native-level freeze survived every in-process guard
+        # (timers die with the event loop) and burned the full 15m silently.
+        # --progress now also arms an out-of-process watchdog that SIGKILLs
+        # the runner after 120s without progress, naming the last scenario/
+        # phase — the step timeout stays as the final backstop.
         timeout-minutes: 15
         run: bun run test:httpapi:ci
 

packages/opencode/test/server/httpapi-exercise/index.ts

@@ -35,7 +35,8 @@ import { coverageResult, parseOptions, routeKey, routeKeys, selectedScenarios }
 import { runScenario } from "./runner"
 import { disposeApps } from "./backend"
 import { runtime } from "./runtime"
-import { type Scenario } from "./types"
+import { type Options, type Scenario } from "./types"
+import { startProgressWatchdog } from "./watchdog"
 
 function cursor(input: Record<string, unknown>) {
   return Buffer.from(JSON.stringify(input)).toString("base64url")
@@ -2091,7 +2092,8 @@ const llmScenarios = new Set([
 
 const main = Effect.gen(function* () {
   yield* Effect.addFinalizer(() => Effect.promise(() => disposeApps()).pipe(Effect.andThen(cleanupExercisePaths)))
-  const options = parseOptions(Bun.argv.slice(2))
+  const parsed = parseOptions(Bun.argv.slice(2))
+  const options: Options = parsed.progress ? { ...parsed, heartbeat: startProgressWatchdog() } : parsed
   const modules = yield* Effect.promise(() => runtime())
   const effectRoutes = routeKeys(OpenApi.fromApi(modules.PublicApi))
   const selected = selectedScenarios(options, scenarios)
@@ -2117,6 +2119,7 @@ const main = Effect.gen(function* () {
           (scenario) =>
             Effect.gen(function* () {
               if (options.progress) console.log(`${color.dim}RUN ${routeKey(scenario)} ${scenario.name}${color.reset}`)
+              options.heartbeat?.(`RUN ${routeKey(scenario)} ${scenario.name}`)
               return yield* runScenario(options)(scenario)
             }),
           { concurrency: 1 },

packages/opencode/test/server/httpapi-exercise/runner.ts

@@ -208,6 +208,7 @@ function withContext<A, E>(
 
 function trace(options: Options, scenario: ActiveScenario, phase: string) {
   return Effect.sync(() => {
+    options.heartbeat?.(`${scenario.name}: ${phase}`)
     if (!options.trace) return
     console.log(`[trace] ${scenario.name}: ${phase}`)
   })

packages/opencode/test/server/httpapi-exercise/types.ts

@@ -27,6 +27,8 @@ export type Options = {
   scenarioTimeout: Duration.Duration
   progress: boolean
   trace: boolean
+  /** Progress heartbeat for the out-of-process watchdog (CI only). */
+  heartbeat?: (label: string) => void
 }
 
 export type RequestSpec = {

packages/opencode/test/server/httpapi-exercise/watchdog.ts

@@ -0,0 +1,74 @@
+import { spawn } from "bun"
+import fs from "node:fs"
+import os from "node:os"
+import path from "node:path"
+
+// Out-of-process progress watchdog for the exerciser.
+//
+// Scenario timeouts and the bounded() cleanup guards are all timer-based —
+// they only fire while the JS event loop runs. A native hang (instance
+// dispose / tree-sitter / sqlite teardown) can freeze the loop so completely
+// that every in-process guard dies with it: the runner then emits nothing
+// until the CI step timeout (2026-08-05: 13 minutes of silence after
+// "worktree.create: shared use done"). A separate process is the only guard
+// that survives a frozen event loop.
+//
+// The runner heartbeats a file on every scenario/phase transition; the child
+// polls it, and if it goes stale the child prints the last recorded activity
+// and SIGKILLs the runner — turning a silent 15-minute freeze into a 2-minute
+// attributed failure.
+
+const WATCHDOG_SCRIPT = `
+const fs = require("node:fs")
+const pid = Number(process.env.WATCHDOG_PID)
+const file = process.env.WATCHDOG_FILE
+const timeoutMs = Number(process.env.WATCHDOG_TIMEOUT_MS)
+const pollMs = Number(process.env.WATCHDOG_POLL_MS)
+setInterval(() => {
+  let alive = true
+  try {
+    process.kill(pid, 0)
+  } catch {
+    alive = false
+  }
+  if (!alive) process.exit(0)
+  let mtime = 0
+  try {
+    mtime = fs.statSync(file).mtimeMs
+  } catch {}
+  if (!mtime || Date.now() - mtime <= timeoutMs) return
+  let last = "<none>"
+  try {
+    last = fs.readFileSync(file, "utf8")
+  } catch {}
+  console.error("[watchdog] no progress for " + Math.round((Date.now() - mtime) / 1000) + "s; last activity: " + last + " — killing pid " + pid)
+  try {
+    process.kill(pid, "SIGKILL")
+  } catch {}
+  process.exit(1)
+}, pollMs)
+`
+
+export function startProgressWatchdog(timeoutMs = 120_000, pollMs = 5_000): (label: string) => void {
+  const heartbeat = path.join(os.tmpdir(), `httpapi-exercise-heartbeat-${process.pid}`)
+  fs.writeFileSync(heartbeat, "startup")
+  const child = spawn(["bun", "-e", WATCHDOG_SCRIPT], {
+    env: {
+      ...process.env,
+      WATCHDOG_PID: String(process.pid),
+      WATCHDOG_FILE: heartbeat,
+      WATCHDOG_TIMEOUT_MS: String(timeoutMs),
+      WATCHDOG_POLL_MS: String(pollMs),
+    },
+    stdout: "inherit",
+    stderr: "inherit",
+  })
+  child.unref()
+  return (label: string) => {
+    try {
+      fs.writeFileSync(heartbeat, label)
+    } catch {
+      // A missed heartbeat only shortens the watchdog's patience margin.
+    }
+  }
+}