diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index f4fc724f01..04ff0a7295 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -143,6 +143,11 @@ jobs: # default 6h. The full gate baseline is ~6m in CI, so 15m is generous; # test:httpapi:ci adds --progress/--trace to effect mode so the log # names the exact scenario and phase if it ever hangs again. + # 2026-08-05: a native-level freeze survived every in-process guard + # (timers die with the event loop) and burned the full 15m silently. + # --progress now also arms an out-of-process watchdog that SIGKILLs + # the runner after 120s without progress, naming the last scenario/ + # phase — the step timeout stays as the final backstop. timeout-minutes: 15 run: bun run test:httpapi:ci diff --git a/packages/opencode/test/server/httpapi-exercise/index.ts b/packages/opencode/test/server/httpapi-exercise/index.ts index f25390c1bd..c308c625a7 100644 --- a/packages/opencode/test/server/httpapi-exercise/index.ts +++ b/packages/opencode/test/server/httpapi-exercise/index.ts @@ -35,7 +35,8 @@ import { coverageResult, parseOptions, routeKey, routeKeys, selectedScenarios } import { runScenario } from "./runner" import { disposeApps } from "./backend" import { runtime } from "./runtime" -import { type Scenario } from "./types" +import { type Options, type Scenario } from "./types" +import { startProgressWatchdog } from "./watchdog" function cursor(input: Record) { return Buffer.from(JSON.stringify(input)).toString("base64url") @@ -2091,7 +2092,8 @@ const llmScenarios = new Set([ const main = Effect.gen(function* () { yield* Effect.addFinalizer(() => Effect.promise(() => disposeApps()).pipe(Effect.andThen(cleanupExercisePaths))) - const options = parseOptions(Bun.argv.slice(2)) + const parsed = parseOptions(Bun.argv.slice(2)) + const options: Options = parsed.progress ? { ...parsed, heartbeat: startProgressWatchdog() } : parsed const modules = yield* Effect.promise(() => runtime()) const effectRoutes = routeKeys(OpenApi.fromApi(modules.PublicApi)) const selected = selectedScenarios(options, scenarios) @@ -2117,6 +2119,7 @@ const main = Effect.gen(function* () { (scenario) => Effect.gen(function* () { if (options.progress) console.log(`${color.dim}RUN ${routeKey(scenario)} ${scenario.name}${color.reset}`) + options.heartbeat?.(`RUN ${routeKey(scenario)} ${scenario.name}`) return yield* runScenario(options)(scenario) }), { concurrency: 1 }, diff --git a/packages/opencode/test/server/httpapi-exercise/runner.ts b/packages/opencode/test/server/httpapi-exercise/runner.ts index 0a84cd5cbf..d98c4ecf26 100644 --- a/packages/opencode/test/server/httpapi-exercise/runner.ts +++ b/packages/opencode/test/server/httpapi-exercise/runner.ts @@ -208,6 +208,7 @@ function withContext( function trace(options: Options, scenario: ActiveScenario, phase: string) { return Effect.sync(() => { + options.heartbeat?.(`${scenario.name}: ${phase}`) if (!options.trace) return console.log(`[trace] ${scenario.name}: ${phase}`) }) diff --git a/packages/opencode/test/server/httpapi-exercise/types.ts b/packages/opencode/test/server/httpapi-exercise/types.ts index 5eed5b5b6a..b87cfdab8c 100644 --- a/packages/opencode/test/server/httpapi-exercise/types.ts +++ b/packages/opencode/test/server/httpapi-exercise/types.ts @@ -27,6 +27,8 @@ export type Options = { scenarioTimeout: Duration.Duration progress: boolean trace: boolean + /** Progress heartbeat for the out-of-process watchdog (CI only). */ + heartbeat?: (label: string) => void } export type RequestSpec = { diff --git a/packages/opencode/test/server/httpapi-exercise/watchdog.ts b/packages/opencode/test/server/httpapi-exercise/watchdog.ts new file mode 100644 index 0000000000..6e704de730 --- /dev/null +++ b/packages/opencode/test/server/httpapi-exercise/watchdog.ts @@ -0,0 +1,74 @@ +import { spawn } from "bun" +import fs from "node:fs" +import os from "node:os" +import path from "node:path" + +// Out-of-process progress watchdog for the exerciser. +// +// Scenario timeouts and the bounded() cleanup guards are all timer-based — +// they only fire while the JS event loop runs. A native hang (instance +// dispose / tree-sitter / sqlite teardown) can freeze the loop so completely +// that every in-process guard dies with it: the runner then emits nothing +// until the CI step timeout (2026-08-05: 13 minutes of silence after +// "worktree.create: shared use done"). A separate process is the only guard +// that survives a frozen event loop. +// +// The runner heartbeats a file on every scenario/phase transition; the child +// polls it, and if it goes stale the child prints the last recorded activity +// and SIGKILLs the runner — turning a silent 15-minute freeze into a 2-minute +// attributed failure. + +const WATCHDOG_SCRIPT = ` +const fs = require("node:fs") +const pid = Number(process.env.WATCHDOG_PID) +const file = process.env.WATCHDOG_FILE +const timeoutMs = Number(process.env.WATCHDOG_TIMEOUT_MS) +const pollMs = Number(process.env.WATCHDOG_POLL_MS) +setInterval(() => { + let alive = true + try { + process.kill(pid, 0) + } catch { + alive = false + } + if (!alive) process.exit(0) + let mtime = 0 + try { + mtime = fs.statSync(file).mtimeMs + } catch {} + if (!mtime || Date.now() - mtime <= timeoutMs) return + let last = "" + try { + last = fs.readFileSync(file, "utf8") + } catch {} + console.error("[watchdog] no progress for " + Math.round((Date.now() - mtime) / 1000) + "s; last activity: " + last + " — killing pid " + pid) + try { + process.kill(pid, "SIGKILL") + } catch {} + process.exit(1) +}, pollMs) +` + +export function startProgressWatchdog(timeoutMs = 120_000, pollMs = 5_000): (label: string) => void { + const heartbeat = path.join(os.tmpdir(), `httpapi-exercise-heartbeat-${process.pid}`) + fs.writeFileSync(heartbeat, "startup") + const child = spawn(["bun", "-e", WATCHDOG_SCRIPT], { + env: { + ...process.env, + WATCHDOG_PID: String(process.pid), + WATCHDOG_FILE: heartbeat, + WATCHDOG_TIMEOUT_MS: String(timeoutMs), + WATCHDOG_POLL_MS: String(pollMs), + }, + stdout: "inherit", + stderr: "inherit", + }) + child.unref() + return (label: string) => { + try { + fs.writeFileSync(heartbeat, label) + } catch { + // A missed heartbeat only shortens the watchdog's patience margin. + } + } +}