diff --git a/.claude/scripts/loop-census.sh b/.claude/scripts/loop-census.sh index 8b44c4f..ece9219 100644 --- a/.claude/scripts/loop-census.sh +++ b/.claude/scripts/loop-census.sh @@ -47,6 +47,18 @@ case "$gates_rel" in /*) gates="$gates_rel" ;; *) gates="$root/$gates_rel" ;; es base=$(node -e 'const g=require(process.argv[1]); console.log((g.merge&&g.merge.baseBranch)||"main")' "$gates") module_labels=$(node -e 'const g=require(process.argv[1]); console.log(g.modules.map(m=>"module:"+m.name).join("\n"))' "$gates") +# --- driver-unit guard (issue #119): never advance an issue whose transient +# driver unit (pr-loop-driver-issue, spawned by loop-daemon.sh's run_driver) +# is currently active — e.g. the driver hasn't reached `git checkout -b` yet, +# so it has no branch for the in_flight check below to catch. No-op (always +# "not active") when systemd/`systemctl --user` is unavailable. +driver_unit_active() { + command -v systemctl >/dev/null 2>&1 || return 1 + local st + st="$(systemctl --user is-active "pr-loop-driver-issue$1" 2>/dev/null || true)" + case "$st" in active|activating) return 0 ;; *) return 1 ;; esac +} + open_prs=$(gh pr list -R "$repo" --state open --base "$base" --json number --jq 'length') echo "open_prs=$open_prs" @@ -81,7 +93,8 @@ while IFS=$'\t' read -r num labels title; do branch=$(git -C "$root" branch -a --list "*feat/issue-$num-*" | head -1 | sed 's/^[* ]*//;s|^remotes/||') || true [ -n "$branch" ] || branch="none" detail+="issue=$num branch=$branch title=$title"$'\n' - if [ "$advance_ready" = "none" ] && [ "$branch" = "none" ] && [ "$open_prs" -eq 0 ]; then + if [ "$advance_ready" = "none" ] && [ "$branch" = "none" ] && [ "$open_prs" -eq 0 ] \ + && ! driver_unit_active "$num"; then advance_ready="$num" fi # in_flight: a branch exists for this issue but no open PR carries it yet diff --git a/.claude/scripts/loop-census.test.sh b/.claude/scripts/loop-census.test.sh index ca8e611..fe0f47b 100644 --- a/.claude/scripts/loop-census.test.sh +++ b/.claude/scripts/loop-census.test.sh @@ -42,6 +42,21 @@ resolve_roots_src="$script_dir/resolve-roots.sh" work="$(mktemp -d "${TMPDIR:-/tmp}/loop-census-test.XXXXXX")" trap 'rm -rf "$work"' EXIT +# --- curated PATH (post-review finding #5) ----------------------------------- +# The driver_unit_active guard scenario (b) below needs systemctl to be +# genuinely ABSENT so it deterministically hits the no-op/fallback branch, +# regardless of what the real host has on /usr/bin:/bin (essentially every +# Linux/CI host, including this sandbox) — mirrors loop-daemon.test.sh's own +# curated_bin technique, with `node` added since loop-census.sh shells out to +# it directly for its adapter-derived facts. +curated_bin="$work/curated-bin" +mkdir -p "$curated_bin" +for tool in bash sh cat sed awk grep head tail tr wc mkdir mktemp rm date printf \ + kill git sleep basename dirname cut sort uniq env true false node; do + real="$(command -v "$tool" 2>/dev/null || true)" + [ -n "$real" ] && ln -sf "$real" "$curated_bin/$tool" +done + fail=0 ok=0 check() { @@ -164,6 +179,89 @@ check "exactly one in_flight line total (only issue 42 qualifies)" bash -c '[ "$ check "planned_issues=4 counted" bash -c 'printf "%s\n" "$1" | grep -qx "planned_issues=4"' _ "$out" check "issue=42 branch line shows the origin-prefixed remote-tracking name" bash -c 'printf "%s\n" "$1" | grep -q "^issue=42 branch=origin/feat/issue-42-y"' _ "$out" +# --------------------------------------------------------------------------- +# driver_unit_active guard (issue #119 post-review finding #5): loop-census.sh +# must never report advance_ready for an issue whose transient driver unit +# (pr-loop-driver-issue, spawned by loop-daemon.sh's run_driver) is +# currently active — the driver may not have reached `git checkout -b` yet, so +# it has no branch for the in_flight check above to catch, and a second tick +# would otherwise double-spawn an orchestrator for the same issue. +# +# build_guard_fixture: like fixture1 above but with two planned issues (5, 6), +# NEITHER with a branch nor an open PR — with the guard disabled, +# advance_ready would always report "5" (lowest-numbered), which is exactly +# what lets scenario (a) below prove the guard actually skips it in favor of +# issue 6. +# --------------------------------------------------------------------------- +build_guard_fixture() { + local name="$1" + local dir="$work/$name" + local scripts="$dir/.claude/scripts" + mkdir -p "$scripts" + cp "$census_src" "$scripts/loop-census.sh" + cp "$resolve_roots_src" "$scripts/resolve-roots.sh" + cat > "$dir/.claude/gates.json" <<'EOF' +{ + "modules": [{ "name": "test", "path": ".", "description": "", "owner": "" }], + "merge": { "baseBranch": "main" } +} +EOF + cat > "$scripts/pr-feedback.sh" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + cat > "$scripts/bot-gh.sh" <<'EOF' +#!/usr/bin/env bash +case "$1" in + repo) echo "acme/repo" ;; + pr) + if printf '%s\n' "$*" | grep -q 'headRefName'; then + : # no open PRs at all + else + echo 0 + fi + ;; + issue) + printf '5\tplanned,module:test\tIssue five\n' + printf '6\tplanned,module:test\tIssue six\n' + ;; + *) echo "fake-bot-gh.sh: unhandled args: $*" >&2; exit 1 ;; +esac +EOF + chmod +x "$scripts"/*.sh + git -C "$dir" init -q -b main + git -C "$dir" -c user.email=t@e.st -c user.name=t commit -q --allow-empty -m init + printf '%s\n' "$dir" +} + +# (a) issue 5's driver unit is active (systemctl stub answers "active") -> +# excluded from advance_ready, which falls through to issue 6 instead. +dirA="$(build_guard_fixture guardA)" +mkdir -p "$dirA/bin" +cat > "$dirA/bin/systemctl" <<'EOF' +#!/usr/bin/env bash +case "$*" in + *"is-active pr-loop-driver-issue5"*) echo "active"; exit 0 ;; + *"is-active"*) echo "inactive"; exit 3 ;; + *) exit 0 ;; +esac +EOF +chmod +x "$dirA/bin/systemctl" +outA="$(env -u GATES_FILE PATH="$dirA/bin:$PATH" bash "$dirA/.claude/scripts/loop-census.sh" "acme/repo")" +check "driver_unit_active guard (a): issue 5's active driver unit excludes it from advance_ready" bash -c ' + ! printf "%s\n" "$1" | grep -qx "advance_ready=5"' _ "$outA" +check "driver_unit_active guard (a): advance_ready falls through to issue 6 instead" bash -c ' + printf "%s\n" "$1" | grep -qx "advance_ready=6"' _ "$outA" + +# (b) systemctl unavailable entirely (curated PATH, no stub) -> the guard +# cleanly no-ops (command -v systemctl fails, driver_unit_active always +# reports "not active"), so advance_ready falls back to prior behavior: +# issue 5 (lowest-numbered, no branch, no open PRs). +dirB="$(build_guard_fixture guardB)" +outB="$(env -u GATES_FILE PATH="$curated_bin" bash "$dirB/.claude/scripts/loop-census.sh" "acme/repo")" +check "driver_unit_active guard (b): systemctl unavailable — guard no-ops, advance_ready falls back to issue 5" bash -c ' + printf "%s\n" "$1" | grep -qx "advance_ready=5"' _ "$outB" + echo "" if [ "$fail" -eq 0 ]; then echo "loop-census.test.sh: PASS ($ok checks)" diff --git a/.claude/scripts/loop-daemon.sh b/.claude/scripts/loop-daemon.sh index 6320092..bc1f7ff 100644 --- a/.claude/scripts/loop-daemon.sh +++ b/.claude/scripts/loop-daemon.sh @@ -45,16 +45,79 @@ # delete — case 1 above) apart from `publishable`/`half-done` (real work, # NEVER deleted — case 2 above). Only the `empty`+no-PR case is destructive. # +# DRIVER LIFETIME DECOUPLED FROM THE DAEMON (issue #119): the DRIVER +# CONTAINMENT setup above still leaves a driver living inside the DAEMON's +# own cgroup — `setsid` gives it an independent process *group*, but a +# process group is not a cgroup, and systemd's default `KillMode=control-group` +# kills the whole cgroup (driver included) whenever the daemon unit stops, be +# it a `systemctl --user restart`, a `Restart=always` crash-bounce, a host +# reboot/sleep, or `wsl --shutdown`. Evidence 2026-07-14/15: five of six +# drivers died ledger-less this way. The fix: when `systemd-run` is on PATH, +# `run_driver` spawns each driver as its OWN transient `--user` unit +# (`pr-loop-driver-issue`/`pr-loop-driver-pr`, derived from the verdict) +# via `systemd-run --user --wait --collect --unit=... -p RuntimeMaxSec=`. +# The driver's real parent becomes the user manager — it lives in ITS OWN +# scope, independent of the daemon's cgroup — so a daemon restart/crash kills +# only the daemon's own `systemd-run --wait` waiter (a disposable client that +# blocks and relays the exit code), never the driver itself. `RuntimeMaxSec` +# (a systemd time-span, e.g. `90m`) replaces the `timeout` wrapper as the hard +# wall-clock ceiling, enforced by the user manager instead of the daemon, so +# an orphaned driver still has a real ceiling even if the daemon never comes +# back. When `systemd-run` is NOT on PATH (legacy-cron / non-systemd +# environments), `run_driver` falls back to the exact `setsid timeout +# --kill-after=30s ...` spawn documented above — unchanged. Both paths feed +# the SAME rc into the SAME ledger/verify code below; only the spawn+wait +# step branches. `main()` also re-attaches to any `pr-loop-driver-*` unit +# still active at startup (left running by a now-dead daemon) instead of +# ticking — see reattach_orphaned_drivers() — and `loop-census.sh` refuses to +# ADVANCE an issue whose driver unit is currently active. Ops helper: +# `.claude/scripts/loop-halt.sh` stops one/all/everything by hand. +# +# POST-REVIEW HARDENING (issue #119, second pass) — three gaps in the above: +# (1) rc NORMALIZATION: `systemd-run --wait` relays `143` (128+SIGTERM) for +# a unit killed by hitting its `RuntimeMaxSec` ceiling — NOT GNU +# timeout's `124`/`137`. Left unnormalized, the `case "$rc" in 124|137)` +# classification below never matches on the systemd path, so a real +# timeout got ledgered as a plain `result=exit rc=143` AND wrongly +# routed into `verify_and_classify_post_exit` (which deliberately skips +# 124/137/127 — a killed driver has no work product to verify yet). +# `run_driver` now queries the unit's own `Result` property right after +# `wait` returns and normalizes rc to 124 when it reads `timeout`, so +# both spawn paths converge on the identical downstream classification. +# (2) SPAWN/CONNECT FAILURE: branch selection was presence-based +# (`command -v systemd-run`), not reachability-based — on a systemd host +# where the `--user` bus is unreachable (classic cron with no +# XDG_RUNTIME_DIR/session, or the user manager not running/lingering), +# `systemd-run --user --wait` fails to connect with no fallback, and the +# driver never spawns at all. `run_driver` now has the wrapped command +# touch a start-marker file as its very first action; if that marker +# never appears after `wait` returns, the driver never actually started +# under the unit (a spawn/connect failure, not a genuine driver exit), +# and `run_driver` falls through to the exact setsid+timeout fallback so +# the driver still actually runs, exactly once. +# (3) UNIT NAME COLLISION: a deterministic unit name can collide with a +# lingering `failed` unit from a previous run (invisible to +# `--state=active` reattach/census checks). `run_driver` now runs +# `systemctl --user reset-failed ` (best-effort, ignored on +# failure) immediately before every spawn — the #2 fallback above still +# catches this failure mode even if a stale unit somehow survives that. +# # Env: # LOOP_MODEL model for the driver (default sonnet; read by loop-event.sh) # GATES_FILE adapter override, passed straight through the environment # (self-hosting: .claude/self/gates.json) -# LOOP_DRIVER_TIMEOUT wall-clock cap per driver (default 90m) +# LOOP_DRIVER_TIMEOUT wall-clock cap per driver (default 90m); becomes +# systemd-run's `-p RuntimeMaxSec=` when systemd-run +# is on PATH, else `timeout`'s duration (issue #119) # LOOP_DAEMON_SLEEP_FAST/WATCH/IDLE/FALLBACK override the adaptive-sleep seconds (test hook) # LOOP_DAEMON_MAX_ITERATIONS bound the forever loop; 0 = unbounded (test/debug hook) +# LOOP_REATTACH_POLL_SECONDS poll interval while re-attaching to an orphaned +# driver unit at startup (default 5; issue #119 pt 3) # CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS forced to 0 for the driver spawn (issue #111 pt 4) # unless the caller already set it — fail-fast on a -# backgrounded driver instead of a silent half-completion +# backgrounded driver instead of a silent half-completion. +# Passed into the transient unit's own environment via +# `--setenv` when spawned through systemd-run (issue #119). # # Sourcing this file (rather than executing it) has ZERO side effects — every # function below only runs when called, and `main` only runs when this file @@ -304,6 +367,105 @@ verify_and_classify_post_exit() { printf '%s' "$out" } +# --- transient systemd unit naming (issue #119 pt 1) ------------------------- +# $1=verdict -> "pr-loop-driver-issue" for "advance issue=N", or +# "pr-loop-driver-pr" for "feedback pr=N". Used both to SPAWN the unit +# (run_driver) and, in reverse (verdict_from_unit_name below), to recover the +# verdict from a unit already running when this daemon process starts up +# (reattach_orphaned_drivers) — the two must stay exact inverses of each other. +driver_unit_name() { + case "$1" in + "advance issue="*) printf 'pr-loop-driver-issue%s' "${1#advance issue=}" ;; + "feedback pr="*) printf 'pr-loop-driver-pr%s' "${1#feedback pr=}" ;; + *) printf 'pr-loop-driver-unknown' ;; + esac +} + +# --- reverse of driver_unit_name: unit name -> verdict (issue #119 pt 3) ---- +# Prints nothing (not an error) for a unit name that doesn't match the +# expected naming convention — reattach_orphaned_drivers skips those rather +# than guessing. +verdict_from_unit_name() { + local unit="${1%.service}" + case "$unit" in + pr-loop-driver-issue*) printf 'advance issue=%s' "${unit#pr-loop-driver-issue}" ;; + pr-loop-driver-pr*) printf 'feedback pr=%s' "${unit#pr-loop-driver-pr}" ;; + *) : ;; + esac +} + +# --- list currently active/activating pr-loop-driver-* units --------------- +# No-op (prints nothing, never fails) when systemd/`systemctl --user` isn't +# usable here — every caller of this treats an empty result as "nothing to +# re-attach to", which is exactly correct in a non-systemd environment. +list_active_driver_units() { + command -v systemctl >/dev/null 2>&1 || return 0 + systemctl --user list-units --no-legend --plain --state=active,activating \ + 'pr-loop-driver-*' 2>/dev/null | awk '{print $1}' | sed 's/\.service$//' +} + +# --- block until a (re-attached) driver unit finishes, print its exit code -- +# $1=unit. Polls `systemctl --user is-active` every LOOP_REATTACH_POLL_SECONDS +# (default 5) while the unit is still active/activating/deactivating, then +# reads back ExecMainCode/ExecMainStatus. Best-effort: a unit garbage-collected +# out from under us (--collect) before this can read it back prints "0" rather +# than guessing — the ledger's own verify_and_classify_post_exit (GitHub + +# pure-git) is what actually tells success apart from a phantom regardless. +wait_for_driver_unit() { + local unit="$1" state + while :; do + state="$(systemctl --user is-active "$unit" 2>/dev/null || true)" + case "$state" in + active|activating|deactivating) sleep "${LOOP_REATTACH_POLL_SECONDS:-5}" ;; + *) break ;; + esac + done + local code status + code="$(systemctl --user show -p ExecMainCode --value "$unit" 2>/dev/null || true)" + status="$(systemctl --user show -p ExecMainStatus --value "$unit" 2>/dev/null || true)" + case "$status" in ''|*[!0-9]*) status=0 ;; esac + if [ "$code" = "killed" ]; then + printf '124' + else + printf '%s' "$status" + fi +} + +# --- startup re-attach (issue #119 pt 3) ------------------------------------ +# Called ONCE from main(), before the first run_once: a driver left running by +# a NOW-DEAD daemon process (the whole point of #119 — its lifetime is no +# longer tied to the daemon's) must never be double-spawned, and must never be +# silently forgotten either (a naive tick would just see no local branch yet +# and re-advance the same issue). Instead: find every still-active +# `pr-loop-driver-*` unit, WAIT for each to finish (blocking, like the +# daemon's own `systemd-run --wait` would have), then run the exact same +# post-exit verify + ledger path a fresh run_driver exit would have. No-op +# when systemd/`systemctl --user` is unavailable. +reattach_orphaned_drivers() { + command -v systemctl >/dev/null 2>&1 || return 0 + local unit + while IFS= read -r unit; do + [ -n "$unit" ] || continue + local verdict; verdict="$(verdict_from_unit_name "$unit")" + if [ -z "$verdict" ]; then + log "startup re-attach: active unit '$unit' doesn't match the pr-loop-driver- naming — leaving it to systemd, not re-attaching" + continue + fi + log "startup re-attach: found active driver unit '$unit' from a previous daemon ($verdict) — waiting instead of spawning a new one" + local rc; rc="$(wait_for_driver_unit "$unit")" + local ts; ts="$(date -u +%FT%TZ)" + local extra="result=exit rc=$rc" + case "$rc" in + 124|137) extra="result=timeout rc=$rc" ;; + esac + case "$verdict" in + "advance issue="*) extra="$(verify_and_classify_post_exit "$verdict" "$rc" "$extra")" ;; + esac + append_ledger "unknown" "" "$verdict" "$ts" "$extra reattached=true" + log "startup re-attach finished ($verdict): $extra reattached=true" + done < <(list_active_driver_units) +} + # --- spawn ONE contained driver, block until it exits/times out, ledger it --- # $1=verdict (e.g. "advance issue=42"), $2=model, $3=prompt-file (plain text). # Returns the driver's exit code (124/137 on timeout). @@ -335,19 +497,107 @@ run_driver() { # of half-completing) so the failure is immediate and visible, not a # phantom success discovered later. Still overridable by the caller's env. export CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS="${CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS:-0}" - # setsid: own session, so the whole tree (claude + any bash children it - # spawns) shares ONE fresh process group independent of this daemon's own — - # timeout's --kill-after below then has a single group to aim at. Backstop - # explicit group kill after `wait` covers anything that outlives timeout's - # own signal delivery (claude-code#29096: a bare SIGTERM to just the - # immediate child has been observed to orphan bash children). - setsid timeout --kill-after=30s "$timeout_dur" \ - claude --model "$model" -p "$prompt" --output-format json \ - >"$out_file" 2>"$out_file.stderr" & - local pgid=$! - wait "$pgid" - local rc=$? - kill -TERM -- "-$pgid" 2>/dev/null || true + + local pgid rc + if command -v systemd-run >/dev/null 2>&1; then + # Transient systemd unit per driver (issue #119 pt 1): the daemon's own + # `systemd-run --wait` invocation below is a DISPOSABLE waiter — it + # blocks and relays the unit's exit code, exactly like `wait` on a + # backgrounded job, but the driver's actual parent is the `--user` + # manager, not this daemon process. A daemon restart/crash kills only + # this waiter; the driver keeps running in its own scope, unaffected. + # --collect unloads the unit right after it stops. RuntimeMaxSec is the + # hard wall-clock ceiling, enforced by the user manager — it replaces + # `timeout` and, unlike `timeout`, survives even if the daemon itself + # never comes back. --setenv threads PATH and the fail-fast bg-wait + # ceiling into the unit's own environment: transient units do NOT inherit + # the caller's shell environment the way a plain backgrounded child would. + local unit; unit="$(driver_unit_name "$verdict")" + # Post-review finding #3: clear any lingering `failed` state under this + # exact deterministic unit name (e.g. a stale unit left behind by a prior + # driver for the same issue/PR) BEFORE spawning — a spawn into a name + # still occupied by a `failed` unit can otherwise hard-fail with no + # fallback. Best-effort; a normal --collect run never leaves one behind. + if command -v systemctl >/dev/null 2>&1; then + systemctl --user reset-failed "$unit" >/dev/null 2>&1 || true + fi + log "spawning driver via transient systemd unit ($unit, RuntimeMaxSec=$timeout_dur)" + # Post-review finding #2: a plain temp file the WRAPPED command touches as + # its very first action, before claude itself runs. Its presence after + # `wait` returns below is how run_driver tells "the driver process + # genuinely started under this unit" apart from "systemd-run itself never + # got to spawn it at all" — e.g. no reachable `--user` bus (classic cron + # with no XDG_RUNTIME_DIR/session, or the user manager not + # running/lingering). Both failure modes otherwise look identical: some + # nonzero rc, no driver output, no other signal to tell them apart. + local start_marker; start_marker="$(mktemp -u "$state_dir/.driver-started.XXXXXX")" + systemd-run --user --wait --collect --quiet \ + --unit="$unit" \ + -p "RuntimeMaxSec=$timeout_dur" \ + --setenv="CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS=$CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS" \ + --setenv="PATH=$PATH" \ + -- bash -c 'touch "$4"; claude --model "$1" -p "$2" --output-format json >"$3" 2>"$3.stderr"' _ \ + "$model" "$prompt" "$out_file" "$start_marker" & + pgid=$! + wait "$pgid" + rc=$? + + if [ ! -f "$start_marker" ]; then + # Post-review finding #2 (continued): the driver never actually started + # under the unit — degrade EXACTLY like the "no systemd-run on PATH" + # branch below: fall through to setsid+timeout so the driver still + # actually runs, exactly once, instead of silently ledgering a phantom + # spawn/connect-error rc. + log "systemd-run failed to spawn the driver unit '$unit' (rc=$rc, no start-marker seen) — falling back to setsid+timeout" + setsid timeout --kill-after=30s "$timeout_dur" \ + claude --model "$model" -p "$prompt" --output-format json \ + >"$out_file" 2>"$out_file.stderr" & + pgid=$! + wait "$pgid" + rc=$? + kill -TERM -- "-$pgid" 2>/dev/null || true + else + rm -f "$start_marker" + # Post-review finding #1: `systemd-run --wait` relays `143` + # (128+SIGTERM) for a unit killed by hitting its RuntimeMaxSec ceiling — + # NOT GNU timeout's `124`/`137`. Left unnormalized, the rc + # classification below (`case 124|137`) never matches on this path: a + # real timeout gets ledgered as a plain `result=exit rc=143` AND + # wrongly routed into verify_and_classify_post_exit (which deliberately + # skips 124/137/127 — there's no work product to verify yet on a killed + # driver). Query the unit's own Result property (best-effort — a + # --collect unit can already be garbage-collected by the time we ask, + # same caveat wait_for_driver_unit documents above) to detect it and + # normalize rc so BOTH spawn paths converge on the identical downstream + # classification. + if [ "$rc" -ne 0 ] && command -v systemctl >/dev/null 2>&1; then + local systemd_result + systemd_result="$(systemctl --user show -p Result --value "$unit" 2>/dev/null || true)" + if [ "$systemd_result" = "timeout" ]; then + log "systemd RuntimeMaxSec ceiling hit for $unit (rc=$rc) — normalizing to rc=124 for ledger/verify classification" + rc=124 + fi + fi + fi + else + # Fallback (legacy-cron / non-systemd environments): setsid gives the + # whole tree (claude + any bash children it spawns) its OWN process + # group, independent of this daemon's own — timeout's --kill-after below + # then has a single group to aim at. Backstop explicit group kill after + # `wait` covers anything that outlives timeout's own signal delivery + # (claude-code#29096: a bare SIGTERM to just the immediate child has been + # observed to orphan bash children). NOTE: a process group is NOT a + # cgroup — this path still dies WITH the daemon's own service cgroup on a + # restart/crash/reboot; that's exactly the gap the systemd-run branch + # above closes when it's available. + setsid timeout --kill-after=30s "$timeout_dur" \ + claude --model "$model" -p "$prompt" --output-format json \ + >"$out_file" 2>"$out_file.stderr" & + pgid=$! + wait "$pgid" + rc=$? + kill -TERM -- "-$pgid" 2>/dev/null || true + fi local session_id; session_id="$(extract_session_id "$out_file")" @@ -431,6 +681,12 @@ main() { append_ledger "unknown" "" "startup" "$(date -u +%FT%TZ)" "result=env-error" fi log "starting (LOOP_MODEL=${LOOP_MODEL:-sonnet} GATES_FILE=${GATES_FILE:-} LOOP_DRIVER_TIMEOUT=${LOOP_DRIVER_TIMEOUT:-90m})" + # Startup re-attach (issue #119 pt 3): BEFORE the first run_once, catch any + # driver a previous (now-dead) daemon process left running as a transient + # systemd unit — never double-spawn it, never let a fresh tick's census + # silently forget it either. No-op when systemd/`systemctl --user` isn't + # usable here. + reattach_orphaned_drivers local iterations=0 local max_iterations="${LOOP_DAEMON_MAX_ITERATIONS:-0}" while :; do diff --git a/.claude/scripts/loop-daemon.test.sh b/.claude/scripts/loop-daemon.test.sh index effd56b..5e0cbbe 100644 --- a/.claude/scripts/loop-daemon.test.sh +++ b/.claude/scripts/loop-daemon.test.sh @@ -25,6 +25,25 @@ resolve_roots_src="$script_dir/resolve-roots.sh" work="$(mktemp -d "${TMPDIR:-/tmp}/loop-daemon-test.XXXXXX")" trap 'rm -rf "$work"' EXIT +# --- curated PATH for the sandboxed daemon subprocess (issue #119) ---------- +# Every "no systemd-run stub installed" scenario below relies on `command -v +# systemd-run` genuinely failing to exercise the FALLBACK (setsid+timeout) +# spawn path — but the real host this test runs on may well have a genuine +# (if non-functional, no user bus) systemd-run/systemctl sitting in /usr/bin, +# which a plain `PATH=".../bin:/usr/bin:/bin"` would still resolve. Build a +# curated bin/ that symlinks in ONLY the standard utilities loop-daemon.sh + +# resolve-roots.sh actually need from the real /usr/bin:/bin, deliberately +# EXCLUDING systemd-run/systemctl — a scenario that wants the systemd path +# stubs one of those itself into its OWN fixture bin/ (which sits earlier on +# PATH and so shadows this curated dir). +curated_bin="$work/curated-bin" +mkdir -p "$curated_bin" +for tool in bash sh cat sed awk grep head tail tr wc mkdir mktemp rm date printf \ + kill git sleep basename dirname cut sort uniq env true false touch; do + real="$(command -v "$tool" 2>/dev/null || true)" + [ -n "$real" ] && ln -sf "$real" "$curated_bin/$tool" +done + fail=0 ok=0 check() { @@ -172,7 +191,18 @@ run_daemon_once() { # points inside the fixture (nothing there) so ensure_claude_on_path's nvm # fallback can never resolve the HOST's ~/.nvm — otherwise scenarios without # a claude stub pass on a dev box with nvm but fail on CI runners without it. - ( cd "$1" && PATH="$1/bin:/usr/bin:/bin" NVM_DIR="$1/no-such-nvm" LOOP_DAEMON_MAX_ITERATIONS=1 LOOP_DAEMON_SLEEP_FAST=0 LOOP_DAEMON_SLEEP_WATCH=0 LOOP_DAEMON_SLEEP_IDLE=0 LOOP_DAEMON_SLEEP_FALLBACK=0 bash .claude/scripts/loop-daemon.sh ) + # PATH uses $curated_bin (not a bare /usr/bin:/bin) so a scenario with no + # systemd-run/systemctl stub of its own genuinely sees them as ABSENT + # (issue #119's fallback contract), regardless of what the real host has. + ( cd "$1" && PATH="$1/bin:$curated_bin" NVM_DIR="$1/no-such-nvm" LOOP_DAEMON_MAX_ITERATIONS=1 LOOP_DAEMON_SLEEP_FAST=0 LOOP_DAEMON_SLEEP_WATCH=0 LOOP_DAEMON_SLEEP_IDLE=0 LOOP_DAEMON_SLEEP_FALLBACK=0 bash .claude/scripts/loop-daemon.sh ) +} + +run_daemon_once_env() { + # $1=fixture root; remaining args are NAME=VALUE pairs exported IN ADDITION + # to run_daemon_once's baseline env (used by the systemd-path scenarios to + # set LOOP_DRIVER_TIMEOUT and exercise RuntimeMaxSec passthrough). + local dir="$1"; shift + ( cd "$dir" && env "$@" PATH="$dir/bin:$curated_bin" NVM_DIR="$dir/no-such-nvm" LOOP_DAEMON_MAX_ITERATIONS=1 LOOP_DAEMON_SLEEP_FAST=0 LOOP_DAEMON_SLEEP_WATCH=0 LOOP_DAEMON_SLEEP_IDLE=0 LOOP_DAEMON_SLEEP_FALLBACK=0 bash .claude/scripts/loop-daemon.sh ) } run_daemon_once_stripped_path() { @@ -536,6 +566,251 @@ check "scenario 10: nothing was deleted on the offline path — branch still exi git -C "$1" rev-parse --verify --quiet refs/heads/feat/issue-100-empty >/dev/null 2>&1' _ "$dir10" check "scenario 10: nothing was deleted on the offline path — worktree still exists" [ -d "$wt10" ] +# --------------------------------------------------------------------------- +# fake_systemd_run: a generic stub that records its full invocation (so a +# scenario can assert on --unit=/RuntimeMaxSec=/--setenv= verbatim) and then +# execs whatever follows the "--" marker, mirroring what real `systemd-run +# --wait` does: block synchronously and relay the wrapped command's own exit +# code as its own (issue #119 pts 1/2: transient-unit spawn + fallback). +# --------------------------------------------------------------------------- +fake_systemd_run() { + local dir="$1" + fake_bin "$dir" systemd-run '#!/usr/bin/env bash +echo "systemd-run-args:$*" >> "'"$dir"'/systemd-run.args" +args=("$@") +i=0 +for a in "${args[@]}"; do + [ "$a" = "--" ] && break + i=$((i + 1)) +done +exec "${args[@]:$((i + 1))}"' +} + +# --------------------------------------------------------------------------- +# 11. Transient systemd unit path (issue #119 pt 1): advance issue=200, a +# systemd-run stub IS installed (so `command -v systemd-run` succeeds). +# Assert: unit naming derived from the verdict (pr-loop-driver-issue200), +# RuntimeMaxSec threaded from LOOP_DRIVER_TIMEOUT, the fail-fast bg-wait +# ceiling passed via --setenv, exit-code propagation into the ledger, and +# that setsid/timeout (the fallback path) were NEVER invoked. +# --------------------------------------------------------------------------- +prompt11_dir="$work/scenario11-support" +mkdir -p "$prompt11_dir" +printf 'Run the ADVANCE step for issue #200.\n' > "$prompt11_dir/prompt.txt" +dir11="$(new_fixture scenario11 "#!/usr/bin/env bash +echo 'cadence=FAST cron=* * * * *' +echo 'loop-event: action=advance issue=200' +echo 'loop-event: model=sonnet' +echo 'loop-event: prompt-file=$prompt11_dir/prompt.txt' +exit 0")" +fake_systemd_run "$dir11" +fake_bin "$dir11" claude '#!/usr/bin/env bash +echo "claude-ran args=$*" >> "'"$dir11"'/claude.marker" +echo "{\"session_id\":\"sess-200\",\"result\":\"ok\"}" +exit 0' +fake_bot_gh "$dir11" '#!/usr/bin/env bash +echo "205" +exit 0' +run_daemon_once_env "$dir11" LOOP_DRIVER_TIMEOUT=45m >/dev/null 2>&1 +check "scenario 11 (systemd-run path): stub was invoked" [ -f "$dir11/systemd-run.args" ] +check "scenario 11: unit name derived from the verdict (pr-loop-driver-issue200)" bash -c ' + grep -qF -- "--unit=pr-loop-driver-issue200" "$1"' _ "$dir11/systemd-run.args" +check "scenario 11: RuntimeMaxSec threaded from LOOP_DRIVER_TIMEOUT=45m" bash -c ' + grep -qF -- "RuntimeMaxSec=45m" "$1"' _ "$dir11/systemd-run.args" +check "scenario 11: CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS passed via --setenv (issue #111 pt 4 preserved)" bash -c ' + grep -qF -- "--setenv=CLAUDE_CODE_PRINT_BG_WAIT_CEILING_MS=0" "$1"' _ "$dir11/systemd-run.args" +check "scenario 11: claude stub was invoked (under the transient unit)" [ -f "$dir11/claude.marker" ] +check "scenario 11: claude stub received the prompt text" bash -c 'grep -qF "issue #200" "$1"' _ "$dir11/claude.marker" +check "scenario 11: setsid (fallback path) was NOT invoked" [ ! -f "$dir11/setsid.marker" ] +check "scenario 11: timeout (fallback path) was NOT invoked" [ ! -f "$dir11/timeout.marker" ] +ledger11="$dir11/.claude/state/loop-runs.log" +check "scenario 11: ledger records rc=0 propagated + pr=205 (post-exit verify)" bash -c ' + grep -Eq "verdict=advance issue=200 ts=[0-9T:Z-]+ result=exit rc=0 pr=205" "$1"' _ "$ledger11" + +# --------------------------------------------------------------------------- +# 12. Transient systemd unit path, feedback verdict: unit naming derived as +# pr-loop-driver-pr (not pr-loop-driver-issue), and exit-code +# propagation still lands in the ledger for a non-advance verdict too. +# --------------------------------------------------------------------------- +prompt12_dir="$work/scenario12-support" +mkdir -p "$prompt12_dir" +printf 'Run the ADDRESS FEEDBACK step for PR #201.\n' > "$prompt12_dir/prompt.txt" +dir12="$(new_fixture scenario12 "#!/usr/bin/env bash +echo 'cadence=FAST cron=* * * * *' +echo 'loop-event: action=feedback pr=201' +echo 'loop-event: model=sonnet' +echo 'loop-event: prompt-file=$prompt12_dir/prompt.txt' +exit 0")" +fake_systemd_run "$dir12" +fake_bin "$dir12" claude '#!/usr/bin/env bash +echo "{\"session_id\":\"sess-201\",\"result\":\"ok\"}" +exit 0' +run_daemon_once "$dir12" >/dev/null 2>&1 +check "scenario 12: unit name derived from a feedback verdict (pr-loop-driver-pr201)" bash -c ' + grep -qF -- "--unit=pr-loop-driver-pr201" "$1"' _ "$dir12/systemd-run.args" +ledger12="$dir12/.claude/state/loop-runs.log" +check "scenario 12: ledger records the feedback verdict with rc=0 propagated" bash -c ' + grep -Eq "verdict=feedback pr=201 ts=[0-9T:Z-]+ result=exit rc=0" "$1"' _ "$ledger12" + +# --------------------------------------------------------------------------- +# 13. Startup re-attach (issue #119 pt 3): a driver unit for issue #300 is +# ALREADY active (left running by a previous, now-dead daemon) when this +# daemon process starts. A fake `systemctl` stub answers list-units with +# that one active unit, then is-active as already finished, then show +# with ExecMainCode=exited/ExecMainStatus=0. Assert: NO new driver is +# spawned (no systemd-run/setsid/timeout/claude invocation at all), the +# re-attach waits and runs the SAME post-exit verify + ledger path a +# fresh spawn would have (pr=301 found via the fake bot-gh.sh), and the +# ledger is tagged reattached=true. +# --------------------------------------------------------------------------- +dir13="$(new_fixture scenario13 "#!/usr/bin/env bash +echo 'cadence=IDLE cron=*/15 * * * *' +echo 'loop-event: action=none' +exit 0")" +fake_bin "$dir13" systemctl '#!/usr/bin/env bash +echo "systemctl-args:$*" >> "'"$dir13"'/systemctl.calls" +case "$*" in + *list-units*) + echo "pr-loop-driver-issue300.service loaded active running Driver for issue 300" + exit 0 + ;; + *is-active*) + echo "failed" + exit 1 + ;; + *"-p ExecMainCode"*) + echo "exited" + exit 0 + ;; + *"-p ExecMainStatus"*) + echo "0" + exit 0 + ;; +esac +exit 0' +# Present but never expected to run — its mere presence lets main()'s startup +# ensure_claude_on_path succeed so the ONLY ledger line is the re-attach one. +fake_bin "$dir13" claude '#!/usr/bin/env bash +echo "claude-should-not-run" >> "'"$dir13"'/claude.should-not-run" +exit 0' +fake_bot_gh "$dir13" '#!/usr/bin/env bash +echo "301" +exit 0' +run_daemon_once "$dir13" >/dev/null 2>&1 +ledger13="$dir13/.claude/state/loop-runs.log" +check "scenario 13 (startup re-attach): claude was NEVER invoked (no duplicate spawn)" [ ! -f "$dir13/claude.should-not-run" ] +check "scenario 13: no systemd-run/setsid/timeout marker (no fresh spawn attempted)" bash -c ' + [ ! -f "$1/systemd-run.args" ] && [ ! -f "$1/setsid.marker" ] && [ ! -f "$1/timeout.marker" ]' _ "$dir13" +check "scenario 13: systemctl was polled (list-units + is-active + show)" bash -c ' + grep -q "list-units" "$1" && grep -q "is-active" "$1" && grep -q "show" "$1"' _ "$dir13/systemctl.calls" +check "scenario 13: exactly one ledger line (the re-attach line only)" [ "$(wc -l < "$ledger13" 2>/dev/null || echo 0)" -eq 1 ] +check "scenario 13: ledger records the reattached verdict, rc=0, pr=301, reattached=true" bash -c ' + grep -Eq "^pid=unknown session=unknown verdict=advance issue=300 ts=[0-9T:Z-]+ result=exit rc=0 pr=301 reattached=true$" "$1"' _ "$ledger13" + +# --------------------------------------------------------------------------- +# 14. rc normalization (issue #119 post-review finding #1): a systemd-enforced +# RuntimeMaxSec timeout relays rc=143 (128+SIGTERM) from `systemd-run +# --wait`, NOT GNU timeout's 124/137. A fake systemd-run stub simulates +# this (it genuinely runs the wrapped command — the start-marker gets +# touched, claude actually runs — but always reports rc=143 regardless of +# the wrapped command's real exit code), and a fake systemctl stub answers +# the unit's own `Result` property as "timeout". Assert: the ledger +# records the NORMALIZED result=timeout rc=124 (not rc=143), and +# verify_and_classify_post_exit was SKIPPED (no pr=/debris= fields) — +# exactly like the fallback path's genuine 124/137 case, converging both +# spawn paths on the identical downstream classification. +# --------------------------------------------------------------------------- +prompt14_dir="$work/scenario14-support" +mkdir -p "$prompt14_dir" +printf 'Run the ADVANCE step for issue #400.\n' > "$prompt14_dir/prompt.txt" +dir14="$(new_fixture scenario14 "#!/usr/bin/env bash +echo 'cadence=FAST cron=* * * * *' +echo 'loop-event: action=advance issue=400' +echo 'loop-event: model=sonnet' +echo 'loop-event: prompt-file=$prompt14_dir/prompt.txt' +exit 0")" +fake_bin "$dir14" systemd-run '#!/usr/bin/env bash +echo "systemd-run-args:$*" >> "'"$dir14"'/systemd-run.args" +args=("$@") +i=0 +for a in "${args[@]}"; do + [ "$a" = "--" ] && break + i=$((i + 1)) +done +"${args[@]:$((i + 1))}" >/dev/null 2>&1 +exit 143' +fake_bin "$dir14" systemctl '#!/usr/bin/env bash +echo "systemctl-args:$*" >> "'"$dir14"'/systemctl.calls" +case "$*" in + *"reset-failed"*) exit 0 ;; + *"-p Result"*) echo "timeout"; exit 0 ;; + *) exit 0 ;; +esac' +fake_bin "$dir14" claude '#!/usr/bin/env bash +echo "claude-ran args=$*" >> "'"$dir14"'/claude.marker" +echo "{\"session_id\":\"sess-400\",\"result\":\"ok\"}" +exit 0' +# bot-gh.sh deliberately NOT stubbed — verify_and_classify_post_exit must +# never even reach it for a (normalized) timeout rc. +run_daemon_once "$dir14" >/dev/null 2>&1 +ledger14="$dir14/.claude/state/loop-runs.log" +check "scenario 14 (rc normalization): claude stub still ran (real work happened before the kill)" [ -f "$dir14/claude.marker" ] +check "scenario 14: systemctl Result property was queried" bash -c 'grep -qF -- "-p Result" "$1"' _ "$dir14/systemctl.calls" +check "scenario 14: ledger normalizes rc=143 -> result=timeout rc=124" bash -c ' + grep -Eq "verdict=advance issue=400 ts=[0-9T:Z-]+ result=timeout rc=124$" "$1"' _ "$ledger14" +check "scenario 14: verify_and_classify_post_exit was skipped (no pr=/debris= fields)" bash -c ' + ! grep -Eq "pr=|debris=" "$1"' _ "$ledger14" + +# --------------------------------------------------------------------------- +# 15. Spawn/connect failure fallback (issue #119 post-review finding #2): a +# fake systemd-run stub simulates a --user bus connect failure — it +# records its invocation but NEVER execs the wrapped command at all (no +# start-marker ever appears), exiting 1 exactly like a real connect +# failure would (e.g. classic cron with no XDG_RUNTIME_DIR/session, or the +# user manager not running/lingering). Assert: run_driver detects the +# missing start-marker and falls through to the setsid+timeout fallback, +# so the driver still actually runs exactly once (setsid/timeout/claude +# stubs all invoked), and the ledger records a genuine result — NOT a +# spawn-error or a phantom. +# --------------------------------------------------------------------------- +prompt15_dir="$work/scenario15-support" +mkdir -p "$prompt15_dir" +printf 'Run the ADVANCE step for issue #500.\n' > "$prompt15_dir/prompt.txt" +dir15="$(new_fixture scenario15 "#!/usr/bin/env bash +echo 'cadence=FAST cron=* * * * *' +echo 'loop-event: action=advance issue=500' +echo 'loop-event: model=sonnet' +echo 'loop-event: prompt-file=$prompt15_dir/prompt.txt' +exit 0")" +fake_bin "$dir15" systemd-run '#!/usr/bin/env bash +echo "systemd-run-args:$*" >> "'"$dir15"'/systemd-run.args" +echo "Failed to connect to bus: no such file or directory" >&2 +exit 1' +fake_bin "$dir15" setsid '#!/usr/bin/env bash +echo "setsid-ran" >> "'"$dir15"'/setsid.marker" +exec "$@"' +fake_bin "$dir15" timeout '#!/usr/bin/env bash +echo "timeout-ran args=$*" >> "'"$dir15"'/timeout.marker" +shift; shift +exec "$@"' +fake_bin "$dir15" claude '#!/usr/bin/env bash +echo "claude-ran args=$*" >> "'"$dir15"'/claude.marker" +echo "{\"session_id\":\"sess-500\",\"result\":\"ok\"}" +exit 0' +fake_bot_gh "$dir15" '#!/usr/bin/env bash +echo "501" +exit 0' +run_daemon_once "$dir15" >/dev/null 2>&1 +check "scenario 15 (spawn-failure fallback): systemd-run stub was invoked (attempted first)" [ -f "$dir15/systemd-run.args" ] +check "scenario 15: fallback setsid stub ran after the connect failure" [ -f "$dir15/setsid.marker" ] +check "scenario 15: fallback timeout stub ran after the connect failure" [ -f "$dir15/timeout.marker" ] +check "scenario 15: fallback claude stub actually ran the driver" [ -f "$dir15/claude.marker" ] +check "scenario 15: claude ran exactly once (no double-spawn)" bash -c '[ "$(wc -l < "$1")" -eq 1 ]' _ "$dir15/claude.marker" +ledger15="$dir15/.claude/state/loop-runs.log" +check "scenario 15: ledger records a genuine result via the fallback (NOT spawn-error/phantom)" bash -c ' + grep -Eq "verdict=advance issue=500 ts=[0-9T:Z-]+ result=exit rc=0 pr=501" "$1"' _ "$ledger15" +check "scenario 15: ledger never records result=spawn-error" bash -c '! grep -q "spawn-error" "$1"' _ "$ledger15" + echo "" if [ "$fail" -eq 0 ]; then echo "loop-daemon.test.sh: PASS ($ok checks)" diff --git a/.claude/scripts/loop-halt.sh b/.claude/scripts/loop-halt.sh new file mode 100755 index 0000000..ba4dd57 --- /dev/null +++ b/.claude/scripts/loop-halt.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# loop-halt.sh — ops helper: stop one driver / all drivers / everything +# (issue #119). Companion to loop-daemon.sh's transient-systemd-unit driver +# spawn (`pr-loop-driver-issue` / `pr-loop-driver-pr`): a daemon +# restart no longer stops an in-flight driver (that is the whole point of +# #119 — its lifetime is decoupled from the daemon's), so an operator needs +# an explicit, obvious way to stop one/all/everything instead of relying on +# `systemctl --user restart pr-loop-.service` to do it as a side effect. +# +# Usage: +# loop-halt.sh stop ONE driver: +# loop-halt.sh issue106 -> stops pr-loop-driver-issue106 +# loop-halt.sh pr42 -> stops pr-loop-driver-pr42 +# loop-halt.sh pr-loop-driver-issue106 -> stops that unit name verbatim +# loop-halt.sh --drivers | all-drivers stop ALL driver units (pr-loop-driver-*) +# loop-halt.sh --all stop the daemon unit AND all driver units +# loop-halt.sh -h | --help show this help +# +# Degrades cleanly when systemd/`systemctl --user` is unavailable (legacy-cron +# / non-systemd environments): logs a message and exits 0 — drivers spawned +# via that fallback are plain daemon children, already covered by stopping +# the daemon itself (see docs/USAGE.md's cron-less loop / failure-contract +# sections). +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=resolve-roots.sh +. "$script_dir/resolve-roots.sh" + +log() { printf '%s loop-halt: %s\n' "$(date -u +%FT%TZ)" "$*" >&2; } + +usage() { + sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' +} + +# --- same repo_slug derivation arm-loop.sh uses for the daemon unit name ---- +repo_slug() { + basename "$root" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9-' '-' | sed -E 's/-+/-/g; s/^-|-$//g' +} + +stop_unit() { + local unit="$1" + log "stopping $unit" + if systemctl --user stop "$unit" 2>&1 | while IFS= read -r line; do log " $line"; done; then + log "stopped (or already stopped) $unit" + else + log "failed to stop $unit (may not exist)" + fi +} + +stop_all_drivers() { + local units + units="$(systemctl --user list-units --no-legend --plain --state=active,activating \ + 'pr-loop-driver-*' 2>/dev/null | awk '{print $1}')" + if [ -z "$units" ]; then + log "no active pr-loop-driver-* units" + return 0 + fi + local u + while IFS= read -r u; do + [ -n "$u" ] || continue + stop_unit "$u" + done <<< "$units" +} + +if [ "$#" -lt 1 ]; then + usage + exit 2 +fi + +case "$1" in + -h|--help) + usage + exit 0 + ;; +esac + +if ! command -v systemctl >/dev/null 2>&1; then + log "systemctl not found on PATH — systemd is unavailable here, nothing to stop (legacy-cron drivers are daemon children; stop/restart the daemon itself to reap them)" + exit 0 +fi + +case "$1" in + --drivers|all-drivers) + stop_all_drivers + ;; + --all) + slug="$(repo_slug)" + stop_unit "pr-loop-$slug.service" + stop_all_drivers + ;; + pr-loop-driver-*) + stop_unit "$1" + ;; + issue*) + n="${1#issue}" + case "$n" in + *[!0-9]*|'') log "invalid argument '$1' — expected issue"; usage; exit 2 ;; + esac + stop_unit "pr-loop-driver-issue$n" + ;; + pr*) + n="${1#pr}" + case "$n" in + *[!0-9]*|'') log "invalid argument '$1' — expected pr"; usage; exit 2 ;; + esac + stop_unit "pr-loop-driver-pr$n" + ;; + *) + log "unrecognized argument '$1' — expected issue, pr, a pr-loop-driver-* unit name, --drivers, or --all" + usage + exit 2 + ;; +esac diff --git a/.claude/scripts/loop-halt.test.sh b/.claude/scripts/loop-halt.test.sh new file mode 100644 index 0000000..792f9e6 --- /dev/null +++ b/.claude/scripts/loop-halt.test.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# loop-halt.test.sh — offline smoke test for loop-halt.sh (issue #119). +# +# Builds a throwaway fixture `.claude/scripts/` (mirroring loop-daemon.test.sh's +# own convention) containing the REAL loop-halt.sh + resolve-roots.sh, with a +# fake `systemctl` stub prepended onto PATH that records every invocation +# instead of touching the real (or, in this sandbox, non-functional) user +# systemd bus. Exit 0 on success, non-zero if any assertion fails. Runnable +# bare: bash .claude/scripts/loop-halt.test.sh +set -uo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +loop_halt_src="$script_dir/loop-halt.sh" +resolve_roots_src="$script_dir/resolve-roots.sh" + +work="$(mktemp -d "${TMPDIR:-/tmp}/loop-halt-test.XXXXXX")" +trap 'rm -rf "$work"' EXIT + +# --- curated PATH (post-review finding #4) ----------------------------------- +# Scenario 2 needs systemctl to be genuinely ABSENT so it deterministically +# hits the "systemd unavailable" degrade branch — a plain `PATH=/usr/bin:/bin` +# still resolves the real host's own systemctl on essentially every Linux/CI +# host, including this sandbox, which made scenario 2's assertion a no-op +# everywhere it actually needed to run. Build a curated bin/ that symlinks in +# ONLY the standard utilities loop-halt.sh + resolve-roots.sh actually need +# from the real /usr/bin:/bin, deliberately EXCLUDING systemctl/systemd-run — +# mirrors loop-daemon.test.sh's own curated_bin technique exactly. +curated_bin="$work/curated-bin" +mkdir -p "$curated_bin" +for tool in bash sh cat sed awk grep head tail tr wc mkdir mktemp rm date printf \ + kill git sleep basename dirname cut sort uniq env true false; do + real="$(command -v "$tool" 2>/dev/null || true)" + [ -n "$real" ] && ln -sf "$real" "$curated_bin/$tool" +done + +fail=0 +ok=0 +check() { + local desc="$1"; shift + if "$@"; then + ok=$((ok + 1)) + echo "ok - $desc" + else + fail=1 + echo "FAIL - $desc" + fi +} + +new_fixture() { + # $1=name -> prints fixture root, containing the real loop-halt.sh + + # resolve-roots.sh under .claude/scripts/ (so resolve-roots.sh's own + # */.claude/scripts detection makes $root == the fixture root). + local name="$1" + local dir="$work/$name" + mkdir -p "$dir/.claude/scripts" "$dir/bin" + cp "$loop_halt_src" "$dir/.claude/scripts/loop-halt.sh" + cp "$resolve_roots_src" "$dir/.claude/scripts/resolve-roots.sh" + chmod +x "$dir/.claude/scripts"/*.sh + printf '%s\n' "$dir" +} + +fake_systemctl() { + # $1=fixture root $2=script body (the case/dispatch logic ONLY — the + # boilerplate arg-logging header is added here) -> installs bin/systemctl. + local dir="$1" body="$2" + { + printf '#!/usr/bin/env bash\n' + printf 'echo "systemctl-args:$*" >> "%s/systemctl.calls"\n' "$dir" + printf '%s\n' "$body" + } > "$dir/bin/systemctl" + chmod +x "$dir/bin/systemctl" +} + +run_halt() { + # $1=fixture root; remaining args passed straight to loop-halt.sh. + local dir="$1"; shift + ( cd "$dir" && PATH="$dir/bin:/usr/bin:/bin" bash .claude/scripts/loop-halt.sh "$@" ) +} + +# --------------------------------------------------------------------------- +# 1. -h / --help: usage text, exit 0, no systemctl call at all. +# --------------------------------------------------------------------------- +dir1="$(new_fixture scenario1)" +fake_systemctl "$dir1" 'exit 0' +out1="$(run_halt "$dir1" -h 2>&1)" +rc1=$? +check "scenario 1 (-h): exit 0" [ "$rc1" -eq 0 ] +check "scenario 1: usage text mentions --drivers" bash -c 'printf "%s" "$1" | grep -q -- "--drivers"' _ "$out1" +check "scenario 1: no systemctl call was made for -h" [ ! -f "$dir1/systemctl.calls" ] + +# --------------------------------------------------------------------------- +# 2. No systemctl on PATH at all: degrades cleanly, exit 0, explanatory log. +# Uses $curated_bin (not a bare /usr/bin:/bin) so this DETERMINISTICALLY hits +# the no-systemctl branch regardless of host — essentially every Linux/CI +# host (including this sandbox) has a real systemctl on /usr/bin:/bin, which +# previously made this assertion a silent no-op everywhere it mattered. +# --------------------------------------------------------------------------- +dir2="$(new_fixture scenario2)" +out2="$( ( cd "$dir2" && PATH="$curated_bin" bash .claude/scripts/loop-halt.sh issue1 2>&1 ) )" +rc2=$? +check "scenario 2 (no systemctl anywhere): exit 0" [ "$rc2" -eq 0 ] +check "scenario 2: explains systemd is unavailable" bash -c 'printf "%s" "$1" | grep -qi "systemctl not found"' _ "$out2" + +# --------------------------------------------------------------------------- +# 3. loop-halt.sh issue106 -> stops pr-loop-driver-issue106. +# --------------------------------------------------------------------------- +dir3="$(new_fixture scenario3)" +fake_systemctl "$dir3" ' +case "$*" in + "--user stop pr-loop-driver-issue106") exit 0 ;; + *) exit 0 ;; +esac' +run_halt "$dir3" issue106 >/dev/null 2>&1 +check "scenario 3 (issue106): stopped exactly pr-loop-driver-issue106" bash -c ' + grep -qF "stop pr-loop-driver-issue106" "$1"' _ "$dir3/systemctl.calls" + +# --------------------------------------------------------------------------- +# 4. loop-halt.sh pr42 -> stops pr-loop-driver-pr42. +# --------------------------------------------------------------------------- +dir4="$(new_fixture scenario4)" +fake_systemctl "$dir4" 'exit 0' +run_halt "$dir4" pr42 >/dev/null 2>&1 +check "scenario 4 (pr42): stopped exactly pr-loop-driver-pr42" bash -c ' + grep -qF "stop pr-loop-driver-pr42" "$1"' _ "$dir4/systemctl.calls" + +# --------------------------------------------------------------------------- +# 5. loop-halt.sh : passed straight through unchanged. +# --------------------------------------------------------------------------- +dir5="$(new_fixture scenario5)" +fake_systemctl "$dir5" 'exit 0' +run_halt "$dir5" pr-loop-driver-issue999 >/dev/null 2>&1 +check "scenario 5 (verbatim unit): stopped exactly the given unit name" bash -c ' + grep -qF "stop pr-loop-driver-issue999" "$1"' _ "$dir5/systemctl.calls" + +# --------------------------------------------------------------------------- +# 6. loop-halt.sh --drivers: lists active pr-loop-driver-* units, stops each. +# --------------------------------------------------------------------------- +dir6="$(new_fixture scenario6)" +fake_systemctl "$dir6" ' +case "$*" in + *list-units*) + echo "pr-loop-driver-issue7.service loaded active running one" + echo "pr-loop-driver-pr9.service loaded active running two" + exit 0 + ;; + *) exit 0 ;; +esac' +run_halt "$dir6" --drivers >/dev/null 2>&1 +check "scenario 6 (--drivers): stopped pr-loop-driver-issue7" bash -c ' + grep -qF "stop pr-loop-driver-issue7" "$1"' _ "$dir6/systemctl.calls" +check "scenario 6 (--drivers): stopped pr-loop-driver-pr9" bash -c ' + grep -qF "stop pr-loop-driver-pr9" "$1"' _ "$dir6/systemctl.calls" + +# --------------------------------------------------------------------------- +# 7. loop-halt.sh --all: stops the daemon unit (repo-slug derived) AND every +# active driver unit. +# --------------------------------------------------------------------------- +dir7="$(new_fixture My-Repo_Fixture7)" +fake_systemctl "$dir7" ' +case "$*" in + *list-units*) + echo "pr-loop-driver-issue3.service loaded active running one" + exit 0 + ;; + *) exit 0 ;; +esac' +run_halt "$dir7" --all >/dev/null 2>&1 +check "scenario 7 (--all): stopped the repo-slug daemon unit" bash -c ' + grep -qF "stop pr-loop-my-repo-fixture7.service" "$1"' _ "$dir7/systemctl.calls" +check "scenario 7 (--all): also stopped the active driver unit" bash -c ' + grep -qF "stop pr-loop-driver-issue3" "$1"' _ "$dir7/systemctl.calls" + +# --------------------------------------------------------------------------- +# 8. loop-halt.sh --drivers with NO active units: no stop call at all. +# --------------------------------------------------------------------------- +dir8="$(new_fixture scenario8)" +fake_systemctl "$dir8" ' +case "$*" in + *list-units*) exit 0 ;; + *) exit 0 ;; +esac' +out8="$(run_halt "$dir8" --drivers 2>&1)" +check "scenario 8 (--drivers, none active): logs nothing-to-stop" bash -c 'printf "%s" "$1" | grep -qi "no active pr-loop-driver"' _ "$out8" +check "scenario 8: no stop call was ever made" bash -c '! grep -q " stop " "$1" 2>/dev/null' _ "$dir8/systemctl.calls" + +# --------------------------------------------------------------------------- +# 9. Argument validation: no args / unknown / malformed issue|pr -> usage, exit 2. +# --------------------------------------------------------------------------- +dir9="$(new_fixture scenario9)" +fake_systemctl "$dir9" 'exit 0' +run_halt "$dir9" >/dev/null 2>&1 +check "scenario 9 (no args): exit 2" [ "$?" -eq 2 ] +run_halt "$dir9" bogus >/dev/null 2>&1 +check "scenario 9 (unknown arg): exit 2" [ "$?" -eq 2 ] +run_halt "$dir9" issueX >/dev/null 2>&1 +check "scenario 9 (malformed issueX): exit 2" [ "$?" -eq 2 ] +run_halt "$dir9" prX >/dev/null 2>&1 +check "scenario 9 (malformed prX): exit 2" [ "$?" -eq 2 ] + +echo "" +if [ "$fail" -eq 0 ]; then + echo "loop-halt.test.sh: PASS ($ok checks)" + exit 0 +else + echo "loop-halt.test.sh: FAIL (see FAIL lines above)" + exit 1 +fi diff --git a/docs/USAGE.md b/docs/USAGE.md index e83b050..bab2c97 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -156,22 +156,38 @@ which have no interactive tty to prompt at all. computed once, in shell, never re-derived by a model). On `action=none` the daemon sleeps and loops — **no model/driver process is ever touched**, so a quiet repo costs nothing beyond the tick's own `gh` calls. On an actionable verdict (`action=advance issue=N` / `action=feedback pr=N`) it spawns exactly - **one** contained driver: `setsid timeout --kill-after=30s claude - --model -p "" --output-format json`. `setsid` gives the driver (and any - bash children it spawns) its own process group, independent of the daemon's; on timeout the whole group - is targeted, not just the immediate child, so a driver's own children can never be orphaned by a bare - `SIGTERM`. **However, a process group is NOT a cgroup**: the driver still lives inside the daemon - *service's* cgroup, and systemd's default `KillMode=control-group` kills everything in it whenever the - unit stops — so `systemctl --user restart pr-loop-`, a daemon crash (`Restart=always` bounce), a - host reboot/sleep, or `wsl --shutdown` all kill an in-flight driver mid-run, with **no ledger line** - (the ledger write is the daemon's last act *after* the driver exits) and the driver's branch/worktree - left as debris (see the failure contract below). Before restarting the daemon or the machine, check - nothing is in flight: the last `.claude/state/loop-runs.log` spawn has a matching `result=` completion, - and `.claude/state/worker-tools.jsonl` has gone quiet. Issue #119 (planned) decouples driver lifetime - from the daemon's (one transient systemd unit per driver + startup re-attach) and retires this caveat. - The daemon itself never runs two drivers concurrently (it's a single-threaded loop), and - `loop-tick.sh`'s own spawn lock additionally guards against a second overlapping tick anywhere else - (e.g. the legacy cron armed at the same time) double-firing the same ADVANCE. + **one** contained driver, as a **transient `systemd --user` unit, decoupled from the daemon's own + lifetime** (issue #119): `systemd-run --user --wait --collect --unit="pr-loop-driver-issue" + -p RuntimeMaxSec= -- bash -c 'claude --model -p "" + --output-format json > 2> '` (unit is `pr-loop-driver-pr` for a `feedback` + verdict). `--wait` makes the daemon's own invocation a **disposable waiter**: it blocks and relays the + unit's exit code exactly like a backgrounded `wait` would, but the driver's REAL parent is the user + manager, which lives in its own scope outside the daemon's cgroup — so `systemctl --user restart + pr-loop-`, a daemon crash (`Restart=always` bounce), or the daemon dying for any other reason kills + only that disposable waiter, **never the driver itself**. `RuntimeMaxSec` (a systemd time-span — `90m` + works as-is) replaces `timeout` as the hard wall-clock ceiling, now enforced by the user manager instead + of the daemon, so an orphaned driver still has a real ceiling even if the daemon never restarts. + **Restart is now driver-safe** — a host reboot/sleep or `wsl --shutdown` still kills the driver along with + everything else (nothing can prevent that), but a plain daemon bounce no longer does. `run_driver`'s + ledger + issue #111's post-exit verification are unchanged; only the spawn+wait step branches. **Fallback** + (legacy-cron / non-systemd environments, detected via `command -v systemd-run`): the exact `setsid timeout + --kill-after=30s claude ...` spawn from before #119, unchanged — `setsid` gives the + driver (and any bash children it spawns) its own process group, independent of the daemon's; on timeout + the whole group is targeted, not just the immediate child, so a driver's own children can never be + orphaned by a bare `SIGTERM`. That fallback path still has the pre-#119 caveat: a process group is not a + cgroup, so it dies WITH the daemon's own service cgroup on a restart/crash/reboot. **Startup re-attach**: + before the first tick, `main()` checks `systemctl --user list-units 'pr-loop-driver-*'` (no-op when + systemd is unavailable) — if a driver unit from a previous, now-dead daemon process is still active, the + daemon WAITS for it instead of ticking (no duplicate spawn, no premature debris classification), then runs + the same post-exit verification + ledger write a fresh spawn would have. `loop-census.sh`'s ADVANCE check + additionally never picks an issue whose driver unit is currently active, even before that driver has + reached `git checkout -b` (so it has no branch yet for the `in_flight` check to catch). **Ops helper**: + `.claude/scripts/loop-halt.sh` stops one driver (`loop-halt.sh issue106` / `loop-halt.sh pr42`), all + drivers (`loop-halt.sh --drivers`), or the daemon + all drivers together (`loop-halt.sh --all`) — see the + failure contract below for when you'd want each. The daemon itself never runs two drivers concurrently + (it's a single-threaded loop), and `loop-tick.sh`'s own spawn lock additionally guards against a second + overlapping tick anywhere else (e.g. the legacy cron armed at the same time) double-firing the same + ADVANCE. - **`claude-rc-.service`** → `claude remote-control` inside a detached tmux session (`rc-`), for spawning **new planning sessions remotely** — from claude.ai or the Claude Code mobile app — decoupled from the loop's own ticking. `arm-loop.sh --capacity N --permission-mode ` controls its @@ -191,13 +207,16 @@ there's something actionable *now*). `loop-daemon.sh`'s `cadence_to_sleep_second Override any of the four via `LOOP_DAEMON_SLEEP_FAST` / `_WATCH` / `_IDLE` / `_FALLBACK` (seconds, test/debug hooks). -**Ledger.** Every driver spawn — successful, timed out, or refused-to-spawn — appends one line to -`.claude/state/loop-runs.log` (gitignored, never committed): +**Ledger.** Every driver spawn — successful, timed out, refused-to-spawn, or re-attached at startup — +appends one line to `.claude/state/loop-runs.log` (gitignored, never committed): ``` -pid= session= verdict= ts= [result=exit|timeout|spawn-error rc=N] +pid= session= verdict= ts= [result=exit|timeout|spawn-error rc=N] [pr=N] [debris=... [action=deleted]] [reattached=true] ``` `session_id` is parsed out of the driver's own `--output-format json` stdout, which is what makes a -hung or already-finished driver resumable later. +hung or already-finished driver resumable later. A `reattached=true` line (issue #119) means the daemon +found this driver already running as a systemd unit at startup — left behind by a previous, now-dead +daemon process — waited for it instead of spawning a new one, and ledgered its real outcome; `pid=unknown` +on that line because the daemon that actually spawned it is gone. **Supervision.** Three read-only windows into a driver, cheapest first: 1. `tail -f .claude/state/loop-runs.log` — the ledger line above, one per spawn. @@ -209,12 +228,18 @@ hung or already-finished driver resumable later. reads the append-only transcript and branches a new one. **Intervention is kill-and-let-it-re-advance, never steer.** A driver is a headless `claude -p` process with -no attach point — there is no "type into it and redirect it" option. To stop one: find its `pid=` (a -process **group** id) in the ledger and `kill -TERM -- -` (the same target `timeout --kill-after` -would eventually use anyway). Do **not** try to nudge a running driver's behavior. Instead, let -`loop-tick.sh`'s own pre-branch spawn lock (`.claude/state/loop-advance.lock`, 15-minute TTL) self-heal so a -later tick can re-advance the same issue cleanly, or fix forward with a normal orchestrator pass once -whatever state the killed driver left behind (a branch, a PR) is visible to a fresh tick. +no attach point — there is no "type into it and redirect it" option. To stop one: **`bash +.claude/scripts/loop-halt.sh issue`** (or `pr` for a feedback driver) — `systemctl --user stop +pr-loop-driver-issue` under the hood (issue #119). This is now the right tool on the systemd path: the +ledger's `pid=` is the daemon's own disposable `systemd-run --wait` waiter, not the driver, so killing that +pid does nothing to the actual driver. `loop-halt.sh --drivers` stops every active driver at once; +`loop-halt.sh --all` also stops the daemon unit. On the legacy fallback path (no `systemd-run` on `PATH`), +`pid=` is still a real process **group** id and `kill -TERM -- -` (the same target `timeout +--kill-after` would eventually use anyway) still works, same as before #119. Do **not** try to nudge a +running driver's behavior either way. Instead, let `loop-tick.sh`'s own pre-branch spawn lock +(`.claude/state/loop-advance.lock`, 15-minute TTL) self-heal so a later tick can re-advance the same issue +cleanly, or fix forward with a normal orchestrator pass once whatever state the killed driver left behind (a +branch, a PR) is visible to a fresh tick. **Remote planning sessions.** `claude-rc-.service` keeps a `claude remote-control` process alive in a detached tmux session, independent of the loop daemon's own ticking, so you can spawn a **new** planning @@ -256,31 +281,47 @@ duplicate-daemon risk. Skipping this is safe: GitHub is the loop's only source of truth, so anything that happened while WSL2 was stopped is simply picked up by the first tick after the next manual WSL2 start. -**WSL2 caveat — going down is safe for the QUEUE, not for a driver in flight.** Autostart + linger bring -the *daemon* back after a reboot, but a Windows reboot, sleep/hibernate, or `wsl --shutdown` that lands -while a driver is mid-run kills that driver with no ledger line, leaving `in_flight` debris (see the -failure contract's "Daemon killed mid-driver" row — this exact pattern killed five of six drivers on -2026-07-14/15). Until #119 lands, prefer rebooting/shutting down when `tail -1 -.claude/state/loop-runs.log` shows the last spawn completed (`result=` present). +**WSL2 caveat — going down is safe for the QUEUE and for a plain daemon restart, not for a HOST-level +shutdown while a driver is mid-run.** Issue #119 made a `systemctl --user restart pr-loop-` (or a +daemon crash) driver-safe: the transient driver unit lives under the `--user` manager, outside the daemon's +own cgroup, so bouncing the daemon no longer kills it (this is what actually killed five of six drivers on +2026-07-14/15, and is now fixed). What #119 can NOT fix: a Windows reboot, sleep/hibernate, or `wsl +--shutdown` tears down the WHOLE WSL2 VM — user manager included — so a driver mid-run at that moment still +dies, now with **no ledger line** the same way a hard `kill -9` on any process would. Check +`systemctl --user list-units 'pr-loop-driver-*'` (or `tail -1 .claude/state/loop-runs.log` for the last +spawn's completion) before a host-level reboot/shutdown; `bash .claude/scripts/loop-halt.sh --drivers` stops +any in-flight drivers cleanly first if you'd rather not wait. Inspect what's armed: ```bash systemctl --user status pr-loop-.service journalctl --user -u pr-loop-.service -f +systemctl --user list-units 'pr-loop-driver-*' # active driver units (issue #119) tail -f .claude/state/loop-runs.log tmux attach -t rc- ``` +Stop things by hand (issue #119 — see `.claude/scripts/loop-halt.sh`): +```bash +bash .claude/scripts/loop-halt.sh issue106 # stop one driver: systemctl --user stop pr-loop-driver-issue106 +bash .claude/scripts/loop-halt.sh pr42 # stop one driver: systemctl --user stop pr-loop-driver-pr42 +bash .claude/scripts/loop-halt.sh --drivers # stop ALL driver units: systemctl --user stop 'pr-loop-driver-*' +bash .claude/scripts/loop-halt.sh --all # stop the daemon unit AND all driver units +``` +Degrades cleanly (logs and exits 0) when `systemctl --user` is unavailable — the legacy fallback path's +drivers are plain daemon children, already covered by stopping the daemon itself. + **Failure contract.** | Failure | Detected as | Self-heals? | Manual fix | |---|---|---|---| | Driver exits non-zero (gate failure, crash mid-run, …) | ledger `result=exit rc=N` | Yes — the next tick's fresh census decides the next action from scratch | none | -| Driver runs past `LOOP_DRIVER_TIMEOUT` (default 90m) | ledger `result=timeout rc=124\|137`; `timeout --kill-after=30s` plus an explicit process-group kill | Yes — same as above | none, unless it left a half-finished branch behind — inspect and fix forward | +| Driver runs past `LOOP_DRIVER_TIMEOUT` (default 90m) | ledger `result=timeout rc=124\|137`; `RuntimeMaxSec` (systemd path) or `timeout --kill-after=30s` plus an explicit process-group kill (fallback path) | Yes — same as above | none, unless it left a half-finished branch behind — inspect and fix forward | | `claude` CLI not found on `PATH` (nor via the `nvm` fallback) | ledger `result=spawn-error rc=127`, logged before any spawn attempt | Partially — the pre-branch spawn lock's 15-minute TTL clears and lets a later tick retry, but every retry hits the same missing-`PATH` wall | fix `PATH`/`nvm` in the daemon's environment (e.g. the systemd unit's `Environment=`), then `systemctl --user restart pr-loop-.service` | | `loop-tick.sh` / `loop-event.sh` itself exits non-zero (broken tick) | daemon logs "not spawning a driver on a broken tick", sleeps the fallback cadence, retries | Yes — retried automatically every tick | investigate only if it persists across many ticks | | **Driver created `feat/issue-N-*` but died before opening the PR** | census reports `N` as `in_flight`; `loop-tick.sh`'s advance check refuses (`# advance refused: issue=N is in_flight`, logged every single tick) and emits `action=none` | **No** — unlike the pre-branch spawn lock, `in_flight` has no TTL/self-heal; it refuses forever until the branch or a PR's state changes | **Classify first, never blind-delete** (`git log main..` + worktree status): **empty** (no commits, clean) → delete the branch/worktree, freeing the issue back to `advance_ready`; **publishable** (commits ahead, clean, reviews/gates were green) → re-run the test gate, push, open the PR by hand via `bot-gh.sh` (proven: #107 → PR #117); **half-done or dirty worktree** → never delete — `wip:`-commit to preserve, then finish via a normal orchestrator pass on the existing branch. Issues #111 (classify + mechanical paths) and #98 (resume half-done work) automate this. | -| **Daemon killed mid-driver** (service restart, daemon crash, host reboot/sleep, `wsl --shutdown`) | ledger shows a spawn with **no `result=` completion line**; `events.jsonl`/`worker-tools.jsonl` activity for the task stops abruptly | **No** — the driver dies with the daemon's cgroup (see the `KillMode` caveat above); the restarted daemon just ticks on, and any branch the driver created wedges as `in_flight` per the row above | avoid restarting the daemon/host while a driver is in flight; recover debris per the row above; #119 (transient unit per driver) removes the restart-collateral case | +| **Daemon killed mid-driver** (service restart, daemon crash) — **fixed by issue #119** | ledger shows a `reattached=true` line once the daemon restarts and re-attaches, with the driver's real outcome (not a missing/phantom line) | **Yes** — the transient driver unit outlives the daemon's own cgroup; `main()`'s startup re-attach waits for it and runs the same post-exit verify + ledger write a fresh spawn would have | none — this is now self-healing. On the legacy fallback path (no `systemd-run` on `PATH`) the pre-#119 behavior still applies: **no** `result=` completion line, driver dies with the daemon's cgroup; recover debris via the row above | +| **Host-level kill while a driver is in flight** (reboot, sleep/hibernate, `wsl --shutdown`) — the one case #119 can't fix, since it tears down the `--user` manager too | ledger shows a spawn with **no `result=` completion line**; `events.jsonl`/`worker-tools.jsonl` activity for the task stops abruptly | **No** — there is no process left anywhere to re-attach to; the restarted daemon just ticks on, and any branch the driver created wedges as `in_flight` per the row above | check `systemctl --user list-units 'pr-loop-driver-*'` (or the ledger's last spawn) before a host-level reboot/shutdown, or `loop-halt.sh --drivers` first; recover debris per the `in_flight` row above | | `claude-rc-.service`'s inner `claude remote-control` process crashes | **not detected by systemd** — the unit is `Type=oneshot`/`RemainAfterExit=yes`; systemd only observes `tmux new -d`'s own (successful) exit, never the health of the process running *inside* that tmux session | No | `tmux attach -t rc-` to check, then `systemctl --user restart claude-rc-.service` | **Never run the daemon and the legacy `/pr-loop` cron against the same repo at the same time** —