From b24e54fee39f521cdfa6d0dd788f79c7dc00ab4c Mon Sep 17 00:00:00 2001 From: Roberto Cano <3525807+robercano@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:40:27 +0200 Subject: [PATCH 1/2] feat(loop): stall detection + bounded resume for hung in_flight issues (issue #98) An in_flight issue (feat/issue-N-* branch, no PR yet) used to be refused forever if its driver hung or died mid-session, silently starving the issue. loop-census.sh now flags stalled=N age_min=M when an in_flight issue's newest events.jsonl activity (task field "N" or "issue-N") is older than budget.stall_minutes (default 30; zero events never counts as stalled). loop-tick.sh resumes a stalled/half-done in_flight candidate (action=resume issue=N branch=...) instead of refusing it, reusing loop-daemon.sh's classify_debris verbatim (issue #111) for the debris check, bounded to 2 attempts via a new .claude/state/loop-resume-attempts.json sibling file; the 3rd stall escalates to needs-human instead of retrying, mirroring the existing per-issue attempt-budget escalation. Every transition (stall detected, resume attempt N, escalated) is best-effort logged via log-event.sh. Co-Authored-By: Claude Sonnet 5 --- .claude/scripts/loop-census.sh | 84 ++++++++++++++- .claude/scripts/loop-census.test.sh | 91 ++++++++++++++++ .claude/scripts/loop-tick.sh | 160 +++++++++++++++++++++++++++- .claude/scripts/loop-tick.test.sh | 158 +++++++++++++++++++++++++++ .claude/self/gates.json | 5 +- 5 files changed, 494 insertions(+), 4 deletions(-) diff --git a/.claude/scripts/loop-census.sh b/.claude/scripts/loop-census.sh index d6d584a..91250ec 100644 --- a/.claude/scripts/loop-census.sh +++ b/.claude/scripts/loop-census.sh @@ -15,6 +15,11 @@ # hasn't reached PR stage. A tick uses this to # avoid double-spawning an orchestrator for an # issue that already has a worktree in progress. +# stalled= age_min= one line PER in_flight issue whose most recent +# events.jsonl activity is older than the stall +# threshold (issue #98) — see STALL DETECTION +# below. A tick uses this to RESUME a hung/dead +# in_flight issue instead of refusing it forever. # blocked= by= one line per candidate that would otherwise be # advance_ready but is skipped because its body # says "Blocked by #N" and issue N is still OPEN @@ -57,6 +62,30 @@ # one candidate was otherwise eligible; with zero eligible candidates, # advance_ready stays "none" exactly as before this feature. # +# --- STALL DETECTION (issue #98) -------------------------------------------- +# An `in_flight` issue (feat/issue-N-* branch exists, no open PR yet) can sit +# forever if the driver that created it hung or died without ever reaching +# loop-daemon.sh's post-exit verification (issue #111) — e.g. the daemon +# process itself was restarted/killed mid-drive. This census can't see driver +# health directly, but it CAN see whether anything has logged progress for +# that issue recently: log-event.sh (issue #52) appends one JSONL line per +# phase transition to events.jsonl, with a `task` field that's been observed +# in BOTH a bare issue number ("42") and an "issue-42" form across this +# project's real history — an in_flight issue is STALLED when the newest +# event naming it (by either form) is older than `budget.stall_minutes` +# (adapter-configurable, default 30; see gates.json). +# +# CONSERVATIVE FALSE-POSITIVE RULE: an issue with ZERO events at all is NEVER +# reported stalled — a branch/worktree just created by a driver that hasn't +# logged its first event yet looks identical to a permanently-abandoned one +# from events.jsonl's point of view alone; treating "no data yet" as "stalled" +# would kill fresh work. Only a issue with AT LEAST ONE event, whose newest is +# past the threshold, is reported. +# +# Events file: defaults to /.claude/state/events.jsonl; override with +# $CLAUDE_EVENTS_FILE (same env var log-event.sh itself honors) for +# testability without touching the real, gitignored state dir. +# # Repo derived from the git remote; override with $1. Bot login via $BOT_LOGIN. # Invoke as `bash .claude/scripts/loop-census.sh` (pre-approve that exact # command). Read-only: advances no cursor, mutates nothing — safe to re-run. @@ -76,6 +105,46 @@ case "$gates_rel" in /*) gates="$gates_rel" ;; *) gates="$root/$gates_rel" ;; es base=$(node -e 'const g=require(process.argv[1]); console.log((g.merge&&g.merge.baseBranch)||"main")' "$gates") module_labels=$(node -e 'const g=require(process.argv[1]); console.log(g.modules.map(m=>"module:"+m.name).join("\n"))' "$gates") +# Stall threshold (issue #98), adapter-overridable via budget.stall_minutes; +# same node -e / require(gates) pattern as base/module_labels above. +stall_minutes=$(node -e ' + const g = require(process.argv[1]); + const v = g.budget && g.budget.stall_minutes; + console.log((Number.isFinite(v) && v > 0) ? v : 30); +' "$gates" 2>/dev/null) +case "$stall_minutes" in ''|*[!0-9]*) stall_minutes=30 ;; esac + +events_file="${CLAUDE_EVENTS_FILE:-$root/.claude/state/events.jsonl}" + +# --- stall detection helper (issue #98) -------------------------------------- +# $1 = issue number. Prints the age in whole minutes of the NEWEST +# events.jsonl line whose `task` field is either "$1" or "issue-$1" (both +# forms occur in this project's real log), or nothing when there is no such +# event at all — callers must treat empty as "do not report stalled" (see the +# conservative false-positive rule in the header comment above), never as 0. +last_event_age_minutes() { + CLAUDE_STALL_EVENTS_FILE="$events_file" CLAUDE_STALL_ISSUE="$1" node -e ' + const fs = require("fs"); + const file = process.env.CLAUDE_STALL_EVENTS_FILE; + const num = process.env.CLAUDE_STALL_ISSUE; + let latest = null; + try { + const text = fs.readFileSync(file, "utf8"); + for (const line of text.split("\n")) { + if (!line.trim()) continue; + let o; + try { o = JSON.parse(line); } catch (e) { continue; } + if (o.task !== num && o.task !== ("issue-" + num)) continue; + const t = Date.parse(o.ts); + if (!Number.isFinite(t)) continue; + if (latest === null || t > latest) latest = t; + } + } catch (e) { /* no file / unreadable -> latest stays null */ } + if (latest === null) process.exit(0); + console.log(Math.floor((Date.now() - latest) / 60000)); + ' 2>/dev/null +} + # --- driver-unit guard (issue #119): never advance an issue whose transient # driver unit (pr-loop-driver-issue, spawned by loop-daemon.sh's run_driver) # is currently active — e.g. the driver hasn't reached `git checkout -b` yet, @@ -135,6 +204,7 @@ advance_ready="none" fallback_ready="none" detail="" in_flight="" +stalled_lines="" blocked_lines="" while IFS=$'\t' read -r num labels title; do [ -z "${num:-}" ] && continue @@ -198,7 +268,18 @@ while IFS=$'\t' read -r num labels title; do "$b"|*"/$b") has_open_pr=1; break ;; esac done <<< "$open_pr_branches" - [ "$has_open_pr" -eq 1 ] || in_flight+="in_flight=$num"$'\n' + if [ "$has_open_pr" -eq 1 ]; then + : # already has an open PR -- never in_flight, never stalled + else + in_flight+="in_flight=$num"$'\n' + # --- stall detection (issue #98): only for genuinely in_flight issues, + # and only when at least one event exists (see the conservative + # false-positive rule in the header comment above) --- + age_min="$(last_event_age_minutes "$num")" + if [ -n "$age_min" ] && [ "$age_min" -ge "$stall_minutes" ]; then + stalled_lines+="stalled=$num age_min=$age_min"$'\n' + fi + fi fi done <<< "$planned" @@ -212,6 +293,7 @@ fi echo "planned_issues=$planned_count" [ -n "$detail" ] && printf '%s' "$detail" [ -n "$in_flight" ] && printf '%s' "$in_flight" +[ -n "$stalled_lines" ] && printf '%s' "$stalled_lines" [ -n "$blocked_lines" ] && printf '%s' "$blocked_lines" echo "advance_ready=$advance_ready" diff --git a/.claude/scripts/loop-census.test.sh b/.claude/scripts/loop-census.test.sh index 0e11b41..dc7f9ed 100644 --- a/.claude/scripts/loop-census.test.sh +++ b/.claude/scripts/loop-census.test.sh @@ -502,6 +502,97 @@ check "(d) task-list ref alone does not gate — advance_ready=40" bash -c \ check "(d) no blocked= line emitted for a task-list-only reference" bash -c \ '! printf "%s\n" "$1" | grep -q "^blocked="' _ "$outTasklist" +# --------------------------------------------------------------------------- +# Stall detection (issue #98): loop-census.sh must emit `stalled=N age_min=M` +# for an in_flight issue whose newest events.jsonl activity (task field either +# "N" or "issue-N" -- both forms occur in real logs) is older than +# budget.stall_minutes, while leaving fresh/zero-event/has-a-PR issues alone. +# +# Fixture: four planned+module:test issues, each with its own +# feat/issue-N-* branch and NO open PR (issue 90 is the one exception, with +# an open PR, to prove "stalled branch + open PR -> NOT stalled"): +# 80 stale events (task="80", well past the threshold) -> stalled +# 81 fresh events (task="issue-81", well within the threshold) -> NOT stalled +# 82 zero events at all for this task -> NOT stalled +# (conservative false-positive rule: never kill a just-created branch) +# 90 stale events (task="90") but an OPEN PR already exists -> NOT stalled +# (never even in_flight, so never even considered for staleness) +# stall_minutes is set to 2 in this fixture's own gates.json so the test +# doesn't need to wait a real 30 minutes -- timestamps below are computed +# relative to the ACTUAL wall clock at test run time via `date -u -d`. +# --------------------------------------------------------------------------- +dirStall="$work/stall" +scriptsStall="$dirStall/.claude/scripts" +mkdir -p "$scriptsStall" +cp "$census_src" "$scriptsStall/loop-census.sh" +cp "$resolve_roots_src" "$scriptsStall/resolve-roots.sh" +cat > "$dirStall/.claude/gates.json" <<'EOF' +{ + "modules": [{ "name": "test", "path": ".", "description": "", "owner": "" }], + "merge": { "baseBranch": "main" }, + "budget": { "stall_minutes": 2 } +} +EOF +cat > "$scriptsStall/pr-feedback.sh" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +cat > "$scriptsStall/bot-gh.sh" <<'EOF' +#!/usr/bin/env bash +case "$1" in + repo) echo "acme/repo" ;; + pr) + if printf '%s\n' "$*" | grep -q 'headRefName'; then + printf '%s\n' "feat/issue-90-x" + else + echo 1 + fi + ;; + issue) + printf '80\tplanned,module:test\tStale issue eighty\n' + printf '81\tplanned,module:test\tFresh issue eighty one\n' + printf '82\tplanned,module:test\tNo-events issue eighty two\n' + printf '90\tplanned,module:test\tStale-but-has-PR issue ninety\n' + ;; + *) echo "fake-bot-gh.sh: unhandled args: $*" >&2; exit 1 ;; +esac +EOF +chmod +x "$scriptsStall"/*.sh +git -C "$dirStall" init -q -b main +git -C "$dirStall" -c user.email=t@e.st -c user.name=t commit -q --allow-empty -m init +git -C "$dirStall" branch feat/issue-80-a main >/dev/null +git -C "$dirStall" branch feat/issue-81-a main >/dev/null +git -C "$dirStall" branch feat/issue-82-a main >/dev/null +git -C "$dirStall" branch feat/issue-90-x main >/dev/null + +eventsStall="$work/stall-events.jsonl" +stale_ts="$(date -u -d '-45 minutes' +%Y-%m-%dT%H:%M:%SZ)" +fresh_ts="$(date -u -d '-1 minutes' +%Y-%m-%dT%H:%M:%SZ)" +{ + printf '{"ts":"%s","role":"implementer","model":"sonnet","task":"80","phase":"implementing","lens":"","detail":""}\n' "$stale_ts" + printf '{"ts":"%s","role":"implementer","model":"sonnet","task":"issue-81","phase":"implementing","lens":"","detail":""}\n' "$fresh_ts" + printf '{"ts":"%s","role":"implementer","model":"sonnet","task":"90","phase":"implementing","lens":"","detail":""}\n' "$stale_ts" +} > "$eventsStall" + +outStall="$(env -u GATES_FILE CLAUDE_EVENTS_FILE="$eventsStall" bash "$scriptsStall/loop-census.sh" "acme/repo")" + +check "stall: stale in_flight issue 80 (task=\"80\" form) IS reported stalled" bash -c \ + 'printf "%s\n" "$1" | grep -q "^stalled=80 age_min="' _ "$outStall" +check "stall: fresh in_flight issue 81 (task=\"issue-81\" form) is NOT stalled" bash -c \ + '! printf "%s\n" "$1" | grep -q "^stalled=81 "' _ "$outStall" +check "stall: issue 82 has zero events -> NOT stalled (conservative false-positive rule)" bash -c \ + '! printf "%s\n" "$1" | grep -q "^stalled=82 "' _ "$outStall" +check "stall: issue 90 has stale events but an OPEN PR -> NOT stalled" bash -c \ + '! printf "%s\n" "$1" | grep -q "^stalled=90 "' _ "$outStall" +check "stall: issue 90 with an open PR is also NOT in_flight" bash -c \ + '! printf "%s\n" "$1" | grep -qx "in_flight=90"' _ "$outStall" +check "stall: exactly one stalled= line total" bash -c \ + '[ "$(printf "%s\n" "$1" | grep -c "^stalled=")" -eq 1 ]' _ "$outStall" +check "stall: age_min on the stalled line is at least the 2-minute threshold" bash -c ' + age="$(printf "%s\n" "$1" | sed -n "s/^stalled=80 age_min=\([0-9]*\)/\1/p")" + [ -n "$age" ] && [ "$age" -ge 2 ] +' _ "$outStall" + echo "" if [ "$fail" -eq 0 ]; then echo "loop-census.test.sh: PASS ($ok checks)" diff --git a/.claude/scripts/loop-tick.sh b/.claude/scripts/loop-tick.sh index eccdec4..d91c3e2 100644 --- a/.claude/scripts/loop-tick.sh +++ b/.claude/scripts/loop-tick.sh @@ -115,6 +115,11 @@ write_tick_record() { case "$verdict" in "action=advance issue="*) action="advance"; issue="${verdict#action=advance issue=}" ;; "action=feedback pr="*) action="feedback"; pr="${verdict#action=feedback pr=}" ;; + "action=resume issue="*) + action="resume" + issue="${verdict#action=resume issue=}" + issue="${issue%% *}" + ;; "action=none") action="none" ;; *) action="${verdict#action=}" @@ -516,6 +521,115 @@ if [ -z "$ceiling_block" ] && [ -n "$attempt_issue" ]; then fi fi +# --------------------------------------------------------------------------- +# STEP 0.5: stall/resume machinery (issue #98). An `in_flight` candidate (a +# feat/issue-N-* branch exists, no open PR yet) used to be refused OUTRIGHT, +# FOREVER -- exactly the incident issue #111's post-exit debris classifier +# deals with AFTER a driver exits cleanly, but with no equivalent for a +# driver that's still nominally "in flight" per the ledger yet has gone quiet +# mid-session (hung, or its session died before loop-daemon.sh's own exit +# handler ever ran verify_and_classify_post_exit on it). This closes that +# gap: an in_flight candidate is RESUMED (verdict points at its existing +# branch instead of refusing) when EITHER of two independent signals fires, +# neither reimplemented here: +# - loop-census.sh's own stall clock (`stalled=N age_min=M`, driven by +# events.jsonl inactivity -- issue #98 pt 1), or +# - the branch's debris classifies as "half-done" via loop-daemon.sh's +# classify_debris (issue #111) -- called VERBATIM in a subshell below so +# the absent/empty/publishable/half-done state vocabulary never diverges +# between the two call sites. +# Bounded to 2 resume attempts, tracked in a SIBLING state file +# (.claude/state/loop-resume-attempts.json, {count,escalated} shape + +# mktemp/atomic-mv discipline mirroring loop-issue-attempts.json) rather than +# folded into loop-issue-attempts.json itself: that file counts advance/ +# feedback DISPATCHES against issue #95's spend ceiling -- a different budget +# than "how many times has THIS stalled branch been resumed"; conflating the +# two would let a resume silently eat into (or be eaten by) the dispatch +# budget. The 3rd stall does not resume again: it escalates to needs-human +# (mirroring the attempt-budget escalation block above -- label create + +# issue edit + issue comment) and sets escalated:true so the loop never +# auto-retries it again. +# --------------------------------------------------------------------------- +resume_attempts_file="$state_dir/loop-resume-attempts.json" + +# Best-effort telemetry only (issue #98) -- must never affect the tick, and +# must never explode in a test fixture that doesn't ship log-event.sh. +log_loop_event() { + [ -f "$script_dir/log-event.sh" ] || return 0 + bash "$script_dir/log-event.sh" --role orchestrator --task "$1" --phase "$2" --detail "${3:-}" >/dev/null 2>&1 || true +} + +# Is issue $1 named on one of census's own `stalled=N age_min=M` lines? +is_census_stalled() { + printf '%s\n' "$census_out" | grep -q "^stalled=$1 " +} + +# The bare branch name census printed for issue $1 on its +# "issue=$1 branch= title=..." detail line -- stripped of any +# "origin/" remote-tracking prefix census may report (classify_debris and +# loop-daemon.sh's worktree_for_branch both expect the BARE local branch +# name). Prints nothing when the issue has no branch, or census's fixture +# output never emitted a detail line for it. +branch_for_issue() { + local b + b="$(printf '%s\n' "$census_out" | sed -n "s/^issue=$1 branch=\([^ ]*\) .*/\1/p" | head -1)" + [ -n "$b" ] && [ "$b" != "none" ] && printf '%s' "${b#origin/}" + return 0 +} + +# classify_debris for issue $1's branch, reusing loop-daemon.sh's classifier +# VERBATIM (issue #111) -- sourced in a SUBSHELL so its top-level state +# (state_dir, ledger, etc.) never leaks into this script's own variables. +# Prints "absent" when there's no branch to classify, or when loop-daemon.sh +# isn't sitting next to this script (a test fixture that never copied it in +# -- guarded so those fixtures degrade to the pre-#98 in_flight-refuses +# behavior instead of erroring). +classify_debris_for_issue() { + local branch; branch="$(branch_for_issue "$1")" + [ -n "$branch" ] || { printf 'absent'; return 0; } + [ -f "$script_dir/loop-daemon.sh" ] || { printf 'absent'; return 0; } + ( + # shellcheck source=loop-daemon.sh + . "$script_dir/loop-daemon.sh" + wt="$(worktree_for_branch "$branch")" + classify_debris "$branch" "$wt" + ) +} + +# $1=issue. Prints "\t" from resume_attempts_file. +read_resume_state() { + local key="$1" + CLAUDE_RES_KEY="$key" node -e ' + const fs = require("fs"); + const key = process.env.CLAUDE_RES_KEY; + try { + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + const e = j[key] || {}; + console.log((e.count||0) + "\t" + (e.escalated?1:0)); + } catch (e) { console.log("0\t0"); } + ' "$resume_attempts_file" 2>/dev/null || printf '0\t0' +} + +# $1=issue $2=count $3=escalated(0|1). Atomic temp+mv, mirroring every other +# state-file writer in this script. +write_resume_state() { + local key="$1" count="$2" escalated="$3" + local tmp; tmp="$(mktemp "$state_dir/.loop-resume-attempts.json.XXXXXX")" + if CLAUDE_RES_KEY="$key" CLAUDE_RES_COUNT="$count" CLAUDE_RES_ESC="$escalated" node -e ' + const fs = require("fs"); + const file = process.argv[1], tmp = process.argv[2]; + const key = process.env.CLAUDE_RES_KEY; + let j = {}; + try { j = JSON.parse(fs.readFileSync(file, "utf8")); } catch (e) {} + j[key] = { count: parseInt(process.env.CLAUDE_RES_COUNT, 10) || 0, escalated: process.env.CLAUDE_RES_ESC === "1" }; + fs.writeFileSync(tmp, JSON.stringify(j, null, 2) + "\n"); + ' "$resume_attempts_file" "$tmp" 2>/dev/null; then + mv -f "$tmp" "$resume_attempts_file" + else + rm -f "$tmp" + fi +} + # --- Decide the verdict ----------------------------------------------------- # The verdict string is captured into a variable (rather than echoed inline) # so it can ALSO be persisted to the tick log below without disturbing the @@ -529,8 +643,46 @@ elif [ -n "$feedback_pr" ]; then verdict="action=feedback pr=$feedback_pr" elif [ "$advance_ready" != "none" ] && [ -n "$advance_ready" ]; then if printf '%s\n' "$in_flight_issues" | grep -qx "$advance_ready"; then - echo "# advance refused: issue=$advance_ready is in_flight (a feat/issue-$advance_ready-* branch already exists with no open PR)" - verdict="action=none" + # --- stall/resume path (issue #98) -------------------------------------- + stalled_now=0 + is_census_stalled "$advance_ready" && stalled_now=1 + debris_now="none" + if [ "$stalled_now" -eq 0 ]; then + debris_now="$(classify_debris_for_issue "$advance_ready")" + fi + if [ "$stalled_now" -eq 1 ] || [ "$debris_now" = "half-done" ]; then + log_loop_event "$advance_ready" "stall-detected" "in_flight issue=$advance_ready flagged stalled=$stalled_now debris=$debris_now" + IFS=$'\t' read -r resume_count resume_escalated <<<"$(read_resume_state "$advance_ready")" + case "$resume_count" in ''|*[!0-9]*) resume_count=0 ;; esac + case "$resume_escalated" in ''|*[!01]*) resume_escalated=0 ;; esac + resume_branch="$(branch_for_issue "$advance_ready")" + if [ "$resume_escalated" = "1" ]; then + echo "# advance refused: issue=$advance_ready is in_flight and stalled, but already escalated to needs-human -- not retrying" + verdict="action=none" + elif [ "$resume_count" -lt 2 ]; then + new_resume_count=$((resume_count + 1)) + write_resume_state "$advance_ready" "$new_resume_count" "0" + echo "# resume: issue=$advance_ready is in_flight and stalled (attempt $new_resume_count/2) -- resuming the existing branch/worktree instead of refusing" + log_loop_event "$advance_ready" "resume-attempt" "resume attempt $new_resume_count/2 for issue=$advance_ready" + if [ -n "$resume_branch" ]; then + verdict="action=resume issue=$advance_ready branch=$resume_branch" + else + verdict="action=resume issue=$advance_ready" + fi + else + write_resume_state "$advance_ready" "$resume_count" "1" + gh label create "needs-human" --color b60205 --description "Loop attempt budget exhausted -- needs a human" --force >/dev/null 2>&1 || true + body="Issue #$advance_ready's feat/issue-$advance_ready-* branch has stalled and already been resumed $resume_count times without landing a PR. The loop will not retry it automatically -- labeling \`needs-human\`. Address it by hand, then either close it out or clear its entry in .claude/state/loop-resume-attempts.json to let the loop resume." + gh issue edit "$advance_ready" --add-label needs-human >/dev/null 2>&1 || true + gh issue comment "$advance_ready" --body "$body" >/dev/null 2>&1 || true + echo "# advance refused: issue=$advance_ready exhausted its 2 resume attempts -- escalated to needs-human" + log_loop_event "$advance_ready" "escalated-to-needs-human" "issue=$advance_ready escalated to needs-human after $resume_count resumes" + verdict="action=none" + fi + else + echo "# advance refused: issue=$advance_ready is in_flight (a feat/issue-$advance_ready-* branch already exists with no open PR)" + verdict="action=none" + fi elif [ "$lock_issue" = "$advance_ready" ]; then echo "# advance refused: spawn lock already held for issue=$advance_ready ($(cat "$lock_file" 2>/dev/null))" verdict="action=none" @@ -551,6 +703,10 @@ fi dispatch_issue="" case "$verdict" in "action=advance issue="*) dispatch_issue="${verdict#action=advance issue=}" ;; + "action=resume issue="*) + dispatch_issue="${verdict#action=resume issue=}" + dispatch_issue="${dispatch_issue%% *}" + ;; "action=feedback pr="*) dispatch_pr="${verdict#action=feedback pr=}" if [ "$dispatch_pr" = "$feedback_pr" ] && [ -n "${feedback_issue:-}" ]; then diff --git a/.claude/scripts/loop-tick.test.sh b/.claude/scripts/loop-tick.test.sh index 5cd16e3..636398f 100644 --- a/.claude/scripts/loop-tick.test.sh +++ b/.claude/scripts/loop-tick.test.sh @@ -378,6 +378,164 @@ rc14=$? check "scenario 14: tick-record write failure never changes the exit status" [ "$rc14" -eq 0 ] check "scenario 14: verdict is still the LAST stdout line despite the write failure" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=none" ]' _ "$out14" +# --------------------------------------------------------------------------- +# 13. Stall/resume machinery (issue #98). An in_flight candidate that census +# flags stalled=N (or whose branch classifies as half-done debris) must get a +# `action=resume issue=N branch=...` verdict instead of a flat refusal, bounded +# to 2 resume attempts before escalating to needs-human on the 3rd stall. +# --------------------------------------------------------------------------- + +# Like new_fixture, but also copies in the REAL log-event.sh (issue #98 +# telemetry) -- needed only by scenarios that actually reach the stall/resume +# path; every earlier scenario above never calls log_loop_event at all +# (in_flight-without-stall short-circuits before it), so this never disturbs +# them. +new_fixture_with_events() { + local dir + dir="$(new_fixture "$@")" + cp "$script_dir/log-event.sh" "$dir/log-event.sh" + chmod +x "$dir/log-event.sh" + printf '%s\n' "$dir" +} + +# Scenario 15: census's own stall clock (stalled=42) fires -> 1st resume +# attempt. Verdict points at the existing branch; resume-attempts.json now +# records count=1; both a stall-detected and a resume-attempt event land in +# events.jsonl. +ticks15_events="$work/scenario15-events.jsonl" +dir15="$(new_fixture_with_events scenario15 'open_prs=0 +feedback_prs=0 +planned_issues=1 +issue=42 branch=feat/issue-42-x title=Stalled thing +in_flight=42 +stalled=42 age_min=45 +advance_ready=42 +cadence=FAST cron=* * * * *' '')" +out15="$(CLAUDE_EVENTS_FILE="$ticks15_events" run_tick "$dir15")" +check "scenario 15 (resume via census stall clock): verdict is action=resume issue=42 branch=feat/issue-42-x" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=42 branch=feat/issue-42-x" ]' _ "$out15" +resume15="$dir15/../state/loop-resume-attempts.json" +check "scenario 15: resume-attempts file records count=1, escalated=false for issue 42" node -e ' + const fs = require("fs"); + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (!j["42"] || j["42"].count !== 1 || j["42"].escalated !== false) throw new Error("got " + JSON.stringify(j)); +' "$resume15" +check "scenario 15: stall-detected event logged (task=42)" bash -c 'grep -q "\"phase\":\"stall-detected\"" "$1" && grep -q "\"task\":\"42\"" "$1"' _ "$ticks15_events" +check "scenario 15: resume-attempt event logged" bash -c 'grep -q "\"phase\":\"resume-attempt\"" "$1"' _ "$ticks15_events" +check "scenario 15: no spawn lock written (resume is not a fresh advance)" [ ! -e "$dir15/../state/loop-advance.lock" ] + +# Scenario 16: SAME fixture/state, tick fires again while still stalled -> +# 2nd resume attempt (bound not yet exhausted). +out16="$(CLAUDE_EVENTS_FILE="$ticks15_events" run_tick "$dir15")" +check "scenario 16 (2nd resume attempt): verdict is still action=resume issue=42" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=42 branch=feat/issue-42-x" ]' _ "$out16" +check "scenario 16: resume-attempts file now records count=2" node -e ' + const fs = require("fs"); + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (!j["42"] || j["42"].count !== 2 || j["42"].escalated !== false) throw new Error("got " + JSON.stringify(j)); +' "$resume15" + +# Scenario 17: 3rd stall (count already at 2) -> escalate to needs-human +# instead of resuming again: label create + issue edit + issue comment (all +# via bot-gh.sh, captured here into a plain log file), escalated:true +# persisted, and an escalated-to-needs-human event logged. +gh_calls17="$work/scenario17-gh-calls.log" +cat > "$dir15/bot-gh.sh" <> "$gh_calls17" +exit 0 +EOF +chmod +x "$dir15/bot-gh.sh" +out17="$(CLAUDE_EVENTS_FILE="$ticks15_events" run_tick "$dir15")" +check "scenario 17 (3rd stall, bound exhausted): verdict is action=none (does not resume again)" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=none" ]' _ "$out17" +check "scenario 17: resume-attempts file now escalated=true (count stays 2)" node -e ' + const fs = require("fs"); + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (!j["42"] || j["42"].count !== 2 || j["42"].escalated !== true) throw new Error("got " + JSON.stringify(j)); +' "$resume15" +check "scenario 17: needs-human label create + issue edit + issue comment all dispatched via bot-gh.sh" bash -c ' + grep -q "^label create needs-human" "$1" && + grep -q "^issue edit 42 --add-label needs-human" "$1" && + grep -q "^issue comment 42 " "$1" +' _ "$gh_calls17" +check "scenario 17: escalated-to-needs-human event logged" bash -c 'grep -q "\"phase\":\"escalated-to-needs-human\"" "$1"' _ "$ticks15_events" + +# Scenario 18: a 4th tick, still stalled, AFTER escalation -> must not retry +# automatically anymore (no further gh calls, no further resume). +gh_calls_before18="$(wc -l < "$gh_calls17" | tr -d ' ')" +out18="$(CLAUDE_EVENTS_FILE="$ticks15_events" run_tick "$dir15")" +check "scenario 18 (already escalated): verdict stays action=none" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=none" ]' _ "$out18" +check "scenario 18: diagnostic cites the prior escalation (not a fresh resume)" bash -c 'printf "%s\n" "$1" | grep -q "already escalated to needs-human"' _ "$out18" +check "scenario 18: no additional gh calls were dispatched (stopped retrying automatically)" bash -c '[ "$(wc -l < "$1" | tr -d " ")" -eq "$2" ]' _ "$gh_calls17" "$gh_calls_before18" + +# Scenario 19: debris-based resume (issue #111's classify_debris reused +# verbatim) -- NO census stalled= line at all, but the branch is genuinely +# "half-done" (uncommitted work sitting in the worktree). Needs a REAL git +# repo (loop-daemon.sh's classify_debris/worktree_for_branch shell out to +# `git`) plus the real loop-daemon.sh copied alongside loop-tick.sh. +new_git_backed_fixture() { + local name="$1" fake_census="$2" fake_feedback="$3" + local dir + dir="$(new_fixture_with_events "$name" "$fake_census" "$fake_feedback")" + cp "$script_dir/loop-daemon.sh" "$dir/loop-daemon.sh" + chmod +x "$dir/loop-daemon.sh" + local root_dir="${dir%/.claude/scripts}" + git -C "$root_dir" init -q -b main + git -C "$root_dir" config user.email t@e.st + git -C "$root_dir" config user.name t + git -C "$root_dir" commit -q --allow-empty -m init + printf '%s\n' "$dir" +} + +dir19="$(new_git_backed_fixture scenario19 'open_prs=0 +feedback_prs=0 +planned_issues=1 +issue=55 branch=feat/issue-55-x title=Debris thing +in_flight=55 +advance_ready=55 +cadence=FAST cron=* * * * *' '')" +root19="${dir19%/.claude/scripts}" +# A SEPARATE linked worktree, not the main checkout -- mirrors real production +# topology (an implementer's branch always lives in its own `.claude/worktrees/` +# checkout, distinct from the root's own `.claude/state/`) and avoids tick's +# OWN bookkeeping files (flock/lock/attempts, written into "$root/.claude/state" +# moments before this classify call) being mistaken for a dirty worktree if the +# candidate branch were checked out directly in root instead. +git -C "$root19" branch feat/issue-55-x main +wt19="$work/scenario19-wt" +git -C "$root19" worktree add -q "$wt19" feat/issue-55-x +echo "wip" > "$wt19/scratch.txt" # uncommitted -> dirty worktree, 0 commits ahead -> half-done +ticks19_events="$work/scenario19-events.jsonl" +out19="$(CLAUDE_EVENTS_FILE="$ticks19_events" run_tick "$dir19")" +check "scenario 19 (resume via half-done debris, no census stall clock): verdict is action=resume issue=55 branch=feat/issue-55-x" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=55 branch=feat/issue-55-x" ]' _ "$out19" +check "scenario 19: stall-detected event records debris=half-done" bash -c 'grep -q "debris=half-done" "$1"' _ "$ticks19_events" +resume19="$dir19/../state/loop-resume-attempts.json" +check "scenario 19: resume-attempts file records count=1 for issue 55" node -e ' + const fs = require("fs"); + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (!j["55"] || j["55"].count !== 1) throw new Error("got " + JSON.stringify(j)); +' "$resume19" + +# Scenario 20 (negative control): in_flight, no census stall clock, and the +# branch's debris is "publishable" (clean, committed, just no PR yet) rather +# than half-done -- must NOT resume; the plain pre-#98 in_flight refusal +# still applies unchanged. +dir20="$(new_git_backed_fixture scenario20 'open_prs=0 +feedback_prs=0 +planned_issues=1 +issue=66 branch=feat/issue-66-x title=Publishable thing +in_flight=66 +advance_ready=66 +cadence=FAST cron=* * * * *' '')" +root20="${dir20%/.claude/scripts}" +# Same separate-worktree topology as scenario 19's fixture above. +git -C "$root20" branch feat/issue-66-x main +wt20="$work/scenario20-wt" +git -C "$root20" worktree add -q "$wt20" feat/issue-66-x +( cd "$wt20" && echo "done" > f.txt && git add f.txt && git -c user.email=t@e.st -c user.name=t commit -q -m work ) +out20="$(run_tick "$dir20")" +check "scenario 20 (in_flight, publishable debris, no stall): still refused, no resume" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=none" ]' _ "$out20" +check "scenario 20: diagnostic is the plain pre-#98 in_flight refusal" bash -c 'printf "%s\n" "$1" | grep -qF "is in_flight (a feat/issue-66-* branch already exists with no open PR)"' _ "$out20" +check "scenario 20: no resume-attempts entry created for issue 66" bash -c '! grep -q "\"66\"" "$1" 2>/dev/null' _ "$dir20/../state/loop-resume-attempts.json" + echo "" if [ "$fail" -eq 0 ]; then echo "loop-tick.test.sh: PASS ($ok checks)" diff --git a/.claude/self/gates.json b/.claude/self/gates.json index ee8d40e..87a0a0b 100644 --- a/.claude/self/gates.json +++ b/.claude/self/gates.json @@ -41,7 +41,10 @@ "_spend_ceilings_note": "issue #95 — loop spend ceilings, read by loop-tick.sh's STEP 0 pre-flight and arm-loop.sh's default --stop-after-days. Documented in docs/TOKEN_BUDGET.md -> 'Loop spend ceilings'.", "stop_after_days": 7, "per_issue_attempts": 5, - "daily_action_ceiling": 50 + "daily_action_ceiling": 50, + + "_stall_resume_note": "issue #98 — stall/resume: stall_minutes is loop-census.sh's threshold (events.jsonl inactivity) for flagging an in_flight issue as stalled=N; loop-tick.sh then resumes it (bounded to 2 attempts, tracked in .claude/state/loop-resume-attempts.json) instead of refusing forever.", + "stall_minutes": 30 }, "merge": { "policy": "pr-per-agent", "baseBranch": "main" } From 6fef3e7e251517b0c36e1655a958d459cc3a939b Mon Sep 17 00:00:00 2001 From: Roberto Cano <3525807+robercano@users.noreply.github.com> Date: Thu, 16 Jul 2026 17:02:24 +0200 Subject: [PATCH 2/2] fix(loop): make stall/resume actually reachable, stop charging #95 budget Review rejected the first cut of issue #98: the resume verdict was gated on advance_ready appearing inside census's in_flight set, but loop-census.sh makes those mutually exclusive (advance_ready requires branch=none, in_flight requires branch!=none) -- so the resume path was dead code, never reachable in production. Rework the verdict decision to scan census's in_flight=/stalled= output directly, independent of advance_ready, once a fresh branchless advance candidate has had first claim on the tick. Also drop action=resume from the dispatch-bookkeeping case that fed loop-issue-attempts.json, matching the header comment's documented intent that resume attempts live only in the sibling loop-resume-attempts.json and never charge issue #95's advance/feedback spend ceiling. Reworks loop-tick.test.sh scenarios 15-20 to respect the real advance_ready/in_flight invariant, and adds precedence, lowest-numbered, escalated-skip, and budget-isolation coverage plus an end-to-end scenario driven by the REAL loop-census.sh (not a hand-fabricated fixture) proving the resume path is reachable through the actual integration. Co-Authored-By: Claude Sonnet 5 --- .claude/scripts/loop-tick.sh | 194 ++++++++++++++++++++-------- .claude/scripts/loop-tick.test.sh | 201 +++++++++++++++++++++++++++++- 2 files changed, 338 insertions(+), 57 deletions(-) diff --git a/.claude/scripts/loop-tick.sh b/.claude/scripts/loop-tick.sh index d91c3e2..3657683 100644 --- a/.claude/scripts/loop-tick.sh +++ b/.claude/scripts/loop-tick.sh @@ -7,6 +7,7 @@ # action=none # action=advance issue=N # action=feedback pr=N +# action=resume issue=N branch= (issue #98 -- see STEP 0.5 below) # # WHY THIS EXISTS (issue #81): the tick used to be a multi-step PROMPT # (.claude/commands/pr-loop.md) that a model re-derived, from scratch, every @@ -29,6 +30,10 @@ # planned+module issue + no existing branch), N is not census's in_flight=N # (a feat/issue-N-* branch with no open PR — someone/something is already # mid-flight on it), and the spawn lock (below) is not already held for N. +# RESUME (issue #98, see STEP 0.5 below) is lowest precedence: it only fires +# when neither FEEDBACK nor a fresh ADVANCE claimed the tick (advance_ready= +# none), and picks the lowest-numbered in_flight issue that census's stall +# clock or debris classifier flags as stuck. # # Spawn lock: .claude/state/loop-advance.lock (root-relative; .claude/state/ # is already gitignored). Written the moment this script emits @@ -538,17 +543,52 @@ fi # classify_debris (issue #111) -- called VERBATIM in a subshell below so # the absent/empty/publishable/half-done state vocabulary never diverges # between the two call sites. -# Bounded to 2 resume attempts, tracked in a SIBLING state file +# +# REACHABILITY (post-review correction): the first cut of this feature gated +# the whole resume path on "advance_ready equals an issue ALSO reported +# in_flight" -- but loop-census.sh makes those mutually exclusive for any +# single issue: advance_ready only ever names a candidate with branch=none +# (eligible=1 requires it), while in_flight only ever names a candidate whose +# branch is NOT none. No real census snapshot can ever satisfy both for the +# same issue, so that gate was dead code -- a genuinely stalled in_flight +# issue was NEVER resumed or escalated in production. Fixed by consuming +# in_flight=/stalled= DIRECTLY off the fresh census output below, entirely +# independent of advance_ready. This block only runs once advance_ready has +# resolved to "none" for this tick (see the verdict decision below) -- i.e. +# a genuinely fresh, branchless advance candidate still has first claim on +# the tick ("fresh advance beats resume", mirroring the existing +# feedback-beats-advance precedence). Among several in_flight candidates +# that qualify (stalled OR half-done debris), the LOWEST-numbered one that +# is not YET escalated is picked (mirrors feedback's "lowest PR wins"); one +# already escalated to needs-human is skipped in favor of a later candidate +# rather than refusing the whole tick. +# +# Bounded to 2 resume attempts PER ISSUE, tracked in a SIBLING state file # (.claude/state/loop-resume-attempts.json, {count,escalated} shape + # mktemp/atomic-mv discipline mirroring loop-issue-attempts.json) rather than # folded into loop-issue-attempts.json itself: that file counts advance/ # feedback DISPATCHES against issue #95's spend ceiling -- a different budget # than "how many times has THIS stalled branch been resumed"; conflating the # two would let a resume silently eat into (or be eaten by) the dispatch -# budget. The 3rd stall does not resume again: it escalates to needs-human -# (mirroring the attempt-budget escalation block above -- label create + -# issue edit + issue comment) and sets escalated:true so the loop never -# auto-retries it again. +# budget (see the dispatch-bookkeeping block near the bottom of this script, +# which deliberately does NOT charge action=resume against +# loop-issue-attempts.json/the daily ceiling). The 3rd stall does not resume +# again: it escalates to needs-human (mirroring the attempt-budget +# escalation block above -- label create + issue edit + issue comment) and +# sets escalated:true so the loop never auto-retries it again. +# +# NOTE (latent risk, acknowledged, follow-up out of scope): driver-side +# resume DISPATCH -- actually reconnecting to / resuming work in the stalled +# branch's worktree -- does not exist yet; this script only emits the +# verdict. Until that dispatch exists, a resume verdict burns one of the 2 +# attempts with no actual resume happening, so a stalled issue whose driver +# is never manually restarted will still escalate to needs-human within 3 +# stalled ticks once this fix makes the path reachable. That is the correct, +# safe default for an unresumable stall (it surfaces to a human instead of +# wedging forever, the bug this fix closes), and the accounting stays +# coherent because an "attempt" is only counted when a resume verdict is +# ACTUALLY emitted below -- never speculatively on a tick that took some +# other action. # --------------------------------------------------------------------------- resume_attempts_file="$state_dir/loop-resume-attempts.json" @@ -643,46 +683,14 @@ elif [ -n "$feedback_pr" ]; then verdict="action=feedback pr=$feedback_pr" elif [ "$advance_ready" != "none" ] && [ -n "$advance_ready" ]; then if printf '%s\n' "$in_flight_issues" | grep -qx "$advance_ready"; then - # --- stall/resume path (issue #98) -------------------------------------- - stalled_now=0 - is_census_stalled "$advance_ready" && stalled_now=1 - debris_now="none" - if [ "$stalled_now" -eq 0 ]; then - debris_now="$(classify_debris_for_issue "$advance_ready")" - fi - if [ "$stalled_now" -eq 1 ] || [ "$debris_now" = "half-done" ]; then - log_loop_event "$advance_ready" "stall-detected" "in_flight issue=$advance_ready flagged stalled=$stalled_now debris=$debris_now" - IFS=$'\t' read -r resume_count resume_escalated <<<"$(read_resume_state "$advance_ready")" - case "$resume_count" in ''|*[!0-9]*) resume_count=0 ;; esac - case "$resume_escalated" in ''|*[!01]*) resume_escalated=0 ;; esac - resume_branch="$(branch_for_issue "$advance_ready")" - if [ "$resume_escalated" = "1" ]; then - echo "# advance refused: issue=$advance_ready is in_flight and stalled, but already escalated to needs-human -- not retrying" - verdict="action=none" - elif [ "$resume_count" -lt 2 ]; then - new_resume_count=$((resume_count + 1)) - write_resume_state "$advance_ready" "$new_resume_count" "0" - echo "# resume: issue=$advance_ready is in_flight and stalled (attempt $new_resume_count/2) -- resuming the existing branch/worktree instead of refusing" - log_loop_event "$advance_ready" "resume-attempt" "resume attempt $new_resume_count/2 for issue=$advance_ready" - if [ -n "$resume_branch" ]; then - verdict="action=resume issue=$advance_ready branch=$resume_branch" - else - verdict="action=resume issue=$advance_ready" - fi - else - write_resume_state "$advance_ready" "$resume_count" "1" - gh label create "needs-human" --color b60205 --description "Loop attempt budget exhausted -- needs a human" --force >/dev/null 2>&1 || true - body="Issue #$advance_ready's feat/issue-$advance_ready-* branch has stalled and already been resumed $resume_count times without landing a PR. The loop will not retry it automatically -- labeling \`needs-human\`. Address it by hand, then either close it out or clear its entry in .claude/state/loop-resume-attempts.json to let the loop resume." - gh issue edit "$advance_ready" --add-label needs-human >/dev/null 2>&1 || true - gh issue comment "$advance_ready" --body "$body" >/dev/null 2>&1 || true - echo "# advance refused: issue=$advance_ready exhausted its 2 resume attempts -- escalated to needs-human" - log_loop_event "$advance_ready" "escalated-to-needs-human" "issue=$advance_ready escalated to needs-human after $resume_count resumes" - verdict="action=none" - fi - else - echo "# advance refused: issue=$advance_ready is in_flight (a feat/issue-$advance_ready-* branch already exists with no open PR)" - verdict="action=none" - fi + # Defensive only: real census can never report the SAME issue as both + # advance_ready (requires branch=none) and in_flight (requires branch!= + # none) -- see the STEP 0.5 comment above. Kept as a belt-and-suspenders + # refusal in case a future census bug (or a hand-built test fixture) + # ever produces this combination; it must never fall through to a fresh + # action=advance for an issue that also looks in_flight. + echo "# advance refused: issue=$advance_ready is in_flight (a feat/issue-$advance_ready-* branch already exists with no open PR)" + verdict="action=none" elif [ "$lock_issue" = "$advance_ready" ]; then echo "# advance refused: spawn lock already held for issue=$advance_ready ($(cat "$lock_file" 2>/dev/null))" verdict="action=none" @@ -693,20 +701,104 @@ elif [ "$advance_ready" != "none" ] && [ -n "$advance_ready" ]; then verdict="action=advance issue=$advance_ready" fi else - verdict="action=none" + # --- stall/resume path (issue #98, reworked) ----------------------------- + # No fresh, branchless candidate is ready this tick (advance_ready=none) -- + # scan census's in_flight= issues DIRECTLY (independent of advance_ready; + # see the reachability note in the STEP 0.5 comment above) for the lowest- + # numbered one that's genuinely stalled (census's own clock) or whose + # branch classifies as half-done debris, and that hasn't already been + # escalated to needs-human. + resume_issue="" resume_branch="" resume_stalled_now=0 resume_debris_now="none" + # Lowest-numbered qualifying candidate that's ALREADY escalated -- tracked + # only so a tick where every qualifying candidate happens to be escalated + # still emits the specific "already escalated" diagnostic below rather than + # the generic in_flight refusal (matches this feature's pre-rework + # behavior for the single-candidate case). + escalated_issue="" + for cand in $(printf '%s\n' "$in_flight_issues" | sort -n -u); do + [ -n "$cand" ] || continue + cand_stalled=0 + is_census_stalled "$cand" && cand_stalled=1 + cand_debris="none" + if [ "$cand_stalled" -eq 0 ]; then + cand_debris="$(classify_debris_for_issue "$cand")" + fi + [ "$cand_stalled" -eq 1 ] || [ "$cand_debris" = "half-done" ] || continue + IFS=$'\t' read -r cand_count cand_escalated <<<"$(read_resume_state "$cand")" + case "$cand_escalated" in ''|*[!01]*) cand_escalated=0 ;; esac + if [ "$cand_escalated" = "1" ]; then + [ -n "$escalated_issue" ] || escalated_issue="$cand" + continue + fi + resume_issue="$cand" + resume_stalled_now="$cand_stalled" + resume_debris_now="$cand_debris" + resume_branch="$(branch_for_issue "$cand")" + break + done + + if [ -n "$resume_issue" ]; then + log_loop_event "$resume_issue" "stall-detected" "in_flight issue=$resume_issue flagged stalled=$resume_stalled_now debris=$resume_debris_now" + IFS=$'\t' read -r resume_count resume_escalated <<<"$(read_resume_state "$resume_issue")" + case "$resume_count" in ''|*[!0-9]*) resume_count=0 ;; esac + case "$resume_escalated" in ''|*[!01]*) resume_escalated=0 ;; esac + if [ "$resume_escalated" = "1" ]; then + # Unreachable given the loop above already skips escalated candidates, + # kept for defense in depth against a race between the read above and + # here (state file changed underneath us mid-tick). + echo "# advance refused: issue=$resume_issue is in_flight and stalled, but already escalated to needs-human -- not retrying" + verdict="action=none" + elif [ "$resume_count" -lt 2 ]; then + new_resume_count=$((resume_count + 1)) + write_resume_state "$resume_issue" "$new_resume_count" "0" + echo "# resume: issue=$resume_issue is in_flight and stalled (attempt $new_resume_count/2) -- resuming the existing branch/worktree instead of refusing" + log_loop_event "$resume_issue" "resume-attempt" "resume attempt $new_resume_count/2 for issue=$resume_issue" + if [ -n "$resume_branch" ]; then + verdict="action=resume issue=$resume_issue branch=$resume_branch" + else + verdict="action=resume issue=$resume_issue" + fi + else + write_resume_state "$resume_issue" "$resume_count" "1" + gh label create "needs-human" --color b60205 --description "Loop attempt budget exhausted -- needs a human" --force >/dev/null 2>&1 || true + body="Issue #$resume_issue's feat/issue-$resume_issue-* branch has stalled and already been resumed $resume_count times without landing a PR. The loop will not retry it automatically -- labeling \`needs-human\`. Address it by hand, then either close it out or clear its entry in .claude/state/loop-resume-attempts.json to let the loop resume." + gh issue edit "$resume_issue" --add-label needs-human >/dev/null 2>&1 || true + gh issue comment "$resume_issue" --body "$body" >/dev/null 2>&1 || true + echo "# advance refused: issue=$resume_issue exhausted its 2 resume attempts -- escalated to needs-human" + log_loop_event "$resume_issue" "escalated-to-needs-human" "issue=$resume_issue escalated to needs-human after $resume_count resumes" + verdict="action=none" + fi + elif [ -n "$escalated_issue" ]; then + # Every stalled/half-done candidate this tick is already escalated to + # needs-human -- do not resume, and do not re-touch its state. + echo "# advance refused: issue=$escalated_issue is in_flight and stalled, but already escalated to needs-human -- not retrying" + verdict="action=none" + else + # No in_flight candidate qualifies for resume this tick (none are + # stalled/half-done, or there are no in_flight issues at all) -- refuse + # each in_flight issue individually, matching the pre-#98 diagnostic + # verbatim so operators/tests can still tell WHICH issue(s) are sitting + # in_flight-but-untouched this tick. + for cand in $(printf '%s\n' "$in_flight_issues" | sort -n -u); do + [ -n "$cand" ] || continue + echo "# advance refused: issue=$cand is in_flight (a feat/issue-$cand-* branch already exists with no open PR)" + done + verdict="action=none" + fi fi # --- Spend-ceiling bookkeeping: increment counts on an ACTUAL dispatch ------ # Only runs when the verdict just decided is a genuine advance/feedback # dispatch (never on action=none, ceiling-blocked or not) -- so a blocked -# tick never itself grows the very counters that blocked it. +# tick never itself grows the very counters that blocked it. action=resume is +# DELIBERATELY excluded here: resume attempts are tracked in the SIBLING +# loop-resume-attempts.json (written above, alongside the verdict decision), +# precisely so a resume never charges issue #95's advance/feedback dispatch +# budget (loop-issue-attempts.json) or its daily action ceiling -- see the +# STEP 0.5 comment above. dispatch_issue="" case "$verdict" in "action=advance issue="*) dispatch_issue="${verdict#action=advance issue=}" ;; - "action=resume issue="*) - dispatch_issue="${verdict#action=resume issue=}" - dispatch_issue="${dispatch_issue%% *}" - ;; "action=feedback pr="*) dispatch_pr="${verdict#action=feedback pr=}" if [ "$dispatch_pr" = "$feedback_pr" ] && [ -n "${feedback_issue:-}" ]; then diff --git a/.claude/scripts/loop-tick.test.sh b/.claude/scripts/loop-tick.test.sh index 636398f..2aa589f 100644 --- a/.claude/scripts/loop-tick.test.sh +++ b/.claude/scripts/loop-tick.test.sh @@ -402,6 +402,15 @@ new_fixture_with_events() { # attempt. Verdict points at the existing branch; resume-attempts.json now # records count=1; both a stall-detected and a resume-attempt event land in # events.jsonl. +# +# NOTE (post-review correction): advance_ready is "none" here, NOT "42" -- +# real loop-census.sh can NEVER report the SAME issue as both advance_ready +# (requires branch=none) and in_flight (requires branch!=none); the earlier +# version of this fixture set advance_ready=42 alongside in_flight=42, which +# is a combination the real census cannot produce and made the resume path +# unreachable in production (it was gated on advance_ready being found +# inside in_flight_issues). See scenario 21 below for an end-to-end +# reachability check driven by the REAL loop-census.sh. ticks15_events="$work/scenario15-events.jsonl" dir15="$(new_fixture_with_events scenario15 'open_prs=0 feedback_prs=0 @@ -409,8 +418,8 @@ planned_issues=1 issue=42 branch=feat/issue-42-x title=Stalled thing in_flight=42 stalled=42 age_min=45 -advance_ready=42 -cadence=FAST cron=* * * * *' '')" +advance_ready=none +cadence=WATCH cron=*/5 * * * *' '')" out15="$(CLAUDE_EVENTS_FILE="$ticks15_events" run_tick "$dir15")" check "scenario 15 (resume via census stall clock): verdict is action=resume issue=42 branch=feat/issue-42-x" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=42 branch=feat/issue-42-x" ]' _ "$out15" resume15="$dir15/../state/loop-resume-attempts.json" @@ -490,8 +499,8 @@ feedback_prs=0 planned_issues=1 issue=55 branch=feat/issue-55-x title=Debris thing in_flight=55 -advance_ready=55 -cadence=FAST cron=* * * * *' '')" +advance_ready=none +cadence=WATCH cron=*/5 * * * *' '')" root19="${dir19%/.claude/scripts}" # A SEPARATE linked worktree, not the main checkout -- mirrors real production # topology (an implementer's branch always lives in its own `.claude/worktrees/` @@ -523,8 +532,8 @@ feedback_prs=0 planned_issues=1 issue=66 branch=feat/issue-66-x title=Publishable thing in_flight=66 -advance_ready=66 -cadence=FAST cron=* * * * *' '')" +advance_ready=none +cadence=WATCH cron=*/5 * * * *' '')" root20="${dir20%/.claude/scripts}" # Same separate-worktree topology as scenario 19's fixture above. git -C "$root20" branch feat/issue-66-x main @@ -536,6 +545,186 @@ check "scenario 20 (in_flight, publishable debris, no stall): still refused, no check "scenario 20: diagnostic is the plain pre-#98 in_flight refusal" bash -c 'printf "%s\n" "$1" | grep -qF "is in_flight (a feat/issue-66-* branch already exists with no open PR)"' _ "$out20" check "scenario 20: no resume-attempts entry created for issue 66" bash -c '! grep -q "\"66\"" "$1" 2>/dev/null' _ "$dir20/../state/loop-resume-attempts.json" +# --------------------------------------------------------------------------- +# 21. Precedence: a FRESH branchless advance candidate (advance_ready=7) still +# wins over a SEPARATE, genuinely stalled in_flight issue (42) in the same +# census snapshot -- matches the documented "fresh advance beats resume" +# precedence and the real invariant that advance_ready and in_flight never +# name the SAME issue (they can, of course, both be populated for DIFFERENT +# issues in one census run). +# --------------------------------------------------------------------------- +dir21="$(new_fixture_with_events scenario21 'open_prs=0 +feedback_prs=0 +planned_issues=2 +issue=7 branch=none title=Fresh branchless issue +issue=42 branch=feat/issue-42-x title=Stalled thing +in_flight=42 +stalled=42 age_min=45 +advance_ready=7 +cadence=FAST cron=* * * * *' '')" +out21="$(run_tick "$dir21")" +check "scenario 21 (fresh advance beats resume): verdict is action=advance issue=7" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=advance issue=7" ]' _ "$out21" +check "scenario 21: no resume-attempts entry created for the stalled-but-deferred issue 42" bash -c '! grep -q "\"42\"" "$1" 2>/dev/null' _ "$dir21/../state/loop-resume-attempts.json" + +# --------------------------------------------------------------------------- +# 22. Lowest-numbered-wins: TWO in_flight issues both stalled (30 and 20) with +# no fresh advance candidate -- resume must pick the LOWER-numbered one (20), +# mirroring feedback's "lowest PR wins" rule. +# --------------------------------------------------------------------------- +dir22="$(new_fixture_with_events scenario22 'open_prs=0 +feedback_prs=0 +planned_issues=2 +issue=30 branch=feat/issue-30-x title=Stalled thing higher +issue=20 branch=feat/issue-20-x title=Stalled thing lower +in_flight=30 +in_flight=20 +stalled=30 age_min=99 +stalled=20 age_min=50 +advance_ready=none +cadence=WATCH cron=*/5 * * * *' '')" +out22="$(run_tick "$dir22")" +check "scenario 22 (lowest-numbered wins): verdict is action=resume issue=20, not 30" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=20 branch=feat/issue-20-x" ]' _ "$out22" + +# --------------------------------------------------------------------------- +# 23. Escalated candidate is SKIPPED in favor of a later, not-yet-escalated +# candidate, instead of refusing the whole tick: issue 10 is stalled but +# already escalated (pre-seeded loop-resume-attempts.json); issue 20 is +# ALSO stalled and not yet escalated -- resume must pick 20, and must NOT +# re-touch issue 10's already-escalated state. +# --------------------------------------------------------------------------- +dir23="$(new_fixture_with_events scenario23 'open_prs=0 +feedback_prs=0 +planned_issues=2 +issue=10 branch=feat/issue-10-x title=Already escalated +issue=20 branch=feat/issue-20-x title=Not yet escalated +in_flight=10 +in_flight=20 +stalled=10 age_min=200 +stalled=20 age_min=50 +advance_ready=none +cadence=WATCH cron=*/5 * * * *' '')" +resume23="$dir23/../state/loop-resume-attempts.json" +mkdir -p "$(dirname "$resume23")" +cat > "$resume23" <<'EOF' +{ "10": { "count": 2, "escalated": true } } +EOF +out23="$(run_tick "$dir23")" +check "scenario 23 (escalated candidate skipped): verdict is action=resume issue=20, not the already-escalated 10" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=20 branch=feat/issue-20-x" ]' _ "$out23" +check "scenario 23: issue 10's escalated state is untouched (still count=2, escalated=true)" node -e ' + const fs = require("fs"); + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (!j["10"] || j["10"].count !== 2 || j["10"].escalated !== true) throw new Error("got " + JSON.stringify(j)); +' "$resume23" +check "scenario 23: issue 20 now has a fresh resume entry (count=1, escalated=false)" node -e ' + const fs = require("fs"); + const j = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); + if (!j["20"] || j["20"].count !== 1 || j["20"].escalated !== false) throw new Error("got " + JSON.stringify(j)); +' "$resume23" + +# --------------------------------------------------------------------------- +# 24. BLOCKER 2 regression guard: a resume verdict must NOT charge issue #95's +# advance/feedback dispatch budget (loop-issue-attempts.json) or its daily +# action ceiling -- those are tracked ONLY in the sibling +# loop-resume-attempts.json (already asserted above). Drive a resume verdict +# and assert loop-issue-attempts.json / loop-daily-ceiling.json are BOTH +# left untouched (absent -- this fixture's state dir starts empty). +# --------------------------------------------------------------------------- +dir24="$(new_fixture_with_events scenario24 'open_prs=0 +feedback_prs=0 +planned_issues=1 +issue=88 branch=feat/issue-88-x title=Resume must not charge issue 95 budget +in_flight=88 +stalled=88 age_min=60 +advance_ready=none +cadence=WATCH cron=*/5 * * * *' '')" +out24="$(run_tick "$dir24")" +check "scenario 24: verdict is action=resume issue=88" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=88 branch=feat/issue-88-x" ]' _ "$out24" +check "scenario 24: loop-issue-attempts.json (issue #95's dispatch budget) was never created" [ ! -e "$dir24/../state/loop-issue-attempts.json" ] +check "scenario 24: loop-daily-ceiling.json (issue #95's daily ceiling) was never created" [ ! -e "$dir24/../state/loop-daily-ceiling.json" ] + +# --------------------------------------------------------------------------- +# 25. REACHABILITY (the bug this fix closes): drive loop-tick.sh against the +# REAL loop-census.sh (not a hand-fabricated fixture) so the resume path is +# proven reachable through the actual integration, not just a census snapshot +# that respects the invariant by construction. A single planned issue (77) +# already has a real local git branch (so census reports it in_flight, never +# advance_ready -- the SAME mutual exclusivity the earlier fixtures above +# were reworked to respect) and a stale events.jsonl entry old enough to trip +# census's own stall clock (default budget.stall_minutes=30). +# +# SABOTAGE CHECK (do this by hand when reviewing, not asserted by the test +# itself): reverting loop-tick.sh's verdict decision to the old +# `if printf '%s\n' "$in_flight_issues" | grep -qx "$advance_ready"` gate +# makes this scenario's verdict regress to action=none, since advance_ready +# is (correctly, per the real census) "none" and can never equal in_flight's +# "77" -- proving this test is non-vacuous. +# --------------------------------------------------------------------------- +build_real_census_fixture() { + local name="$1" + local dir="$work/$name" + local scripts="$dir/.claude/scripts" + mkdir -p "$scripts" + cp "$script_dir/loop-tick.sh" "$scripts/loop-tick.sh" + cp "$script_dir/loop-census.sh" "$scripts/loop-census.sh" + cp "$script_dir/resolve-roots.sh" "$scripts/resolve-roots.sh" + cp "$script_dir/loop-daemon.sh" "$scripts/loop-daemon.sh" + cp "$script_dir/log-event.sh" "$scripts/log-event.sh" + cat > "$dir/.claude/gates.json" <<'EOF' +{ + "modules": [{ "name": "test", "path": ".", "description": "", "owner": "" }], + "merge": { "baseBranch": "main" } +} +EOF + cat > "$scripts/pr-feedback.sh" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + cat > "$scripts/notify-poll.sh" <<'EOF' +#!/usr/bin/env bash +echo "=== fake notify-poll output ===" +EOF + cat > "$scripts/merge-ready.sh" <<'EOF' +#!/usr/bin/env bash +echo "=== merge-ready: merged=0 skipped=0 ===" +EOF + # Fake bot-gh.sh: real loop-census.sh's ACTUAL gh call shapes -- one open + # planned issue (77), zero open PRs, no open-issue set needed (issue 77 has + # a branch so it's never eligible and its body/blockers are never fetched). + cat > "$scripts/bot-gh.sh" <<'EOF' +#!/usr/bin/env bash +case "$1" in + repo) echo "acme/repo" ;; + pr) + if printf '%s\n' "$*" | grep -q 'headRefName'; then + : # no open PRs -> no branches + else + echo 0 + fi + ;; + issue) + if printf '%s\n' "$*" | grep -q -- '--label'; then + printf '77\tplanned,module:test\tStalled real thing\n' + else + : # open_issue_set -- unused by this fixture (issue 77 is never eligible) + fi + ;; + *) echo "fake-bot-gh.sh: unhandled args: $*" >&2; exit 1 ;; +esac +EOF + chmod +x "$scripts"/*.sh + git -C "$dir" init -q -b main + git -C "$dir" -c user.email=t@e.st -c user.name=t commit -q --allow-empty -m init + git -C "$dir" branch feat/issue-77-x main + printf '%s\n' "$scripts" +} + +dir25="$(build_real_census_fixture scenario25)" +events25="$work/scenario25-events.jsonl" +printf '%s\n' '{"ts":"2020-01-01T00:00:00Z","task":"77","phase":"driver-start","role":"orchestrator"}' > "$events25" +out25="$(env -u GATES_FILE CLAUDE_EVENTS_FILE="$events25" bash "$dir25/loop-tick.sh" "acme/repo")" +check "scenario 25 (real loop-census.sh reports advance_ready=none, issue 77 in_flight+stalled)" bash -c 'printf "%s\n" "$1" | grep -qx "advance_ready=none" && printf "%s\n" "$1" | grep -qx "in_flight=77" && printf "%s\n" "$1" | grep -q "^stalled=77 "' _ "$out25" +check "scenario 25 (end-to-end reachability via REAL census): verdict is action=resume issue=77 branch=feat/issue-77-x" bash -c '[ "$(printf "%s\n" "$1" | tail -1)" = "action=resume issue=77 branch=feat/issue-77-x" ]' _ "$out25" + echo "" if [ "$fail" -eq 0 ]; then echo "loop-tick.test.sh: PASS ($ok checks)"