Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions src/context_system/prompt_assembly.py
Original file line number Diff line number Diff line change
Expand Up @@ -911,12 +911,20 @@ def _emit_group(
"investigation only when results rule them out or reveal new evidence; "
"do not replace focused diagnosis with an exhaustive search of the "
"environment.\n"
# The trailing sentence here used to be: "When the user asks for all,
# every, multiple, or an exhaustive set of results, actively search for
# additional valid results instead of stopping after the first one."
# It keyed off bare quantifiers, the same over-broad signal as the
# (now removed) exhaustive-audit nudge, and pushed extra verification
# rounds on any request that merely contained "each" or "all". What
# remains is the part that holds regardless of phrasing: check the
# requirements you were actually given. If a request genuinely asks for
# every qualifying result, that IS an explicit requirement and the
# first sentence already covers it.
"- Before finishing, audit the result against every explicit requirement "
"in the user's request and verify the requirements that can be checked. "
"Do not treat producing a plausible result as proof that the task is "
"complete. When the user asks for all, every, multiple, or an exhaustive "
"set of results, actively search for additional valid results instead of "
"stopping after the first one.\n"
"complete.\n"
"- Use the narrowest sufficient verification for the change. Prefer an "
"existing focused check, add a regression test when it provides lasting "
"value, and run broader checks when the scope or risk warrants them. "
Expand Down
21 changes: 0 additions & 21 deletions src/query/continuation_nudge.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,27 +26,6 @@
"explicitly and state what you did."
)

EXHAUSTIVE_AUDIT_NUDGE = (
"Before finishing, audit the result against the user's exhaustive-result "
"requirement. Actively search for additional valid results; do not assume "
"the first valid result is the only one. For an enumerable or mechanically "
"checkable state space, you MUST use a tool to enumerate or independently "
"validate candidates rather than declaring a prose-only audit exhaustive. "
"Evaluate the state after each candidate action, including candidates that "
"do not have the relevant property before they move. Update the deliverable "
"if the audit finds more."
)

_EXHAUSTIVE_REQUIREMENT = re.compile(
r"\b(all|every|multiple|each|exhaustive|complete set|any other|others)\b",
re.IGNORECASE,
)


def requests_exhaustive_results(text: str) -> bool:
"""Whether a user request explicitly asks for a result set audit."""
return bool(_EXHAUSTIVE_REQUIREMENT.search(text))

# Don't nudge when the model signaled completion (query.ts:1487).
_COMPLETION_MARKERS = re.compile(
r"\b(done|finished|completed|complete|summary|that's all|that is all"
Expand Down
44 changes: 0 additions & 44 deletions src/query/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -2171,49 +2171,6 @@ def _marking_chunk_cb(text: str) -> None:
for b in content
if getattr(b, "type", None) == "text"
)
# A completion claim is not sufficient when the user asked
# for an exhaustive result set. Require one bounded audit
# pass before accepting the clean exit. This closes a common
# harness failure mode where the model finds one plausible
# answer and stops despite an explicit "all"/"multiple"
# requirement. A dedicated latch survives tool rounds so the
# guard cannot create an unbounded self-review loop.
from .continuation_nudge import (
EXHAUSTIVE_AUDIT_NUDGE,
requests_exhaustive_results,
)

exhaustive_request = any(
not getattr(msg, "isMeta", False)
and getattr(msg, "role", None) == "user"
and requests_exhaustive_results(
getattr(msg, "content", "")
if isinstance(getattr(msg, "content", ""), str)
else ""
)
for msg in messages
)
if exhaustive_request and not state.exhaustive_audit_performed:
logger.debug("Exhaustive-result audit nudge triggered")
state = QueryState(
messages=[
*messages,
*assistant_messages,
UserMessage(content=EXHAUSTIVE_AUDIT_NUDGE, isMeta=True),
],
tool_use_context=tool_use_context,
auto_compact_tracking=state.auto_compact_tracking,
max_output_tokens_recovery_count=0,
has_attempted_reactive_compact=False,
max_output_tokens_override=None,
stop_hook_active=None,
turn_count=turn_count,
pending_tool_use_summary=None,
continuation_nudge_count=state.continuation_nudge_count,
exhaustive_audit_performed=True,
transition=Transition(reason="continuation_nudge"),
)
continue
# An assistant turn with NO tool calls and NO text is not a
# completion — it is a degenerate response, and accepting it
# ends the run with an empty answer while every caller
Expand Down Expand Up @@ -2512,7 +2469,6 @@ def _marking_chunk_cb(text: str) -> None:
stop_hook_active=state.stop_hook_active,
# Phase A: reset per-turn counter; carry pending summary forward.
continuation_nudge_count=0,
exhaustive_audit_performed=state.exhaustive_audit_performed,
pending_tool_use_summary=state.pending_tool_use_summary,
transition=Transition(reason="next_turn"),
)
Expand Down
3 changes: 0 additions & 3 deletions src/query/transitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,4 @@ class QueryState:
# when the model keeps matching continuation signals without tool calls.
# Mirrors TS State.continuationNudgeCount at query.ts:218.
continuation_nudge_count: int = 0
# Session-chain latch for the one-time exhaustive-result audit. Unlike
# continuation_nudge_count, this survives successful tool rounds.
exhaustive_audit_performed: bool = False
transition: Transition | None = None
36 changes: 24 additions & 12 deletions tests/test_query_loop_wires_round3.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,8 @@ def get_pending_post_compaction() -> bool:
return _bs._STATE.pending_post_compaction
from src.providers.base import ChatResponse
from src.query.continuation_nudge import (
EXHAUSTIVE_AUDIT_NUDGE,
MAX_CONTINUATION_NUDGES,
detect_continuation_signal,
requests_exhaustive_results,
)
from src.query.query import QueryParams, run_query
from src.query.stop_hooks import StopHookResult
Expand Down Expand Up @@ -623,10 +621,30 @@ def test_completion_text_no_nudge(self):
sum(1 for m in msgs if isinstance(m, AssistantMessage)), 1
)

def test_exhaustive_request_forces_one_audit_pass(self):
def test_no_forced_audit_pass_on_a_completion_claim(self):
"""A completion claim ends the turn — no extra audit round-trip.

#740 added a one-shot "exhaustive audit" nudge: when the request
matched a bare quantifier (all|every|multiple|each|...), the loop
appended a MUST-enumerate instruction and forced another model
turn. Removed, on evidence from the latest Claude Code rather than
from the older vendored fork:

* CC scores 1.0 on the two tasks the mechanism was built for
(chess-best-move, regex-chess) with no such machinery;
* only 1 of its 89 trials writes more than two verification files,
against clawcodex's 57 temporary verification scripts overall;
* the trigger was a plain-English quantifier match, so it fired on
49% of terminal-bench 2.1 — "each line of the file", "where each
task is", "in all years" — and in regex-log it demanded
exhaustive enumeration of a task that asks to match only ONE.

Verification guidance now lives entirely in the system prompt,
which is where the model can weigh it against the request.
"""
provider = _provider([
_completion("I found one result and finished."),
_completion("I audited all candidates and updated the result."),
_completion("second turn should never happen"),
])
params = _params(self.workspace, provider)
params.messages = [
Expand All @@ -637,16 +655,10 @@ def test_exhaustive_request_forces_one_audit_pass(self):

self.assertEqual(terminal.reason, "completed")
self.assertEqual(
sum(1 for m in msgs if isinstance(m, AssistantMessage)), 2
sum(1 for m in msgs if isinstance(m, AssistantMessage)), 1,
"an exhaustive-sounding request must not buy an extra turn",
)

def test_exhaustive_request_detector(self):
self.assertTrue(requests_exhaustive_results("print them all"))
self.assertTrue(requests_exhaustive_results("multiple winning moves"))
self.assertFalse(requests_exhaustive_results("find the best move"))
self.assertIn("MUST use a tool", EXHAUSTIVE_AUDIT_NUDGE)
self.assertIn("state after each candidate action", EXHAUSTIVE_AUDIT_NUDGE)


# ---------------------------------------------------------------------------
# G3 + G4 — /clear resets; compaction marks
Expand Down
20 changes: 17 additions & 3 deletions tests/test_system_prompt_full.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,25 @@ def test_basic_prompt_has_intro(self):
assert "interactive agent" in prompt
assert "software engineering tasks" in prompt

def test_task_prompt_requires_requirement_audit_and_exhaustive_results(self):
def test_task_prompt_audits_requirements_without_a_quantifier_trigger(self):
"""Keep the requirement audit; drop the quantifier-keyed clause.

The removed sentence told the model to search for additional
results "when the user asks for all, every, multiple…" — keying off
bare quantifiers, which appear in ordinary task descriptions
("each line of the file", "in all years"). Paired with the
exhaustive-audit nudge it drove verification rounds the latest
Claude Code does not spend: 1 of its 89 terminal-bench 2.1 trials
writes more than two verification files.

A genuine "give me every match" request is an explicit requirement,
so the surviving sentence already covers it.
"""
prompt = build_full_system_prompt(use_cache=False)
assert "audit the result against every explicit requirement" in prompt
assert "all, every, multiple, or an exhaustive set" in prompt
assert "stopping after the first one" in prompt
assert "plausible result as proof" in prompt
assert "all, every, multiple, or an exhaustive set" not in prompt
assert "stopping after the first one" not in prompt

def test_task_prompt_prioritizes_evidence_and_cheap_discriminating_checks(self):
prompt = build_full_system_prompt(use_cache=False)
Expand Down
Loading