From a8308d6f5e4cdf40c792cbb0bb10e6a3bee4539d Mon Sep 17 00:00:00 2001 From: Kevin Simback <16635828+ksimback@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:16:29 -0700 Subject: [PATCH 1/2] Close cmd-output redaction hole and surface artifact-leak scrubs Two-layer redaction, honestly documented: 1. cmd context-source output was inlined verbatim into context.md, so a command that printed a flagged file (cat .env-style, env dumps, git log) leaked it to every downstream member. Output is now scrubbed against redaction-glob file contents like everything else. 2. Content-scrubbing is no longer silent. scrub_flagged_content() returns which flagged files were caught, and the runner logs a redaction_applied event to run-log.md and appends a state.json warning naming the sources and destination - so a leak into loop artifacts is visible instead of quietly masked. README now describes the real posture: path-based non-send for flagged files (first layer, guaranteed), best-effort content scrub with surfacing for derived content (second layer), local models recommended when redaction-sensitive paths exist. 2 new tests (43 total). Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 14 ++++++++ README.md | 2 +- templates/run-loop.py | 53 ++++++++++++++++++++++----- tests/test_looper.py | 83 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3265853..07eb0d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ separately via `version:` in `loop.yaml` (currently `1`). ## Unreleased +### Fixed — artifact-level redaction (runner) +- `cmd` context-source output is now scrubbed against redaction-glob file + contents before it enters `context.md` or any prompt. Previously a + context command that printed a flagged file (`cat .env`-style, env dumps, + `git log`) leaked it verbatim to every downstream member. +- Scrubbing is no longer silent: when flagged-file content is caught in a + prompt or command output, the runner appends a `redaction_applied` event + to `run-log.md` and a warning to `state.json` naming the source files and + destination, so a leak into loop artifacts is visible instead of quietly + masked. +- README documents the two-layer posture honestly: path-based non-send for + flagged files, best-effort content scrub (with surfacing) for everything + derived; local models recommended when redaction-sensitive paths exist. + ### Changed — positioning vs Claude Code's loop taxonomy - README's `/goal`-`/loop` comparison rewritten around the Claude Code team's official loop taxonomy ("Getting started with loops"): turn-based / diff --git a/README.md b/README.md index 2f80d81..51a76d7 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,7 @@ rm -rf "$HOME/.looper" # optional: model registry 6. **Confirm** — review the loop as an ASCII flow preview. 7. **Run or emit** — Looper writes `RUN_IN_SESSION.md`, `loop.yaml`, `loop.resolved.json`, `run-loop.py`, an empty workspace, and a README. The default is to offer to run the loop in the current session; the Python runner is there for external control. -A council sends your project context to another model's CLI. Looper makes that explicit, applies default redactions, lets you scope what's sent, and asks for consent before the first cross-vendor send. Pick a local model (e.g. via `ollama`) to keep the council in-house. +A council sends your project context to another model's CLI. Looper makes that explicit, lets you scope what's sent, and asks for consent before the first cross-vendor send. Redaction is two layers: files matching the redaction globs are **never read into prompts** (context sources show a `[redacted]` marker instead), and content from those files that re-surfaces anywhere else — a context command that printed it, an artifact the host copied it into — is scrubbed before any send, with the catch logged in `run-log.md` and surfaced in `state.json` warnings. The second layer is best-effort by design (reformatted content or very short values can survive), which is why the first layer refuses the read outright — and why a local model (e.g. via `ollama`) is the strongest option when redaction-sensitive paths exist. ## License diff --git a/templates/run-loop.py b/templates/run-loop.py index da46b58..948c1d1 100644 --- a/templates/run-loop.py +++ b/templates/run-loop.py @@ -327,22 +327,54 @@ def iter_redaction_files(self, globs: list[str]): if is_redacted(path, self.base_dir, globs): yield path - def redact_prompt_for_member(self, member_id: str, prompt: str) -> str: - redacted = prompt - for path in self.iter_redaction_files(self.redactions_for(member_id)): + def scrub_flagged_content(self, text: str, globs: list[str]) -> tuple[str, list[str]]: + """Remove content originating from redaction-glob files. + + The flagged files themselves are never read into prompts (path-based + non-send in gather_context); this second layer catches their content + when it re-surfaces elsewhere - a cmd context source that printed it, + or an artifact the host copied it into. Returns the scrubbed text and + the relative paths whose content was found, so callers can surface + that a leak was caught. Best effort by design: reformatted content or + lines shorter than 8 characters can survive. + """ + hits: list[str] = [] + for path in self.iter_redaction_files(globs): try: secret_text = path.read_text(encoding="utf-8") except (OSError, UnicodeDecodeError): continue if not secret_text.strip() or len(secret_text) > 1_000_000: continue - marker = f"[redacted:{path.relative_to(self.base_dir).as_posix()}]" - redacted = redacted.replace(secret_text, marker) + rel = path.relative_to(self.base_dir).as_posix() + marker = f"[redacted:{rel}]" + scrubbed = text.replace(secret_text, marker) for line in secret_text.splitlines(): stripped = line.strip() if len(stripped) >= 8: - redacted = redacted.replace(stripped, marker) - return redacted + scrubbed = scrubbed.replace(stripped, marker) + if scrubbed != text: + hits.append(rel) + text = scrubbed + return text, hits + + def surface_redaction(self, where: str, hits: list[str]) -> None: + if not hits: + return + self.append_log("redaction_applied", where=where, sources=hits) + warnings = list(self.state.get("warnings", []) or []) + note = ( + f"flagged content from {', '.join(hits)} appeared in {where}; " + "scrubbed before use" + ) + if note not in warnings: + warnings.append(note) + self.save_state(warnings=warnings) + + def redact_prompt_for_member(self, member_id: str, prompt: str) -> str: + scrubbed, hits = self.scrub_flagged_content(prompt, self.redactions_for(member_id)) + self.surface_redaction(f"prompt for {member_id}", hits) + return scrubbed def ensure_consent(self, member_id: str) -> None: member = self.member(member_id) @@ -400,9 +432,14 @@ def gather_context(self) -> str: elif "cmd" in source: argv = ensure_argv(source["cmd"], f"context_sources[{index}].cmd") result = run_argv(argv, cwd=self.base_dir, timeout_sec=int(source.get("timeout_sec", 60))) + # Command output can reproduce flagged-file content (cat, git + # log, env dumps); scrub it before it enters any prompt. + stdout_text, stdout_hits = self.scrub_flagged_content(result.stdout, redaction_globs) + stderr_text, stderr_hits = self.scrub_flagged_content(result.stderr, redaction_globs) + self.surface_redaction(f"context command output ({' '.join(argv)})", sorted(set(stdout_hits + stderr_hits))) chunks.append( f"## Context source {index}: {' '.join(argv)}\n" - f"exit={result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}\n" + f"exit={result.returncode}\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}\n" ) self.append_log("context_cmd", argv=argv, returncode=result.returncode) context = "\n".join(chunks).strip() diff --git a/tests/test_looper.py b/tests/test_looper.py index 2c2f651..76db6eb 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -274,6 +274,89 @@ def test_runner_redacts_configured_files_before_judge_send(self) -> None: self.assertNotIn("SUPERSECRET-LOOPER-VALUE", judge_prompt) self.assertIn("[redacted:inputs/secret.txt]", judge_prompt) + def test_cmd_context_source_output_is_scrubbed_and_surfaced(self) -> None: + # Regression: cmd context-source output used to be inlined verbatim, + # so a command that printed a flagged file leaked it into context.md + # and every downstream prompt. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + (work / "inputs" / "secret.txt").write_text("SUPERSECRET-LOOPER-VALUE\n", encoding="utf-8") + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + python = Path(sys.executable).as_posix() + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + loop_yaml = loop_yaml.replace( + "- file: ./inputs/process-notes.md", + "- file: ./inputs/process-notes.md\n" + f' - cmd: ["{python}", "-c", "import pathlib; print(pathlib.Path(\'inputs/secret.txt\').read_text())"]', + ) + loop_yaml = loop_yaml.replace( + "privacy:\n egress: []", + "privacy:\n egress:\n - to: reviewer-1\n sends: [plan, deliveries]\n" + " redact: [\"inputs/secret.txt\"]\n consent: required", + ) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + context = (work / "loop-workspace" / "context.md").read_text(encoding="utf-8") + self.assertNotIn("SUPERSECRET-LOOPER-VALUE", context) + self.assertIn("[redacted:inputs/secret.txt]", context) + run_log = (work / "loop-workspace" / "run-log.md").read_text(encoding="utf-8") + self.assertIn("redaction_applied", run_log) + state = json.loads((work / "loop-workspace" / "state.json").read_text(encoding="utf-8")) + self.assertTrue( + any("context command output" in item for item in state.get("warnings", [])), + state.get("warnings"), + ) + + def test_artifact_leak_scrub_is_surfaced_in_state_and_log(self) -> None: + # Content-scrubbing an artifact send is no longer silent: the runner + # records which flagged files leaked into which prompt. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + (work / "inputs" / "secret.txt").write_text("SUPERSECRET-LOOPER-VALUE\n", encoding="utf-8") + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + loop_yaml = loop_yaml.replace( + "privacy:\n egress: []", + "privacy:\n egress:\n - to: reviewer-1\n sends: [plan, deliveries]\n" + " redact: [\"inputs/**\"]\n consent: required", + ) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + workspace = work / "loop-workspace" + workspace.mkdir() + (workspace / "plan.md").write_text( + "Plan leaked SUPERSECRET-LOOPER-VALUE before redaction.\n", encoding="utf-8" + ) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + run_log = (workspace / "run-log.md").read_text(encoding="utf-8") + self.assertIn("redaction_applied", run_log) + state = json.loads((workspace / "state.json").read_text(encoding="utf-8")) + self.assertTrue( + any("prompt for reviewer-1" in item for item in state.get("warnings", [])), + state.get("warnings"), + ) + def test_fixed_passes_reviewer_gate_completes_cleanly(self) -> None: # Regression: the synthetic "fixed_passes reviewer pass" marker used to # feed the no-progress detector and fail the run before its passes From 49b166e967a4efc70d368ee89d7ce141d6d842e1 Mon Sep 17 00:00:00 2001 From: Kevin Simback <16635828+ksimback@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:41:25 -0700 Subject: [PATCH 2/2] Fix redaction defects found in high-effort review The review's core finding: the branch's "scrubbed before any send" claim was false for the largest egress path - host prompts (plan/delivery/revise) were never scrubbed, so an artifact leak went verbatim to the host CLI while state.json said it was contained. Fixes: - Host prompts now pass through the same scrub as every other send (call_host helper used by run_host and the revise path), with "prompt for host" surfacing. Flagged means flagged for every recipient; the escape hatch is not flagging files the loop needs. - Scrub runs before the first-send consent question, and the consent prompt prints any leak warnings for that member, so the one irreversible decision is made with the leak signal visible. - Flagged files the scrub cannot read (>1MB, non-UTF-8) are surfaced as blind spots (redaction_unscrubbable event + state warning) instead of silently skipped. - Leak attribution detects against the original text, so two flagged files sharing content are both named, not just the first replaced. - redaction_applied events dedupe per (destination, sources); flagged-file contents are read once per run instead of per prompt; cmd stdout/stderr scrub as one block. - CHANGELOG rewritten to state old behavior accurately (member prompts were already scrubbed on main; the verbatim path was the host) and README documents the over-redaction bias explicitly. 4 new tests (47 total). Conformance harness still 8/8. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 40 ++++++---- README.md | 2 +- templates/run-loop.py | 139 ++++++++++++++++++++++----------- tests/test_looper.py | 174 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 295 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07eb0d4..8248219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,19 +6,33 @@ separately via `version:` in `loop.yaml` (currently `1`). ## Unreleased -### Fixed — artifact-level redaction (runner) -- `cmd` context-source output is now scrubbed against redaction-glob file - contents before it enters `context.md` or any prompt. Previously a - context command that printed a flagged file (`cat .env`-style, env dumps, - `git log`) leaked it verbatim to every downstream member. -- Scrubbing is no longer silent: when flagged-file content is caught in a - prompt or command output, the runner appends a `redaction_applied` event - to `run-log.md` and a warning to `state.json` naming the source files and - destination, so a leak into loop artifacts is visible instead of quietly - masked. -- README documents the two-layer posture honestly: path-based non-send for - flagged files, best-effort content scrub (with surfacing) for everything - derived; local models recommended when redaction-sensitive paths exist. +### Fixed — redaction covers every send (runner) +- **Host prompts are now scrubbed.** The host was the one recipient whose + prompts never passed through the content scrub: flagged-file content that + leaked into an artifact went verbatim to the host CLI on every + delivery/revise prompt (council prompts were already best-effort + scrubbed). Every send — host included — now uses the same scrub. +- **`cmd` context-source output is scrubbed** before it enters `context.md` + or any prompt. Previously a context command that printed a flagged file + (`cat .env`-style, env dumps, `git log`) flowed verbatim into + `context.md` and from there onward. +- **Scrubbing is no longer silent.** A caught leak appends a + `redaction_applied` event to `run-log.md` (deduplicated per destination) + and a `state.json` warning naming every source file whose content + matched — not just the first — and the destination. The scrub now runs + *before* the first-send consent question, and the consent prompt displays + any leak warnings for that member, so consent is decided with the leak + signal visible. +- **Unscrubbable flagged files are surfaced.** A redaction-glob file the + scrub cannot read (over 1MB, not valid UTF-8) is reported as a blind spot + in `run-log.md` and `state.json` instead of being silently skipped. +- Flagged-file contents are read once per run (not re-walked per prompt), + and stdout/stderr of a context command are scrubbed as one block. +- README documents the posture honestly: path-based non-send is the first + layer; the content scrub is best-effort and errs toward over-redaction (a + flagged-file line that legitimately appears elsewhere is masked too); + flagged means flagged for every recipient, host included; local models + recommended when redaction-sensitive paths exist. ### Changed — positioning vs Claude Code's loop taxonomy - README's `/goal`-`/loop` comparison rewritten around the Claude Code diff --git a/README.md b/README.md index 51a76d7..648b332 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,7 @@ rm -rf "$HOME/.looper" # optional: model registry 6. **Confirm** — review the loop as an ASCII flow preview. 7. **Run or emit** — Looper writes `RUN_IN_SESSION.md`, `loop.yaml`, `loop.resolved.json`, `run-loop.py`, an empty workspace, and a README. The default is to offer to run the loop in the current session; the Python runner is there for external control. -A council sends your project context to another model's CLI. Looper makes that explicit, lets you scope what's sent, and asks for consent before the first cross-vendor send. Redaction is two layers: files matching the redaction globs are **never read into prompts** (context sources show a `[redacted]` marker instead), and content from those files that re-surfaces anywhere else — a context command that printed it, an artifact the host copied it into — is scrubbed before any send, with the catch logged in `run-log.md` and surfaced in `state.json` warnings. The second layer is best-effort by design (reformatted content or very short values can survive), which is why the first layer refuses the read outright — and why a local model (e.g. via `ollama`) is the strongest option when redaction-sensitive paths exist. +A council sends your project context to another model's CLI. Looper makes that explicit, lets you scope what's sent, and asks for consent before the first cross-vendor send — with any caught leak warning shown before the consent question. Redaction is two layers: files matching the redaction globs are **never read into prompts** (context sources show a `[redacted]` marker instead), and content from those files that re-surfaces anywhere else — a context command that printed it, an artifact a model copied it into — is scrubbed before **every** send, host included, with the catch logged in `run-log.md` and surfaced in `state.json` warnings. The second layer is best-effort and errs toward over-redaction: reformatted content or very short values can survive, a flagged-file line that legitimately appears elsewhere gets masked too, and a flagged file the scrub cannot read (over 1MB, non-UTF-8) is reported as a blind spot rather than silently skipped. Flagged means flagged for every recipient — if the loop genuinely needs a file's content, don't match it with a redaction glob. When redaction-sensitive paths exist, a local model (e.g. via `ollama`) remains the strongest option. ## License diff --git a/templates/run-loop.py b/templates/run-loop.py index 948c1d1..12d4a04 100644 --- a/templates/run-loop.py +++ b/templates/run-loop.py @@ -218,6 +218,11 @@ def __init__(self, spec_path: Path) -> None: self.prior_elapsed = float(self.state.get("wall_clock_used_sec", 0) or 0) self.started = time.monotonic() self.stop_reason: str | None = None + # Per-run caches for the scrub layer: flagged-file contents are read + # once per glob set, surfaced events are logged once per destination. + self._flagged_cache: dict[tuple[str, ...], list[tuple[str, str]]] = {} + self._unscrubbable_reported: set[str] = set() + self._surfaced_events: set[tuple[str, tuple[str, ...]]] = set() def load_state(self) -> dict[str, Any]: if self.state_path.exists(): @@ -327,49 +332,85 @@ def iter_redaction_files(self, globs: list[str]): if is_redacted(path, self.base_dir, globs): yield path + def flagged_file_contents(self, globs: list[str]) -> list[tuple[str, str]]: + """Read flagged files once per glob set; surface the unreadable ones. + + A flagged file the scrub cannot read (too large, not UTF-8) cannot be + detected if its content leaks, so that blind spot is reported instead + of silently skipped. + """ + key = tuple(sorted(globs)) + if key in self._flagged_cache: + return self._flagged_cache[key] + contents: list[tuple[str, str]] = [] + for path in self.iter_redaction_files(globs): + rel = path.relative_to(self.base_dir).as_posix() + reason = "" + try: + if path.stat().st_size > 1_000_000: + reason = "larger than 1MB" + else: + secret_text = path.read_text(encoding="utf-8") + if secret_text.strip(): + contents.append((rel, secret_text)) + continue + except UnicodeDecodeError: + reason = "not valid UTF-8" + except OSError as exc: + reason = f"unreadable ({exc})" + if reason and rel not in self._unscrubbable_reported: + self._unscrubbable_reported.add(rel) + self.append_log("redaction_unscrubbable", source=rel, reason=reason) + self.add_state_warning( + f"flagged file {rel} is {reason}; its content cannot be " + "detected by the scrub layer if it leaks into artifacts" + ) + self._flagged_cache[key] = contents + return contents + def scrub_flagged_content(self, text: str, globs: list[str]) -> tuple[str, list[str]]: """Remove content originating from redaction-glob files. The flagged files themselves are never read into prompts (path-based non-send in gather_context); this second layer catches their content when it re-surfaces elsewhere - a cmd context source that printed it, - or an artifact the host copied it into. Returns the scrubbed text and - the relative paths whose content was found, so callers can surface - that a leak was caught. Best effort by design: reformatted content or - lines shorter than 8 characters can survive. + or an artifact a model copied it into. Returns the scrubbed text and + the relative paths whose content was found. Best effort by design, + erring toward over-redaction: reformatted content or lines shorter + than 8 characters can survive, and a flagged-file line that + legitimately appears elsewhere is masked too. """ + original = text hits: list[str] = [] - for path in self.iter_redaction_files(globs): - try: - secret_text = path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError): - continue - if not secret_text.strip() or len(secret_text) > 1_000_000: - continue - rel = path.relative_to(self.base_dir).as_posix() + for rel, secret_text in self.flagged_file_contents(globs): marker = f"[redacted:{rel}]" - scrubbed = text.replace(secret_text, marker) - for line in secret_text.splitlines(): - stripped = line.strip() - if len(stripped) >= 8: - scrubbed = scrubbed.replace(stripped, marker) - if scrubbed != text: + # Detect against the original text so a secret shared by two + # flagged files attributes both, not just the first replaced. + lines = [line.strip() for line in secret_text.splitlines() if len(line.strip()) >= 8] + if secret_text in original or any(line in original for line in lines): hits.append(rel) - text = scrubbed + text = text.replace(secret_text, marker) + for line in lines: + text = text.replace(line, marker) return text, hits + def add_state_warning(self, note: str) -> None: + warnings = list(self.state.get("warnings", []) or []) + if note not in warnings: + warnings.append(note) + self.save_state(warnings=warnings) + def surface_redaction(self, where: str, hits: list[str]) -> None: if not hits: return - self.append_log("redaction_applied", where=where, sources=hits) - warnings = list(self.state.get("warnings", []) or []) - note = ( + event_key = (where, tuple(hits)) + if event_key not in self._surfaced_events: + self._surfaced_events.add(event_key) + self.append_log("redaction_applied", where=where, sources=hits) + self.add_state_warning( f"flagged content from {', '.join(hits)} appeared in {where}; " "scrubbed before use" ) - if note not in warnings: - warnings.append(note) - self.save_state(warnings=warnings) def redact_prompt_for_member(self, member_id: str, prompt: str) -> str: scrubbed, hits = self.scrub_flagged_content(prompt, self.redactions_for(member_id)) @@ -397,6 +438,9 @@ def ensure_consent(self, member_id: str) -> None: print(f"Looper is about to send {', '.join(sends) or 'loop artifacts'} to {member_id}.") print(f"CLI: {member.get('cli')} / model: {member.get('model', 'default')}") print(f"Redactions: {', '.join(redactions)}") + for note in self.state.get("warnings", []) or []: + if f"prompt for {member_id}" in note: + print(f"Warning: {note}") if not matching: print("No privacy.egress entry covers this member; consent is required by default.") answer = ask("Type 'yes' to consent to this first send: ").strip().lower() @@ -434,13 +478,13 @@ def gather_context(self) -> str: result = run_argv(argv, cwd=self.base_dir, timeout_sec=int(source.get("timeout_sec", 60))) # Command output can reproduce flagged-file content (cat, git # log, env dumps); scrub it before it enters any prompt. - stdout_text, stdout_hits = self.scrub_flagged_content(result.stdout, redaction_globs) - stderr_text, stderr_hits = self.scrub_flagged_content(result.stderr, redaction_globs) - self.surface_redaction(f"context command output ({' '.join(argv)})", sorted(set(stdout_hits + stderr_hits))) - chunks.append( - f"## Context source {index}: {' '.join(argv)}\n" - f"exit={result.returncode}\nstdout:\n{stdout_text}\nstderr:\n{stderr_text}\n" + block = ( + f"exit={result.returncode}\nstdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}\n" ) + block, hits = self.scrub_flagged_content(block, redaction_globs) + self.surface_redaction(f"context command output ({' '.join(argv)})", hits) + chunks.append(f"## Context source {index}: {' '.join(argv)}\n{block}") self.append_log("context_cmd", argv=argv, returncode=result.returncode) context = "\n".join(chunks).strip() write_text(self.workspace / "context.md", context or "No context sources configured.") @@ -470,10 +514,18 @@ def host_prompt(self, phase: str, artifact: str = "", review: str = "") -> str: ) raise RunnerError(f"Unknown host phase: {phase}") + def call_host(self, prompt: str) -> str: + # The host is a send like any other: flagged content is scrubbed for + # every recipient, host included. Files a loop genuinely needs must + # not match redaction globs. + scrubbed, hits = self.scrub_flagged_content(prompt, self.all_redaction_globs()) + self.surface_redaction("prompt for host", hits) + return call_model(spec_get(self.spec, "host"), scrubbed, self.base_dir) + def run_host(self, phase: str, target: Path, artifact: str = "", review: str = "") -> None: self.enforce_wall_clock() self.append_log("host_start", phase=phase, target=target.name) - output = call_model(spec_get(self.spec, "host"), self.host_prompt(phase, artifact, review), self.base_dir) + output = self.call_host(self.host_prompt(phase, artifact, review)) write_text(target, output) self.append_log("host_done", phase=phase, target=target.name) @@ -535,15 +587,14 @@ def run_judge( artifact_text: str, criteria: list[dict[str, Any]], ) -> dict[str, Any]: - self.ensure_consent(member_id) - output = call_model( - self.member(member_id), - self.redact_prompt_for_member( - member_id, - self.judge_prompt(gate_name, artifact_label, artifact_text, criteria), - ), - self.base_dir, + # Scrub (and surface any leak) before asking for consent, so the + # consent decision is made with the leak warning already visible. + prompt = self.redact_prompt_for_member( + member_id, + self.judge_prompt(gate_name, artifact_label, artifact_text, criteria), ) + self.ensure_consent(member_id) + output = call_model(self.member(member_id), prompt, self.base_dir) verdict = parse_judge_output(output) verdict["member"] = member_id self.append_log("judge_verdict", gate=gate_name, member=member_id, verdict=verdict.get("verdict")) @@ -561,13 +612,13 @@ def run_reviewers( member = self.member(member_id) if member.get("role") != "reviewer": continue - self.ensure_consent(member_id) prompt = ( "You are a Looper reviewer. Return concise blocking and non-blocking notes. " "Do not return a verdict.\n\n" f"Gate: {gate_name}\nArtifact: {artifact_label}\n\n{artifact_text}\n" ) prompt = self.redact_prompt_for_member(member_id, prompt) + self.ensure_consent(member_id) notes.append(f"## {member_id}\n\n{call_model(member, prompt, self.base_dir)}") self.append_log("reviewer_notes", gate=gate_name, member=member_id) return notes @@ -668,11 +719,7 @@ def run_gate(self, gate_name: str, artifact_path: Path, artifact_label: str) -> self.append_log("stop", reason=f"{gate_name}_max_revisions_reached") return False - revised = call_model( - spec_get(self.spec, "host"), - self.host_prompt("revise", artifact_text, review_text), - self.base_dir, - ) + revised = self.call_host(self.host_prompt("revise", artifact_text, review_text)) write_text(artifact_path, revised) revision += 1 self.save_state(status=f"{gate_name}_revision_{revision}", last_review=str(review_path)) diff --git a/tests/test_looper.py b/tests/test_looper.py index 76db6eb..7a5aa45 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -357,6 +357,180 @@ def test_artifact_leak_scrub_is_surfaced_in_state_and_log(self) -> None: state.get("warnings"), ) + def test_host_prompts_are_scrubbed_of_flagged_content(self) -> None: + # Regression: the host was the one recipient whose prompts were never + # scrubbed - an artifact leak went verbatim to the host CLI. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + (work / "inputs" / "secret.txt").write_text("SUPERSECRET-LOOPER-VALUE\n", encoding="utf-8") + capture_file = work / "host-stdin.txt" + spy_host = work / "spy_host.py" + spy_host.write_text( + "import pathlib\n" + "import sys\n" + "\n" + "prompt = sys.stdin.read()\n" + f"capture = pathlib.Path({str(capture_file)!r})\n" + "previous = capture.read_text(encoding='utf-8') if capture.exists() else ''\n" + "capture.write_text(previous + '\\n---CALL---\\n' + prompt, encoding='utf-8')\n" + "print('# Artifact')\n" + "print()\n" + "print('Owner: host')\n" + "print('No TBD')\n", + encoding="utf-8", + ) + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + python = Path(sys.executable).as_posix() + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + loop_yaml = loop_yaml.replace( + f'invoke: ["{python}", "{(FIXTURES / "fake_host.py").as_posix()}"]', + f'invoke: ["{python}", "{spy_host.as_posix()}"]', + ) + loop_yaml = loop_yaml.replace( + "privacy:\n egress: []", + "privacy:\n egress:\n - to: reviewer-1\n sends: [plan, deliveries]\n" + " redact: [\"inputs/**\"]\n consent: required", + ) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + workspace = work / "loop-workspace" + workspace.mkdir() + (workspace / "plan.md").write_text( + "Plan leaked SUPERSECRET-LOOPER-VALUE before redaction.\nOwner: host\nNo TBD\n", + encoding="utf-8", + ) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + host_prompts = capture_file.read_text(encoding="utf-8") + self.assertNotIn("SUPERSECRET-LOOPER-VALUE", host_prompts) + self.assertIn("[redacted:inputs/secret.txt]", host_prompts) + state = json.loads((workspace / "state.json").read_text(encoding="utf-8")) + self.assertTrue( + any("prompt for host" in item for item in state.get("warnings", [])), + state.get("warnings"), + ) + + def test_leak_attribution_names_all_matching_files_and_dedupes_log(self) -> None: + # Two flagged files sharing the same content must both be named in + # the warning, and repeated identical scrubs log one event. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + (work / "inputs" / "secret-a.txt").write_text("SUPERSECRET-LOOPER-VALUE\n", encoding="utf-8") + (work / "inputs" / "secret-b.txt").write_text("SUPERSECRET-LOOPER-VALUE\n", encoding="utf-8") + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + loop_yaml = loop_yaml.replace( + "privacy:\n egress: []", + "privacy:\n egress:\n - to: reviewer-1\n sends: [plan, deliveries]\n" + " redact: [\"inputs/secret-a.txt\", \"inputs/secret-b.txt\"]\n consent: required", + ) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + workspace = work / "loop-workspace" + workspace.mkdir() + (workspace / "plan.md").write_text( + "Plan leaked SUPERSECRET-LOOPER-VALUE before redaction.\nOwner: host\nNo TBD\n", + encoding="utf-8", + ) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + state = json.loads((workspace / "state.json").read_text(encoding="utf-8")) + warnings = [item for item in state.get("warnings", []) if "prompt for reviewer-1" in item] + self.assertTrue(warnings, state.get("warnings")) + self.assertIn("inputs/secret-a.txt", warnings[0]) + self.assertIn("inputs/secret-b.txt", warnings[0]) + run_log = (workspace / "run-log.md").read_text(encoding="utf-8") + reviewer_events = [ + line for line in run_log.splitlines() + if "redaction_applied" in line and "prompt for reviewer-1" in line + ] + self.assertEqual(len(reviewer_events), 1, run_log) + + def test_unscrubbable_flagged_file_is_surfaced_as_blind_spot(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + (work / "inputs" / "huge.bin").write_text("A" * 1_100_000, encoding="utf-8") + write_loop_yaml(work / "loop.yaml") + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + loop_yaml = loop_yaml.replace( + "privacy:\n egress: []", + "privacy:\n egress:\n - to: reviewer-1\n sends: [plan, deliveries]\n" + " redact: [\"inputs/huge.bin\"]\n consent: required", + ) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + result = run_cmd([sys.executable, "run-loop.py"], work) + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + state = json.loads((work / "loop-workspace" / "state.json").read_text(encoding="utf-8")) + self.assertTrue( + any("cannot be detected" in item and "huge.bin" in item for item in state.get("warnings", [])), + state.get("warnings"), + ) + run_log = (work / "loop-workspace" / "run-log.md").read_text(encoding="utf-8") + self.assertIn("redaction_unscrubbable", run_log) + + def test_consent_prompt_shows_leak_warning_before_asking(self) -> None: + # The scrub runs before the consent question, and the consent prompt + # displays the leak warning for that member. + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Need a useful loop.\n", encoding="utf-8") + (work / "inputs" / "secret.txt").write_text("SUPERSECRET-LOOPER-VALUE\n", encoding="utf-8") + write_loop_yaml(work / "loop.yaml", judge_local=False) + shutil.copyfile(RUNNER_TEMPLATE, work / "run-loop.py") + + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + loop_yaml = loop_yaml.replace( + "privacy:\n egress: []", + "privacy:\n egress:\n - to: reviewer-1\n sends: [plan, deliveries]\n" + " redact: [\"inputs/**\"]\n consent: required", + ) + (work / "loop.yaml").write_text(loop_yaml, encoding="utf-8") + + compiled = run_cmd( + [sys.executable, str(LOOPER), "compile", "loop.yaml", "--out", "loop.resolved.json"], + work, + ) + self.assertEqual(compiled.returncode, 0, compiled.stderr) + workspace = work / "loop-workspace" + workspace.mkdir() + (workspace / "plan.md").write_text( + "Plan leaked SUPERSECRET-LOOPER-VALUE before redaction.\nOwner: host\nNo TBD\n", + encoding="utf-8", + ) + result = run_cmd([sys.executable, "run-loop.py"], work, stdin="yes\n") + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + consent_block = result.stdout.split("Type 'yes' to consent")[0] + self.assertIn("Warning: flagged content", consent_block) + def test_fixed_passes_reviewer_gate_completes_cleanly(self) -> None: # Regression: the synthetic "fixed_passes reviewer pass" marker used to # feed the no-progress detector and fail the run before its passes