From 3b05b2eb53030674a54023f69e5ce1cd1c05dbf0 Mon Sep 17 00:00:00 2001 From: Kevin Simback <16635828+ksimback@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:42:40 -0700 Subject: [PATCH 1/2] Add loop pattern library: five templates + wizard Template Mode (v0.3.0) templates/loops/ ships five named, pre-designed loops the wizard customizes instead of starting blank: security-scan (promoted from the real hermes-ecosystem run), code-review, bug-hunt, docs-sync, and research-synthesis. Each is a complete, compiler-validated loop.yaml with {{PLACEHOLDER}} slots, a README (use-when, placeholder table, customization notes), and helper check scripts where needed. - /looper [target-dir] --template : compressed interview that asks only for placeholders, models, and paths; critique, structural rules, privacy statement, and preview still run. - looper.py compile now warns on unresolved {{PLACEHOLDER}} tokens; the wizard treats the warning as an emit blocker. - scan-secrets.py: streaming secret/PII sweep (worktree + git history), vendor paths skipped, masked excerpts only. - Tests 16 -> 18: every template must compile with the expected placeholder warning and appear in the catalog; the warning must disappear after substitution. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 24 ++ README.md | 22 + SKILL.md | 51 ++- commands/looper.md | 14 +- pyproject.toml | 2 +- scripts/looper.py | 26 ++ templates/loops/README.md | 41 ++ templates/loops/bug-hunt/README.md | 39 ++ templates/loops/bug-hunt/loop.yaml | 148 +++++++ .../bug-hunt/scripts/check-fix-report.py | 69 ++++ templates/loops/code-review/README.md | 35 ++ templates/loops/code-review/loop.yaml | 155 ++++++++ .../loops/code-review/scripts/check-review.py | 85 ++++ templates/loops/docs-sync/README.md | 36 ++ templates/loops/docs-sync/loop.yaml | 141 +++++++ .../docs-sync/scripts/check-drift-report.py | 79 ++++ templates/loops/research-synthesis/README.md | 41 ++ templates/loops/research-synthesis/loop.yaml | 136 +++++++ .../scripts/check-citations.py | 90 +++++ templates/loops/security-scan/README.md | 36 ++ templates/loops/security-scan/loop.yaml | 176 ++++++++ .../security-scan/scripts/check-findings.py | 78 ++++ .../security-scan/scripts/scan-secrets.py | 375 ++++++++++++++++++ tests/test_looper.py | 68 ++++ 24 files changed, 1954 insertions(+), 13 deletions(-) create mode 100644 templates/loops/README.md create mode 100644 templates/loops/bug-hunt/README.md create mode 100644 templates/loops/bug-hunt/loop.yaml create mode 100644 templates/loops/bug-hunt/scripts/check-fix-report.py create mode 100644 templates/loops/code-review/README.md create mode 100644 templates/loops/code-review/loop.yaml create mode 100644 templates/loops/code-review/scripts/check-review.py create mode 100644 templates/loops/docs-sync/README.md create mode 100644 templates/loops/docs-sync/loop.yaml create mode 100644 templates/loops/docs-sync/scripts/check-drift-report.py create mode 100644 templates/loops/research-synthesis/README.md create mode 100644 templates/loops/research-synthesis/loop.yaml create mode 100644 templates/loops/research-synthesis/scripts/check-citations.py create mode 100644 templates/loops/security-scan/README.md create mode 100644 templates/loops/security-scan/loop.yaml create mode 100644 templates/loops/security-scan/scripts/check-findings.py create mode 100644 templates/loops/security-scan/scripts/scan-secrets.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b703a24..e586b08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,30 @@ All notable changes to Looper are documented here. Versions follow [Semantic Versioning](https://semver.org/); the loop spec format is versioned separately via `version:` in `loop.yaml` (currently `1`). +## 0.3.0 — 2026-07-05 + +### Added — loop pattern library +- `templates/loops/` — five named, pre-designed loops the wizard customizes + instead of starting blank: `security-scan` (promoted from the real run + that produced hermes-ecosystem's security fixes), `code-review`, + `bug-hunt`, `docs-sync`, and `research-synthesis`. Each is a complete, + compiler-validated `loop.yaml` with `{{PLACEHOLDER}}` slots, a README + (use-when, placeholder table, customization notes), and helper check + scripts where the pattern needs them. +- `/looper [target-dir] --template ` — Template Mode in the wizard: + a compressed interview that asks only for the placeholder slots, model + selection, and paths, while keeping the full critique, structural-rule, + privacy, and preview flow. +- `looper.py compile` warns when unresolved `{{PLACEHOLDER}}` tokens remain + in the resolved spec; the wizard treats the warning as an emit blocker. +- `scan-secrets.py` (security-scan template): deterministic secret/PII + candidate sweep over working tree + full git history — streaming reads, + generated/vendor paths skipped, masked excerpts only, placeholder-value + suppression. +- 2 new tests (18 total): every template must compile (with the expected + placeholder warning) and be listed in the catalog; the warning must + disappear after substitution. + ## 0.2.1 — 2026-07-05 ### Fixed diff --git a/README.md b/README.md index 294fab0..875cdf1 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,28 @@ then offers to run the loop right there in the same Claude Code session. If you want a different folder name, pass it after `/looper`, for example `/looper client-onboarding-loop`. +### Start from a pattern template + +Instead of a blank interview, start from a named, pre-designed loop: + +```text +/looper my-review --template code-review +``` + +| Template | Use when | +|----------|----------| +| `security-scan` | Read-only sweep of a repo for secrets, PII, and vulnerabilities → triaged `SECURITY-FINDINGS.md`. | +| `code-review` | Review a branch's diff against its base → typed, severity-rated `REVIEW.md` grounded in the diff. | +| `bug-hunt` | Reproduce a reported bug, fix the root cause, prove it with before/after repro evidence. | +| `docs-sync` | Find and fix doc/code drift → per-item `DRIFT-REPORT.md`; docs follow code, code untouched. | +| `research-synthesis` | Synthesize collected sources into a cited `REPORT.md`; every claim traceable to a file. | + +Each template is a complete, compiler-validated `loop.yaml` with a handful of +`{{PLACEHOLDER}}` slots; the wizard asks only for those, picks models from +what's installed, and still runs its full critique, privacy, and preview flow +before emitting. See [`templates/loops/`](templates/loops/) for the catalog +and per-template docs — including how to add your own. + ### Easy: run in the same session The default path is to let Looper continue in the same conversation. It follows diff --git a/SKILL.md b/SKILL.md index a975d07..f577592 100644 --- a/SKILL.md +++ b/SKILL.md @@ -5,12 +5,14 @@ description: > cross-model review council. Use when the user wants to design, build, or set up an agent loop, iterative agent workflow, self-review loop, LLM-as-judge loop, multi-model council, reviewer/judge gate, or /goal-style looping - process. Guide goal refinement, typed verification criteria, reviewer and - judge selection, privacy boundaries, termination guards, no-progress stops, - and lightweight observability, then emit a RUN_IN_SESSION.md handoff prompt - plus portable loop.yaml, loop.resolved.json, LOOP.md, and run-loop.py. + process. Start from a named pattern template (security-scan, code-review, + bug-hunt, docs-sync, research-synthesis) or from a blank interview. Guide + goal refinement, typed verification criteria, reviewer and judge selection, + privacy boundaries, termination guards, no-progress stops, and lightweight + observability, then emit a RUN_IN_SESSION.md handoff prompt plus portable + loop.yaml, loop.resolved.json, LOOP.md, and run-loop.py. disable-model-invocation: true -argument-hint: "[target-dir]" +argument-hint: "[target-dir] [--template ]" allowed-tools: Read, Write, Bash --- @@ -23,9 +25,11 @@ advanced external runner. ## Workflow -1. Resolve the target path from the `/looper` argument. If no target is given, - use `./looper-output`. If the target contains an existing `loop.yaml`, treat - the task as an edit/resume instead of a fresh scaffold. +1. Resolve the target path and optional `--template ` from the + `/looper` arguments. If no target is given, use `./looper-output`. If the + target contains an existing `loop.yaml`, treat the task as an edit/resume + instead of a fresh scaffold. If a template was requested, follow Template + Mode below instead of the blank-slate interview in step 3. 2. Load the relevant rubric only when entering that stage: - Goal stage: `references/goal-rubric.md`. - Verification stage: `references/verification-rubric.md`. @@ -67,6 +71,37 @@ advanced external runner. the same file is the easy restart path and `run-loop.py` is available for advanced external execution. +## Template Mode + +The pattern library lives at `${CLAUDE_SKILL_DIR}/templates/loops/` — one +directory per template containing a complete, compilable `loop.yaml` (with +`{{PLACEHOLDER}}` tokens marking project-specific slots), a `README.md` +(use-when, placeholder table, customization notes), and optionally +`scripts/` with helper checkers. The catalog index is +`templates/loops/README.md`. + +A template is a pre-answered interview, not a bypass of design review: + +1. If `--template` has no name, an unknown name, or the user asks what is + available, show the catalog table (template + use-when) and let them pick. +2. Read the template's `loop.yaml` and `README.md`. Use the template as the + seed instead of a blank spec. +3. Run a compressed interview in place of the seven blank-slate stages: + ask for each `{{PLACEHOLDER}}` slot named in the template README, run + the host-model stage against detected CLIs (`detect-models`) and swap + `host` / `council` invocations to what is actually installed and authed, + then confirm target and workspace paths. +4. Everything after the interview still applies unchanged: critique each + pre-filled stage against its rubric (step 4), the structural rules + (steps 5–8) including the cross-vendor egress statement, the ASCII flow + preview, confirmation, emission, and compile. +5. Never emit while any `{{` token remains in `loop.yaml`. The compiler + prints `looper: warning: unresolved template placeholders remain ...` + for this case — treat that warning as a blocker, not advice. +6. At emission, copy the template's `scripts/` directory (when present) + into `/scripts/` alongside the standard emitted files, before + running compile. + ## File Rules - Write argv arrays, never shell command strings, for all model and check diff --git a/commands/looper.md b/commands/looper.md index bf2dd4e..3b767d4 100644 --- a/commands/looper.md +++ b/commands/looper.md @@ -1,6 +1,6 @@ --- description: Design and scaffold a Looper agent loop. -argument-hint: [target-dir] +argument-hint: [target-dir] [--template ] allowed-tools: Read, Write, Bash --- @@ -23,9 +23,15 @@ Find the Looper skill root before doing any loop-design work: exactly. Treat the located directory as `CLAUDE_SKILL_DIR` when running helper scripts. -## Target Directory +## Arguments -- If `$ARGUMENTS` is empty, use `./looper-output`. -- Otherwise, treat `$ARGUMENTS` as the target directory argument. +Parse `$ARGUMENTS` as `[target-dir] [--template ]`, in any order: + +- If `--template` is present, take the next token as the template name and + follow the skill's Template Mode (the catalog lives at + `templates/loops/` inside the skill directory). `--template` with no + name means: show the catalog and ask. +- The remaining token, if any, is the target directory. If none is left, + use `./looper-output`. Then continue with the Looper skill workflow. diff --git a/pyproject.toml b/pyproject.toml index 492c750..403b459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "looper-skill" -version = "0.2.1" +version = "0.3.0" description = "A Claude Code skill that scaffolds well-designed agent loops." readme = "README.md" requires-python = ">=3.9" diff --git a/scripts/looper.py b/scripts/looper.py index fb92c10..a149bab 100644 --- a/scripts/looper.py +++ b/scripts/looper.py @@ -35,6 +35,25 @@ ] +# Loop templates under templates/loops/ mark project-specific slots with +# {{TOKEN}} placeholders. They compile as ordinary strings; the warning below +# keeps a half-customized template from being run by accident. +PLACEHOLDER_PATTERN = re.compile(r"\{\{[A-Za-z0-9_]+\}\}") + + +def find_placeholders(value: Any) -> set[str]: + found: set[str] = set() + if isinstance(value, str): + found.update(PLACEHOLDER_PATTERN.findall(value)) + elif isinstance(value, list): + for item in value: + found.update(find_placeholders(item)) + elif isinstance(value, dict): + for item in value.values(): + found.update(find_placeholders(item)) + return found + + def reject_secret_material(values: list[str], context: str) -> None: for value in values: for pattern in SECRET_PATTERNS: @@ -831,6 +850,13 @@ def cmd_compile(args: argparse.Namespace) -> int: source = args.loop_yaml.resolve() spec = load_yaml(source) resolved = normalize_spec(spec, source) + placeholders = sorted(find_placeholders(resolved)) + if placeholders: + print( + "looper: warning: unresolved template placeholders remain " + f"({', '.join(placeholders)}); fill them in before running this loop", + file=sys.stderr, + ) out = args.out or source.with_name("loop.resolved.json") write_json(out, resolved) if args.render: diff --git a/templates/loops/README.md b/templates/loops/README.md new file mode 100644 index 0000000..6a5f7f0 --- /dev/null +++ b/templates/loops/README.md @@ -0,0 +1,41 @@ +# Loop pattern library + +Named, pre-designed loops the Looper wizard customizes instead of starting +from a blank interview: + +``` +/looper my-loop-dir --template +``` + +Each template is a complete, compilable `loop.yaml` plus a README describing +when to use it and which `{{PLACEHOLDER}}` slots to fill. Templates with a +`scripts/` directory ship helper checkers that the wizard copies next to the +emitted loop. A template is a pre-answered interview, not a bypass: the +wizard still critiques every stage, enforces the structural rules, and shows +the flow preview before emitting. + +| Template | Use when | +|----------|----------| +| [security-scan](security-scan/) | Read-only sweep of a repo for secrets, PII, and vulnerabilities → triaged `SECURITY-FINDINGS.md`. | +| [code-review](code-review/) | Review a branch's diff against its base → typed, severity-rated `REVIEW.md` grounded in the diff. | +| [bug-hunt](bug-hunt/) | Reproduce a reported bug, fix the root cause, prove it with before/after repro evidence. | +| [docs-sync](docs-sync/) | Find and fix doc/code drift → per-item `DRIFT-REPORT.md`; docs follow code, code untouched. | +| [research-synthesis](research-synthesis/) | Synthesize collected sources into a cited `REPORT.md`; every claim traceable to a file. | + +## Conventions + +- `{{PLACEHOLDER}}` tokens mark the project-specific slots; the README table + in each template says what goes where. `looper.py compile` warns if any + remain, and the wizard refuses to emit while one survives. +- Every template compiles as-is (CI enforces this), so the catalog can't + drift from the compiler's contract. +- Templates that edit files (`bug-hunt`, `docs-sync`) do so under + side-effect approval and never commit; read-only templates say so in + their `execution.side_effects.notes`. + +## Adding a template + +Add `templates/loops//` with `loop.yaml` (compilable, placeholders +only inside string values), `README.md` (use-when, emits, placeholder +table), and optional `scripts/`. Add a row to the table above. The test +suite compiles every template automatically. diff --git a/templates/loops/bug-hunt/README.md b/templates/loops/bug-hunt/README.md new file mode 100644 index 0000000..4bc37ed --- /dev/null +++ b/templates/loops/bug-hunt/README.md @@ -0,0 +1,39 @@ +# bug-hunt + +Reproduce a reported bug, fix the root cause, and prove it: the repro command +is observed failing before the change (plan gate) and passing after it +(delivery gate), with the full test suite still green. + +## Use when + +You have a bug report and a way to trigger the bug from the command line, and +you want the fix held to a before/after evidence standard instead of "the +error went away". + +## What it emits + +- `loop-workspace/fix-plan.md` — includes the verbatim failing repro output. +- `loop-workspace/FIX-REPORT.md` — root cause, fix at file:line, + before/after evidence. +- The fix itself, applied to the target repo's working tree (never + committed — committing stays your call). + +## Placeholders + +| Token | Replace with | +|-------|--------------| +| `{{REPO_DIR}}` | Path to the repository containing the bug. | +| `{{REPRO_CMD}}` | Argv array that triggers the bug, e.g. `["python", "-m", "pytest", "tests/test_x.py::test_bug", "-x"]`. Replace the whole one-element list. | +| `{{TEST_CMD}}` | Argv array for the full test suite, e.g. `["python", "-m", "pytest"]`. Replace the whole one-element list. | + +Also: put the bug report (the issue text, stack trace, expected vs actual) +in `inputs/bug-report.md` inside the emitted loop directory. + +## Customization notes + +- **No command-line repro yet?** Write the repro as a small script first and + point `{{REPRO_CMD}}` at it — a bug you can't trigger deterministically + can't be gated programmatically, and this template's value collapses to + judge-only. +- The loop edits the target repo's working tree under side-effect approval; + run it on a clean branch so the diff is easy to review and revert. diff --git a/templates/loops/bug-hunt/loop.yaml b/templates/loops/bug-hunt/loop.yaml new file mode 100644 index 0000000..d72c050 --- /dev/null +++ b/templates/loops/bug-hunt/loop.yaml @@ -0,0 +1,148 @@ +version: 1 +meta: + name: bug-hunt + description: > + Reproduce a reported bug, fix its root cause, and prove the fix: the + repro command must be observed failing before the change and passing + after it, with the full test suite still green. + author: looper-pattern-library + created: 2026-07-05 + +goal: + statement: > + Fix the bug described in inputs/bug-report.md in the repository at + {{REPO_DIR}}. First reproduce it by running the repro command and + recording the failing output in the plan. Then find and fix the root + cause - not the symptom - and produce FIX-REPORT.md stating the root + cause, the change made (file:line), the before/after repro evidence, and + why the fix addresses the cause. + context_sources: + - file: ./inputs/bug-report.md + - cmd: "git -C {{REPO_DIR}} log --oneline -10" + timeout_sec: 60 + definition_of_done: > + FIX-REPORT.md exists with root cause, fix location, and before/after + evidence; the repro command exits zero; the test suite passes; the diff + contains no changes unrelated to the fix; no TBD or placeholder text. + verification: + - id: repro-passes + type: programmatic + # The recorded repro. Must have been observed FAILING at plan time + # (plan gate checks that); passing here proves the fix. + check: ["{{REPRO_CMD}}"] + expect: exit_zero + timeout_sec: 300 + - id: tests-pass + type: programmatic + check: ["{{TEST_CMD}}"] + expect: exit_zero + timeout_sec: 600 + - id: report-schema + type: programmatic + # Structural check: root cause, fix location, before/after evidence; no TBDs. + check: ["python", "scripts/check-fix-report.py", "loop-workspace/FIX-REPORT.md"] + expect: exit_zero + - id: plan-repro-observed + type: judge + rubric: > + The fix-plan.md contains the verbatim output of the repro command + observed FAILING before any change was made, names the suspected + root-cause area (file or module), and states how the fix will be + verified. Verdict revise if the failure was never actually + reproduced or the plan jumps to a fix without evidence. + - id: root-cause-sound + type: judge + rubric: > + Judge FIX-REPORT.md against the actual diff of the fix. The stated + root cause must explain the failing behavior in the bug report, the + fix must plausibly remove that cause (not mask the symptom, not + special-case the repro input), and the before/after evidence must + show the same repro command failing then passing. Mark verdict + revise if the fix suppresses the failure without addressing the + cause, or if the diff includes changes unrelated to the fix. + +host: + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 900 + +council: + - id: judge-1 + role: judge + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + scope: [plan, delivery] + local: false + +gates: + plan_gate: + when: after_plan + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [plan-repro-observed] + max_revisions: 2 + delivery_gate: + when: after_each_delivery + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [repro-passes, tests-pass, report-schema, root-cause-sound] + max_revisions: 3 + +loop_control: + max_iterations: 10 + budget: + usd: 5.0 + tokens: 2000000 + wall_clock_min: 45 + no_progress: + max_stalled_iterations: 2 + signals: + - same blocking issue repeats across iterations + - the repro command fails with an unchanged error after a revise verdict + - FIX-REPORT.md has no material change after a revise verdict + action: stop + human_checkpoints: [] + stop_conditions: + - delivery_gate passes clean (all four criteria) on FIX-REPORT.md + - max_iterations reached + - same blocker repeats for 2 iterations + - any budget cap exceeded + +execution: + mode: in_session + # This loop EDITS the target repository - that is its job. Every write + # outside loop-workspace/ goes through side-effect approval. + isolation: current_workspace + side_effects: + requires_approval: true + duplicate_action_check: true + notes: > + The fix is applied to the working tree at the target repo. The loop + never commits or pushes; committing is the user's call after review. + +observability: + state_file: state.json + run_log: run-log.md + checkpoint_granularity: gate + +privacy: + egress: + - to: judge-1 + sends: [plan, fix-report, fix-diff] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + +workspace: + dir: ./loop-workspace + layout: + - fix-plan.md # host plan incl. observed failing repro output + - "fix-draft-{n}.md" + - "review-{n}.md" # judge notes per iteration + - FIX-REPORT.md # final deliverable + - state.json + - run-log.md diff --git a/templates/loops/bug-hunt/scripts/check-fix-report.py b/templates/loops/bug-hunt/scripts/check-fix-report.py new file mode 100644 index 0000000..5eedeb4 --- /dev/null +++ b/templates/loops/bug-hunt/scripts/check-fix-report.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Structural gate check for FIX-REPORT.md. + +Verifies the fix report is well-formed before the judge spends tokens on it: + - the file exists and is non-empty, + - it states a root cause, the fix location, and before/after evidence, + - the fix location includes at least one file:line reference, + - there are no leftover TBD / TODO / FIXME placeholders. + +Exit 0 = structurally complete. Exit 1 = revise. This is deliberately +format-tolerant: it accepts either sections or labeled blocks, so the host +is not forced into one exact layout. + +Usage: + python check-fix-report.py +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REQUIRED_LABELS = ["root cause", "fix", "before", "after"] +FILE_LINE = re.compile(r"\S+:\d+") +PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: check-fix-report.py ", file=sys.stderr) + return 1 + + path = Path(sys.argv[1]) + if not path.is_file(): + print(f"FAIL: report not found: {path}", file=sys.stderr) + return 1 + + text = path.read_text(encoding="utf-8", errors="ignore") + if len(text.strip()) < 50: + print("FAIL: report is empty or trivially short", file=sys.stderr) + return 1 + + lower = text.lower() + problems = [] + + for label in REQUIRED_LABELS: + if label not in lower: + problems.append(f"missing required label: '{label}'") + + if not FILE_LINE.search(text): + problems.append("no file:line reference for the fix location") + + placeholder_hits = PLACEHOLDERS.findall(text) + if placeholder_hits: + problems.append(f"contains {len(placeholder_hits)} placeholder token(s): " + f"{sorted(set(h.upper() for h in placeholder_hits))}") + + if problems: + print("FAIL: FIX-REPORT.md is not structurally complete:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + print("PASS: FIX-REPORT.md structurally complete.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/loops/code-review/README.md b/templates/loops/code-review/README.md new file mode 100644 index 0000000..f23ff08 --- /dev/null +++ b/templates/loops/code-review/README.md @@ -0,0 +1,35 @@ +# code-review + +Review a branch's diff against its base and produce a typed, severity-rated +`REVIEW.md`, with a cross-model second opinion and a judge that rejects +speculation and style-nit padding. + +## Use when + +You want a repeatable, gated review of a feature branch before merge — every +finding located at file:line in the diff and defended by a rationale, and an +explicit statement of what was examined when the review comes back clean. + +## What it emits + +- `loop-workspace/review-plan.md` — file/hunk inventory, gated before review. +- `loop-workspace/REVIEW.md` — the final review report. + +## Placeholders + +| Token | Replace with | +|-------|--------------| +| `{{REPO_DIR}}` | Path to the repository containing the branch. | +| `{{BRANCH}}` | The branch under review. | +| `{{BASE_BRANCH}}` | The base to diff against (usually `main`). | + +## Customization notes + +- **PR instead of a branch:** point the context commands at + `git -C diff ...` after fetching the PR head, or + replace them with `gh pr diff `. +- **Dropping the cross-vendor reviewer:** delete the `second-opinion` council + member, its gate membership, and its egress entry if the diff must not + leave the machine. +- **Review focus:** tighten the goal statement to the dimension you care + about (e.g. security-only) rather than adding more judges. diff --git a/templates/loops/code-review/loop.yaml b/templates/loops/code-review/loop.yaml new file mode 100644 index 0000000..16caf57 --- /dev/null +++ b/templates/loops/code-review/loop.yaml @@ -0,0 +1,155 @@ +version: 1 +meta: + name: code-review + description: > + Review the diff of a branch against its base and produce a typed, + severity-rated review report grounded in the actual changes. Read-only: + the loop never modifies the target repository. + author: looper-pattern-library + created: 2026-07-05 + +goal: + statement: > + Review the changes on branch {{BRANCH}} relative to {{BASE_BRANCH}} in the + repository at {{REPO_DIR}} for correctness bugs, security issues, and + maintainability problems, and produce REVIEW.md where every finding is + typed (bug / security / maintainability), severity-rated (critical / + major / minor), located at file:line in the new code, and justified by a + rationale grounded in the diff - not in speculation about unseen code. + context_sources: + # The diff and the commit list are captured into the host prompt directly. + - cmd: "git -C {{REPO_DIR}} diff {{BASE_BRANCH}}...{{BRANCH}}" + timeout_sec: 60 + - cmd: "git -C {{REPO_DIR}} log --oneline {{BASE_BRANCH}}..{{BRANCH}}" + timeout_sec: 60 + definition_of_done: > + REVIEW.md exists, names the reviewed range, and (1) every finding carries + type, severity, file:line, and rationale, (2) every hunk that changes + behavior has been examined - anything skipped is listed with a reason, + (3) there is no TBD or placeholder text, and (4) a report with zero + findings says so explicitly and states what was checked. + verification: + - id: review-schema + type: programmatic + # Structural check: findings carry type/severity/location/rationale; no TBDs. + check: ["python", "scripts/check-review.py", "loop-workspace/REVIEW.md"] + expect: exit_zero + - id: plan-sound + type: judge + rubric: > + The review-plan.md lists the files and hunks in the diff, flags which + ones are behavior-changing and therefore need close review, and + depends only on the captured diff and log. Verdict revise if any + changed file is absent from the plan or the plan invents files not + in the diff. + - id: findings-grounded + type: judge + rubric: > + Judge REVIEW.md against the diff in the context. Every finding must + point at lines that exist in the diff and describe a defect visible + there. Mark verdict revise for any finding that is speculative, + mislocated, restates a style preference as a defect, or pads the + report with nits while a behavior-changing hunk goes unexamined. + +host: + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + +council: + # Local-family judge - gates the plan and the report. + - id: judge-1 + role: judge + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + scope: [plan, delivery] + local: false + # Cross-vendor second opinion - NOTES ONLY, cannot clean a gate. First send + # is consent-gated at runtime; redaction-glob files never reach it. + - id: second-opinion + role: reviewer + cli: codex + model: default + invoke: ["codex", "exec"] + timeout_sec: 600 + scope: [delivery] + local: false + +gates: + plan_gate: + when: after_plan + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [plan-sound] + max_revisions: 2 + delivery_gate: + when: after_each_delivery + members: [judge-1, second-opinion] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [review-schema, findings-grounded] + max_revisions: 3 + +loop_control: + max_iterations: 8 + budget: + usd: 5.0 + tokens: 1500000 + wall_clock_min: 30 + no_progress: + max_stalled_iterations: 2 + signals: + - same blocking issue repeats across iterations + - REVIEW.md has no material change after a revise verdict + - the same hunk stays unexamined + action: stop + human_checkpoints: + - before first cross-vendor send to second-opinion + stop_conditions: + - delivery_gate passes clean on REVIEW.md + - max_iterations reached + - same blocker repeats for 2 iterations + - any budget cap exceeded + +execution: + mode: in_session + # The loop writes only inside ./loop-workspace; the reviewed repo is + # untouched (diff and log are read via git, no checkout changes). + isolation: current_workspace + side_effects: + requires_approval: true + duplicate_action_check: true + notes: > + Only side effect is the consent-gated cross-vendor send. The reviewed + repository is never modified. + +observability: + state_file: state.json + run_log: run-log.md + checkpoint_granularity: gate + +privacy: + egress: + - to: judge-1 + sends: [plan, review-report] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + - to: second-opinion + # The diff under review plus the draft findings - never redaction-glob files. + sends: [diff, review-report] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + +workspace: + dir: ./loop-workspace + layout: + - review-plan.md # host plan, gated by plan_gate + - "review-draft-{n}.md" + - "review-{n}.md" # judge + reviewer notes per iteration + - REVIEW.md # final deliverable + - state.json + - run-log.md diff --git a/templates/loops/code-review/scripts/check-review.py b/templates/loops/code-review/scripts/check-review.py new file mode 100644 index 0000000..d3f80a5 --- /dev/null +++ b/templates/loops/code-review/scripts/check-review.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Structural gate check for REVIEW.md. + +Verifies the review report is well-formed before the judge spends tokens on it: + - the file exists and is non-empty, + - it names the reviewed range (base and branch), + - every finding carries Type, Severity, Location (file:line), and Rationale, + - severities use the agreed scale (critical / major / minor), + - there are no leftover TBD / TODO / FIXME placeholders. + +Exit 0 = structurally complete. Exit 1 = revise. This is deliberately +format-tolerant: it accepts either a Markdown table or repeated labeled +blocks, so the host is not forced into one exact layout. A review that +explicitly declares zero findings passes without finding fields, as long as +it states what was checked. + +Usage: + python check-review.py +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REQUIRED_FIELDS = ["type", "severity", "location", "rationale"] +SEVERITY_SCALE = re.compile(r"\b(critical|major|minor)\b", re.IGNORECASE) +FILE_LINE = re.compile(r"\S+:\d+") +PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: check-review.py ", file=sys.stderr) + return 1 + + path = Path(sys.argv[1]) + if not path.is_file(): + print(f"FAIL: report not found: {path}", file=sys.stderr) + return 1 + + text = path.read_text(encoding="utf-8", errors="ignore") + if len(text.strip()) < 50: + print("FAIL: report is empty or trivially short", file=sys.stderr) + return 1 + + lower = text.lower() + problems = [] + + # 1. The reviewed range is named. + if not re.search(r"\b(diff|branch|range|against|\.\.\.?)\b", lower): + problems.append("report never names the reviewed range (branch/base/diff)") + + # 2. A report with findings must expose all required fields somewhere. + # If the report explicitly states no findings, fields are not required. + declares_no_findings = bool(re.search( + r"no (confirmed )?(findings|issues|defects|problems)", lower)) + if not declares_no_findings: + for field in REQUIRED_FIELDS: + if field not in lower: + problems.append(f"missing required field label: '{field}'") + if not SEVERITY_SCALE.search(text): + problems.append("no severity from the scale critical/major/minor") + if not FILE_LINE.search(text): + problems.append("no file:line location anywhere in the report") + + # 3. No placeholder text. + placeholder_hits = PLACEHOLDERS.findall(text) + if placeholder_hits: + problems.append(f"contains {len(placeholder_hits)} placeholder token(s): " + f"{sorted(set(h.upper() for h in placeholder_hits))}") + + if problems: + print("FAIL: REVIEW.md is not structurally complete:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + status = "no-findings report" if declares_no_findings else "findings report" + print(f"PASS: REVIEW.md structurally complete ({status}).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/loops/docs-sync/README.md b/templates/loops/docs-sync/README.md new file mode 100644 index 0000000..1d43687 --- /dev/null +++ b/templates/loops/docs-sync/README.md @@ -0,0 +1,36 @@ +# docs-sync + +Detect drift between documentation and the code it describes, fix the docs, +and account for every drift item — fixed, or deferred with a reason. Docs +follow code; the code is never modified. + +## Use when + +Documentation has fallen behind the code — commands, flags, defaults, paths, +or behavior claims no longer match — and you want the cleanup done with an +auditable per-item report instead of a silent rewrite. + +## What it emits + +- `loop-workspace/sync-plan.md` — doc inventory + likely drift hot spots. +- `loop-workspace/DRIFT-REPORT.md` — every drift item with doc location, + code location, description, and status. +- Doc fixes, applied to the target repo's working tree (never committed). + +## Placeholders + +| Token | Replace with | +|-------|--------------| +| `{{REPO_DIR}}` | Path to the repository. | +| `{{DOCS_DIR}}` | Docs path relative to the repo root, e.g. `docs` or `README.md`. | +| `{{SRC_DIR}}` | Code path relative to the repo root, e.g. `src`. | + +## Customization notes + +- **Multiple doc surfaces** (README + docs/ + CLI help text): list them all + in the `ls-files` context command and name them in the goal statement. +- The judge verifies fixed items against the code itself, not the report's + claims — keep `{{SRC_DIR}}` scoped to what the docs actually describe so + verification stays cheap. +- Run on a clean branch; doc edits go through side-effect approval and stay + uncommitted for your review. diff --git a/templates/loops/docs-sync/loop.yaml b/templates/loops/docs-sync/loop.yaml new file mode 100644 index 0000000..d640b72 --- /dev/null +++ b/templates/loops/docs-sync/loop.yaml @@ -0,0 +1,141 @@ +version: 1 +meta: + name: docs-sync + description: > + Detect drift between documentation and the code it describes, fix the + docs, and produce a drift report where every item is either corrected or + deferred with a reason. The code is never modified - docs follow code, + not the other way around. + author: looper-pattern-library + created: 2026-07-05 + +goal: + statement: > + Compare the documentation under {{DOCS_DIR}} with the code under + {{SRC_DIR}} in the repository at {{REPO_DIR}}. Find every place the docs + say something the code no longer does - commands, flags, file paths, + defaults, API shapes, behavior claims - update the docs to match the + code, and produce DRIFT-REPORT.md listing every drift item with its doc + location, code location, what drifted, and its status (fixed or + deferred with a reason). + context_sources: + # Inventory of both trees; the host reads individual files from disk. + - cmd: "git -C {{REPO_DIR}} ls-files {{DOCS_DIR}} {{SRC_DIR}}" + timeout_sec: 60 + definition_of_done: > + DRIFT-REPORT.md exists and every item carries doc location, code + location, drift description, and status; every fixed item's doc text now + matches the code; every deferred item has a concrete reason; no doc file + in scope was skipped without a note; no TBD or placeholder text. + verification: + - id: report-schema + type: programmatic + # Structural check: doc/code locations, drift description, status; no TBDs. + check: ["python", "scripts/check-drift-report.py", "loop-workspace/DRIFT-REPORT.md"] + expect: exit_zero + - id: plan-inventory + type: judge + rubric: > + The sync-plan.md lists every documentation file in scope (from the + captured inventory), names the code area each one describes, and + flags the doc surfaces most likely to drift (CLI help, install + steps, config references, examples). Verdict revise if a doc file + from the inventory is missing without a stated reason. + - id: drift-verified + type: judge + rubric: > + Judge DRIFT-REPORT.md against the current code. For each item marked + fixed, the updated doc text must now match what the code actually + does - verify against the named code location, not the report's own + claim. Nothing may be invented: every statement added to the docs + must be traceable to code. Mark verdict revise for any fixed item + that still mismatches, any invented behavior, or any deferred item + whose reason is vague. + +host: + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 900 + +council: + # Single local-family judge; docs and code stay on-machine by default. + - id: judge-1 + role: judge + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + scope: [plan, delivery] + local: false + +gates: + plan_gate: + when: after_plan + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [plan-inventory] + max_revisions: 2 + delivery_gate: + when: after_each_delivery + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [report-schema, drift-verified] + max_revisions: 3 + +loop_control: + max_iterations: 10 + budget: + usd: 5.0 + tokens: 2000000 + wall_clock_min: 45 + no_progress: + max_stalled_iterations: 2 + signals: + - same blocking issue repeats across iterations + - DRIFT-REPORT.md has no material change after a revise verdict + - the same drift item stays unresolved + action: stop + human_checkpoints: [] + stop_conditions: + - delivery_gate passes clean on DRIFT-REPORT.md + - max_iterations reached + - same blocker repeats for 2 iterations + - any budget cap exceeded + +execution: + mode: in_session + # This loop EDITS documentation files in the target repo - that is its + # job. It never modifies code. Every write outside loop-workspace/ goes + # through side-effect approval. + isolation: current_workspace + side_effects: + requires_approval: true + duplicate_action_check: true + notes: > + Doc edits are applied to the working tree at the target repo. The loop + never touches files under the source dir and never commits or pushes. + +observability: + state_file: state.json + run_log: run-log.md + checkpoint_granularity: gate + +privacy: + egress: + - to: judge-1 + sends: [plan, drift-report, doc-diffs] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + +workspace: + dir: ./loop-workspace + layout: + - sync-plan.md # doc inventory + drift hot spots, gated by plan_gate + - "drift-draft-{n}.md" + - "review-{n}.md" # judge notes per iteration + - DRIFT-REPORT.md # final deliverable + - state.json + - run-log.md diff --git a/templates/loops/docs-sync/scripts/check-drift-report.py b/templates/loops/docs-sync/scripts/check-drift-report.py new file mode 100644 index 0000000..d173376 --- /dev/null +++ b/templates/loops/docs-sync/scripts/check-drift-report.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Structural gate check for DRIFT-REPORT.md. + +Verifies the drift report is well-formed before the judge spends tokens on it: + - the file exists and is non-empty, + - every drift item carries a doc location, a code location, a drift + description, and a status, + - statuses use the agreed vocabulary (fixed / deferred), + - a deferred item carries a reason, + - there are no leftover TBD / TODO / FIXME placeholders. + +Exit 0 = structurally complete. Exit 1 = revise. This is deliberately +format-tolerant: it accepts either a Markdown table or repeated labeled +blocks. A report that explicitly declares zero drift passes without item +fields, as long as it states what was compared. + +Usage: + python check-drift-report.py +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REQUIRED_FIELDS = ["doc", "code", "status"] +STATUS_SCALE = re.compile(r"\b(fixed|deferred)\b", re.IGNORECASE) +PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: check-drift-report.py ", file=sys.stderr) + return 1 + + path = Path(sys.argv[1]) + if not path.is_file(): + print(f"FAIL: report not found: {path}", file=sys.stderr) + return 1 + + text = path.read_text(encoding="utf-8", errors="ignore") + if len(text.strip()) < 50: + print("FAIL: report is empty or trivially short", file=sys.stderr) + return 1 + + lower = text.lower() + problems = [] + + declares_no_drift = bool(re.search( + r"no (drift|mismatches|discrepancies)( items| found| detected)?", lower)) + if not declares_no_drift: + for field in REQUIRED_FIELDS: + if field not in lower: + problems.append(f"missing required field label: '{field}'") + if not STATUS_SCALE.search(text): + problems.append("no status from the scale fixed/deferred") + deferred_count = len(re.findall(r"\bdeferred\b", lower)) + reason_count = len(re.findall(r"\b(reason|because)\b", lower)) + if deferred_count and not reason_count: + problems.append("deferred item(s) present but no reason given anywhere") + + placeholder_hits = PLACEHOLDERS.findall(text) + if placeholder_hits: + problems.append(f"contains {len(placeholder_hits)} placeholder token(s): " + f"{sorted(set(h.upper() for h in placeholder_hits))}") + + if problems: + print("FAIL: DRIFT-REPORT.md is not structurally complete:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + status = "no-drift report" if declares_no_drift else "drift report" + print(f"PASS: DRIFT-REPORT.md structurally complete ({status}).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/loops/research-synthesis/README.md b/templates/loops/research-synthesis/README.md new file mode 100644 index 0000000..8345c17 --- /dev/null +++ b/templates/loops/research-synthesis/README.md @@ -0,0 +1,41 @@ +# research-synthesis + +Synthesize a folder of source documents into a cited report that answers a +research question. Citation discipline is enforced programmatically; a judge +spot-checks that citations actually support their claims and that source +disagreements surface instead of being averaged away. + +## Use when + +You have collected source material (notes, papers, transcripts, exported +docs) and want a synthesis you can trust — every claim traceable to a file, +gaps stated instead of papered over. + +## What it emits + +- `loop-workspace/synthesis-plan.md` — per-source notes + report structure. +- `loop-workspace/REPORT.md` — the cited synthesis. + +## Placeholders + +| Token | Replace with | +|-------|--------------| +| `{{RESEARCH_QUESTION}}` | The question the report must answer, in one sentence. | + +Also: put the source documents in `inputs/sources/` inside the emitted loop +directory (plain-text formats work best), and create +`inputs/sources/MANIFEST.md` — a one-line-per-file list of what each source +is and where it came from. The manifest is the loop's context anchor; the +host reads the sources themselves from disk. + +## Customization notes + +- **Citation format** is `[source: inputs/sources/]`, enforced by + `scripts/check-citations.py`. If you change the format, change it in both + the goal statement and the script. +- **Web research is out of scope by design** — this loop synthesizes what + you already collected, which keeps verification honest. Gather first, + then loop. +- If sources are sensitive, note the judge egress entry: the plan, report, + and source excerpts go to the judge CLI under the default redaction + globs, consent-gated on first send. diff --git a/templates/loops/research-synthesis/loop.yaml b/templates/loops/research-synthesis/loop.yaml new file mode 100644 index 0000000..2de6101 --- /dev/null +++ b/templates/loops/research-synthesis/loop.yaml @@ -0,0 +1,136 @@ +version: 1 +meta: + name: research-synthesis + description: > + Synthesize a set of source documents into a cited report that answers a + research question. Every substantive claim carries an inline citation to + a source file; the judge rejects uncited claims and anything a source + contradicts. + author: looper-pattern-library + created: 2026-07-05 + +goal: + statement: > + Answer the research question "{{RESEARCH_QUESTION}}" using only the + source documents under inputs/sources/. Read every source, then produce + REPORT.md: a structured synthesis where every substantive claim carries + an inline citation in the form [source: inputs/sources/], sources + that disagree are surfaced rather than averaged, and questions the + sources cannot answer are listed explicitly as gaps. + context_sources: + - file: ./inputs/sources/MANIFEST.md + definition_of_done: > + REPORT.md exists, answers the research question or states precisely why + it cannot, cites only files that exist under inputs/sources/, leaves no + substantive paragraph uncited, notes disagreements between sources, and + contains no TBD or placeholder text. + verification: + - id: citations-valid + type: programmatic + # Every cited path exists; every substantive paragraph is cited; no TBDs. + check: ["python", "scripts/check-citations.py", "loop-workspace/REPORT.md", "--sources", "inputs/sources"] + expect: exit_zero + - id: plan-covers-sources + type: judge + rubric: > + The synthesis-plan.md lists every file under inputs/sources/ with a + one-line note on what it contributes to the question, states the + intended report structure, and flags sources that look contradictory + or off-topic. Verdict revise if any source file is missing from the + plan without a stated reason. + - id: faithful-to-sources + type: judge + rubric: > + Spot-check REPORT.md against the cited sources. Every checked claim + must be supported by the file it cites - not by general knowledge, + not by another file. Disagreements between sources must appear as + disagreements, not be silently resolved. Mark verdict revise for any + claim its citation does not support, any material contradicted by a + source, or any answer to the question that goes beyond what the + sources establish without flagging it as a gap. + +host: + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 900 + +council: + # Single local-family judge; source material stays on-machine by default. + - id: judge-1 + role: judge + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + scope: [plan, delivery] + local: false + +gates: + plan_gate: + when: after_plan + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [plan-covers-sources] + max_revisions: 2 + delivery_gate: + when: after_each_delivery + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [citations-valid, faithful-to-sources] + max_revisions: 3 + +loop_control: + max_iterations: 8 + budget: + usd: 5.0 + tokens: 2000000 + wall_clock_min: 45 + no_progress: + max_stalled_iterations: 2 + signals: + - same blocking issue repeats across iterations + - REPORT.md has no material change after a revise verdict + - the same uncited or unsupported claim persists + action: stop + human_checkpoints: [] + stop_conditions: + - delivery_gate passes clean on REPORT.md + - max_iterations reached + - same blocker repeats for 2 iterations + - any budget cap exceeded + +execution: + mode: in_session + # Read-only toward inputs/; the loop writes only inside ./loop-workspace. + isolation: current_workspace + side_effects: + requires_approval: true + duplicate_action_check: true + notes: > + No side effects outside the loop directory. Sources are read in place + and never modified. + +observability: + state_file: state.json + run_log: run-log.md + checkpoint_granularity: gate + +privacy: + egress: + - to: judge-1 + sends: [plan, report, source-excerpts] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + +workspace: + dir: ./loop-workspace + layout: + - synthesis-plan.md # per-source notes + report structure, gated by plan_gate + - "report-draft-{n}.md" + - "review-{n}.md" # judge notes per iteration + - REPORT.md # final deliverable + - state.json + - run-log.md diff --git a/templates/loops/research-synthesis/scripts/check-citations.py b/templates/loops/research-synthesis/scripts/check-citations.py new file mode 100644 index 0000000..03e5c09 --- /dev/null +++ b/templates/loops/research-synthesis/scripts/check-citations.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Structural gate check for REPORT.md citations. + +Verifies the synthesis report's citation discipline before the judge spends +tokens on it: + - the file exists and is non-empty, + - it contains at least one [source: ] citation, + - every cited path exists under the sources directory, + - every substantive paragraph (prose of ~40+ words outside headings and + code blocks) carries at least one citation, + - there are no leftover TBD / TODO / FIXME placeholders. + +Exit 0 = structurally complete. Exit 1 = revise. Whether the citations +actually SUPPORT the claims is the judge's job, not this script's. + +Usage: + python check-citations.py --sources +""" +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +CITATION = re.compile(r"\[source:\s*([^\]]+?)\s*\]") +PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) +SUBSTANTIVE_WORDS = 40 + + +def paragraphs(text: str) -> list[str]: + """Prose paragraphs, with fenced code blocks and headings removed.""" + text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) + parts = re.split(r"\n\s*\n", text) + return [p.strip() for p in parts + if p.strip() and not p.lstrip().startswith("#")] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("report", type=Path) + parser.add_argument("--sources", type=Path, required=True) + args = parser.parse_args() + + if not args.report.is_file(): + print(f"FAIL: report not found: {args.report}", file=sys.stderr) + return 1 + + text = args.report.read_text(encoding="utf-8", errors="ignore") + if len(text.strip()) < 50: + print("FAIL: report is empty or trivially short", file=sys.stderr) + return 1 + + problems = [] + cited = CITATION.findall(text) + if not cited: + problems.append("no [source: ] citations anywhere in the report") + + # 1. Every cited path must exist. Citations are written relative to the + # loop dir (inputs/sources/); accept sources-dir-relative too. + for raw in sorted(set(cited)): + candidate = Path(raw) + if not (candidate.is_file() or (args.sources / candidate.name).is_file()): + problems.append(f"cited source does not exist: '{raw}'") + + # 2. Substantive paragraphs must be cited. + for para in paragraphs(text): + if len(para.split()) >= SUBSTANTIVE_WORDS and not CITATION.search(para): + problems.append( + f"uncited substantive paragraph starting: '{para[:60]}...'") + + # 3. No placeholder text. + placeholder_hits = PLACEHOLDERS.findall(text) + if placeholder_hits: + problems.append(f"contains {len(placeholder_hits)} placeholder token(s): " + f"{sorted(set(h.upper() for h in placeholder_hits))}") + + if problems: + print("FAIL: REPORT.md citation discipline is incomplete:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + print(f"PASS: REPORT.md cites {len(set(cited))} source(s); " + "all paths resolve and substantive paragraphs are cited.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/loops/security-scan/README.md b/templates/loops/security-scan/README.md new file mode 100644 index 0000000..11c67f3 --- /dev/null +++ b/templates/loops/security-scan/README.md @@ -0,0 +1,36 @@ +# security-scan + +Read-only security sweep of a repository: secrets, PII, and exploitable +vulnerabilities, triaged into a findings report. This is the pattern Looper's +own proof-of-concept run used against a real repo. + +## Use when + +You want a bounded, evidence-first security review of a repo — every candidate +from a deterministic sweep either confirmed (severity + location + remediation) +or dismissed with a reason, and every claimed vulnerability defended as +actually exploitable. + +## What it emits + +- `loop-workspace/candidates.json` — deterministic secret/PII sweep output + (working tree + git history, masked excerpts). +- `loop-workspace/SECURITY-FINDINGS.md` — the triaged final report. + +## Placeholders + +| Token | Replace with | +|-------|--------------| +| `{{TARGET_REPO}}` | Clone URL (or local path) of the repository to scan. | + +## Customization notes + +- **Local repo instead of a URL:** replace the `git clone` context command + with a clone from a local path, or point the scan at an existing checkout — + keep it inside `loop-workspace/` so the loop stays read-only toward the + original. +- **Dropping the cross-vendor reviewer:** delete the `vuln-reviewer` council + member, its gate membership, and its egress entry if the code must not + leave the machine. The judge alone still gates all four criteria. +- The gate criteria assume the two helper scripts in `scripts/` are copied + next to the emitted loop (the wizard does this automatically). diff --git a/templates/loops/security-scan/loop.yaml b/templates/loops/security-scan/loop.yaml new file mode 100644 index 0000000..f12a8c4 --- /dev/null +++ b/templates/loops/security-scan/loop.yaml @@ -0,0 +1,176 @@ +version: 1 +meta: + name: security-scan + description: > + Scan a repository for leaked secrets, personal information, and exploitable + vulnerabilities, and produce a triaged findings report. Read-only: the loop + never modifies the target repository. + author: looper-pattern-library + created: 2026-07-05 + +goal: + statement: > + Review the source of {{TARGET_REPO}} (working tree AND full git history) + for security exposures - hardcoded API keys / tokens / credentials, + personal information (PII), and exploitable code vulnerabilities - and + produce SECURITY-FINDINGS.md where every finding is typed, severity-rated, + located at file:line (and commit for history hits), and given a + remediation note. + context_sources: + # The repo is not on disk yet. This clone (not shallow, so history is + # available) lands in the throwaway workspace dir before the plan draft. + - cmd: "git clone {{TARGET_REPO}} loop-workspace/repo" + timeout_sec: 300 + definition_of_done: > + SECURITY-FINDINGS.md exists and (1) covers secrets, PII, and + vulnerabilities, (2) has zero unresolved candidates - each is either + confirmed with severity + location + remediation, or dismissed with a + stated reason, (3) contains no TBD or placeholder text, and (4) every + confirmed vulnerability is exploitable as described, not speculative. + verification: + - id: prog-scan-ran + type: programmatic + # Deterministic secret/PII candidate sweep over working tree + history. + check: ["python", "scripts/scan-secrets.py", "loop-workspace/repo", "--out", "loop-workspace/candidates.json"] + expect: exit_zero + - id: findings-schema + type: programmatic + # Structural check: every finding has type, severity, location, remediation; no TBDs. + check: ["python", "scripts/check-findings.py", "loop-workspace/SECURITY-FINDINGS.md"] + expect: exit_zero + - id: plan-sound + type: judge + rubric: > + The scan-plan.md names every area to inspect (secrets, PII, + vulnerabilities), states how git history will be covered, lists the + file groups / languages present in loop-workspace/repo, and depends + only on context the loop actually gathers. Verdict revise if any of + the three security areas is unaddressed or history coverage is + undefined. + - id: triage-complete + type: judge + rubric: > + Judge SECURITY-FINDINGS.md against loop-workspace/candidates.json. + Every candidate from the programmatic sweep appears in the report as + either CONFIRMED (with severity, file:line/commit, and remediation) + or DISMISSED (with a concrete reason such as "example value in + README" or "test fixture"). No candidate is left untriaged and no + entry contains TBD. Return verdict pass only when zero candidates + are unresolved. + - id: vuln-soundness + type: judge + rubric: > + For each vulnerability in SECURITY-FINDINGS.md, confirm it is + exploitable as described given the surrounding code (e.g. unsanitized + input reaching a sink, missing authz check on a real route). Mark + verdict revise for any finding that is speculative, mislocated, or + already mitigated by nearby code. + +host: + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + +council: + # Local-family judge - gates the plan and final report. Same host family is + # acceptable here because the judge task (real secret vs placeholder, + # exploitable vs speculative) is near-objective and keeps content on-machine. + - id: judge-1 + role: judge + cli: claude + model: default + invoke: ["claude", "-p"] + timeout_sec: 600 + scope: [plan, delivery] + local: false + # Cross-vendor adversarial reviewer - NOTES ONLY, scoped to vulnerability + # snippets. Cannot clean a gate. First send is consent-gated at runtime; + # secret/PII values and redaction-glob files never reach it. + - id: vuln-reviewer + role: reviewer + cli: codex + model: default + invoke: ["codex", "exec"] + timeout_sec: 600 + scope: [delivery] + local: false + +gates: + plan_gate: + when: after_plan + members: [judge-1] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [plan-sound] + max_revisions: 2 + delivery_gate: + when: after_each_delivery + members: [judge-1, vuln-reviewer] + verdict_policy: revise_until_clean + verdict_source: judge-1 + criteria: [prog-scan-ran, findings-schema, triage-complete, vuln-soundness] + max_revisions: 2 + +loop_control: + max_iterations: 10 + budget: + usd: 5.0 + tokens: 2000000 + wall_clock_min: 30 + no_progress: + max_stalled_iterations: 2 + signals: + - same blocking issue repeats across iterations + - SECURITY-FINDINGS.md has no material change after a revise verdict + - the same candidate stays untriaged + action: stop + human_checkpoints: + - before first cross-vendor send to vuln-reviewer + stop_conditions: + - delivery_gate passes clean (all four criteria) on SECURITY-FINDINGS.md + - max_iterations reached + - same blocker repeats for 2 iterations + - any budget cap exceeded + +execution: + mode: in_session + # The loop writes only inside ./loop-workspace. The target repo is cloned + # into the gitignored throwaway loop-workspace/repo and is never modified. + isolation: current_workspace + side_effects: + requires_approval: true + duplicate_action_check: true + notes: > + Only side effects are (1) git clone of the target repo into the + throwaway dir and (2) the consent-gated cross-vendor send. No pushes, + no writes to the target repo. + +observability: + state_file: state.json + run_log: run-log.md + checkpoint_granularity: gate + +privacy: + egress: + - to: judge-1 + sends: [plan, findings-report] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + - to: vuln-reviewer + # Only vulnerability code snippets - never raw secret/PII values. + sends: [vulnerability-snippets] + redact: [".env", ".env.*", "secrets/**", "**/*.key"] + consent: required + +workspace: + dir: ./loop-workspace + layout: + - repo/ # throwaway clone of the target (gitignored, read-only) + - scan-plan.md # host plan, gated by plan_gate + - candidates.json # programmatic secret/PII sweep output + - "findings-draft-{n}.md" + - "review-{n}.md" # judge + reviewer notes per iteration + - SECURITY-FINDINGS.md # final deliverable + - state.json + - run-log.md diff --git a/templates/loops/security-scan/scripts/check-findings.py b/templates/loops/security-scan/scripts/check-findings.py new file mode 100644 index 0000000..663cf54 --- /dev/null +++ b/templates/loops/security-scan/scripts/check-findings.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Structural gate check for SECURITY-FINDINGS.md. + +Verifies the report is well-formed before the judge spends tokens on it: + - the file exists and is non-empty, + - it covers all three required areas (secrets, PII, vulnerabilities), + - every finding row carries Type, Severity, Location, and Remediation, + - there are no leftover TBD / TODO / FIXME placeholders. + +Exit 0 = structurally complete. Exit 1 = revise. This is deliberately format-tolerant: +it accepts either a Markdown table or repeated labeled blocks, so the host is not forced +into one exact layout. + +Usage: + python check-findings.py +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +REQUIRED_AREAS = ["secret", "pii", "vulnerab"] # substrings, case-insensitive +REQUIRED_FIELDS = ["type", "severity", "location", "remediation"] +PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: check-findings.py ", file=sys.stderr) + return 1 + + path = Path(sys.argv[1]) + if not path.is_file(): + print(f"FAIL: report not found: {path}", file=sys.stderr) + return 1 + + text = path.read_text(encoding="utf-8", errors="ignore") + if len(text.strip()) < 50: + print("FAIL: report is empty or trivially short", file=sys.stderr) + return 1 + + lower = text.lower() + problems = [] + + # 1. All three security areas mentioned. + for area in REQUIRED_AREAS: + if area not in lower: + problems.append(f"missing required area keyword: '{area}'") + + # 2. A report with confirmed findings must expose all required fields somewhere. + # If the report explicitly states no findings, fields are not required. + declares_no_findings = bool(re.search( + r"no (confirmed )?(findings|secrets|vulnerabilities|issues)", lower)) + if not declares_no_findings: + for field in REQUIRED_FIELDS: + if field not in lower: + problems.append(f"missing required field label: '{field}'") + + # 3. No placeholder text. + placeholder_hits = PLACEHOLDERS.findall(text) + if placeholder_hits: + problems.append(f"contains {len(placeholder_hits)} placeholder token(s): " + f"{sorted(set(h.upper() for h in placeholder_hits))}") + + if problems: + print("FAIL: SECURITY-FINDINGS.md is not structurally complete:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + status = "no-findings report" if declares_no_findings else "findings report" + print(f"PASS: SECURITY-FINDINGS.md structurally complete ({status}).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/templates/loops/security-scan/scripts/scan-secrets.py b/templates/loops/security-scan/scripts/scan-secrets.py new file mode 100644 index 0000000..fd2aa59 --- /dev/null +++ b/templates/loops/security-scan/scripts/scan-secrets.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +"""Deterministic secret / PII candidate sweep for the security-scan loop. + +This is a *sweep*, not a gate. It streams the working tree (and, by default, the +git history) of a repository line by line, applies a fixed battery of regexes for +common credential shapes and PII, suppresses obvious placeholders, and writes a +JSON file of MASKED candidates for a downstream reviewer to triage. + +Everything it emits is a *candidate*: a lead worth a human/agent look, not a +confirmed leak. Matched values are always masked before they touch disk or the +terminal — the full secret is never written out. + +Usage: + python scan-secrets.py [--out .json] [--no-history] + +Exit codes: + 0 = the sweep completed. This does NOT mean "no secrets" — candidates may or + may not have been found. Zero and non-zero candidate counts both exit 0. + 2 = bad usage / missing repo dir. + +Stdlib only, Python 3.9+, cross-platform. +""" +from __future__ import annotations + +import argparse +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Pattern, Tuple + +# --- Tunable skip lists ------------------------------------------------------ +# Directory names that, anywhere in a path, exclude a file from the worktree scan. +SKIP_DIRS = { + ".git", + "node_modules", + "dist", + "build", + "out", + "vendor", + "third_party", + "__pycache__", + ".venv", + "venv", + ".tox", + ".mypy_cache", + "coverage", + "target", +} + +# Filename glob patterns that exclude a file from the worktree scan. +SKIP_FILE_GLOBS = [ + "*.min.js", + "*.min.css", + "*.map", + "*.lock", + "package-lock.json", + "yarn.lock", + "pnpm-lock.yaml", + "*.svg", + "*.ipynb checkpoints", +] + +MAX_FILE_BYTES = 10 * 1024 * 1024 # 10 MB +BINARY_SNIFF_BYTES = 8 * 1024 # 8 KB + +# Case-insensitive substrings that mark a captured value as an obvious placeholder. +PLACEHOLDER_MARKERS = [ + "example", + "sample", + "placeholder", + "changeme", + "your_key", + "your-key", + "yourkey", + "xxxx", + "dummy", + "test", + "fake", + "<", + "{{", +] +_PLACEHOLDER_RE = re.compile(r"your[_-]?key", re.IGNORECASE) + +# --- Detection patterns ------------------------------------------------------ +# Each entry is (pattern_id, compiled regex). Group 1, when present, is the +# specific value to mask/dedupe on; otherwise the whole match is used. +PATTERNS: List[Tuple[str, Pattern[str]]] = [ + ("aws_access_key_id", re.compile(r"\b(AKIA[0-9A-Z]{16})\b")), + ( + "aws_secret_access_key", + re.compile( + r"(?i)aws.{0,20}?(?:secret|key).{0,20}?[=:]\s*['\"]?([A-Za-z0-9/+]{40})['\"]?" + ), + ), + ( + "generic_secret_assignment", + re.compile( + r"(?i)\b(?:api[_-]?key|apikey|token|secret|passwd|password)\b\s*[=:]\s*" + r"['\"]?([^\s'\"]{8,})['\"]?" + ), + ), + ("bearer_token", re.compile(r"(?i)bearer\s+([A-Za-z0-9._\-+/=]{8,})")), + ("private_key_pem", re.compile(r"(-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----)")), + ( + "github_token", + re.compile(r"\b((?:ghp|gho|ghs|github_pat)_[A-Za-z0-9_]{20,})\b"), + ), + ("slack_token", re.compile(r"\b(xox[baprs]-[A-Za-z0-9-]{10,})\b")), + ("google_api_key", re.compile(r"\b(AIza[0-9A-Za-z_\-]{35})\b")), + ( + "jwt", + re.compile(r"\b(eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b"), + ), + ( + "connection_string_credentials", + re.compile(r"\b([a-zA-Z][a-zA-Z0-9+.\-]*://[^\s:/@]+:[^\s:/@]+@[^\s/]+)"), + ), + ( + "email_address", + re.compile(r"\b([A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,})\b"), + ), + # Phone-like sequences are only reported when the line also mentions a + # phone-ish keyword (checked separately) to keep noise down. + ( + "phone_number", + re.compile(r"(\+?\d[\d\s().\-]{7,}\d)"), + ), +] + +_PHONE_KEYWORD_RE = re.compile(r"(?i)\b(phone|tel|mobile|cell)\b") + + +def is_placeholder(value: str) -> bool: + """True when the captured value looks like an obvious placeholder/example.""" + low = value.lower() + for marker in PLACEHOLDER_MARKERS: + if marker in low: + return True + if _PLACEHOLDER_RE.search(value): + return True + return False + + +def mask(value: str) -> str: + """Mask a secret value: keep first 4 and last 4 chars, elide the middle. + + Anything shorter than 12 chars is fully masked so short values don't leak. + """ + if len(value) < 12: + return "…" * len(value) if value else "…" + return f"{value[:4]}…{value[-4:]}" + + +def _skip_by_glob(name: str) -> bool: + return any(Path(name).match(pat) for pat in SKIP_FILE_GLOBS) + + +def iter_worktree_files(repo: Path) -> Iterable[Path]: + """Yield candidate files under repo, honoring the skip lists.""" + for path in repo.rglob("*"): + if not path.is_file(): + continue + rel_parts = set(path.relative_to(repo).parts) + if rel_parts & SKIP_DIRS: + continue + if _skip_by_glob(path.name): + continue + yield path + + +def looks_binary(path: Path) -> bool: + """Sniff the first 8 KB for a NUL byte.""" + try: + with path.open("rb") as fh: + chunk = fh.read(BINARY_SNIFF_BYTES) + except OSError: + return True + return b"\x00" in chunk + + +def scan_line(line: str) -> Iterable[Tuple[str, str]]: + """Yield (pattern_id, captured_value) for every match in a single line.""" + for pattern_id, regex in PATTERNS: + for m in regex.finditer(line): + value = m.group(1) if m.groups() else m.group(0) + if not value: + continue + if pattern_id == "phone_number" and not _PHONE_KEYWORD_RE.search(line): + continue + if is_placeholder(value): + continue + yield pattern_id, value + + +class Sweeper: + """Accumulates deduped, masked candidates from worktree and history.""" + + def __init__(self) -> None: + self._seen: set = set() + self.candidates: List[Dict[str, object]] = [] + self._counter = 0 + + def add( + self, + pattern_id: str, + value: str, + source: str, + path: str, + line: Optional[int] = None, + commit: Optional[str] = None, + ) -> None: + masked = mask(value) + key = (pattern_id, path, masked) + if key in self._seen: + return + self._seen.add(key) + self._counter += 1 + entry: Dict[str, object] = { + "id": f"C{self._counter}", + "pattern": pattern_id, + "source": source, + "path": path, + "excerpt": masked, + } + if source == "worktree": + entry["line"] = line + else: + entry["commit"] = commit + self.candidates.append(entry) + + +def sweep_worktree(repo: Path, sweeper: Sweeper) -> None: + for path in iter_worktree_files(repo): + try: + if path.stat().st_size > MAX_FILE_BYTES: + continue + except OSError: + continue + if looks_binary(path): + continue + rel = path.relative_to(repo).as_posix() + try: + with path.open("r", encoding="utf-8", errors="ignore") as fh: + for lineno, line in enumerate(fh, start=1): + for pattern_id, value in scan_line(line): + sweeper.add(pattern_id, value, "worktree", rel, line=lineno) + except OSError: + continue + + +def sweep_history(repo: Path, sweeper: Sweeper) -> bool: + """Stream `git log --all -p` and scan added lines. Returns True if scanned.""" + if not (repo / ".git").exists(): + return False + if shutil.which("git") is None: + return False + + cmd = ["git", "-C", str(repo), "log", "--all", "-p", "--unified=0", "--no-color"] + try: + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + encoding="utf-8", + errors="ignore", + shell=False, + ) + except OSError: + return False + + commit = "" + cur_path = "" + assert proc.stdout is not None + try: + for raw in proc.stdout: + line = raw.rstrip("\n") + if line.startswith("commit "): + commit = line.split(" ", 1)[1].strip()[:12] + continue + if line.startswith("+++ "): + # +++ b/path/to/file (or /dev/null on deletion) + target = line[4:].strip() + if target == "/dev/null": + cur_path = "" + elif target.startswith("b/"): + cur_path = target[2:] + else: + cur_path = target + continue + if line.startswith("+") and not line.startswith("+++"): + added = line[1:] + for pattern_id, value in scan_line(added): + sweeper.add( + pattern_id, value, "history", cur_path, commit=commit + ) + finally: + if proc.stdout is not None: + proc.stdout.close() + proc.wait() + return True + + +def summarize(sweeper: Sweeper, history_scanned: bool) -> str: + by_source: Dict[str, int] = {} + by_pattern: Dict[str, int] = {} + for c in sweeper.candidates: + by_source[str(c["source"])] = by_source.get(str(c["source"]), 0) + 1 + by_pattern[str(c["pattern"])] = by_pattern.get(str(c["pattern"]), 0) + 1 + total = len(sweeper.candidates) + src = ", ".join(f"{k}={v}" for k, v in sorted(by_source.items())) or "none" + pat = ", ".join(f"{k}={v}" for k, v in sorted(by_pattern.items())) or "none" + hist = "on" if history_scanned else "off" + return ( + f"sweep complete: {total} candidate(s) [history={hist}] | " + f"by source: {src} | by pattern: {pat}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="scan-secrets.py", + description="Deterministic secret / PII candidate sweep (sweep, not a gate).", + ) + parser.add_argument("repo", help="path to the repository to sweep") + parser.add_argument( + "--out", + default="candidates.json", + help="output JSON path (default: candidates.json in CWD)", + ) + parser.add_argument( + "--no-history", + action="store_true", + help="skip the git-history scan (worktree only)", + ) + args = parser.parse_args() + + repo = Path(args.repo) + if not repo.is_dir(): + print(f"error: repo dir not found: {repo}", file=sys.stderr) + return 2 + + sweeper = Sweeper() + sweep_worktree(repo, sweeper) + + history_scanned = False + if not args.no_history: + history_scanned = sweep_history(repo, sweeper) + + out_path = Path(args.out) + payload = { + "version": 1, + "repo": str(repo), + "history_scanned": history_scanned, + "candidates": sweeper.candidates, + } + try: + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + fh.write("\n") + except OSError as exc: + print(f"error: could not write {out_path}: {exc}", file=sys.stderr) + return 2 + + print(summarize(sweeper, history_scanned)) + print(f"wrote {out_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_looper.py b/tests/test_looper.py index 87530da..4fc4a53 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -602,6 +602,74 @@ def test_session_prompt_command_renders_from_resolved_json(self) -> None: self.assertIn("covers-goal", prompt) self.assertIn("Execution Boundary", prompt) + def test_pattern_library_templates_compile_with_placeholder_warning(self) -> None: + loops_dir = ROOT / "templates" / "loops" + template_dirs = sorted( + path for path in loops_dir.iterdir() if path.is_dir() + ) + self.assertTrue(template_dirs, "templates/loops/ has no template directories") + catalog = (loops_dir / "README.md").read_text(encoding="utf-8") + for template in template_dirs: + with self.subTest(template.name): + self.assertTrue( + (template / "loop.yaml").is_file(), + f"{template.name} is missing loop.yaml", + ) + self.assertTrue( + (template / "README.md").is_file(), + f"{template.name} is missing README.md", + ) + self.assertIn( + f"[{template.name}]({template.name}/)", + catalog, + f"{template.name} is not listed in templates/loops/README.md", + ) + with tempfile.TemporaryDirectory() as tmp: + result = run_cmd( + [ + sys.executable, + str(LOOPER), + "compile", + str(template / "loop.yaml"), + "--out", + str(Path(tmp) / "loop.resolved.json"), + ], + ROOT, + ) + self.assertEqual(result.returncode, 0, f"{template.name}: {result.stderr}") + self.assertIn( + "unresolved template placeholders remain", + result.stderr, + f"{template.name} should carry {{{{PLACEHOLDER}}}} tokens", + ) + + def test_compile_placeholder_warning_absent_after_substitution(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + (work / "inputs").mkdir() + (work / "inputs" / "process-notes.md").write_text("Notes.\n", encoding="utf-8") + write_loop_yaml(work / "loop.yaml") + + loop_yaml = (work / "loop.yaml").read_text(encoding="utf-8") + mutated = loop_yaml.replace( + "statement: Produce a checked LOOP.md.", + "statement: Produce a checked LOOP.md for {{PROJECT_NAME}}.", + 1, + ) + self.assertNotEqual(mutated, loop_yaml) + (work / "loop.yaml").write_text(mutated, encoding="utf-8") + with_token = run_cmd([sys.executable, str(LOOPER), "compile", "loop.yaml"], work) + self.assertEqual(with_token.returncode, 0, with_token.stderr) + self.assertIn("unresolved template placeholders remain", with_token.stderr) + self.assertIn("{{PROJECT_NAME}}", with_token.stderr) + + (work / "loop.yaml").write_text( + mutated.replace("{{PROJECT_NAME}}", "demo-project"), encoding="utf-8" + ) + filled = run_cmd([sys.executable, str(LOOPER), "compile", "loop.yaml"], work) + self.assertEqual(filled.returncode, 0, filled.stderr) + self.assertNotIn("unresolved template placeholders", filled.stderr) + if __name__ == "__main__": unittest.main() From 9cd039a983c172781eeb6ebc11c554383880c0ac Mon Sep 17 00:00:00 2001 From: Kevin Simback <16635828+ksimback@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:30:09 -0700 Subject: [PATCH 2/2] Harden pattern-library scanner and checkers (code-review fixes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit High-effort review of PR #15 surfaced real defects in the new template scripts. Fixes: scan-secrets.py - Match underscore-joined credential names (SECRET_KEY, DB_PASSWORD, client_secret) — the trailing \b rejected exactly those. - Drop over-broad "test"/"sample"/"dummy"/"fake" placeholder markers that suppressed real secrets (e.g. Contest2024!). - Dedupe on a hash of the raw value, not the masked form, so distinct secrets that mask alike are no longer silently collapsed; mask no longer leaks exact length. - Prune skipped dirs in the walk (os.walk) instead of rglob-then-filter; drop dead "*.ipynb checkpoints" glob (skip .ipynb_checkpoints dir). Checker scripts (all five) - Fix PLACEHOLDERS regex so a standalone "???" is actually caught (\b can't bound a non-word char). - Waive required-field validation only for a genuine empty report — a standalone no-findings line AND no finding signal (severity/file:line/ status) — closing the incidental-substring bypass in check-findings, check-review, check-drift-report. - check-citations: require cited paths resolve to a file UNDER the sources dir (reject drafts/same-basename files elsewhere). Templates / compiler / docs - bug-hunt: REPRO_CMD/TEST_CMD are string-form checks (compiler shlex-splits) so a substituted multi-word command yields a valid argv, not an unexecutable one-element list. - find_placeholders scans dict keys too. - SKILL.md: existing loop.yaml (edit/resume) takes precedence over --template; commands/looper.md: target dir may contain spaces. Tests 18 -> 27: scanner detection/dedupe/masking and checker gate-bypass regressions. Co-Authored-By: Claude Fable 5 --- CHANGELOG.md | 11 +- SKILL.md | 4 + commands/looper.md | 6 +- scripts/looper.py | 3 +- templates/loops/README.md | 3 + templates/loops/bug-hunt/README.md | 4 +- templates/loops/bug-hunt/loop.yaml | 7 +- .../bug-hunt/scripts/check-fix-report.py | 4 +- .../loops/code-review/scripts/check-review.py | 16 ++- .../docs-sync/scripts/check-drift-report.py | 12 +- .../scripts/check-citations.py | 35 ++++- .../security-scan/scripts/check-findings.py | 20 ++- .../security-scan/scripts/scan-secrets.py | 79 +++++++----- tests/test_looper.py | 122 ++++++++++++++++++ 14 files changed, 265 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e586b08..ea7fa7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,8 +22,15 @@ separately via `version:` in `loop.yaml` (currently `1`). in the resolved spec; the wizard treats the warning as an emit blocker. - `scan-secrets.py` (security-scan template): deterministic secret/PII candidate sweep over working tree + full git history — streaming reads, - generated/vendor paths skipped, masked excerpts only, placeholder-value - suppression. + directory-pruned walk, masked excerpts only, placeholder-value + suppression. Detects underscore-joined credential names (`SECRET_KEY`, + `DB_PASSWORD`, `client_secret`), does not suppress real secrets whose + value merely contains `test`, and dedupes on a hash of the raw value so + distinct secrets that mask alike are never dropped. +- Template checker scripts reject `???` placeholders, only waive + required-field validation for a genuinely empty report (standalone + no-findings line with no finding signal), and validate citations resolve + to a file under the sources directory. - 2 new tests (18 total): every template must compile (with the expected placeholder warning) and be listed in the catalog; the warning must disappear after substitution. diff --git a/SKILL.md b/SKILL.md index f577592..f2ae295 100644 --- a/SKILL.md +++ b/SKILL.md @@ -82,6 +82,10 @@ directory per template containing a complete, compilable `loop.yaml` (with A template is a pre-answered interview, not a bypass of design review: +0. If the target directory already contains a `loop.yaml`, the edit/resume + rule in step 1 wins: do **not** overwrite it with a template. Say the + directory already has a loop and ask the user to pick an empty target or + confirm they want it replaced before continuing. 1. If `--template` has no name, an unknown name, or the user asks what is available, show the catalog table (template + use-when) and let them pick. 2. Read the template's `loop.yaml` and `README.md`. Use the template as the diff --git a/commands/looper.md b/commands/looper.md index 3b767d4..ef5ac0a 100644 --- a/commands/looper.md +++ b/commands/looper.md @@ -31,7 +31,9 @@ Parse `$ARGUMENTS` as `[target-dir] [--template ]`, in any order: follow the skill's Template Mode (the catalog lives at `templates/loops/` inside the skill directory). `--template` with no name means: show the catalog and ask. -- The remaining token, if any, is the target directory. If none is left, - use `./looper-output`. +- Remove `--template ` from the arguments first; everything left is the + target directory — treat it as a single path even if it contains spaces + (e.g. `C:\My Loops\out`), and do not split it into multiple tokens. If + nothing is left, use `./looper-output`. Then continue with the Looper skill workflow. diff --git a/scripts/looper.py b/scripts/looper.py index a149bab..fea2edb 100644 --- a/scripts/looper.py +++ b/scripts/looper.py @@ -49,7 +49,8 @@ def find_placeholders(value: Any) -> set[str]: for item in value: found.update(find_placeholders(item)) elif isinstance(value, dict): - for item in value.values(): + for key, item in value.items(): + found.update(find_placeholders(key)) found.update(find_placeholders(item)) return found diff --git a/templates/loops/README.md b/templates/loops/README.md index 6a5f7f0..a7530be 100644 --- a/templates/loops/README.md +++ b/templates/loops/README.md @@ -32,6 +32,9 @@ the flow preview before emitting. - Templates that edit files (`bug-hunt`, `docs-sync`) do so under side-effect approval and never commit; read-only templates say so in their `execution.side_effects.notes`. +- Programmatic checks invoke the interpreter as `python` (matching the + repo's example loop). On a machine where only `python3` is on PATH, the + wizard should swap the interpreter token when it customizes the template. ## Adding a template diff --git a/templates/loops/bug-hunt/README.md b/templates/loops/bug-hunt/README.md index 4bc37ed..5cb85a8 100644 --- a/templates/loops/bug-hunt/README.md +++ b/templates/loops/bug-hunt/README.md @@ -23,8 +23,8 @@ error went away". | Token | Replace with | |-------|--------------| | `{{REPO_DIR}}` | Path to the repository containing the bug. | -| `{{REPRO_CMD}}` | Argv array that triggers the bug, e.g. `["python", "-m", "pytest", "tests/test_x.py::test_bug", "-x"]`. Replace the whole one-element list. | -| `{{TEST_CMD}}` | Argv array for the full test suite, e.g. `["python", "-m", "pytest"]`. Replace the whole one-element list. | +| `{{REPRO_CMD}}` | Command that triggers the bug, written as a plain string, e.g. `python -m pytest tests/test_x.py::test_bug -x`. The compiler splits it into an argv for you. | +| `{{TEST_CMD}}` | Command for the full test suite, e.g. `python -m pytest`. Plain string, split for you. | Also: put the bug report (the issue text, stack trace, expected vs actual) in `inputs/bug-report.md` inside the emitted loop directory. diff --git a/templates/loops/bug-hunt/loop.yaml b/templates/loops/bug-hunt/loop.yaml index d72c050..2cc9cde 100644 --- a/templates/loops/bug-hunt/loop.yaml +++ b/templates/loops/bug-hunt/loop.yaml @@ -28,13 +28,14 @@ goal: - id: repro-passes type: programmatic # The recorded repro. Must have been observed FAILING at plan time - # (plan gate checks that); passing here proves the fix. - check: ["{{REPRO_CMD}}"] + # (plan gate checks that); passing here proves the fix. String form: the + # compiler shlex-splits it, so a multi-word command works as written. + check: "{{REPRO_CMD}}" expect: exit_zero timeout_sec: 300 - id: tests-pass type: programmatic - check: ["{{TEST_CMD}}"] + check: "{{TEST_CMD}}" expect: exit_zero timeout_sec: 600 - id: report-schema diff --git a/templates/loops/bug-hunt/scripts/check-fix-report.py b/templates/loops/bug-hunt/scripts/check-fix-report.py index 5eedeb4..09acb95 100644 --- a/templates/loops/bug-hunt/scripts/check-fix-report.py +++ b/templates/loops/bug-hunt/scripts/check-fix-report.py @@ -22,7 +22,9 @@ REQUIRED_LABELS = ["root cause", "fix", "before", "after"] FILE_LINE = re.compile(r"\S+:\d+") -PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) +# `?` is not a word char, so `\b\?\?\?\b` can never match; keep `???` as its +# own alternative outside the word-boundary group. +PLACEHOLDERS = re.compile(r"\b(?:TBD|TODO|FIXME|XXX)\b|\?\?\?", re.IGNORECASE) def main() -> int: diff --git a/templates/loops/code-review/scripts/check-review.py b/templates/loops/code-review/scripts/check-review.py index d3f80a5..62b52dd 100644 --- a/templates/loops/code-review/scripts/check-review.py +++ b/templates/loops/code-review/scripts/check-review.py @@ -26,7 +26,9 @@ REQUIRED_FIELDS = ["type", "severity", "location", "rationale"] SEVERITY_SCALE = re.compile(r"\b(critical|major|minor)\b", re.IGNORECASE) FILE_LINE = re.compile(r"\S+:\d+") -PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) +# `?` is not a word char, so `\b\?\?\?\b` can never match; keep `???` as its +# own alternative outside the word-boundary group. +PLACEHOLDERS = re.compile(r"\b(?:TBD|TODO|FIXME|XXX)\b|\?\?\?", re.IGNORECASE) def main() -> int: @@ -52,9 +54,15 @@ def main() -> int: problems.append("report never names the reviewed range (branch/base/diff)") # 2. A report with findings must expose all required fields somewhere. - # If the report explicitly states no findings, fields are not required. - declares_no_findings = bool(re.search( - r"no (confirmed )?(findings|issues|defects|problems)", lower)) + # If the report explicitly declares no findings, fields are not required — + # but only when it's a genuine empty report: a standalone declaration line + # AND no finding signal (a severity term or a file:line). Otherwise an + # incidental "no issues in module X" sentence would waive validation. + declares_empty = bool(re.search( + r"(?m)^[\s>*#-]*no (?:confirmed |material )?(?:findings|issues|defects|problems)\b", + lower)) + has_finding_signal = bool(SEVERITY_SCALE.search(text) or FILE_LINE.search(text)) + declares_no_findings = declares_empty and not has_finding_signal if not declares_no_findings: for field in REQUIRED_FIELDS: if field not in lower: diff --git a/templates/loops/docs-sync/scripts/check-drift-report.py b/templates/loops/docs-sync/scripts/check-drift-report.py index d173376..595f144 100644 --- a/templates/loops/docs-sync/scripts/check-drift-report.py +++ b/templates/loops/docs-sync/scripts/check-drift-report.py @@ -25,7 +25,9 @@ REQUIRED_FIELDS = ["doc", "code", "status"] STATUS_SCALE = re.compile(r"\b(fixed|deferred)\b", re.IGNORECASE) -PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) +# `?` is not a word char, so `\b\?\?\?\b` can never match; keep `???` as its +# own alternative outside the word-boundary group. +PLACEHOLDERS = re.compile(r"\b(?:TBD|TODO|FIXME|XXX)\b|\?\?\?", re.IGNORECASE) def main() -> int: @@ -46,8 +48,12 @@ def main() -> int: lower = text.lower() problems = [] - declares_no_drift = bool(re.search( - r"no (drift|mismatches|discrepancies)( items| found| detected)?", lower)) + # Waive the per-item fields only for a genuine no-drift report: a standalone + # declaration line AND no drift signal (a fixed/deferred status). A "no drift + # in section X" line inside a populated report must not waive validation. + declares_empty = bool(re.search( + r"(?m)^[\s>*#-]*no (?:drift|mismatches|discrepancies)\b", lower)) + declares_no_drift = declares_empty and not STATUS_SCALE.search(text) if not declares_no_drift: for field in REQUIRED_FIELDS: if field not in lower: diff --git a/templates/loops/research-synthesis/scripts/check-citations.py b/templates/loops/research-synthesis/scripts/check-citations.py index 03e5c09..3af8c44 100644 --- a/templates/loops/research-synthesis/scripts/check-citations.py +++ b/templates/loops/research-synthesis/scripts/check-citations.py @@ -24,10 +24,33 @@ from pathlib import Path CITATION = re.compile(r"\[source:\s*([^\]]+?)\s*\]") -PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) +# `?` is not a word char, so `\b\?\?\?\b` can never match; keep `???` as its +# own alternative outside the word-boundary group. +PLACEHOLDERS = re.compile(r"\b(?:TBD|TODO|FIXME|XXX)\b|\?\?\?", re.IGNORECASE) SUBSTANTIVE_WORDS = 40 +def resolves_under_sources(raw: str, sources: Path) -> bool: + """True only when the cited path is a real file located under `sources`. + + Guards against citing a file outside the sources tree (e.g. the model's own + draft, or an unrelated same-basename file elsewhere): a citation of + `inputs/sources/foo.md` (loop-dir-relative) or a bare `foo.md` must both + land inside the sources directory to count. + """ + sources_root = sources.resolve() + for candidate in (Path(raw), sources / Path(raw).name): + try: + resolved = candidate.resolve() + except OSError: + continue + if resolved.is_file() and ( + resolved.parent == sources_root or sources_root in resolved.parents + ): + return True + return False + + def paragraphs(text: str) -> list[str]: """Prose paragraphs, with fenced code blocks and headings removed.""" text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) @@ -56,12 +79,12 @@ def main() -> int: if not cited: problems.append("no [source: ] citations anywhere in the report") - # 1. Every cited path must exist. Citations are written relative to the - # loop dir (inputs/sources/); accept sources-dir-relative too. + # 1. Every cited path must resolve to a real file UNDER the sources dir. + # A path that exists elsewhere (a draft, an unrelated same-name file) is + # not a valid citation. for raw in sorted(set(cited)): - candidate = Path(raw) - if not (candidate.is_file() or (args.sources / candidate.name).is_file()): - problems.append(f"cited source does not exist: '{raw}'") + if not resolves_under_sources(raw, args.sources): + problems.append(f"cited source is not a file under {args.sources}: '{raw}'") # 2. Substantive paragraphs must be cited. for para in paragraphs(text): diff --git a/templates/loops/security-scan/scripts/check-findings.py b/templates/loops/security-scan/scripts/check-findings.py index 663cf54..02ebe53 100644 --- a/templates/loops/security-scan/scripts/check-findings.py +++ b/templates/loops/security-scan/scripts/check-findings.py @@ -22,7 +22,11 @@ REQUIRED_AREAS = ["secret", "pii", "vulnerab"] # substrings, case-insensitive REQUIRED_FIELDS = ["type", "severity", "location", "remediation"] -PLACEHOLDERS = re.compile(r"\b(TBD|TODO|FIXME|XXX|\?\?\?)\b", re.IGNORECASE) +SEVERITY_SCALE = re.compile(r"\b(critical|high|medium|low|major|minor)\b", re.IGNORECASE) +FILE_LINE = re.compile(r"\S+:\d+") +# `?` is not a word char, so `\b\?\?\?\b` can never match; keep `???` as its +# own alternative outside the word-boundary group. +PLACEHOLDERS = re.compile(r"\b(?:TBD|TODO|FIXME|XXX)\b|\?\?\?", re.IGNORECASE) def main() -> int: @@ -48,10 +52,16 @@ def main() -> int: if area not in lower: problems.append(f"missing required area keyword: '{area}'") - # 2. A report with confirmed findings must expose all required fields somewhere. - # If the report explicitly states no findings, fields are not required. - declares_no_findings = bool(re.search( - r"no (confirmed )?(findings|secrets|vulnerabilities|issues)", lower)) + # 2. A report with confirmed findings must expose all required fields + # somewhere. Fields are waived only for a genuine empty report: a + # standalone no-findings declaration line AND no finding signal (a + # severity term or a file:line). A prose "no secrets in git history" + # inside a report full of findings must not waive validation. + declares_empty = bool(re.search( + r"(?m)^[\s>*#-]*no (?:confirmed |material )?(?:findings|secrets|vulnerabilities|issues)\b", + lower)) + has_finding_signal = bool(SEVERITY_SCALE.search(text) or FILE_LINE.search(text)) + declares_no_findings = declares_empty and not has_finding_signal if not declares_no_findings: for field in REQUIRED_FIELDS: if field not in lower: diff --git a/templates/loops/security-scan/scripts/scan-secrets.py b/templates/loops/security-scan/scripts/scan-secrets.py index fd2aa59..91b80af 100644 --- a/templates/loops/security-scan/scripts/scan-secrets.py +++ b/templates/loops/security-scan/scripts/scan-secrets.py @@ -23,7 +23,9 @@ from __future__ import annotations import argparse +import hashlib import json +import os import re import shutil import subprocess @@ -46,6 +48,7 @@ "venv", ".tox", ".mypy_cache", + ".ipynb_checkpoints", "coverage", "target", } @@ -60,29 +63,26 @@ "yarn.lock", "pnpm-lock.yaml", "*.svg", - "*.ipynb checkpoints", ] MAX_FILE_BYTES = 10 * 1024 * 1024 # 10 MB BINARY_SNIFF_BYTES = 8 * 1024 # 8 KB # Case-insensitive substrings that mark a captured value as an obvious placeholder. +# Kept deliberately specific: a marker that also appears inside real credentials +# (e.g. "test", which suppresses "Contest2024!" or "prodtest-…") causes false +# negatives, which for a secret sweep is the worst failure mode. PLACEHOLDER_MARKERS = [ "example", - "sample", "placeholder", "changeme", "your_key", "your-key", "yourkey", "xxxx", - "dummy", - "test", - "fake", "<", "{{", ] -_PLACEHOLDER_RE = re.compile(r"your[_-]?key", re.IGNORECASE) # --- Detection patterns ------------------------------------------------------ # Each entry is (pattern_id, compiled regex). Group 1, when present, is the @@ -97,9 +97,13 @@ ), ( "generic_secret_assignment", + # The keyword may be embedded in a larger identifier: SECRET_KEY, + # DB_PASSWORD, client_secret, POSTGRES_PASSWORD. Underscore is a word + # char, so a trailing \b after the keyword would reject exactly those + # common names; instead allow identifier chars on either side. re.compile( - r"(?i)\b(?:api[_-]?key|apikey|token|secret|passwd|password)\b\s*[=:]\s*" - r"['\"]?([^\s'\"]{8,})['\"]?" + r"(?i)[A-Za-z0-9_]*(?:api[_-]?key|apikey|token|secret|passwd|password)" + r"[A-Za-z0-9_]*\s*[=:]\s*['\"]?([^\s'\"]{8,})['\"]?" ), ), ("bearer_token", re.compile(r"(?i)bearer\s+([A-Za-z0-9._\-+/=]{8,})")), @@ -136,22 +140,27 @@ def is_placeholder(value: str) -> bool: """True when the captured value looks like an obvious placeholder/example.""" low = value.lower() - for marker in PLACEHOLDER_MARKERS: - if marker in low: - return True - if _PLACEHOLDER_RE.search(value): - return True - return False + return any(marker in low for marker in PLACEHOLDER_MARKERS) def mask(value: str) -> str: - """Mask a secret value: keep first 4 and last 4 chars, elide the middle. + """Mask a secret value for display: keep a few edge chars, elide the middle. - Anything shorter than 12 chars is fully masked so short values don't leak. + The exact length is never revealed (a fixed elision, not one dot per char), + and the mask is display-only — dedupe keys off a hash of the raw value, not + this string, so two distinct secrets never collapse just because they mask + the same way. """ - if len(value) < 12: - return "…" * len(value) if value else "…" - return f"{value[:4]}…{value[-4:]}" + if len(value) >= 12: + return f"{value[:4]}…{value[-4:]}" + if len(value) >= 6: + return f"{value[:2]}…{value[-2:]}" + return "…" + + +def value_fingerprint(value: str) -> str: + """Stable, non-reversible fingerprint of a raw value for deduping.""" + return hashlib.sha256(value.encode("utf-8", "ignore")).hexdigest()[:16] def _skip_by_glob(name: str) -> bool: @@ -159,16 +168,20 @@ def _skip_by_glob(name: str) -> bool: def iter_worktree_files(repo: Path) -> Iterable[Path]: - """Yield candidate files under repo, honoring the skip lists.""" - for path in repo.rglob("*"): - if not path.is_file(): - continue - rel_parts = set(path.relative_to(repo).parts) - if rel_parts & SKIP_DIRS: - continue - if _skip_by_glob(path.name): - continue - yield path + """Yield candidate files under repo, honoring the skip lists. + + Prunes skipped directories in place so the walk never descends into + .git / node_modules / .venv / etc. (rglob would stat every entry first). + """ + for dirpath, dirnames, filenames in os.walk(repo): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + base = Path(dirpath) + for name in filenames: + if _skip_by_glob(name): + continue + path = base / name + if path.is_file(): + yield path def looks_binary(path: Path) -> bool: @@ -212,8 +225,10 @@ def add( line: Optional[int] = None, commit: Optional[str] = None, ) -> None: - masked = mask(value) - key = (pattern_id, path, masked) + # Dedupe on a hash of the RAW value, not the masked form — masking is + # lossy (short values elide to the same string), so masked-keyed dedupe + # silently drops distinct secrets that happen to mask alike. + key = (pattern_id, path, value_fingerprint(value)) if key in self._seen: return self._seen.add(key) @@ -223,7 +238,7 @@ def add( "pattern": pattern_id, "source": source, "path": path, - "excerpt": masked, + "excerpt": mask(value), } if source == "worktree": entry["line"] = line diff --git a/tests/test_looper.py b/tests/test_looper.py index 4fc4a53..c524d70 100644 --- a/tests/test_looper.py +++ b/tests/test_looper.py @@ -671,5 +671,127 @@ def test_compile_placeholder_warning_absent_after_substitution(self) -> None: self.assertNotIn("unresolved template placeholders", filled.stderr) +LOOPS = ROOT / "templates" / "loops" + + +class SecurityScanTemplateTests(unittest.TestCase): + SCANNER = LOOPS / "security-scan" / "scripts" / "scan-secrets.py" + + def _sweep(self, files: dict[str, str]) -> list[dict]: + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) / "repo" + repo.mkdir() + for name, body in files.items(): + (repo / name).write_text(body, encoding="utf-8") + out = Path(tmp) / "cand.json" + result = run_cmd( + [sys.executable, str(self.SCANNER), str(repo), "--out", str(out), "--no-history"], + ROOT, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(out.read_text(encoding="utf-8"))["candidates"] + + def test_detects_underscore_joined_credential_names(self) -> None: + cands = self._sweep({"config.py": ( + 'SECRET_KEY = "Zq9rT2vXw8LmNp4c"\n' + "DB_PASSWORD=Corr3ctHorseBat\n" + "client_secret: NineCharsX\n" + )}) + self.assertEqual(len(cands), 3, cands) + + def test_value_containing_test_is_not_suppressed(self) -> None: + cands = self._sweep({"config.py": "password = Contest2024xyz\n"}) + self.assertEqual(len(cands), 1, cands) + + def test_distinct_short_secrets_not_collapsed_by_mask(self) -> None: + cands = self._sweep({"config.py": "password = hunter22aa\ntoken: swordfish9xy\n"}) + self.assertEqual(len(cands), 2, cands) + + def test_obvious_placeholder_values_are_suppressed(self) -> None: + cands = self._sweep({"config.py": 'api_key = "your_key_placeholder"\n'}) + self.assertEqual(cands, []) + + def test_masked_excerpt_never_contains_full_value(self) -> None: + cands = self._sweep({"config.py": "password = hunter22aa\n"}) + self.assertEqual(len(cands), 1) + self.assertNotIn("hunter22aa", cands[0]["excerpt"]) + + +class TemplateCheckerScriptTests(unittest.TestCase): + def _run(self, script: Path, *args: str) -> subprocess.CompletedProcess[str]: + return run_cmd([sys.executable, str(script), *args], ROOT) + + def test_findings_rejects_triple_question_placeholder(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report = Path(tmp) / "SECURITY-FINDINGS.md" + report.write_text( + "# Findings\n\nsecret pii vulnerability\nType: bug\nSeverity: high\n" + "Location: a.py:1\nRemediation: ???\n", + encoding="utf-8", + ) + result = self._run( + LOOPS / "security-scan" / "scripts" / "check-findings.py", str(report) + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("placeholder", result.stderr) + + def test_findings_incidental_no_secrets_phrase_does_not_waive_fields(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report = Path(tmp) / "SECURITY-FINDINGS.md" + report.write_text( + "# Report\n\nsecret pii vulnerability scan.\n" + "No secrets were detected in git history.\n" + "Critical issue at server.py:42 but no field labels.\n", + encoding="utf-8", + ) + result = self._run( + LOOPS / "security-scan" / "scripts" / "check-findings.py", str(report) + ) + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("missing required field label", result.stderr) + + def test_findings_genuine_empty_report_passes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + report = Path(tmp) / "SECURITY-FINDINGS.md" + report.write_text( + "# Report\n\nChecked secret, pii, vulnerability areas.\nNo findings.\n", + encoding="utf-8", + ) + result = self._run( + LOOPS / "security-scan" / "scripts" / "check-findings.py", str(report) + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_citations_reject_path_outside_sources_dir(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + loop = Path(tmp) + (loop / "inputs" / "sources").mkdir(parents=True) + (loop / "inputs" / "sources" / "a.md").write_text("notes\n", encoding="utf-8") + (loop / "loop-workspace").mkdir() + (loop / "loop-workspace" / "report-draft-1.md").write_text("draft\n", encoding="utf-8") + report = loop / "loop-workspace" / "REPORT.md" + checker = LOOPS / "research-synthesis" / "scripts" / "check-citations.py" + + report.write_text( + "# A\n\nClaim supported here indeed. [source: loop-workspace/report-draft-1.md]\n", + encoding="utf-8", + ) + outside = run_cmd( + [sys.executable, str(checker), "loop-workspace/REPORT.md", "--sources", "inputs/sources"], + loop, + ) + self.assertEqual(outside.returncode, 1, outside.stdout) + + report.write_text( + "# A\n\nClaim supported here indeed. [source: inputs/sources/a.md]\n", + encoding="utf-8", + ) + inside = run_cmd( + [sys.executable, str(checker), "loop-workspace/REPORT.md", "--sources", "inputs/sources"], + loop, + ) + self.assertEqual(inside.returncode, 0, inside.stderr) + + if __name__ == "__main__": unittest.main()