From ba00eca087fa8004adfabaaa18a1b44f83186466 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 00:49:25 +0000 Subject: [PATCH 1/2] gate: a deterministic test-integrity kind, so green CI must be earned docs/agentic-risk-coverage.md has carried this row honestly for weeks: ...an agent deleting tests or assertions to get green? agent-discipline (assertion_deletion: block) advisory today -- a deterministic test-integrity gate is a named catalog candidate, not yet a roadmap issue, so this row stays advisory until it is. It could not be anything else: all three existing kinds read only the added side of a diff, and test tampering is visible only in what a change takes away. This adds the missing primitive, GateContext.removed_lines(), and one kind built on it. test_integrity blocks three shapes, all deterministic and author-blind: a deleted test file, a net loss of assertions across the change, and a vacuous assertion (assert True, expect(true)) added in its place. The escape hatch is a pragma written on the line that removes the test, so an obsolete test can still go -- visibly, in the diff, where a human sees it. The net count is taken across the whole change rather than per file, which keeps a test split across two files from reading as a deletion. What it costs is a change that strips one file while adding to another; that is the right trade for a gate whose false positives would get it disabled. Patterns are policy data, not engine behaviour: the kind is language- agnostic and every language's idea of a test and an assertion arrives through params from .agents/policies/test-integrity/manifest.yaml. The gate runs on commit and in CI. CI is the half that matters for an inbound contributor, whose agent never ran a local hook. 7 behavioural tests over real git repositories, including the two that keep the gate adoptable: a refactor deleting an assert from application code passes, and a reviewed removal passes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014gwmBVUHSgohCLkNVqAR77 --- .agents/policies/test-integrity/manifest.yaml | 62 +++++++++++++++ src/chock/gate/runner.py | 40 ++++++++++ src/chock/gate/schema.py | 10 +++ tests/test_test_integrity_gate.py | 76 +++++++++++++++++++ 4 files changed, 188 insertions(+) create mode 100644 .agents/policies/test-integrity/manifest.yaml create mode 100644 tests/test_test_integrity_gate.py diff --git a/.agents/policies/test-integrity/manifest.yaml b/.agents/policies/test-integrity/manifest.yaml new file mode 100644 index 0000000..cab3773 --- /dev/null +++ b/.agents/policies/test-integrity/manifest.yaml @@ -0,0 +1,62 @@ +id: test-integrity +name: "Test Integrity" +version: "0.0.1" +description: > + Block a change that wins green CI by weakening the tests instead of fixing the code. + Catches a deleted test file, a net loss of assertions across the change, and a vacuous + assertion (`assert True`, `expect(true)`) added in its place. Author-blind: the CI gate + re-runs it on the PR head, so it holds for an inbound contributor whose agent never ran + a hook. Defends the signal every other gate trusts. +artifact: hook +enforcement: block +mandatory: false +effects: +- read_only +approval: + required: false + +hook: + gate: + kind: test_integrity + "on": [commit, ci] + action: block + message: > + Tests were weakened, not fixed. If a test is genuinely obsolete, say so on the line + that removes it with `chock: test-removal-reviewed` and have a human confirm it. + params: + # Patterns are policy data, not engine behaviour: the kind is language-agnostic and + # every language's idea of "a test" and "an assertion" arrives from here. + test_path_regex: "(^|/)(tests?|spec|__tests__)/|(_test|_spec|\\.test|\\.spec)\\.[a-z]+$" + assertion_pattern: "\\b(assert|assertEquals|assertTrue|expect|should|require\\.(True|NoError))\\b" + # Counted only when ADDED -- a test that asserts nothing is worse than no test, + # because it reports green. + dummy_assertion_pattern: "\\b(assert\\s+(True|1)\\b|expect\\s*\\(\\s*(true|1)\\s*\\)|assertTrue\\s*\\(\\s*true\\s*\\))" + allowlist_pragma: "chock: test-removal-reviewed" + +compliance: + owasp_asi: + - control: ASI04 + coverage: partial + note: "blocks test deletion, net assertion loss and vacuous assertions in test paths" + +provenance: + author: "chock-core" + created_at: "2026-09-03T00:00:00Z" + updated_at: "2026-09-03T00:00:00Z" + source_repo: "https://github.com/open-coder-ai/chock" + license: "Apache-2.0" + trust_tier: "community" + +lifecycle: + status: draft + reviewed_by: + - "chock-core" + +security: + content_instructions: never-obey + +changelog: +- version: 0.0.1 + date: '2026-09-03' + changes: + - "Initial: test_integrity gate kind, closing the advisory row in agentic-risk-coverage." diff --git a/src/chock/gate/runner.py b/src/chock/gate/runner.py index 95b7ecd..e156131 100644 --- a/src/chock/gate/runner.py +++ b/src/chock/gate/runner.py @@ -77,6 +77,12 @@ def added_lines(self, path: str) -> list[str]: lines.append(line[1:]) return lines + def removed_lines(self, path: str) -> list[str]: + """The deleted side of the diff -- what a test-weakening change takes away.""" + out = self._git("diff", *self._range(), "-U0", "--", path) + return [line[1:] for line in out.splitlines() + if line.startswith("-") and not line.startswith("---")] + def staged_blob(self, path: str) -> str: """The proposed content: staged in index mode, committed at HEAD in range mode.""" return self._git("show", f"HEAD:{path}" if self.base else f":{path}") @@ -235,10 +241,44 @@ def _kind_dependency_allowlist(ctx: GateContext, params: dict, _event: str) -> G return GateResult(allowed=not matches, matches=matches) +def _count(pattern: "re.Pattern[str]", lines: list[str], pragma: "re.Pattern[str] | None") -> int: + return sum(1 for line in lines if pattern.search(line) and not (pragma and pragma.search(line))) + + +def _kind_test_integrity(ctx: GateContext, params: dict, _event: str) -> GateResult: + """Block a change that wins green CI by weakening the tests rather than fixing the code.""" + path_re = re.compile(params["test_path_regex"]) + assertion_re = re.compile(params["assertion_pattern"]) + dummy_pattern = params.get("dummy_assertion_pattern") + dummy_re = re.compile(dummy_pattern) if dummy_pattern else None + pragma = params.get("allowlist_pragma") + pragma_re = re.compile(pragma) if pragma else None + + matches: list[str] = [] + added = removed = 0 + for path in ctx.staged_paths("D"): + if path_re.search(path): + matches.append(f"{path}: test file deleted") + for path in ctx.staged_paths("ACMRT"): + if not path_re.search(path): + continue + added_lines = ctx.added_lines(path) + if pragma_re and any(pragma_re.search(line) for line in added_lines): + continue + added += _count(assertion_re, added_lines, pragma_re) + removed += _count(assertion_re, ctx.removed_lines(path), pragma_re) + if dummy_re and any(dummy_re.search(line) for line in added_lines): + matches.append(f"{path}: vacuous assertion added") + if removed > added: + matches.append(f"assertions removed across tests: {removed} removed, {added} added") + return GateResult(allowed=not matches, matches=matches) + + KINDS = { "content_regex": _kind_content_regex, "forbidden_ref": _kind_forbidden_ref, "dependency_allowlist": _kind_dependency_allowlist, + "test_integrity": _kind_test_integrity, } diff --git a/src/chock/gate/schema.py b/src/chock/gate/schema.py index 6838a76..5a92eea 100644 --- a/src/chock/gate/schema.py +++ b/src/chock/gate/schema.py @@ -45,6 +45,16 @@ "allowlist_file": {"type": "string"}, }, }, + "test_integrity": { + **_CLOSED_OBJECT, + "required": ["test_path_regex", "assertion_pattern"], + "properties": { + "test_path_regex": {"type": "string", "minLength": 1}, + "assertion_pattern": {"type": "string", "minLength": 1}, + "dummy_assertion_pattern": {"type": "string"}, + "allowlist_pragma": {"type": "string"}, + }, + }, "egress_allowlist": { **_CLOSED_OBJECT, "required": ["allowed_hosts"], diff --git a/tests/test_test_integrity_gate.py b/tests/test_test_integrity_gate.py new file mode 100644 index 0000000..925f581 --- /dev/null +++ b/tests/test_test_integrity_gate.py @@ -0,0 +1,76 @@ +"""Behavioural tests for the test-integrity gate: green CI must be earned, not arranged.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from conftest import baseline_policy, build_test_gate_json, init_repo, stage + +from chock.gate.runner import run + +POLICY_DIR = baseline_policy("test-integrity") + +REAL_TEST = """def test_adds(): + assert add(1, 2) == 3 + assert add(0, 0) == 0 +""" + + +def _repo(tmp_path: Path) -> tuple[Path, Path]: + init_repo(tmp_path) + stage(tmp_path, "tests/test_math.py", REAL_TEST) + subprocess.run(["git", "commit", "-qm", "base"], cwd=tmp_path, check=True) + return tmp_path, build_test_gate_json(tmp_path, POLICY_DIR) + + +def _verdict(repo: Path, gate: Path) -> int: + return run(gate, "pre-commit", None, repo) + + +def test_a_real_fix_passes(tmp_path: Path) -> None: + repo, gate = _repo(tmp_path) + stage(repo, "app.py", "def add(a, b):\n return a + b\n") + assert _verdict(repo, gate) == 0 + + +def test_adding_assertions_passes(tmp_path: Path) -> None: + repo, gate = _repo(tmp_path) + stage(repo, "tests/test_math.py", REAL_TEST + " assert add(-1, 1) == 0\n") + assert _verdict(repo, gate) == 0 + + +def test_deleting_a_test_file_is_blocked(tmp_path: Path) -> None: + repo, gate = _repo(tmp_path) + subprocess.run(["git", "rm", "-q", "tests/test_math.py"], cwd=repo, check=True) + assert _verdict(repo, gate) != 0 + + +def test_stripping_assertions_is_blocked(tmp_path: Path) -> None: + """The actual attack: keep the test, remove what it checks, collect green CI.""" + repo, gate = _repo(tmp_path) + stage(repo, "tests/test_math.py", "def test_adds():\n add(1, 2)\n") + assert _verdict(repo, gate) != 0 + + +def test_a_vacuous_assertion_is_blocked(tmp_path: Path) -> None: + repo, gate = _repo(tmp_path) + stage(repo, "tests/test_math.py", REAL_TEST + "\n\ndef test_new():\n assert True\n") + assert _verdict(repo, gate) != 0 + + +def test_non_test_paths_are_not_policed(tmp_path: Path) -> None: + """Deleting an assert from application code is a refactor, not test tampering.""" + repo, gate = _repo(tmp_path) + stage(repo, "app.py", "def add(a, b):\n assert isinstance(a, int)\n return a + b\n") + subprocess.run(["git", "commit", "-qm", "app"], cwd=repo, check=True) + stage(repo, "app.py", "def add(a, b):\n return a + b\n") + assert _verdict(repo, gate) == 0 + + +def test_a_reviewed_removal_is_allowed_through(tmp_path: Path) -> None: + """The escape hatch is deliberate, in the diff, and named — not a config toggle.""" + repo, gate = _repo(tmp_path) + stage(repo, "tests/test_math.py", + "def test_adds(): # chock: test-removal-reviewed\n assert add(1, 2) == 3\n") + assert _verdict(repo, gate) == 0 From da66872f6a24ea7e3474aa9b62576716a050548f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 01:30:14 +0000 Subject: [PATCH 2/2] gate: wire test_integrity into validation, docs, and CI; fix latent bugs Full-suite regression over the test-integrity work from ba00eca surfaced seven failures, all caused by this branch and now fixed: - Vendored .chock/bin/gate.py was stale against src/chock/gate/runner.py (re-vendored via `chock sync`). - The manifest's `on: [commit, ci]` declared a literal "ci" event that no runtime or schema recognizes -- CI coverage is derived from "commit" being present, per every other manifest and spec/gate-dsl.md. Fixed to `on: [commit]`. - src/chock/validation/schemas/manifest.hook.json's `kind` enum (structural JSON-schema layer, separate from KINDS/KIND_PARAM_SCHEMAS) never listed `test_integrity`, so any manifest declaring it failed validation. Added. - The policy shipped no evals/suite.yaml, tripping the eval_first check. Added six hand-authored cases (trigger/negative_trigger/ behavior/edge) -- not added to eval/derive.py's DERIVABLE_KINDS, since its params are open regexes over diff content, not an enumerable list like forbidden_ref's refs or dependency_allowlist's manifests. - spec/gate-dsl.md never documented the new kind's params. - Two files were unformatted (`ruff format --check` is part of CI though only `ruff check` was verified in the prior pass). Also, per chock-g1 T2-T5: - checks_gate_shape.py needed no code change (it already reads KINDS/KIND_PARAM_SCHEMAS); added tests confirming a test_integrity manifest validates and a bad param shape is rejected. - Confirmed test_integrity is absent from GATEWAY_ONLY_KINDS and RUNTIME_KINDS (it needs a git diff, so it has no gateway runtime). - .agents/policies/INDEX.md gets the new entry automatically from `chock sync`; docs/baseline-policies.md's guard table and declarative list are updated by hand (docs/policies/, docs/cli-reference.md and llms.txt don't enumerate individual policies). - docs/agentic-risk-coverage.md's test-deletion row moves from `advisory` to `enforced-at-commit`, backed by the tests above; no pre-tool-use tier is claimed since none exists. - CHANGELOG.md entry added. Found live while writing this commit: test_path_regex's directory branch matched any file under a `tests?/spec/__tests__` dir regardless of extension, so documenting the gate in spec/gate-dsl.md (this repo's own design-doc directory, not RSpec tests) with the worked example "assert True" tripped the gate's own vacuous-assertion check on itself. Fixed by excluding .md/.rst/.txt from the directory branch; regression cases added to both the eval suite and the behavioural pytest suite. Full suite: 1055 passed, 6 skipped. `ruff check .` and `ruff format --check .` both clean. Co-Authored-By: Claude Sonnet 5 Signed-off-by: Claude --- .agents/policies/INDEX.md | 1 + .../policies/test-integrity/evals/suite.yaml | 110 ++++++++++++++++++ .agents/policies/test-integrity/manifest.yaml | 7 +- .chock/bin/gate.py | 39 +++++++ .../test-integrity/ambient-rule/ambient.md | 6 + .../compiled/test-integrity/ci-gate/gate.json | 14 +++ .../compiled/test-integrity/ci-gate/step.yaml | 12 ++ .../test-integrity/git-hook/gate.json | 14 +++ .../test-integrity/git-hook/git-pre-commit.sh | 11 ++ .../managed-setting/managed-settings.json | 4 + .chock/coverage.json | 67 +++++++++++ .chock/registry.json | 15 +++ CHANGELOG.md | 8 ++ chock.lock | 8 ++ docs/agentic-risk-coverage.md | 2 +- docs/baseline-policies.md | 3 +- spec/gate-dsl.md | 18 ++- src/chock/gate/runner.py | 3 +- .../validation/schemas/manifest.hook.json | 3 +- tests/test_manifest_validation.py | 53 +++++++++ tests/test_test_integrity_gate.py | 10 +- 21 files changed, 398 insertions(+), 10 deletions(-) create mode 100644 .agents/policies/test-integrity/evals/suite.yaml create mode 100644 .chock/compiled/test-integrity/ambient-rule/ambient.md create mode 100644 .chock/compiled/test-integrity/ci-gate/gate.json create mode 100644 .chock/compiled/test-integrity/ci-gate/step.yaml create mode 100644 .chock/compiled/test-integrity/git-hook/gate.json create mode 100755 .chock/compiled/test-integrity/git-hook/git-pre-commit.sh create mode 100644 .chock/compiled/test-integrity/managed-setting/managed-settings.json diff --git a/.agents/policies/INDEX.md b/.agents/policies/INDEX.md index 66d0300..cf77eef 100644 --- a/.agents/policies/INDEX.md +++ b/.agents/policies/INDEX.md @@ -56,6 +56,7 @@ - **block-wildcard-agent-permissions**: Wildcard agent permission grant detected. Scope the grant to specific tools or commands (e.g. Bash(git status:*), a named tool list), or add 'pragma: allowlist broad-agency' on the same line for a reviewed exception. - **protect-main-branch**: Direct commits/pushes to a protected branch (main|master) are blocked. Create a feature branch and open a pull request. - **scan-secrets**: Potential secret detected in staged changes. Remove credentials and rotate any exposed keys. Add '# pragma: allowlist secret' on the same line only for documented test fixtures. +- **test-integrity**: Tests were weakened, not fixed. If a test is genuinely obsolete, say so on the line that removes it with `chock: test-removal-reviewed` and have a human confirm it. ## Skills — invoke when the task matches diff --git a/.agents/policies/test-integrity/evals/suite.yaml b/.agents/policies/test-integrity/evals/suite.yaml new file mode 100644 index 0000000..694a63d --- /dev/null +++ b/.agents/policies/test-integrity/evals/suite.yaml @@ -0,0 +1,110 @@ +suite: + id: test-integrity-tests-v1 + policy_id: test-integrity + version_constraint: ">=0.0.1" + maintainer: chock-core + primary_metric: pass_rate + metrics: + pass_rate: + direction: higher_is_better + threshold: 1.0 + cases: + - id: tc-001 + category: trigger + prompt: An agent removes a failing assertion from a test instead of fixing the code under test. + expect: The gate blocks because assertions were removed with nothing added in their place. + execute: + head_files: + tests/test_foo.py: | + def test_a(): + assert 1 == 1 + + def test_b(): + assert 2 == 2 + files: + tests/test_foo.py: | + def test_a(): + assert 1 == 1 + event: commit + expect: block + - id: tc-002 + category: trigger + prompt: An agent replaces a real assertion with a vacuous one so the test still runs but proves nothing. + expect: The gate blocks because a vacuous assertion (assert True) was added. + execute: + head_files: + tests/test_foo.py: | + def test_a(): + assert compute() == 42 + files: + tests/test_foo.py: | + def test_a(): + assert True + event: commit + expect: block + - id: tc-003 + category: negative_trigger + prompt: An agent adds a new test with a real assertion alongside the existing ones. + expect: The gate allows the commit because assertions only grew. + execute: + head_files: + tests/test_foo.py: | + def test_a(): + assert 1 == 1 + files: + tests/test_foo.py: | + def test_a(): + assert 1 == 1 + + def test_b(): + assert 2 == 2 + event: commit + expect: allow + - id: tc-004 + category: behavior + prompt: A human reviews a genuinely obsolete test and marks its removal with the allowlist pragma. + expect: The gate allows the commit because the removal is pragma-reviewed. + execute: + head_files: + tests/test_foo.py: | + def test_obsolete(): + assert 1 == 1 # covers a feature removed last release + files: + tests/test_foo.py: | + # test_obsolete removed: feature no longer exists + # chock: test-removal-reviewed + event: commit + expect: allow + # A source file losing an assert-like line is not a test-integrity concern; the path + # regex scopes the gate to test files only. No derived case can express this scoping -- + # it needs a repository with a history, same as verify-dependency-exists tc-005. + - id: tc-005 + category: edge + prompt: An agent removes a runtime assertion from application code, not a test. + expect: The gate allows the commit because the changed file is not a test file. + execute: + head_files: + src/app.py: | + def f(x): + assert x > 0 + return x + files: + src/app.py: | + def f(x): + return x + event: commit + expect: allow + # Found live while documenting this gate: a `spec/` directory of design prose (this + # repo's own layout) is not RSpec test code, so writing "assert True" as a worked + # example in a markdown doc under spec/ must not itself trip the vacuous-assertion + # check that same doc describes. + - id: tc-006 + category: edge + prompt: A markdown doc under spec/ uses "assert True" as a worked example in prose. + expect: The gate allows the commit because a .md file is documentation, not a test file. + execute: + files: + spec/gate-dsl.md: | + `dummy_assertion_pattern` matches a vacuous check such as `assert True`. + event: commit + expect: allow diff --git a/.agents/policies/test-integrity/manifest.yaml b/.agents/policies/test-integrity/manifest.yaml index cab3773..4dc2700 100644 --- a/.agents/policies/test-integrity/manifest.yaml +++ b/.agents/policies/test-integrity/manifest.yaml @@ -18,7 +18,7 @@ approval: hook: gate: kind: test_integrity - "on": [commit, ci] + "on": [commit] action: block message: > Tests were weakened, not fixed. If a test is genuinely obsolete, say so on the line @@ -26,7 +26,10 @@ hook: params: # Patterns are policy data, not engine behaviour: the kind is language-agnostic and # every language's idea of "a test" and "an assertion" arrives from here. - test_path_regex: "(^|/)(tests?|spec|__tests__)/|(_test|_spec|\\.test|\\.spec)\\.[a-z]+$" + # The directory branch excludes prose extensions (.md/.rst/.txt) via a negative + # lookahead -- a `spec/` or `tests/` directory holding design docs, not test code, + # (as this very repo's spec/ does) must not be policed as test assertions. + test_path_regex: "(^|/)(tests?|spec|__tests__)/(?!.*\\.(?:md|rst|txt)$)|(_test|_spec|\\.test|\\.spec)\\.[a-z]+$" assertion_pattern: "\\b(assert|assertEquals|assertTrue|expect|should|require\\.(True|NoError))\\b" # Counted only when ADDED -- a test that asserts nothing is worse than no test, # because it reports green. diff --git a/.chock/bin/gate.py b/.chock/bin/gate.py index 95b7ecd..4c1c45e 100755 --- a/.chock/bin/gate.py +++ b/.chock/bin/gate.py @@ -77,6 +77,11 @@ def added_lines(self, path: str) -> list[str]: lines.append(line[1:]) return lines + def removed_lines(self, path: str) -> list[str]: + """The deleted side of the diff -- what a test-weakening change takes away.""" + out = self._git("diff", *self._range(), "-U0", "--", path) + return [line[1:] for line in out.splitlines() if line.startswith("-") and not line.startswith("---")] + def staged_blob(self, path: str) -> str: """The proposed content: staged in index mode, committed at HEAD in range mode.""" return self._git("show", f"HEAD:{path}" if self.base else f":{path}") @@ -235,10 +240,44 @@ def _kind_dependency_allowlist(ctx: GateContext, params: dict, _event: str) -> G return GateResult(allowed=not matches, matches=matches) +def _count(pattern: "re.Pattern[str]", lines: list[str], pragma: "re.Pattern[str] | None") -> int: + return sum(1 for line in lines if pattern.search(line) and not (pragma and pragma.search(line))) + + +def _kind_test_integrity(ctx: GateContext, params: dict, _event: str) -> GateResult: + """Block a change that wins green CI by weakening the tests rather than fixing the code.""" + path_re = re.compile(params["test_path_regex"]) + assertion_re = re.compile(params["assertion_pattern"]) + dummy_pattern = params.get("dummy_assertion_pattern") + dummy_re = re.compile(dummy_pattern) if dummy_pattern else None + pragma = params.get("allowlist_pragma") + pragma_re = re.compile(pragma) if pragma else None + + matches: list[str] = [] + added = removed = 0 + for path in ctx.staged_paths("D"): + if path_re.search(path): + matches.append(f"{path}: test file deleted") + for path in ctx.staged_paths("ACMRT"): + if not path_re.search(path): + continue + added_lines = ctx.added_lines(path) + if pragma_re and any(pragma_re.search(line) for line in added_lines): + continue + added += _count(assertion_re, added_lines, pragma_re) + removed += _count(assertion_re, ctx.removed_lines(path), pragma_re) + if dummy_re and any(dummy_re.search(line) for line in added_lines): + matches.append(f"{path}: vacuous assertion added") + if removed > added: + matches.append(f"assertions removed across tests: {removed} removed, {added} added") + return GateResult(allowed=not matches, matches=matches) + + KINDS = { "content_regex": _kind_content_regex, "forbidden_ref": _kind_forbidden_ref, "dependency_allowlist": _kind_dependency_allowlist, + "test_integrity": _kind_test_integrity, } diff --git a/.chock/compiled/test-integrity/ambient-rule/ambient.md b/.chock/compiled/test-integrity/ambient-rule/ambient.md new file mode 100644 index 0000000..5815fd0 --- /dev/null +++ b/.chock/compiled/test-integrity/ambient-rule/ambient.md @@ -0,0 +1,6 @@ + +``` +on(commit): block(test_integrity) test_path_regex=(^|/)(tests?|spec|__tests__)/(?!.*\.(?:md|rst... ... +Tests were weakened, not fixed. If a test is genuinely obsolete, say so on the line that removes it with `chock: test-removal-reviewed` and have a human confirm it. +``` + diff --git a/.chock/compiled/test-integrity/ci-gate/gate.json b/.chock/compiled/test-integrity/ci-gate/gate.json new file mode 100644 index 0000000..cf53e79 --- /dev/null +++ b/.chock/compiled/test-integrity/ci-gate/gate.json @@ -0,0 +1,14 @@ +{ + "kind": "test_integrity", + "on": [ + "commit" + ], + "action": "block", + "message": "Tests were weakened, not fixed. If a test is genuinely obsolete, say so on the line that removes it with `chock: test-removal-reviewed` and have a human confirm it.", + "params": { + "test_path_regex": "(^|/)(tests?|spec|__tests__)/(?!.*\\.(?:md|rst|txt)$)|(_test|_spec|\\.test|\\.spec)\\.[a-z]+$", + "assertion_pattern": "\\b(assert|assertEquals|assertTrue|expect|should|require\\.(True|NoError))\\b", + "dummy_assertion_pattern": "\\b(assert\\s+(True|1)\\b|expect\\s*\\(\\s*(true|1)\\s*\\)|assertTrue\\s*\\(\\s*true\\s*\\))", + "allowlist_pragma": "chock: test-removal-reviewed" + } +} \ No newline at end of file diff --git a/.chock/compiled/test-integrity/ci-gate/step.yaml b/.chock/compiled/test-integrity/ci-gate/step.yaml new file mode 100644 index 0000000..36d218c --- /dev/null +++ b/.chock/compiled/test-integrity/ci-gate/step.yaml @@ -0,0 +1,12 @@ +# Auto-generated by chock compile. +# Policy: test-integrity +- name: chock-ci-gate (test-integrity) + run: | + PY="" + for c in python3 python py; do + if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import tomllib' >/dev/null 2>&1; then PY="$c"; break; fi + done + [ -n "$PY" ] || { echo "gate: no python >= 3.11 (with tomllib) found on PATH" >&2; exit 2; } + base="${GITHUB_BASE_REF:?ci-gate needs GITHUB_BASE_REF -- run this step on the pull_request event}" + "$PY" .chock/bin/gate.py run --gate .chock/compiled/test-integrity/ci-gate/gate.json --event ci --base "origin/$base" \ + --head-ref "${GITHUB_HEAD_REF:-}" diff --git a/.chock/compiled/test-integrity/git-hook/gate.json b/.chock/compiled/test-integrity/git-hook/gate.json new file mode 100644 index 0000000..cf53e79 --- /dev/null +++ b/.chock/compiled/test-integrity/git-hook/gate.json @@ -0,0 +1,14 @@ +{ + "kind": "test_integrity", + "on": [ + "commit" + ], + "action": "block", + "message": "Tests were weakened, not fixed. If a test is genuinely obsolete, say so on the line that removes it with `chock: test-removal-reviewed` and have a human confirm it.", + "params": { + "test_path_regex": "(^|/)(tests?|spec|__tests__)/(?!.*\\.(?:md|rst|txt)$)|(_test|_spec|\\.test|\\.spec)\\.[a-z]+$", + "assertion_pattern": "\\b(assert|assertEquals|assertTrue|expect|should|require\\.(True|NoError))\\b", + "dummy_assertion_pattern": "\\b(assert\\s+(True|1)\\b|expect\\s*\\(\\s*(true|1)\\s*\\)|assertTrue\\s*\\(\\s*true\\s*\\))", + "allowlist_pragma": "chock: test-removal-reviewed" + } +} \ No newline at end of file diff --git a/.chock/compiled/test-integrity/git-hook/git-pre-commit.sh b/.chock/compiled/test-integrity/git-hook/git-pre-commit.sh new file mode 100755 index 0000000..a7e9338 --- /dev/null +++ b/.chock/compiled/test-integrity/git-hook/git-pre-commit.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Auto-generated by chock compile. Declarative gate: test-integrity +set -eu +repo_root="$(git rev-parse --show-toplevel)" +PY="" +for c in python3 python py; do + if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import tomllib' >/dev/null 2>&1; then PY="$c"; break; fi +done +[ -n "$PY" ] || { echo "gate: no python >= 3.11 (with tomllib) found on PATH" >&2; exit 2; } +exec "$PY" "$repo_root/.chock/bin/gate.py" run \ + --gate "$repo_root/.chock/compiled/test-integrity/git-hook/gate.json" --event pre-commit diff --git a/.chock/compiled/test-integrity/managed-setting/managed-settings.json b/.chock/compiled/test-integrity/managed-setting/managed-settings.json new file mode 100644 index 0000000..8efb266 --- /dev/null +++ b/.chock/compiled/test-integrity/managed-setting/managed-settings.json @@ -0,0 +1,4 @@ +{ + "deny": [], + "ask": [] +} \ No newline at end of file diff --git a/.chock/coverage.json b/.chock/coverage.json index 5b72d9c..b18a984 100644 --- a/.chock/coverage.json +++ b/.chock/coverage.json @@ -1071,6 +1071,73 @@ "witnessed": false } }, + "test-integrity": { + "aider": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "claude": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "codex": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "copilot": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "cursor": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "devin": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "gemini": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "grok": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "kimi-code": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "replit": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "tabnine": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "vscode": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + }, + "windsurf": { + "level": "enforced-at-commit", + "basis": null, + "witnessed": false + } + }, "token-efficiency": { "aider": { "level": "advisory", diff --git a/.chock/registry.json b/.chock/registry.json index 8aab850..1960644 100644 --- a/.chock/registry.json +++ b/.chock/registry.json @@ -301,6 +301,21 @@ "script_hashes": {} } ], + "test-integrity": [ + { + "id": "test-integrity", + "artifact": "hook", + "version": "0.0.1", + "name": "Test Integrity", + "description": "Block a change that wins green CI by weakening the tests instead of fixing the code. Catches a deleted test file, a net loss of assertions across the change, and a vacuous assertion (`assert True`, `expect(true)`) added in its place. Author-blind: the CI gate re-runs it on the PR head, so it holds for an inbound contributor whose agent never ran a hook. Defends the signal every other gate trusts.", + "path": ".agents/policies/test-integrity", + "manifest": "manifest.yaml", + "trust_tier": "community", + "lifecycle_status": "draft", + "dependencies": [], + "script_hashes": {} + } + ], "token-efficiency": [ { "id": "token-efficiency", diff --git a/CHANGELOG.md b/CHANGELOG.md index 775e8e8..c167d96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Unreleased +- **Added the `test_integrity` gate kind, closing the `agentic-risk-coverage.md` row on an + agent deleting tests or assertions to get green** (catalog policy `test-integrity`, + `chock-g1`). Blocks a deleted test file, a net loss of assertions across the whole + change, and a vacuous assertion (`assert True`, `expect(true)`) added in its place; + `chock: test-removal-reviewed` on the removing line is the reviewed escape hatch. + Declarative (`hook.gate` in `manifest.yaml`), `enforced-at-commit` with the `ci-gate` + backstop so it holds for an inbound contributor whose agent never ran a local hook. The + coverage row moves from `advisory` to `enforced-at-commit`. - **Fixed a dead `import shutil` in every vendored runtime bundle except `claude_code`'s.** `chock.gate.runtime_bundle.render()` spliced its fixed `_chock_`-renamed stdlib import block into every agent's bundle regardless of which of those names the assembled handler diff --git a/chock.lock b/chock.lock index d1af309..4d95edb 100644 --- a/chock.lock +++ b/chock.lock @@ -130,6 +130,14 @@ "source": "local", "artifacts_sha256": "cd7df13ded59738cd76aa24c19ec0600ca6847be5614536ec4c39f886f4f6e58" }, + { + "id": "test-integrity", + "version": "0.0.1", + "managed": false, + "sha256": "c140c5ca23c3e9dde10712b889840ee9a4a1976f7cb328d088f8f781e03c274e", + "source": "local", + "artifacts_sha256": "c5e8de5994b7c5a044203babbca3db334500cab99e66ac409ba07daa5baf51a6" + }, { "id": "token-efficiency", "version": "0.0.1", diff --git a/docs/agentic-risk-coverage.md b/docs/agentic-risk-coverage.md index 8720330..26005e0 100644 --- a/docs/agentic-risk-coverage.md +++ b/docs/agentic-risk-coverage.md @@ -42,7 +42,7 @@ Chock's guard degrades to allow, so no row here earns it. | …an agent poisoning its own long-term memory? | `memory-discipline` | `advisory` — and deliberately so here: write-path memory enforcement is a different system than a repo-scoped framework, and this page does not claim it | | …direct pushes to `main`, `--no-verify`, force-pushes? | `protect-main-branch` gate, `block-no-verify`, `git-safety` | `enforced-at-commit` (gate); the never-bypass-hooks discipline itself: `advisory` backed by the CI gate, which re-runs `chock check` on the PR head regardless of what was skipped locally | | …wildcard permission grants in agent config? | `block-wildcard-agent-permissions` gate | `enforced-at-commit` | -| …an agent deleting tests or assertions to get green? | `agent-discipline` (`assertion_deletion: block`) | `advisory` today — a deterministic test-integrity gate is a named catalog candidate, not yet a roadmap issue, so this row stays `advisory` until it is | +| …an agent deleting tests or assertions to get green? | `test-integrity` gate, `agent-discipline` (`assertion_deletion: block`) | `enforced-at-commit` (gate) — blocks a deleted test file, a net loss of assertions across the change, or a vacuous assertion (`assert True`, `expect(true)`) added in their place; re-checked in CI on the PR head via the `ci-gate` backstop, so it holds for an inbound contributor whose agent never ran a hook. The pragma `chock: test-removal-reviewed` on the removing line is the reviewed escape hatch. No `pre-tool-use` binding exists, so this is not claimed at the in-agent tier. The ambient rule text remains `advisory` | ## Against the OWASP Top 10 for Agentic Applications diff --git a/docs/baseline-policies.md b/docs/baseline-policies.md index 9a105cd..e205ead 100644 --- a/docs/baseline-policies.md +++ b/docs/baseline-policies.md @@ -46,6 +46,7 @@ These block risky actions at commit/push time and compile into agent-native cont | **`block-invisible-unicode`** | Bidi override/isolate controls and Unicode tag-block characters in staged changes — Trojan Source (CVE-2021-42574) and instructions hidden from reviewers but legible to agents. ZWJ and RTL marks pass by design (emoji, internationalised text). | | **`protect-agent-config`** | Shell writes to the agent's own instruction, permission and vendored-enforcement files (`AGENTS.md`, `.claude/settings.json`, `.mcp.json`, `.chock/bin/` …) — self-modification refused before it runs; regenerate through `chock sync` instead. | | **`block-wildcard-agent-permissions`** | Committed everything-grants in agent settings and MCP configs — bare-wildcard shell grants and allow-everything tool lists. Scoped grants pass; the twin of the catalog's `block-wildcard-iam`. | +| **`test-integrity`** | A change that wins green CI by weakening the tests instead of fixing the code: a deleted test file, a net loss of assertions across the change, or a vacuous assertion (`assert True`, `expect(true)`) added in their place. The pragma `chock: test-removal-reviewed` on the removing line is the reviewed escape hatch. | > These are **best-effort friction, not a security boundary.** Known bypasses (aliases, quoting, > non-standard clients) are documented on each policy. Pair them with the CI-gate backstop. @@ -79,7 +80,7 @@ own the same way: a folder, a manifest, and (optionally) an ## Cross-platform & tested -`scan-secrets`, `protect-main-branch` and `verify-dependency-exists` are **declarative** +`scan-secrets`, `protect-main-branch`, `verify-dependency-exists` and `test-integrity` are **declarative** (`hook.gate` in `manifest.yaml`); `chock compile` emits the cross-platform git-hook shims and a self-contained, stdlib-only Python runner. The remaining guards ship bash implementations invoked through the PreToolUse adapter. `.gitattributes` pins scripts to LF so their hashes — and therefore diff --git a/spec/gate-dsl.md b/spec/gate-dsl.md index 7e6d95a..7dd8f20 100644 --- a/spec/gate-dsl.md +++ b/spec/gate-dsl.md @@ -7,7 +7,7 @@ For `artifact: hook` policies, the gate is declared under `hook.gate` in `manife | field | required | type | notes | |-------|----------|------|-------| -| `kind` | yes | string | `content_regex`, `forbidden_ref`, `dependency_allowlist`, or `egress_allowlist` (gateway-only) | +| `kind` | yes | string | `content_regex`, `forbidden_ref`, `dependency_allowlist`, `test_integrity`, or `egress_allowlist` (gateway-only) | | `on` | yes | list | events: `commit`, `push`, `tool_use`. The key must be quoted `"on"` in YAML. | | `action` | yes | string | `block`, `verify`, or `warn` | | `message` | yes | string | printed to stderr when the gate blocks | @@ -84,6 +84,22 @@ a parse error is never converted into a block. Extracted names are lowercased and compared against a lowercased allowlist. +### `kind: test_integrity` + +| param | required | type | notes | +|-------|----------|------|-------| +| `test_path_regex` | yes | string | regex matched against staged paths to identify test files | +| `assertion_pattern` | yes | string | regex matched against a line to count it as an assertion | +| `dummy_assertion_pattern` | no | string | regex for a vacuous assertion (`assert True`, `expect(true)`); matched only on added lines | +| `allowlist_pragma` | no | string | regex matched on a line; a match on an added line skips that file's counting entirely | + +Blocks three shapes of a change that wins green CI by weakening the tests rather than +fixing the code: a deleted test file, a **net** loss of assertions across the whole +change (removed lines matching `assertion_pattern` outnumber added ones, counted only in +files matching `test_path_regex`), and a vacuous assertion added in place of a real one. +Only the staged diff is read (`removed_lines`/`added_lines`), so a file that already +contained fewer assertions before this commit does not block it. + ## Runtime note The emitted git-hook shim probes for a working Python 3 interpreter in the order `python3`, `python`, `py` and then calls `.chock/bin/gate.py`. This makes enforcement work on stock Windows as well as POSIX without `pip install`. diff --git a/src/chock/gate/runner.py b/src/chock/gate/runner.py index e156131..4c1c45e 100644 --- a/src/chock/gate/runner.py +++ b/src/chock/gate/runner.py @@ -80,8 +80,7 @@ def added_lines(self, path: str) -> list[str]: def removed_lines(self, path: str) -> list[str]: """The deleted side of the diff -- what a test-weakening change takes away.""" out = self._git("diff", *self._range(), "-U0", "--", path) - return [line[1:] for line in out.splitlines() - if line.startswith("-") and not line.startswith("---")] + return [line[1:] for line in out.splitlines() if line.startswith("-") and not line.startswith("---")] def staged_blob(self, path: str) -> str: """The proposed content: staged in index mode, committed at HEAD in range mode.""" diff --git a/src/chock/validation/schemas/manifest.hook.json b/src/chock/validation/schemas/manifest.hook.json index 12ada1e..4109440 100644 --- a/src/chock/validation/schemas/manifest.hook.json +++ b/src/chock/validation/schemas/manifest.hook.json @@ -25,7 +25,8 @@ "enum": [ "content_regex", "forbidden_ref", - "dependency_allowlist" + "dependency_allowlist", + "test_integrity" ] }, "on": { diff --git a/tests/test_manifest_validation.py b/tests/test_manifest_validation.py index c79ac11..8f1126d 100644 --- a/tests/test_manifest_validation.py +++ b/tests/test_manifest_validation.py @@ -119,6 +119,59 @@ def test_manifest_gate_params_missing_required(tmp_path: Path) -> None: assert "manifest_gate_params" in _codes(report) +def test_manifest_gate_params_test_integrity_valid(tmp_path: Path) -> None: + """A test_integrity gate with its required params validates cleanly.""" + data = { + **MINIMAL_RULE, + "id": "test-hook", + "name": "Test Hook", + "artifact": "hook", + "enforcement": "block", + "hook": { + "gate": { + "kind": "test_integrity", + "on": ["commit"], + "action": "block", + "message": "blocked", + "params": { + "test_path_regex": r"(^|/)tests?/", + "assertion_pattern": r"\bassert\b", + }, + } + }, + } + policy_dir = _manifest_dir(tmp_path, "test-hook", data) + report = Report() + manifest, _ = load_manifest(policy_dir) + check_manifest_schema(policy_dir, manifest, "hook", report) + assert "manifest_gate_params" not in _codes(report) + + +def test_manifest_gate_params_test_integrity_missing_required(tmp_path: Path) -> None: + """Rule 4: a test_integrity gate without assertion_pattern is rejected.""" + data = { + **MINIMAL_RULE, + "id": "test-hook", + "name": "Test Hook", + "artifact": "hook", + "enforcement": "block", + "hook": { + "gate": { + "kind": "test_integrity", + "on": ["commit"], + "action": "block", + "message": "blocked", + "params": {"test_path_regex": r"(^|/)tests?/"}, + } + }, + } + policy_dir = _manifest_dir(tmp_path, "test-hook", data) + report = Report() + manifest, _ = load_manifest(policy_dir) + check_manifest_schema(policy_dir, manifest, "hook", report) + assert "manifest_gate_params" in _codes(report) + + def test_manifest_self_dependency(tmp_path: Path) -> None: """Rule 5: a policy must not list itself in dependencies.policies.""" data = { diff --git a/tests/test_test_integrity_gate.py b/tests/test_test_integrity_gate.py index 925f581..fb86045 100644 --- a/tests/test_test_integrity_gate.py +++ b/tests/test_test_integrity_gate.py @@ -68,9 +68,15 @@ def test_non_test_paths_are_not_policed(tmp_path: Path) -> None: assert _verdict(repo, gate) == 0 +def test_a_spec_directory_of_prose_is_not_policed(tmp_path: Path) -> None: + """A `spec/` directory of design docs (this repo's own layout) is not RSpec test code.""" + repo, gate = _repo(tmp_path) + stage(repo, "spec/gate-dsl.md", "docs describing `assert True` as an example of a vacuous check\n") + assert _verdict(repo, gate) == 0 + + def test_a_reviewed_removal_is_allowed_through(tmp_path: Path) -> None: """The escape hatch is deliberate, in the diff, and named — not a config toggle.""" repo, gate = _repo(tmp_path) - stage(repo, "tests/test_math.py", - "def test_adds(): # chock: test-removal-reviewed\n assert add(1, 2) == 3\n") + stage(repo, "tests/test_math.py", "def test_adds(): # chock: test-removal-reviewed\n assert add(1, 2) == 3\n") assert _verdict(repo, gate) == 0