From 1feafce5417aaddea709c0663da209bd9814547d Mon Sep 17 00:00:00 2001 From: Daniel Ochoa Date: Tue, 11 Aug 2026 11:13:46 -0500 Subject: [PATCH] fix(code): harden decision-table gates - Treat Not aligned as a terminal workflow stop - Require bounded source interactions and executable-twin parity - Map required tests to decision rows and negative cases Testing: 521 code-plugin tests; Ruff; Pyright; skill validation Risks: Prompt-only behavior change guarded by contract tests --- CHANGELOG.md | 11 ++++ plugins/code/.claude-plugin/plugin.json | 2 +- plugins/code/skills/decision-table/SKILL.md | 10 ++-- .../references/artifact-format.md | 23 +++++--- .../decision-table/references/edge-cases.md | 22 +++++++ .../references/review-prevention.md | 9 +++ .../test_decision_table_skill_contract.py | 57 +++++++++++++++++++ 7 files changed, 122 insertions(+), 12 deletions(-) create mode 100644 plugins/code/tools/python/test_decision_table_skill_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index df86e39..e7507bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to the claude-plugins project will be documented in this fil The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Entries are listed newest-first; each plugin section is treated as released when merged to `main`. +### code v1.14.8 + +#### Added +- Contract test suite `test_decision_table_skill_contract.py` pinning the new decision-table skill guarantees: `Not aligned` as a terminal workflow stop, required coexisting-source interaction and precedence rows, shared-corpus production-boundary parity for executable twins, and row-ID-to-test traceability in `Required Tests`. + +#### Changed +- Hardened the `decision-table` skill's alignment gate: `Final Alignment Status: Not aligned` is now a terminal workflow stop, not a report-only status. No PR creation, merge, completion signal, or success closeout may follow until the artifact is re-verified as `Aligned`; fixable repo-local findings remain unresolved work and cannot be normalized into a successful handoff. The human-facing closeout for a `Not aligned` run must state that downstream PR/merge/completion is blocked, name every unresolved blocker, and give the exact next action and owner. A matching common-miss item ("Not-aligned status treated as success") and a contract-heavy review bullet enforce the stop during review-prevention passes. +- Decision rows now carry stable row IDs, and `Required Tests` is a traceability table instead of a prose list. The `Current Code` and `Intended Change` tables gain a `Row ID` column, and each required test maps to its decision-row IDs with a positive control, a wrong-input or mixed-state negative case, the real production boundary exercised, and expected evidence. Every material row must be covered by at least one mapped test or carried into `Not aligned` with a blocker; a test listed without row IDs or without the negative case needed to distinguish the row's branch is a named review-prevention anti-pattern. +- New "Executable policy twins and parity" edge-case category: when one behavior or policy has multiple executable twins (a pure helper, SQL predicate, route, worker, producer, batch path, or recovery path), one shared scenario corpus must exercise every twin through its real production boundary and assert identical decisions. Source-string, AST-presence, and SQL-shape assertions are supplemental only and never establish parity, and a negative shape assertion that requires a policy predicate, identity term, join, or branch to be absent is flagged as pinning missing policy: if it permits divergence, the row is `Not aligned` until corrected. Two new review-prevention anti-patterns cover parity inferred from separate tests and shape assertions that pin missing policy. +- New "Coexisting sources and precedence" edge-case category: when multiple evidence, authority, history, cache, or fallback sources can coexist, singleton rows are insufficient. The table must add a bounded interaction set (pairwise plus code-identified high-risk intersections, never an unbounded Cartesian product) covering legacy or absent evidence alongside fresh valid evidence, corrupt or undated evidence alongside fresh valid evidence, irrelevant historical evidence alongside a current authoritative record, tied or conflicting current records, and source/state precedence, stating which source wins and why. Singleton-only source coverage is a named review-prevention anti-pattern. + ### platform v1.1.4 #### Fixed diff --git a/plugins/code/.claude-plugin/plugin.json b/plugins/code/.claude-plugin/plugin.json index bc05a88..2e4c6cc 100644 --- a/plugins/code/.claude-plugin/plugin.json +++ b/plugins/code/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "code", "description": "Code and planning framework plugin", - "version": "1.14.7", + "version": "1.14.8", "author": { "name": "ClosedLoop", "email": "support@closedloop.ai" diff --git a/plugins/code/skills/decision-table/SKILL.md b/plugins/code/skills/decision-table/SKILL.md index 85b56e9..6915879 100644 --- a/plugins/code/skills/decision-table/SKILL.md +++ b/plugins/code/skills/decision-table/SKILL.md @@ -29,13 +29,13 @@ Keep multiple behavior areas as sections inside the same artifact. Only split in 5. Read the actual code paths. Build the table from code, not expectations. 6. For shared routes, handlers, helpers, contracts, or policy surfaces, build a call-site inventory before choosing axes. Search for literal route paths, exported helper names, feature flag keys, rollout keys, query parameters, cache key segments, environment variable names, storage keys, event names, command names, plugin or marketplace identifiers, header/reason/status strings, and shared types. For each caller, record what data it can supply, what response shapes/statuses it expects, peer version skew, and how missing/unknown fields degrade. Classify each literal by semantic purpose and source of truth; do not treat similar-looking strings as aliases unless a shared constant, documented contract, or existing compatibility path proves they are aliases. 7. For dependencies, model success, null/absent, validation failure, and thrown/rejected branches whenever externally visible behavior depends on them. -8. Run the behavioral edge-case expansion pass. Apply every category in `references/edge-cases.md`. Each must be represented by rows or an explicit non-applicability note with source-backed evidence. For distributed command, signing, key, capability, or cross-process state work, treat web app, backend, Electron, local store, OS notification, cache, and remote peer behavior as separate surfaces unless code proves they are the same surface. +8. Run the behavioral edge-case expansion pass. Apply every category in `references/edge-cases.md`. Each must be represented by rows or an explicit non-applicability note with source-backed evidence. When multiple evidence, authority, history, or fallback sources can coexist, add a bounded interaction pass: cover pairwise and high-risk intersections instead of an unbounded Cartesian product, including legacy/absent plus fresh valid, corrupt/undated plus fresh valid, irrelevant historical plus current authoritative, tied/conflicting current records, and source/state precedence. For distributed command, signing, key, capability, or cross-process state work, treat web app, backend, Electron, local store, OS notification, cache, and remote peer behavior as separate surfaces unless code proves they are the same surface. 9. Choose a small set of state axes that explain the branch behavior. Reuse the same axes within a behavior area across `Current Code` and `Intended Change`. 10. Write the artifact using `references/artifact-format.md`. 11. When a plan is in scope, include `Current Code`, `Intended Change`, `Delta Checklist`, and `Required Tests`. When no plan is in scope, omit `Intended Change` and focus on the current-state table plus gaps or suspicious branches. -12. For `Required Tests`, name the invariant being proved, the positive path, and the failure or compatibility mutation. A test must prove the specific binding/fallback/diagnostic the row claims; it cannot just trigger a generic rejection. When a row depends on an exact external contract literal, the test oracle must fail closed for the wrong literal: feature-flag mocks enable only the exact expected key, query/header/event assertions check exact names and values, and cache/storage/command/plugin identifiers are asserted by semantic type rather than broad substring or "any key" matching. +12. Give every material decision row a stable row ID. For `Required Tests`, map each test to one or more row IDs and name the invariant being proved, the positive path, and the wrong-input, mixed-state, failure, or compatibility mutation. A test must prove the specific binding/fallback/diagnostic the row claims; it cannot just trigger a generic rejection. When a row depends on an exact external contract literal, the test oracle must fail closed for the wrong literal: feature-flag mocks enable only the exact expected key, query/header/event assertions check exact names and values, and cache/storage/command/plugin identifiers are asserted by semantic type rather than broad substring or "any key" matching. 13. Once implementation begins, freeze `Current Code` and `Intended Change`. All later updates are append-only in `Verification Findings`, `Fixes Applied`, `Final Alignment Status`, and optional `Plan Clarifications`. -14. After implementation, verify the final code against the intended behavior. If drift, missing edges, missing tests, or guardrail violations exist, fix them, append the verification/fix sections, and re-verify until aligned. +14. After implementation, verify the final code against the intended behavior. If drift, missing edges, missing tests, or guardrail violations exist, fix them, append the verification/fix sections, and re-verify until aligned. `Final Alignment Status: Not aligned` is a terminal stop: do not proceed to PR creation, merge, completion, or any downstream success state while it remains. Fixable repo-local findings remain unresolved work and must be fixed and re-verified; they cannot be normalized into a successful handoff. 15. Record evidence artifacts for high-yield coverage and non-applicability claims. Use `references/artifact-format.md` and capture the actual command or source evidence for changed exports, package subpaths, CLI flags, route/query/header/event/cache/storage/command literals, path or filesystem writes, untrusted input fields, persisted schema fields, replay/idempotency behavior, and test-boundary coverage. Do not accept an assertion such as "no consumers", "not externally visible", or "covered by tests" without the corresponding evidence. 16. Run the adversarial responsibility split described below. When delegation is available, this is a post-implementation review by independent lane workers. When delegation is unavailable, run the applicable lanes sequentially and record that the pass was not independent. 17. Group `Fixes Applied` by discovery source when more than one source exists (e.g., `Initial verification`, `Adversarial abuse/filesystem lane`, `Adversarial compatibility lane`, `Runtime testing`, `Review findings`, `Validation failures`, `Repo guardrails`, `Plan clarification`, `Final hygiene`). Do not leave a broad `During verification` bucket once other sources have produced fixes. @@ -45,7 +45,7 @@ Keep multiple behavior areas as sections inside the same artifact. Only split in - **Review-prevention:** for every touched externally visible surface, walk `references/review-prevention.md`. Each item must be fixed, already covered by a named row/test, marked not applicable with source-backed evidence, or carried into `Not aligned`. Do not mark `Aligned` while any item is merely assumed covered. - **Coverage and evidence disposition rule (hard rule):** every `Covered` or `already covered` disposition, whether it appears in `Verification Findings`, in the Behavioral Edge-Case Expansion, in the adversarial lanes, or in the review-prevention pass, must cite a specific test name and the wrong-input or negative case that test fails closed on. Every `not applicable` disposition must cite source-backed evidence such as grep output, export/package inventory, call-site inventory, schema/query inventory, or code references proving the surface is absent or out of scope. A coverage claim backed only by a happy-path assertion, a pure helper test for an integration boundary, or no evidence is treated as `Not aligned`, not as covered. This applies to security findings: do not mark a security finding `Covered` without a named test proving the rejected or blocked case. 20. Only change `Intended Change` post-implementation if the plan itself was ambiguous or wrong. Record this as `Plan Clarifications` with reason and source. Never silently rewrite the target. -21. Always give the user a human-facing closeout summary outside the artifact. Separate: what was verified and fixed, important nuances, and anything still requiring user input or external action. If the agent cannot complete something autonomously (product decision, credentials, deployment access, independent adversarial review), ask the user directly. If no user action is required, say so. If the user also asked for review, use the artifact as a first-class input rather than recreating the analysis. +21. Always give the user a human-facing closeout summary outside the artifact. Separate: what was verified and fixed, important nuances, and anything still requiring user input or external action. When status is `Not aligned`, explicitly state that downstream PR/merge/completion is blocked, name every unresolved blocker, and give the exact next action and owner; never phrase it as completion. If the agent cannot complete something autonomously (product decision, credentials, deployment access, independent adversarial review), ask the user directly. If no user action is required, say so. If the user also asked for review, use the artifact as a first-class input rather than recreating the analysis. ## Adversarial Responsibility Split @@ -75,6 +75,7 @@ Do not mark `Final Alignment Status: Aligned` solely on the basis of unavailable - Keep wording compact and behaviorally specific. Use clickable file links and plan IDs/URLs for non-obvious rows. - Treat an entry path as actual caller plus capabilities, not just a route or module name. Include rows for callers that cannot supply newly required headers, proof material, payload fields, or response handling. - Call out parity requirements between entry paths (live vs recovery, upload vs replay, retry vs terminal, middleware vs direct, internal vs external). When two entry paths must enforce the same policy, final verification should confirm a shared helper or focused parity tests unless duplication is intentionally documented. +- When one behavior or policy has multiple executable twins (for example a pure helper, SQL predicate, route, worker, producer, or recovery path), use one shared scenario corpus to exercise every twin through its real production boundary and assert identical decisions. Source-string, AST-presence, and SQL-shape assertions are supplemental only and never prove behavioral parity. Treat a negative source-shape assertion that requires a predicate or policy term to be absent as suspicious: if it pins a missing predicate or permits divergence, record it as `Not aligned` until corrected. - For cross-surface state changes, include how each affected replica or process learns about the write, what durable source of truth resolves disagreement, and how missed events, offline peers, refreshes, reconnects, polling, heartbeats, and startup recovery converge. - Separate data visibility from side effects. A table row proving data appears in a list, cache, or refresh result does not prove notifications, telemetry, command dispatch, revocation, deduplication, or other effects fired. - For cached peer capabilities or supported operations, include fresh cache, stale false negative, stale false positive, old peer, retry, fallback, and reconciliation rows before relying on the cache to skip a command or safety action. @@ -88,6 +89,7 @@ Do not mark `Final Alignment Status: Aligned` solely on the basis of unavailable - If the same state or failure reason appears in multiple rows with different intended outcomes, add the missing axis or flag the contradiction before implementation. - Every `Covered` or `already covered` disposition must cite a specific test name and the wrong-input or negative case that test fails closed on. Coverage for a CLI, route, package export, worker/job, replay, ingest, attribution, or other integration surface must cite a test through that real boundary; a pure helper test only covers helper-local invariants. Every `not applicable` disposition must cite source-backed evidence such as grep output, export/package inventory, call-site inventory, schema/query inventory, or code references proving the surface is absent or out of scope. - Do not use soft final states (`Partially aligned`, `Mostly aligned`, `Recorded Gaps`). Use `Aligned` only when no known fixable drift or required-test gap remains; otherwise `Not aligned` with the blocker. +- Treat `Not aligned` as a hard workflow gate, not a report-only status. No PR, merge, completion signal, or success closeout may follow until the artifact is re-verified as `Aligned`. - Do not mark `Aligned` if repo guardrails are violated, a required backward-compatible fallback is missing, plan/guardrail tension is unresolved, throw-capable preparation or framework-managed retry/reconnect paths are unrepresented, required evidence artifacts are missing, required independent adversarial review was not run, or the review-prevention pass is incomplete. - The decision table is an LLM working artifact. Never bury user-actionable information only in the artifact; surface it in the final response. diff --git a/plugins/code/skills/decision-table/references/artifact-format.md b/plugins/code/skills/decision-table/references/artifact-format.md index 9b5d367..d5cb25c 100644 --- a/plugins/code/skills/decision-table/references/artifact-format.md +++ b/plugins/code/skills/decision-table/references/artifact-format.md @@ -74,17 +74,17 @@ Apply every category in [`edge-cases.md`](edge-cases.md). Each must be represent Frozen pre-implementation baseline. Do not rewrite after implementation begins. -| Entry Path | State Inputs | Decision / Branch | Actions / Side Effects | External Outcome | Source | -| --- | --- | --- | --- | --- | --- | -| ... | ... | ... | ... | ... | ... | +| Row ID | Entry Path | State Inputs | Decision / Branch | Actions / Side Effects | External Outcome | Source | +| --- | --- | --- | --- | --- | --- | --- | +| DT-01 | ... | ... | ... | ... | ... | ... | #### Intended Change Frozen target behavior derived from the plan or work item. Only change after implementation if you are explicitly recording a plan clarification. -| Entry Path | State Inputs | Decision / Branch | Actions / Side Effects | External Outcome | Source | -| --- | --- | --- | --- | --- | --- | -| ... | ... | ... | ... | ... | ... | +| Row ID | Entry Path | State Inputs | Decision / Branch | Actions / Side Effects | External Outcome | Source | +| --- | --- | --- | --- | --- | --- | --- | +| DT-01 | ... | ... | ... | ... | ... | ... | ### @@ -98,7 +98,12 @@ Repeat as needed. ## Required Tests -- Exact scenario coverage implied by the changed rows +| Test / Command | Decision Row IDs | Positive Control | Wrong-Input / Mixed-State Negative Case | Production Boundary | Expected Evidence | +| --- | --- | --- | --- | --- | --- | +| ... | DT-01, DT-04 | ... | ... | route / SQL / worker / producer / etc. | ... | + +- Every material row must be covered by at least one mapped test or carried into `Not aligned` with a blocker. +- When executable twins implement the same policy, use one shared scenario corpus through every real production boundary. Source-string, AST-presence, and SQL-shape assertions are supplemental and cannot establish parity. - Exact contract-literal binding tests for feature flags, query parameters, headers, events, cache/storage keys, command names, plugin identifiers, or other external keys. Mocks should fail closed unless the exact intended literal is used. ## Verification Findings @@ -129,6 +134,7 @@ Repeat as needed. - Short explanation with source links - Do not use soft statuses such as `Partially aligned`, `Mostly aligned`, or `Recorded Gaps`. - Use `Aligned` only when no known fixable drift or required-test gap remains. Otherwise use `Not aligned` and state the blocker or user action. +- `Not aligned` is a terminal workflow stop. Name the unresolved blocker, required next action, and owner; do not proceed to PR creation, merge, completion, or a success closeout until re-verification produces `Aligned`. ## Plan Clarifications @@ -148,9 +154,12 @@ Guidelines: - Use `Contract Literal Inventory` to prevent similar-looking strings from being conflated. Classify each literal by semantic role, record the source of truth, and state whether wrong or legacy literals are rejected, ignored, or supported through an explicit compatibility path. - Use `Evidence Artifacts` for claims that are easy for an author to wave through: changed exports or subpaths, CLI flags, filesystem writes, untrusted or persisted fields, replay/idempotency behavior, and integration-boundary coverage. A `Covered` claim needs a named fail-closed test; a `not applicable` claim needs source-backed evidence such as grep output, export/package inventory, call-site inventory, schema/query inventory, or code references. - Keep the same state axes and column meanings across `Current Code` and `Intended Change` within each behavior area. +- Give every material row a stable row ID and map each required test back to those row IDs. - Use additive rows rather than prose for behavior changes whenever possible. - For distributed workflows, include rows for how each replica/process learns about writes, how stale or offline replicas reconcile, and which durable source of truth wins. - When the same eligibility, completion, or terminal predicate is enforced at one site (a gate) but re-derived independently at another (a list filter, a terminal or disposition check, a batch or unscoped routing path, or an early short-circuit), include rows for both sites and confirm they share the predicate through a shared helper or a parity test, because the two derivations can disagree (for example a predicate at a gate vs. re-derived in a list filter, or scoped vs. unscoped routing). +- When those paths are executable twins, run one shared scenario corpus through their real production boundaries. Do not treat source-string, AST-presence, or SQL-shape assertions as parity evidence, and flag negative shape assertions that preserve an absent policy predicate. +- When multiple evidence, authority, history, cache, or fallback sources can coexist, include bounded pairwise/high-risk intersections rather than only singleton rows or an unbounded Cartesian product. - Keep data visibility rows separate from side-effect rows such as notifications, dispatches, telemetry, cleanup, and deduplication. - For capability- or operation-gated behavior, include fresh cache, stale false negative, stale false positive, old peer, fallback, retry, and reconciliation rows. - For legacy persisted records missing new fields, include conservative defaults, evidence-backed promotion/backfill, downgrade behavior, and manual-record protection. diff --git a/plugins/code/skills/decision-table/references/edge-cases.md b/plugins/code/skills/decision-table/references/edge-cases.md index 2284cb8..1bd41ca 100644 --- a/plugins/code/skills/decision-table/references/edge-cases.md +++ b/plugins/code/skills/decision-table/references/edge-cases.md @@ -18,6 +18,28 @@ Include rows for synchronous preparation failures before fetch/await/return: URL When a helper, service, adapter, route, command, job, or handler can be called by more than one path, include rows for the invariants it must enforce itself even when current callers validate first, especially before network I/O, persistence, credentials, filesystem mutation, or other durable side effects. Do not rely only on caller-side validation: either the boundary enforces its own invariants, or record why it is intentionally private/single-caller and how that is kept true. +## Executable policy twins and parity + +When the same behavior or policy is implemented by more than one executable path (for example a pure helper, SQL predicate, route, worker, producer, batch path, or recovery path), inventory every twin and identify its real production boundary. Build one shared scenario corpus and run every twin against it so the test proves identical decisions for identical inputs. + +Source-string, AST-presence, and SQL-shape assertions may supplement behavioral tests, but they never establish parity. Explicitly inspect negative shape assertions that require a policy predicate, identity term, join, or branch to be absent. If such an assertion pins a missing predicate or permits two twins to diverge, record the row as `Not aligned` until the implementation and assertion are corrected. + +**Tests:** require a shared-corpus parity test through each production boundary, with named decision-row coverage, a positive control, a wrong-input case, and a mixed-state case. A helper-only test plus an independent query-shape test is not sufficient. + +## Coexisting sources and precedence + +When multiple evidence, authority, history, cache, or fallback sources can coexist, singleton rows are insufficient. Add a bounded interaction set using pairwise coverage plus any high-risk intersections identified by the code. At minimum model: + +- legacy or absent evidence alongside fresh valid evidence; +- corrupt or undated evidence alongside fresh valid evidence; +- irrelevant historical evidence alongside a current authoritative record; +- tied or conflicting current records; and +- state/source precedence when otherwise-valid sources disagree. + +State which source wins, why it wins, and how irrelevant or malformed records are prevented from suppressing a valid authority or granting access on their own. Do not generate an unbounded Cartesian product; select pairwise and risk-driven intersections and record why they cover the precedence rules. + +**Tests:** require a positive singleton control, at least one wrong-input case, and the applicable mixed-state intersections above through the real decision boundary. Map each test to stable decision-row IDs. + ## Contract-signal precedence When a dependency or peer returns multiple signals affecting the same decision (transport status, structured body fields, error codes, reason strings, headers, metadata, exit status, sentinel files), include rows for each signal and for conflicts between signals. State which signal wins and how unknown/missing signals degrade. Cover transport-vs-payload conflicts, older peers omitting newer fields, and newer peers sending unknown values. diff --git a/plugins/code/skills/decision-table/references/review-prevention.md b/plugins/code/skills/decision-table/references/review-prevention.md index cda447e..6680c09 100644 --- a/plugins/code/skills/decision-table/references/review-prevention.md +++ b/plugins/code/skills/decision-table/references/review-prevention.md @@ -39,6 +39,11 @@ Coverage and evidence disposition rule: any `covered` or `already covered` dispo 29. **Coverage claim without a fail-closed test** where a `covered` or `already covered` disposition cites no test, or cites only a happy-path assertion, instead of a named test that fails closed on the wrong-input or negative case (including security findings). 30. **Integration-boundary coverage backed only by helper tests** where a CLI, route handler, package export, worker/job, replay path, ingest pipeline, attribution pipeline, or public API is claimed covered by a pure helper test that never exercises production wiring through the real boundary. 31. **Non-applicability claim without source evidence** where a `not applicable` disposition says a surface has no consumers, no exported contract, no filesystem path, no legacy records, no compatibility path, or no untrusted input without grep output, export/package inventory, call-site inventory, schema/query inventory, or exact code references proving the claim. +32. **Singleton-only source coverage** where multiple evidence, authority, history, cache, or fallback sources can coexist but the table never exercises legacy/absent plus fresh valid, corrupt/undated plus fresh valid, irrelevant historical plus current authoritative, tied/conflicting current records, or source/state precedence. +33. **Executable-twin parity inferred from separate tests** where a helper, SQL predicate, route, worker, producer, batch path, or recovery path reimplements one policy but no shared scenario corpus exercises every twin through its real production boundary. +34. **Source-shape assertion pins missing policy** where a negative source-string, AST, or SQL-shape assertion requires a predicate, identity term, join, or branch to be absent and thereby preserves divergence instead of testing behavior. +35. **Required test not traceable to decision rows** where a test is listed without stable row IDs, or lacks the wrong-input or mixed-state negative case needed to distinguish the row's branch. +36. **Not-aligned status treated as success** where a caller proceeds to PR creation, merge, completion, or a success closeout despite unresolved `Final Alignment Status: Not aligned`. ## Contract-Heavy Review Surface @@ -83,3 +88,7 @@ For contract-heavy work, also explicitly review: - duplicated policy logic or wire-contract constants that can drift between intended-parity entry paths - the same eligibility, completion, or terminal predicate enforced at a gate but re-derived independently at another site (predicate at a gate vs. re-derived in a list filter, scoped vs. unscoped routing, a terminal or disposition check, a batch path, or an early short-circuit), where the two can disagree unless they share a helper or are tied together by a parity test - whether `Final Alignment Status` is still defensible given the implemented compatibility and failure behavior +- bounded pairwise/high-risk coexistence coverage when multiple evidence, authority, historical, cache, or fallback sources can be present at once +- shared-corpus production-boundary parity for every executable twin; source-string, AST-presence, and SQL-shape assertions are supplemental only +- stable decision-row IDs mapped to every required test, including wrong-input and mixed-state negative cases +- a hard stop on PR, merge, and completion whenever `Final Alignment Status` is `Not aligned`, with the blocker and exact next action surfaced in the closeout diff --git a/plugins/code/tools/python/test_decision_table_skill_contract.py b/plugins/code/tools/python/test_decision_table_skill_contract.py new file mode 100644 index 0000000..f6635a1 --- /dev/null +++ b/plugins/code/tools/python/test_decision_table_skill_contract.py @@ -0,0 +1,57 @@ +"""Contract checks for decision-table workflow hard stops and test realism.""" + +from __future__ import annotations + +from pathlib import Path + + +PLUGIN_ROOT = Path(__file__).resolve().parents[2] +SKILL_ROOT = PLUGIN_ROOT / "skills" / "decision-table" + + +def read_skill_file(relative_path: str) -> str: + return (SKILL_ROOT / relative_path).read_text() + + +def test_not_aligned_is_a_terminal_workflow_stop() -> None: + skill = read_skill_file("SKILL.md") + artifact_format = read_skill_file("references/artifact-format.md") + + for text in (skill, artifact_format): + assert "Not aligned" in text + assert "terminal" in text + assert "PR" in text + assert "merge" in text + assert "completion" in text + assert "next action" in text + + +def test_source_interactions_and_precedence_are_required() -> None: + edge_cases = read_skill_file("references/edge-cases.md") + + assert "legacy or absent evidence alongside fresh valid evidence" in edge_cases + assert "corrupt or undated evidence alongside fresh valid evidence" in edge_cases + assert "irrelevant historical evidence alongside a current authoritative record" in edge_cases + assert "tied or conflicting current records" in edge_cases + assert "state/source precedence" in edge_cases + assert "unbounded Cartesian product" in edge_cases + + +def test_executable_twins_require_real_boundary_shared_corpus() -> None: + edge_cases = read_skill_file("references/edge-cases.md") + review_prevention = read_skill_file("references/review-prevention.md") + + for text in (edge_cases, review_prevention): + assert "shared scenario corpus" in text + assert "real production boundary" in text + assert "Source-string" in text or "source-string" in text + assert "SQL-shape" in text + assert "supplement" in text + + +def test_required_tests_map_to_rows_and_negative_cases() -> None: + artifact_format = read_skill_file("references/artifact-format.md") + + assert "Row ID" in artifact_format + assert "Decision Row IDs" in artifact_format + assert "Wrong-Input / Mixed-State Negative Case" in artifact_format