tests: property-based tests for the parsers and id derivations

.chock/dependency-allowlist.txt

@@ -24,3 +24,4 @@ ruff
 hatchling
 matplotlib
 markdown
+hypothesis

.github/dependabot.yml

@@ -20,6 +20,11 @@ updates:
       default-days: 7
     open-pull-requests-limit: 5
     labels: [dependencies]
+    ignore:
+      # matplotlib 3.11 dropped Python 3.10, which this package still supports; a
+      # feature bump would also rewrite the floor in ci.in. Patch releases still flow.
+      - dependency-name: matplotlib
+        update-types: ["version-update:semver-major", "version-update:semver-minor"]
     groups:
       # One PR for the routine patch churn, individual PRs for anything that could
       # change behaviour.

pyproject.toml

@@ -47,7 +47,7 @@ dependencies = ["jsonschema>=4.18,<5", "pyyaml>=6"]
 
 [project.optional-dependencies]
 efficacy = ["anthropic>=1.0"]
-dev = ["pytest>=8", "ruff>=0.6", "matplotlib>=3.8", "markdown>=3.7"]
+dev = ["pytest>=8", "ruff>=0.6", "matplotlib>=3.8", "markdown>=3.7", "hypothesis>=6.100"]
 
 [project.urls]
 Homepage = "https://github.com/open-coder-ai/context-report"

requirements/ci.in

@@ -2,5 +2,7 @@ jsonschema>=4.18,<5
 pyyaml>=6
 pytest>=8
 ruff>=0.6
-matplotlib>=3.8
+# matplotlib 3.11 dropped Python 3.10; lift the ceiling when requires-python does.
+matplotlib>=3.8,<3.11
 markdown>=3.7
+hypothesis>=6.100

requirements/ci.txt

@@ -72,7 +72,9 @@ cycler==0.12.1 \
 exceptiongroup==1.3.1 \
     --hash=sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219 \
     --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598
-    # via pytest
+    # via
+    #   hypothesis
+    #   pytest
 fonttools==4.64.0 \
     --hash=sha256:043f6c572bf236f2a76e762c25f841daea11e8fc03e78088d7be66e0c5b4e4c0 \
     --hash=sha256:06b6409b868494556a831ae33b2d9a090476c37516b38d70f45a9720b460d423 \
@@ -141,14 +143,97 @@ fonttools==4.64.0 \
     --hash=sha256:fa75c7970bc6bca340cc6e20f20f069201bfcb50094c31a536fd99724d1d01ca \
     --hash=sha256:ff7aff4637fbf71394df139c63ccfe08a47aa4252d2f91224ddb3335c716c925
     # via matplotlib
+hypothesis==6.167.1 \
+    --hash=sha256:04f807b85d425a7005e8a24498ca832bf5590f0d306737471d94c842569cecef \
+    --hash=sha256:0807ae8d399162827fc1c396ab4c697a41921c2a2baacfada439771e9dc2b867 \
+    --hash=sha256:0ff5ad833480c1e34ae902cb52fc00802b08ac6c87bd22d7e5f04fd925869608 \
+    --hash=sha256:1434bbd25d05aaf75c4e6829b4cad8e9931b690f840d92b747b2e6e5af575922 \
+    --hash=sha256:163e4cebd4b2380ff92f5d36bd04697e82b13973794440694da768521e1e2eab \
+    --hash=sha256:1937ae4e23f7dde6d4202d3d08c2633bcd535a091bdf866b8799abaabcb1e6f0 \
+    --hash=sha256:19b334180260de636a0b3017dd96d63709da0c4182c9cb28f52dd6cda78dc933 \
+    --hash=sha256:21a7122ddf072906083e3704fe3961ecbc49d7d30a9b51bcd525d977b3afe65e \
+    --hash=sha256:22b6c3def5446016148523b74ef9da4948bec5ef05865d56c99ba187f4092663 \
+    --hash=sha256:26f8cec74c4fad7aeb0852cb34c2134b16db05d878ad3946a53337dace7016f4 \
+    --hash=sha256:27829aa89fe2e47c5c8d13b3ce31e0f53a01a98f76a4f99cfa3369ab35362f33 \
+    --hash=sha256:27ca252991fdbe2ff5c611a1cc4d972d4e009eb45292c7802faa2190f995dc50 \
+    --hash=sha256:2de4e67289f86e358732eb16b345f9d1f987c33e44b10d4ffa3ccd715dea9316 \
+    --hash=sha256:35e90c121b1518d7428a45e6b0d5c6d06e0ed9eaa567f1106e1f09dae006d6da \
+    --hash=sha256:36e83e1d7e97aaacbf6cd778e14a841344f848a674b20dfe4fe997546a6a2151 \
+    --hash=sha256:3ad7206de9c398c8da5745b69b5ba2ef45100082eeb174656490bc4f262b112c \
+    --hash=sha256:3b596efec5bd714588e3bb269544d993c5258c979f3a26f51fadf62c215d0e68 \
+    --hash=sha256:3e04f6001299708b6fd4512267b189c0b029ef1e34500deb4e4c9639023598d7 \
+    --hash=sha256:40cd5de7dd252942a08480639f5850594b1aca4a463e8a7f15e1fb6c2c3760c1 \
+    --hash=sha256:436027c9a00eb11a2ca3d608147ca2d0d623b4f02878c56a50fc3f3f58c2b41b \
+    --hash=sha256:47c99256df28555ecc2aed0e22ca17cd61c63c8c44207a07b4e402cc49661fae \
+    --hash=sha256:4819adbc5911648f6bfaeb574f276add184b4b49f54731dbde46fd71256bb157 \
+    --hash=sha256:495989cf0a5ee03f7f9598ee9efeaabf15fd861ec52b5a9d6435849453e17e5d \
+    --hash=sha256:546fef39c7aadba74bf3e592585694a71340d1775e9b3274bb3f94106dbde4b7 \
+    --hash=sha256:5c5a26d4d3dca0c84e01bde41df4cabaa5a373c7393f9eef372d19fe93b07ccd \
+    --hash=sha256:5d6614e88fd267bbd870e3ec02f8a5387897d2d573626a02f5ac06d81533afa6 \
+    --hash=sha256:5e35f98b427bf438a946203426b485dd5b62485f3d5a69a0e0862870a545e518 \
+    --hash=sha256:5ef7dd225f7df7d74d1c5a905592cd8b4cd348e6be639b189a43def8b0b5dd79 \
+    --hash=sha256:613bf10e6e490daaa88eb4bf06fb3aacf6572b887e2f9fa5d0bac1be96a18c00 \
+    --hash=sha256:627ce3bd166799a6c0ddcf1351049be5b9a772d5bce436216d42b41a935f42c0 \
+    --hash=sha256:62eefcb4d2791423626e9901c3027a6e0c5ffda2ac0b44b3c7e797ab9d2d5a4c \
+    --hash=sha256:630eb37df80b5bc4ec6f391b13caaecc06942ff5da3aadebacf85f53bbc55757 \
+    --hash=sha256:6478d19a7887731cc2afaa1ec15f62811c9ceb6fd18e5b7563e0a18399a9528f \
+    --hash=sha256:64cf8b7ac9a0cc80dad8884f81a2e50f0b79956694927d40e21e0cfa48830b8a \
+    --hash=sha256:69b92f037080cefb949c5f9683873abac12283e0dc77201df089369c1ef67e4c \
+    --hash=sha256:6c91f6f2f15b8bc6e474b824f931247c39711231e7b1b2f68277726e0ae1c728 \
+    --hash=sha256:742be00d7bb53d10634e6435e5b98f51fcdbe7ed377d473ab7387d9499c87169 \
+    --hash=sha256:769bdd9aa0af08c063327730ab6dc18b7a23837a2912f2aeaab3912f11a7e3ad \
+    --hash=sha256:769d0242531067e6daf16b6c9894fd9584139884de887023328e3c5658993597 \
+    --hash=sha256:7d7585429f2263d3ceeb3474bae3871024630a7a598e71eeb4b0dcf03e291623 \
+    --hash=sha256:803c6a98ff66cee4caf03245bfd00e442a907264031b994a3a650dc6e4786f51 \
+    --hash=sha256:8b1e393ab01b71f683ba2a783785871cc6b81a6e41017780c64a5bc0b99759ae \
+    --hash=sha256:8c385c7741893404306f9e5559ab3835432e85e7c153e25f854c502c410bbcbb \
+    --hash=sha256:8cdd2fc0232b47910e9621f8c1b5732381438055b03fcc69180e1ef3659b7e70 \
+    --hash=sha256:8f13167a4b81c93e7e051d1f02790814a6495fb79cacf3fb89560a796a2f7d00 \
+    --hash=sha256:94920ca1fae70c26b0bd3fabbeef9437ffc17a39fe85696fb9a86187d92f6dba \
+    --hash=sha256:96d5e8017a9508f06c8a61a6130cb0d0b4810847ed5c76923cb5cfb9952b31af \
+    --hash=sha256:971ce0d8a367a37c4690b83a2e7f6ef832fa3543357eb6da0da27ba078e5088a \
+    --hash=sha256:9c903b4f1c8531736fc7e8e537f47ef509756b731a32a5e5e7014e5291343acb \
+    --hash=sha256:a17a5618b6a5b84f17c8acb3bce37122647cf7a3e48b660a39d68a773bd627dd \
+    --hash=sha256:a2837c60d782eb0b8a910c541264675b9d11486e186af8c82a5e2920b5fe4fe8 \
+    --hash=sha256:a4e4de36a397cba49d949d89cbc26135977c15f9d797caa95317962ceb5b5674 \
+    --hash=sha256:af3c09428e553b1dd2f9abbc4738377c58bf6d74cb0b8b528cc1dde3a9cdfbe8 \
+    --hash=sha256:af84ce2416be2a65bc0ea18e64d2dbb9796b7692593b5b2064d60ea1d52ec1e2 \
+    --hash=sha256:b57e950f9d5c93ca335bc612e8fa8fb49abb187c3fc9d5e7d9966d52eb27d747 \
+    --hash=sha256:b6fcdc8d03b37a902262be13112d113bb4ac87edf3b08afb47f3d1210deb038a \
+    --hash=sha256:b8d90ded2ffdc7e56b5e571993f384b52fade0a7b424e614f999cc2491789970 \
+    --hash=sha256:b9c33f921ddc7fea93660eca408b25fe755516e22ec7ab21cb9951031f1cd608 \
+    --hash=sha256:bbf4f0cad201d0b8e821e82ad828b2aec99ce6d9967779eecb2cad4d4a93debd \
+    --hash=sha256:bc4e65f7c43b187f7a40964706b5ded1073e0c1839e9fb5e041d7ed973bb65fe \
+    --hash=sha256:bc73c46ce8ff93b0eb220f2b75adcbd9fcc9112078a74d3522d2532ad8069bad \
+    --hash=sha256:c25c556d51d55d94988dc0a2c716d471ff19cf9632cd33f5e6db2914d802428a \
+    --hash=sha256:c63a0d292a5dde3c0fe999892e76d8375a003ca40c0c00d763f3360f91be5b96 \
+    --hash=sha256:cbbb6bc17a5a04120a5bfd830335b8b301a22d21d2262804be330c235031b4c5 \
+    --hash=sha256:cdc7e20161f21f14c2d7a057054521db0a8c4bbb647d2e50c90c84f28e4621a0 \
+    --hash=sha256:d28118fd70e4e15ff9c308a98b312b544b6145ae45aaa3b566328c1fdee8058f \
+    --hash=sha256:d75d44bdead6679b6ee9a7c90d10207db865ca0c77c5212103b5ff421379f99e \
+    --hash=sha256:dd6a0808a2eb8b5b1ac06bca4244eee18ed2c0e7b105599e1662203d164317b5 \
+    --hash=sha256:e517be7f82a0a917758cc489a88b826b5371f56381fd94b4a8a09ce82d8de406 \
+    --hash=sha256:e56e7841514276c308c2bb4d033cf01860d0fc8c76e2b79ce748a9f123eaf83b \
+    --hash=sha256:e849f518cbc4e76ab15f2f1473c60dd3103da8d32399187325ceb84309105976 \
+    --hash=sha256:e936777d92ae27393b4a941839bbb43c1f339b5a0e394c7f3730454cdf091b3a \
+    --hash=sha256:ebb841d21156039d7da0a41fa9de4ccf468510a4e4d8144f4fe2b3f31239ef3b \
+    --hash=sha256:f6fe9c40ab14def363d9e7ab22863fa31652bd5e08f8495b34ff7bd0062b3f8d \
+    --hash=sha256:f715d912560dd4df3daab4c1fd98c01132eea7ab292f5b9e1d28fc419fe63348 \
+    --hash=sha256:fb4d87454d2459c2ccb541a4c61c92ce13058b91305ed3304695a409a1d886e4 \
+    --hash=sha256:fb9194f450417cf35f66b6c72737cc6b8f21f20567ba4d39822c64f0d1075784 \
+    --hash=sha256:fcfc2a78fc1025644f889a74684b3201f4652ce8e6694c2a01af0f100d0348cf \
+    --hash=sha256:fcfd20792f62d65729f850ea50328c1f8b09874d5960b122715b9e6784a7a547 \
+    --hash=sha256:fd202d02d129197a5e771f8a11c7d30559927284c23ec3a8bd4f37a7955964d1 \
+    --hash=sha256:ff07f98a0b230632bb2836b5dad3e94d85c114ae155a316afd251c58760958ae
+    # via -r requirements/ci.in
 iniconfig==2.3.0 \
     --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \
     --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12
     # via pytest
 jsonschema==4.26.0 \
     --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \
     --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce
-    # via -r ci.in
+    # via -r requirements/ci.in
 jsonschema-specifications==2025.9.1 \
     --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \
     --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d
@@ -299,7 +384,7 @@ kiwisolver==1.5.1 \
 markdown==3.10.3 \
     --hash=sha256:3589362618f743188b4d955b874402bc814f4f83f544dc207719f4baa7d9c45f \
     --hash=sha256:fa6c92a00a4a3c98b22728c64a935ae1928250ae65058a6ded814d2cc29a4cea
-    # via -r ci.in
+    # via -r requirements/ci.in
 matplotlib==3.10.9 \
     --hash=sha256:09218df8a93712bd6ea133e83a153c755448cf7868316c531cffcc43f69d1cc9 \
     --hash=sha256:10cc5ce06d10231c36f40e875f3c7e8050362a4ee8f0ee5d29a6b3277d57bb42 \
@@ -356,7 +441,7 @@ matplotlib==3.10.9 \
     --hash=sha256:f4399f64b3e94cd500195490972ae1ee81170df1636fa15364d157d5bdd7b921 \
     --hash=sha256:f76e640a5268850bfda54b5131b1b1941cc685e42c5fa98ed9f2d64038308cba \
     --hash=sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358
-    # via -r ci.in
+    # via -r requirements/ci.in
 numpy==2.2.6 \
     --hash=sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff \
     --hash=sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47 \
@@ -526,7 +611,7 @@ pyparsing==3.3.2 \
 pytest==9.1.1 \
     --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \
     --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c
-    # via -r ci.in
+    # via -r requirements/ci.in
 python-dateutil==2.9.0.post0 \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
@@ -605,7 +690,7 @@ pyyaml==6.0.3 \
     --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \
     --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \
     --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0
-    # via -r ci.in
+    # via -r requirements/ci.in
 referencing==0.37.0 \
     --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \
     --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8
@@ -750,11 +835,15 @@ ruff==0.16.6 \
     --hash=sha256:dcf8a73d2ff77e99dde91244b4da16feba7f14e6beeb4015dee7c5a909e99050 \
     --hash=sha256:e25cc89174874b176a157e4428d66761c2c0c006654419bf384f967f361ff1b1 \
     --hash=sha256:ecf4f068e2e123e43a26e9db4e19524cc56563912404e83bbfca375757e45a32
-    # via -r ci.in
+    # via -r requirements/ci.in
 six==1.17.0 \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
     # via python-dateutil
+sortedcontainers==2.4.0 \
+    --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \
+    --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0
+    # via hypothesis
 tomli==2.4.1 \
     --hash=sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853 \
     --hash=sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe \

tests/test_properties.py

@@ -0,0 +1,124 @@
+"""Property-based tests: the parsers and id derivations hold for any input, not only examples."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+from hypothesis import given, settings
+from hypothesis import strategies as st
+
+from context_report.efficacy import rules
+from context_report.rows import input_hash
+from context_report.run import evalcases, layout
+
+SLUG_CHARS = set("abcdefghijklmnopqrstuvwxyz0123456789-")
+
+
+@settings(max_examples=200, deadline=None)
+@given(st.text())
+def test_slug_is_a_stable_id_of_bounded_shape(text: str) -> None:
+    """Whatever the rule says, its id is short, lowercase, hyphenated, and re-slugs to itself."""
+    out = rules.slug(text)
+    assert out and set(out) <= SLUG_CHARS
+    assert len(out) <= rules._SLUG_CHARS
+    assert not out.startswith("-") and not out.endswith("-")
+    assert rules.slug(out) == out
+
+
+@settings(max_examples=100, deadline=None)
+@given(st.text())
+def test_extract_never_raises_and_ids_are_unique(tmp_path_factory: pytest.TempPathFactory, text):
+    """Any file is either rules or skips; no crash, and ids never collide within one file."""
+    path = tmp_path_factory.mktemp("subject") / "AGENTS.md"
+    path.write_text(text, encoding="utf-8")
+    found = rules.extract(path)
+    ids = [r.id for r in found.rules]
+    assert len(ids) == len(set(ids))
+    assert all(set(i) <= SLUG_CHARS for i in ids)
+    assert all(c.id and set(c.id) <= SLUG_CHARS for c in found.candidates)
+
+
+@settings(max_examples=100, deadline=None)
+@given(st.lists(st.text(), max_size=6), st.lists(st.text(), max_size=6))
+def test_input_hash_is_deterministic_and_order_sensitive(a: list[str], b: list[str]) -> None:
+    assert input_hash(*a) == input_hash(*a)
+    if a != b:
+        assert input_hash(*a) != input_hash(*b)
+
+
+_key = st.from_regex(r"\A[a-z][a-z0-9_]{0,15}\Z")
+# Plain YAML scalars only: letters, digits, spaces and a few safe marks, starting with a letter,
+# so the property is about the frontmatter split, not about YAML's own scalar grammar.
+_value = st.from_regex(r"\A[A-Za-z][A-Za-z0-9 _.-]{0,39}\Z")
+
+
+@settings(max_examples=100, deadline=None)
+@given(
+    st.dictionaries(_key, _value, max_size=4),
+    st.text(max_size=200).filter(lambda b: "\r" not in b),  # read_text normalises newlines
+)
+def test_frontmatter_round_trips_meta_and_body(tmp_path_factory, meta: dict, body: str) -> None:
+    """A prompt.md written with a YAML mapping and any body reads back as that mapping and body."""
+    path = tmp_path_factory.mktemp("case") / "prompt.md"
+    yaml_lines = "\n".join(f"{k}: {v}" for k, v in meta.items())
+    path.write_text(f"---\n{yaml_lines}\n---\n{body}", encoding="utf-8")
+    parsed_meta, parsed_body = evalcases._frontmatter(path)
+    assert set(parsed_meta) == set(meta)
+    assert parsed_body == body.strip()
+
+
+@settings(max_examples=100, deadline=None)
+@given(
+    st.lists(
+        st.fixed_dictionaries(
+            {
+                "runId": st.from_regex(r"\A[0-9]{8}T[0-9]{6}Z\Z"),
+                "startedOn": st.just("2026-09-07T00:00:00Z"),
+                "finishedOn": st.just("2026-09-07T00:01:00Z"),
+                "rows": st.lists(
+                    st.fixed_dictionaries(
+                        {
+                            "subject": st.from_regex(r"\A[a-z][a-z0-9-]{0,10}\Z"),
+                            "model": st.from_regex(r"\A[a-z]+/[a-z0-9.-]{1,12}\Z"),
+                            "result": st.sampled_from(
+                                ["PASSED", "WARNED", "FAILED", "NotAvailable"]
+                            ),
+                            "estimate": st.none()
+                            | st.tuples(st.floats(-1, 1), st.floats(-1, 1), st.floats(-1, 1)).map(
+                                list
+                            ),
+                        }
+                    ),
+                    max_size=4,
+                ),
+            }
+        ),
+        max_size=4,
+        unique_by=lambda e: e["runId"],
+    )
+)
+def test_history_table_names_every_run_and_pair(index: list[dict]) -> None:
+    table = layout.history_markdown(index)
+    for entry in index:
+        assert entry["runId"] in table
+        for row in entry["rows"]:
+            assert row["subject"] in table and row["model"] in table
+
+
+_SCRIPT = Path(__file__).resolve().parents[1] / ".github" / "scripts" / "render_pages.py"
+
+
+@settings(max_examples=100, deadline=None)
+@given(st.from_regex(r"\A[A-Za-z0-9_./-]{1,30}\Z"))
+def test_rendered_links_to_markdown_point_at_pages(target: str) -> None:
+    """Every `.md` link becomes a `.html` link; links to anything else are untouched."""
+    pytest.importorskip("markdown")
+    spec = importlib.util.spec_from_file_location("render_pages", _SCRIPT)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    html = module.render_markdown(f"[a]({target}.md) [b]({target}.json)", "t")
+    assert f'href="{target}.html"' in html
+    assert f'href="{target}.json"' in html
+    assert f'href="{target}.md"' not in html