From 2139aea1e89b057171e892242dd6c6d0f9a285c3 Mon Sep 17 00:00:00 2001 From: "Ioannis L." <44038245+blitzcrieg1@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:25:59 +0300 Subject: [PATCH] docs: the README still said fifteen rules, and now a test says otherwise 0.7.0 delisted `autonomous-unapproved-write` because it could not fire on any capture surface a user installs. That release existed to make published claims true. It updated docs/detection-rules.md, the Sigma pack and the whitepaper, and missed the most read document in the repository. The number was spelled out as a word, "Fifteen built-in rules ship today", so it matched no numeric search and survived three passes over the file. It is now fourteen published plus one experimental. The Sigma count was worse: the README said four rules while the pack ships 23. That one drifted because the pack is generated, so it grows without anyone editing prose about it. Both are now pinned. test_readme_claims.py already stopped the corpus counts rotting; it now covers the rule total and the Sigma pack size, and I checked that reintroducing either stale number fails the suite rather than trusting that it would. Also links the OWASP AST10 coverage table from the detection section, since a reader following the rules link is exactly the reader who wants to know which risks this does not cover. Co-Authored-By: Claude Opus 5 --- README.md | 9 +++- apps/orchestrator/tests/test_readme_claims.py | 43 +++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9258a9d..d2b98f0 100644 --- a/README.md +++ b/README.md @@ -552,7 +552,7 @@ Hardening beyond these honesty notes is tracked in | **Agent frameworks** | [CrewAI](adapters/crewai/) · [OpenSRE](adapters/opensre/) · Microsoft AGT audit files (Semantic Kernel, AutoGen, LangGraph via AGT) | LangChain · AutoGen native | | **MCP transport** | Stdio audit proxy (wrap any MCP server command) | SSE / streamable HTTP proxy | | **Observability / SIEM** | Loki · Grafana · Elastic ECS · Splunk HEC · CloudEvents v1.0 (Knative, EventBridge, Event Grid, Dapr, Kafka) · generic webhook | Datadog · New Relic | -| **Detection formats** | In-engine sequence rules · LogQL · Elastic · Splunk · [Sigma pack](docs/integrations/sigma/README.md) (4 rules) | STIX/TAXII export | +| **Detection formats** | In-engine sequence rules · LogQL · Elastic · Splunk · [Sigma pack](docs/integrations/sigma/README.md) (23 rules) | STIX/TAXII export | | **Policy engines** | Regex DLP manifest (`agentmetry/policies/dlp/`) · tool allow/deny YAML (`agentmetry/policies/tool/`) | OPA / Rego policy-as-code | | **Compliance docs** | [ISO 42001 mapping](docs/compliance/iso-42001-mapping.md) · [AI Act checklist](docs/compliance/ai-act-deployer-checklist.md) | SOC 2 evidence templates | @@ -570,12 +570,17 @@ Matching is **ordered within a session**, not a threshold on one row. `credential-exfil` requires credential access (T1552) *then* network egress (TA0011), in that order. Reversed, it does not fire. -Fifteen built-in rules ship today (plus YAML count rules), covering credential exfiltration, guardrail bypass, +Fourteen published rules ship today, plus one held back as experimental and +the YAML count rules. They cover credential exfiltration, guardrail bypass, download cradles, supply-chain merges, recon-then-collect, and both published [Agent Data Injection](https://arxiv.org/abs/2607.05120) chains. **[Every rule, how ordered matching works, and the research behind it →](docs/detection-rules.md)** +That page also maps this project against the +[OWASP Agentic Skills Top 10](https://owasp.org/www-project-agentic-skills-top-10/): +two risks covered, three partial, and five that are somebody else's job. + ```http GET /api/v1/audit/detections/{correlation_id} ``` diff --git a/apps/orchestrator/tests/test_readme_claims.py b/apps/orchestrator/tests/test_readme_claims.py index a52ae4d..d415644 100644 --- a/apps/orchestrator/tests/test_readme_claims.py +++ b/apps/orchestrator/tests/test_readme_claims.py @@ -179,3 +179,46 @@ def test_readme_does_not_reintroduce_the_overstatement(): # happens without updating the README, the invitation stops working. assert "agentmetry/api/routes/audit.py" in text assert "agentmetry/core/audit/identity.py" in text + +def test_readme_rule_count_matches_the_registry(): + """0.7.0 delisted a rule and the README kept saying fifteen. + + That release existed to make published claims true. It fixed + `detection-rules.md`, the Sigma pack and the whitepaper, and missed the most + read document in the repository, where the number sat spelled out as a word + and so matched no numeric search. + """ + from agentmetry.core.audit.detection.rules import BUILTIN_RULE_IDS + + text = _readme().lower() + words = { + 12: "twelve", 13: "thirteen", 14: "fourteen", + 15: "fifteen", 16: "sixteen", 17: "seventeen", + } + published = len(BUILTIN_RULE_IDS) + correct = words[published] + + for count, word in words.items(): + if count == published: + continue + for phrase in (f"{word} built-in rules", f"{word} published rules", f"{count} built-in rules"): + assert phrase not in text, ( + f"README says {phrase!r} but the registry publishes {published}. " + f"Use {correct!r}, and remember an experimental rule is not published." + ) + + +def test_readme_sigma_count_matches_the_pack(): + """The pack is generated, so its size moves without anyone editing prose.""" + import re + + sigma_dir = Path(__file__).resolve().parents[3] / "docs" / "integrations" / "sigma" + if not sigma_dir.is_dir(): + pytest.skip("sigma pack not present") + shipped = len(list(sigma_dir.glob("*.yml"))) + + match = re.search(r"Sigma pack\]\([^)]+\)\s*\((\d+) rules\)", _readme()) + assert match, "README no longer states a Sigma rule count; update this test or the README" + assert int(match.group(1)) == shipped, ( + f"README claims {match.group(1)} Sigma rules, pack ships {shipped}" + )