From b41f9bf3deeccb0b7c8b86fb4f3ae6f103f54926 Mon Sep 17 00:00:00 2001 From: "Ioannis L." <44038245+blitzcrieg1@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:26:07 +0300 Subject: [PATCH] fix(funding): the manifest guard billed contributors for my public copy `test_funding_manifest.py` pinned the benchmark case count in `funding.json` to `len(load_corpus())`. The next contributor pull request to add corpus cases, #145, went red on both platforms with `assert 54 == 57`, against a file the contributor had never touched and had no business editing. The guard worked. It was in the wrong place. A public claim of mine is not a contributor's problem to fix, and a red check on their branch for it is a tax they cannot pay. So the manifest no longer quotes counts at all, and this no longer checks them. What stays pinned is what moves on a release rather than on a contribution: the version and the licence, both of which change in a commit I make anyway. The counts live in the README, where `test_readme_claims.py` guards them and the change and the claim arrive in the same pull request. This is the same rule CONTRIBUTORS.md landed on yesterday, applied consistently this time: stamp a number with a date when it moves on its own, pin it only when the person changing it is the person who can fix it. Co-Authored-By: Claude Opus 5 --- .../tests/test_funding_manifest.py | 35 ++++++++++++------- funding.json | 2 +- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/apps/orchestrator/tests/test_funding_manifest.py b/apps/orchestrator/tests/test_funding_manifest.py index 045d7b6..edde7d6 100644 --- a/apps/orchestrator/tests/test_funding_manifest.py +++ b/apps/orchestrator/tests/test_funding_manifest.py @@ -10,9 +10,17 @@ for the version string. This does the same for the manifest, so a figure that drifts fails here instead of in front of a funder. -Only the claims that a command can settle are pinned. The download count and -the test count are stamped with a date in the manifest itself and are left -alone, because a number that says when it was true does not become false. +Only claims that move on a *release* are pinned: the version and the licence. +Anything that moves when somebody contributes is not. + +That distinction was learned the expensive way. The first version of this file +pinned the benchmark case count too, and the next contributor PR to add corpus +cases went red on `funding.json`, a file they had never touched and had no +business editing. A guard that taxes contributors for the maintainer's public +copy is worse than the drift it prevents, so the manifest no longer quotes +counts at all and this no longer checks them. The README quotes them, and +`test_readme_claims.py` guards that, where the change and the claim live in the +same pull request. """ from __future__ import annotations @@ -23,7 +31,6 @@ import pytest -from agentmetry.core.audit.detection.benchmark import load_corpus from agentmetry.core.version import __version__ MANIFEST = Path(__file__).resolve().parents[3] / "funding.json" @@ -64,14 +71,6 @@ def test_quoted_version_matches_the_package(): ) -def test_quoted_benchmark_case_count_matches_the_corpus(): - quoted = re.search(r"(\d+) case detection benchmark", _project_description()) - assert quoted, "funding.json no longer quotes a case count; update this test or the manifest" - assert int(quoted.group(1)) == len(load_corpus()), ( - "funding.json quotes a different case count than the corpus holds" - ) - - def test_declared_licence_matches_the_package_metadata(): pyproject = (Path(__file__).resolve().parents[1] / "pyproject.toml").read_text( encoding="utf-8" @@ -83,3 +82,15 @@ def test_declared_licence_matches_the_package_metadata(): assert licences == [f"spdx:{declared.group(1)}"], ( "funding.json declares a different licence than the package does" ) + + +def test_the_manifest_quotes_no_count_that_a_contribution_can_move(): + """The regression that made this change necessary. + + A number here is a number somebody else's pull request can invalidate. + """ + description = _project_description() + assert not re.search(r"\d[\d,]* (?:tests|cases|case )", description), ( + "funding.json quotes a count that moves when somebody contributes; the " + "README is the place for those" + ) diff --git a/funding.json b/funding.json index 8d08bb4..df0285d 100644 --- a/funding.json +++ b/funding.json @@ -15,7 +15,7 @@ { "guid": "agentmetry", "name": "Agentmetry", - "description": "AI coding agents run shell commands, edit files and call third-party tools hundreds of times an hour, largely unattended. What they actually did is recorded nowhere by default. The agent transcript is a conversation rather than a record of effects, the editor keeps no durable log, and endpoint security sees only that one process spawned another, which is what a coding agent looks like when it is working correctly.\n\nAgentmetry records what an agent did at the tool boundary, correlates sequences into detections, and writes a hash-chained JSONL trail the operator owns, forwarding into whatever SIEM they already run. It is local-first: there is no vendor cloud, and no telemetry reaches the maintainer. Tool arguments are hashed inside the hook process, so plaintext never crosses the wire.\n\nVerifiable as of 2026-09-02: version 0.7.0 on PyPI, Apache-2.0, 1,186 tests, a 54 case detection benchmark at zero misses and zero false positives, MITRE ATT&CK and ATLAS mappings, and four SIEM adapters. Every one of those figures is printed by a command in the README.\n\nFunding would go to making the format underneath it common rather than ours: a standalone agent session record specification, mappings into Elastic Common Schema, OCSF, Splunk CIM and Google SecOps UDM, and a conformance suite so a third party can verify their own mapping without reading our Python. The success condition is other vendors adopting the fields, including ones who compete with us.", + "description": "AI coding agents run shell commands, edit files and call third-party tools hundreds of times an hour, largely unattended. What they actually did is recorded nowhere by default. The agent transcript is a conversation rather than a record of effects, the editor keeps no durable log, and endpoint security sees only that one process spawned another, which is what a coding agent looks like when it is working correctly.\n\nAgentmetry records what an agent did at the tool boundary, correlates sequences into detections, and writes a hash-chained JSONL trail the operator owns, forwarding into whatever SIEM they already run. It is local-first: there is no vendor cloud, and no telemetry reaches the maintainer. Tool arguments are hashed inside the hook process, so plaintext never crosses the wire.\n\nVerifiable as of 2026-09-04: version 0.7.0 on PyPI, Apache-2.0, a detection benchmark that runs at zero misses and zero false positives, MITRE ATT&CK and ATLAS mappings, and four SIEM adapters. Case and test counts are deliberately not quoted here, because they move every time somebody contributes. The README prints the current ones and a test checks that it still prints them correctly.\n\nFunding would go to making the format underneath it common rather than ours: a standalone agent session record specification, mappings into Elastic Common Schema, OCSF, Splunk CIM and Google SecOps UDM, and a conformance suite so a third party can verify their own mapping without reading our Python. The success condition is other vendors adopting the fields, including ones who compete with us.", "webpageUrl": { "url": "https://agentmetry.ai", "wellKnown": "https://agentmetry.ai/.well-known/funding-manifest-urls"