diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4107431..bfac7db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,7 @@ jobs: assert "archsift/docs/architecture-knowledge-v1.md" in names assert "archsift/docs/architecture-knowledge-v2.md" in names assert "archsift/docs/authoring-check-v1.md" in names + assert "archsift/docs/authoring-check-v1.0.1.md" in names assert "archsift/docs/graph-change-v1.md" in names assert "archsift/docs/graph-snapshot-v1.md" in names assert "archsift/docs/method-v1.0.0.md" in names @@ -860,7 +861,7 @@ jobs: authoring_path = root / "authoring-results.json" authoring_payload = { "schema_version": 1, - "protocol_version": "1.0.0", + "protocol_version": "1.0.1", "archsift_source_commit": "a" * 40, "material_set_content_identity": ( "sha256:deca6741b7c69fbb313ed1292caa55a7a698eecb60f4a42aed849b3dcffd57ee" diff --git a/README.md b/README.md index fa34e7d..7e474b5 100644 --- a/README.md +++ b/README.md @@ -115,10 +115,13 @@ three pass. The initial simulated cohort has run and its committed run and no human-participant claim is made. A completed cohort is checked offline with exactly `archsift usability-results usability-results.json`. -The [simulated assisted-authoring protocol](docs/authoring-check-v1.md) freezes a synthetic material +The [simulated assisted-authoring protocol 1.0.1](docs/authoring-check-v1.0.1.md) freezes a synthetic material set, six objective CLI milestones, exactly four fresh sessions using distinct agent products, and a three-of-four success threshold. The protocol publishes no answer dossier or expected verdict and -retains only privacy-bounded outcome metadata. No cohort has been run yet. A future completed result +retains only privacy-bounded outcome metadata. ArchSift and every CLI milestone remain offline with +outbound sockets blocked; only the separately controlled external agent product's ordinary model +transport is permitted, with browsing, search, retrieval plugins, and private context prohibited. +No cohort has been run yet. A future completed result will be checked offline with exactly `archsift authoring-results authoring-results.json`; until such a record exists and validates, no assisted-authoring success claim is made. diff --git a/SECURITY.md b/SECURITY.md index 5762029..fe543eb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -33,3 +33,9 @@ records, prompts, command output, and transcripts are temporary private data and committed. `authoring-results` reads only one regular JSON file beneath the current directory, rejects links and escaping paths, performs no network access, writes nothing, and accepts only the strict privacy-bounded result fields in its packaged schema. + +Protocol 1.0.1 distinguishes ArchSift's offline boundary from the external author's transport. +Every ArchSift milestone runs with outbound sockets blocked. A cohort operator may separately +authorise an agent product's ordinary model transport outside ArchSift, but browser/search access, +retrieval plugins, network source lookup, private repositories, and continued private context are +not eligible protocol inputs. diff --git a/docs/authoring-check-v1.0.1.md b/docs/authoring-check-v1.0.1.md new file mode 100644 index 0000000..a4f709d --- /dev/null +++ b/docs/authoring-check-v1.0.1.md @@ -0,0 +1,123 @@ +# Simulated assisted-authoring check — protocol 1.0.1 + +Protocol 1.0.1 supersedes protocol 1.0.0 before the first cohort. It preserves the material set, +milestones, privacy contract, and success threshold while clarifying the boundary between +ArchSift's offline runtime and an external authoring product's ordinary user-controlled model +transport. Protocol 1.0.0 remains frozen at [authoring-check-v1.md](authoring-check-v1.md). At +publication, no cohort has been run and no success claim is made. + +This is an independent, fully synthetic check of whether agent products can use ArchSift's public +authoring surfaces to produce and assess a structurally valid decision dossier. It does not measure +human usability, decision quality in a real organisation, or the truth of an architecture +recommendation. + +## Frozen criterion + +Run exactly four fresh sessions using four distinct agent products. Each session starts in a fresh +environment, receives only the instructions and synthetic material named below, and binds to the +same full 40-character commit of ArchSift on which this protocol and material set have landed. The +criterion is met when at least three of the four sessions complete every objective milestone with +no maintainer intervention. + +A failed or interrupted session remains failed. Do not repair, rerun, replace, or selectively omit +a completed session. A later attempt is a new precommitted four-session cohort and result record. + +## Transport and source boundary + +ArchSift, every ArchSift CLI subprocess, and all local validation run with outbound sockets blocked. +ArchSift never invokes, embeds, imports, or depends on an agent product, model client, model API, or +network service. + +The external agent product may use only its ordinary user-controlled model transport. That +transport is selected and disclosed by the cohort operator, runs outside ArchSift, and is not a +product dependency or an ArchSift transmission. The session must not use a browser, web search, +network source lookup, retrieval plugin, remote shell, private repository, continued conversation, +or any material beyond the frozen public inputs. This distinction follows FR-018: an external +author may operate under the user's control without making ArchSift an online authoring service. + +## Frozen material + +The only case source material is the packaged `authoring-material/` set: + +- `brief.md`: a fictional domain-neutral review brief; +- `repository/required_fields.py`: inert illustrative source text; +- `repository/routing_rules.py`: inert illustrative source text; and +- `manifest-v1.json`: byte lengths, SHA-256 identities, a synthetic repository commit, and the + canonical material-set identity. + +The repository snippets are data for `register-repository`; they are not an executable project. +They contain no completed dossier, expected verdict, answer key, or reference output. Sessions +receive no private case, real repository, personal data, previous result, transcript, or +maintainer-authored dossier. + +## Session instructions + +Give every agent product these instructions without product-specific hints: + +1. Start in a fresh temporary working directory with ArchSift installed from the cohort's bound + source checkout or built wheel. Do not browse, search, fetch sources, or inspect other + workspaces. Run ArchSift commands through the supplied socket-blocked environment. +2. Read `authoring-material/brief.md` and `authoring-material/manifest-v1.json`. Treat every fact + not stated by the material as an assumption, estimate, missing item, or unknown. Never invent an + observation or artefact. +3. Run `archsift init synthetic-routing`. +4. Register the brief with `archsift register-document`, and register exactly the two repository + files with `archsift register-repository` using the full synthetic commit identity in the + manifest. Registration is inert: do not run or import the snippets. +5. Inspect the complete schema-version-3 contract with + `archsift dossier-schema --schema-version 3 --json`. +6. Author `synthetic-routing/case.yaml` from the frozen material. Cite registrations only where the + schema permits them and preserve the evidence authorship and attestation boundary. +7. Run `archsift prerequisites synthetic-routing --json` and complete every prerequisite that the + synthetic material supports. Leave unsupported facts explicitly missing or unknown. +8. Run `archsift validate synthetic-routing` and then `archsift assess synthetic-routing`. +9. Stop. Report only milestone exit outcomes to the cohort harness. Do not return dossier text, + reports, record bytes, paths, prompts, command output, or transcripts. + +The harness supplies the installed executable path, frozen material path, temporary workspace, +socket-blocking environment, and these instructions. Any additional advice, correction, command, +file edit, retry after a milestone failure, or interpretation from a maintainer is intervention and +makes that session fail. + +## Objective milestones + +Each session records exactly these six milestones: + +- `register_material`: both registrations return exit `0` and bind the manifest identities; +- `inspect_schema`: schema version 3 is emitted successfully before dossier authoring completes; +- `author_dossier`: one schema-version-3 dossier is written without a supplied answer key; +- `complete_prerequisites`: the command runs successfully and the agent responds to its worklist; +- `validate`: final validation returns exit `0`; and +- `assess`: final assessment returns exit `0` and creates the canonical outputs. + +A session passes only if every milestone is `pass` and `maintainer_intervention` is `false`. +Milestone assessment is mechanical from command outcomes and file existence; it does not score the +recommendation or authored prose. Any valid verdict is acceptable. + +## Privacy-bounded result + +Record only the fields allowed by +`src/archsift/schemas/authoring-results-v1.schema.json`: pseudonymous session ID, public agent +product/model and harness version, coarse operating-system/Python/install-mode metadata, six +milestone outcomes, intervention flag, derived session result, and a short non-sensitive failure +reason. The cohort record also binds protocol 1.0.1, the exact ArchSift source commit, and the +frozen material-set identity. + +Never commit prompts, transcripts, generated dossiers, workspaces, command output, absolute paths, +user or account names, machine identifiers, credentials, real organisation data, or personal data. +Delete temporary session workspaces after extracting the bounded fields. The strict schema rejects +all extra fields and limits every retained string. + +## Offline validation + +Place the completed record at `authoring-results.json` beneath the current directory and run: + +```console +archsift authoring-results authoring-results.json +``` + +The validator performs no network access and writes nothing. A conforming record with three or +four passing sessions exits `0` with `criterion-met`. A valid record below threshold exits `12` +with `criterion-not-met`. Malformed JSON exits `10`, unsupported schema/protocol versions exit +`11`, contract or consistency failures exit `12`, unsafe paths exit `13`, and unavailable files +exit `14`. `--json` emits a stable machine-readable summary; `--quiet` emits nothing. diff --git a/docs/usage.md b/docs/usage.md index 250f344..b5bd71a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -376,7 +376,7 @@ errors are not accepted cohort evidence. ### `archsift authoring-results ` Validates one completed simulated assisted-authoring cohort against -[protocol 1.0.0](authoring-check-v1.md) and the packaged privacy-bounded result schema. Exactly four +[protocol 1.0.1](authoring-check-v1.0.1.md) and the packaged privacy-bounded result schema. Exactly four fresh sessions from four distinct agent products are required. A session passes only when material registration, schema inspection, dossier authoring, prerequisite completion, validation, and assessment all pass without maintainer intervention. Three or four passing sessions exit `0` with @@ -389,6 +389,12 @@ links and escaping paths, performs no network access, writes nothing, and never workspace. `--json` returns protocol, session, pass-count, criterion, diagnostic, and exit-code fields; `--quiet` returns only the exit status. +Protocol 1.0.1 requires outbound sockets to be blocked for ArchSift and every local milestone. The +external authoring product may use its ordinary user-controlled model transport outside ArchSift, +but may not browse, search, retrieve network sources, load plugins or private context, or continue a +prior session. The superseded pre-cohort [protocol 1.0.0](authoring-check-v1.md) remains frozen and +its result version remains supported. + ### `archsift method-review-results ` Validates one completed architecture-method review result against the frozen diff --git a/pyproject.toml b/pyproject.toml index 64cb5dd..2bc6895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ exclude = [ "docs/usability-check-v1.md" = "archsift/docs/usability-check-v1.md" "docs/usability-check-v2.md" = "archsift/docs/usability-check-v2.md" "docs/authoring-check-v1.md" = "archsift/docs/authoring-check-v1.md" +"docs/authoring-check-v1.0.1.md" = "archsift/docs/authoring-check-v1.0.1.md" "authoring-material/brief.md" = "archsift/authoring-material/brief.md" "authoring-material/manifest-v1.json" = "archsift/authoring-material/manifest-v1.json" "authoring-material/repository/required_fields.py" = "archsift/authoring-material/repository/required_fields.py" diff --git a/src/archsift/authoring_results.py b/src/archsift/authoring_results.py index 32544ca..fae9f3e 100644 --- a/src/archsift/authoring_results.py +++ b/src/archsift/authoring_results.py @@ -16,7 +16,9 @@ from archsift.diagnostics import Diagnostic, ExitCode -PROTOCOL_VERSION = "1.0.0" +PROTOCOL_VERSION_1_0_0 = "1.0.0" +PROTOCOL_VERSION = "1.0.1" +SUPPORTED_PROTOCOL_VERSIONS = (PROTOCOL_VERSION_1_0_0, PROTOCOL_VERSION) RESULT_SCHEMA_VERSION = 1 REQUIRED_SESSION_COUNT = 4 REQUIRED_PASS_COUNT = 3 @@ -32,7 +34,7 @@ "sha256:deca6741b7c69fbb313ed1292caa55a7a698eecb60f4a42aed849b3dcffd57ee" ) MAX_RESULT_BYTES = 64 * 1024 -_REQUIREMENT = "AUTHORING-1.0.0" +_REQUIREMENT = "AUTHORING-RESULTS-v1" @dataclass(frozen=True, slots=True) @@ -296,7 +298,10 @@ def _unsupported(payload: dict[str, object]) -> AuthoringValidationResult: "authoring-results-version-unsupported", "The declared authoring-result schema and protocol versions are unsupported.", field, - f"Use schema version {RESULT_SCHEMA_VERSION} with protocol {PROTOCOL_VERSION}.", + ( + f"Use schema version {RESULT_SCHEMA_VERSION} with protocol " + f"{PROTOCOL_VERSION_1_0_0} or {PROTOCOL_VERSION}." + ), ), ), protocol_version=protocol if type(protocol) is str else None, @@ -304,6 +309,11 @@ def _unsupported(payload: dict[str, object]) -> AuthoringValidationResult: def _validate_payload(payload: object) -> AuthoringValidationResult: + declared_protocol = ( + payload.get("protocol_version") + if type(payload) is dict and type(payload.get("protocol_version")) is str + else None + ) errors = sorted( _schema_validator().iter_errors(payload), key=lambda error: tuple((type(part).__name__, repr(part)) for part in error.absolute_path), @@ -316,10 +326,10 @@ def _validate_payload(payload: object) -> AuthoringValidationResult: "authoring-results-contract", "The result data does not match the authoring-results-v1 contract.", _json_path(errors[0].absolute_path), - "Correct the named field using protocol 1.0.0 and the packaged JSON schema.", + "Correct the named field using a supported protocol and the packaged schema.", ), ), - protocol_version=PROTOCOL_VERSION, + protocol_version=cast(str | None, declared_protocol), ) result_payload = cast(dict[str, object], payload) sessions = cast(list[dict[str, object]], result_payload["sessions"]) @@ -341,7 +351,7 @@ def _validate_payload(payload: object) -> AuthoringValidationResult: "authoring-agent-product-duplicate", "Agent product names must be unique within the four-session cohort.", "$.sessions", - "Use four distinct agent products as required by protocol 1.0.0.", + "Use four distinct agent products as required by the supported protocol.", ) ) if result_payload["material_set_content_identity"] != MATERIAL_SET_CONTENT_IDENTITY: @@ -393,7 +403,7 @@ def _validate_payload(payload: object) -> AuthoringValidationResult: return _result( ExitCode.VALIDATION_FAILED if diagnostics else ExitCode.SUCCESS, diagnostics, - protocol_version=PROTOCOL_VERSION, + protocol_version=cast(str, result_payload["protocol_version"]), session_count=len(sessions), passed_session_count=passed, ) @@ -419,7 +429,7 @@ def validate_authoring_results(path: Path) -> AuthoringValidationResult: return _malformed("The authoring-result file is not strict JSON.") if type(payload) is dict and ( payload.get("schema_version") != RESULT_SCHEMA_VERSION - or payload.get("protocol_version") != PROTOCOL_VERSION + or payload.get("protocol_version") not in SUPPORTED_PROTOCOL_VERSIONS ): return _unsupported(payload) return _validate_payload(payload) @@ -428,10 +438,12 @@ def validate_authoring_results(path: Path) -> AuthoringValidationResult: __all__ = [ "MATERIAL_SET_CONTENT_IDENTITY", "PROTOCOL_VERSION", + "PROTOCOL_VERSION_1_0_0", "REQUIRED_MILESTONES", "REQUIRED_PASS_COUNT", "REQUIRED_SESSION_COUNT", "RESULT_SCHEMA_VERSION", + "SUPPORTED_PROTOCOL_VERSIONS", "AuthoringValidationResult", "validate_authoring_results", ] diff --git a/src/archsift/schemas/authoring-results-v1.schema.json b/src/archsift/schemas/authoring-results-v1.schema.json index f79a24a..7e26357 100644 --- a/src/archsift/schemas/authoring-results-v1.schema.json +++ b/src/archsift/schemas/authoring-results-v1.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/maximalfocus/archsift/schemas/authoring-results-v1.schema.json", - "title": "ArchSift simulated assisted-authoring results 1.0.0", + "title": "ArchSift simulated assisted-authoring results v1", "type": "object", "additionalProperties": false, "required": [ @@ -14,7 +14,7 @@ ], "properties": { "schema_version": { "const": 1 }, - "protocol_version": { "const": "1.0.0" }, + "protocol_version": { "enum": ["1.0.0", "1.0.1"] }, "archsift_source_commit": { "type": "string", "pattern": "^[0-9a-f]{40}$" diff --git a/tests/test_authoring_results.py b/tests/test_authoring_results.py index 6106579..b9e5fdc 100644 --- a/tests/test_authoring_results.py +++ b/tests/test_authoring_results.py @@ -12,6 +12,7 @@ from archsift.authoring_results import ( MATERIAL_SET_CONTENT_IDENTITY, PROTOCOL_VERSION, + PROTOCOL_VERSION_1_0_0, REQUIRED_MILESTONES, REQUIRED_PASS_COUNT, REQUIRED_SESSION_COUNT, @@ -74,7 +75,10 @@ def test_packaged_schema_and_frozen_material_manifest_are_consistent() -> None: ) Draft202012Validator.check_schema(schema) assert schema["properties"]["schema_version"]["const"] == RESULT_SCHEMA_VERSION - assert schema["properties"]["protocol_version"]["const"] == PROTOCOL_VERSION + assert schema["properties"]["protocol_version"]["enum"] == [ + PROTOCOL_VERSION_1_0_0, + PROTOCOL_VERSION, + ] assert schema["properties"]["sessions"]["minItems"] == REQUIRED_SESSION_COUNT assert schema["properties"]["sessions"]["maxItems"] == REQUIRED_SESSION_COUNT milestones = schema["$defs"]["session"]["properties"]["milestones"] @@ -113,6 +117,18 @@ def test_three_of_four_distinct_sessions_meets_criterion(_authorised_root: Path) assert result.diagnostics == () +def test_protocol_1_0_0_result_remains_supported(_authorised_root: Path) -> None: + payload = _cohort() + payload["protocol_version"] = PROTOCOL_VERSION_1_0_0 + path = _authorised_root / "results.json" + _write(path, payload) + + result = validate_authoring_results(path) + + assert result.exit_code is ExitCode.SUCCESS + assert result.protocol_version == PROTOCOL_VERSION_1_0_0 + + def test_two_of_four_sessions_is_criterion_not_met(_authorised_root: Path) -> None: path = _authorised_root / "results.json" _write(path, _cohort(2)) @@ -269,7 +285,7 @@ def test_cli_human_json_quiet_and_failure_modes( assert main(["authoring-results", "results.json"]) == ExitCode.SUCCESS assert capsys.readouterr().out == ( - "Assisted-authoring criterion met: 3 of 4 sessions passed (protocol 1.0.0)\n" + "Assisted-authoring criterion met: 3 of 4 sessions passed (protocol 1.0.1)\n" ) assert main(["authoring-results", "results.json", "--json"]) == ExitCode.SUCCESS @@ -279,7 +295,7 @@ def test_cli_human_json_quiet_and_failure_modes( "diagnostics": [], "exit_code": 0, "passed_session_count": 3, - "protocol_version": "1.0.0", + "protocol_version": "1.0.1", "session_count": 4, "status": "criterion-met", } @@ -298,13 +314,15 @@ def test_cli_human_json_quiet_and_failure_modes( def test_public_protocol_freezes_authoring_contract() -> None: root = Path(__file__).parents[1] - protocol = (root / "docs/authoring-check-v1.md").read_text(encoding="utf-8") + protocol = (root / "docs/authoring-check-v1.0.1.md").read_text(encoding="utf-8") words = " ".join(protocol.split()) - assert "protocol 1.0.0" in words + assert "protocol 1.0.1" in words assert "exactly four fresh sessions" in words assert "four distinct agent products" in words assert "at least three of the four sessions" in words assert "no cohort has been run" in words assert "archsift authoring-results authoring-results.json" in protocol assert "transcripts" in protocol + assert "ordinary user-controlled model transport" in words + assert "outbound sockets blocked" in words