From b2ed074dbda032344a6d7941e6181152ae69fb8c Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:04:46 -0700 Subject: [PATCH 1/4] refactor(tools): decompose the DSL language-evaluation checker check_dsl_language_evaluation.py was the last 3,000-line governance checker (3,096 lines; single functions of ~700 and ~635 lines). Any future edit would trip the strict Sonar gate's file and function thresholds the way the other three checkers did. It is now an import-stable entry point over a tools/dsl_language_evaluation/ support package (keys, shape primitives, claim scope/binding, measure recomputation, protocol validation with a plans sibling, snapshot validation with a subjects/review/completion sibling, analysis validation), every module under the 500-line cap. _protocol_records_by_id moves to the shape module where its consumers expect it. Pure restructuring: the checker emits byte-identical output and exit status on this repo in both text and --json modes, and the import surface the test suite relies on is re-exported unchanged. Verification: test_dsl_language_evaluation (40 tests) and the full hermetic suite pass with the coverage gate green; ruff format and lint clean; check_repo_policy pass. Co-Authored-By: Claude Fable 5 --- tools/check_dsl_language_evaluation.py | 2949 +---------------- tools/dsl_language_evaluation/__init__.py | 10 + tools/dsl_language_evaluation/_analysis.py | 375 +++ tools/dsl_language_evaluation/_claims.py | 435 +++ tools/dsl_language_evaluation/_keys.py | 324 ++ tools/dsl_language_evaluation/_measures.py | 352 ++ tools/dsl_language_evaluation/_protocol.py | 444 +++ .../_protocol_plans.py | 343 ++ tools/dsl_language_evaluation/_shape.py | 159 + tools/dsl_language_evaluation/_snapshot.py | 464 +++ .../_snapshot_completion.py | 296 ++ 11 files changed, 3251 insertions(+), 2900 deletions(-) create mode 100644 tools/dsl_language_evaluation/__init__.py create mode 100644 tools/dsl_language_evaluation/_analysis.py create mode 100644 tools/dsl_language_evaluation/_claims.py create mode 100644 tools/dsl_language_evaluation/_keys.py create mode 100644 tools/dsl_language_evaluation/_measures.py create mode 100644 tools/dsl_language_evaluation/_protocol.py create mode 100644 tools/dsl_language_evaluation/_protocol_plans.py create mode 100644 tools/dsl_language_evaluation/_shape.py create mode 100644 tools/dsl_language_evaluation/_snapshot.py create mode 100644 tools/dsl_language_evaluation/_snapshot_completion.py diff --git a/tools/check_dsl_language_evaluation.py b/tools/check_dsl_language_evaluation.py index a71873c99..df0548423 100644 --- a/tools/check_dsl_language_evaluation.py +++ b/tools/check_dsl_language_evaluation.py @@ -1,2921 +1,70 @@ #!/usr/bin/env python3 -"""Validate the preregistered RAES SDL language-evaluation evidence bundle.""" +# ruff: noqa: E402, I001 +"""Validate the preregistered RAES SDL language-evaluation evidence bundle. + +The closed key sets, shape primitives, claim-scope machinery, measure +recomputation, and per-surface validators live in the +``tools/dsl_language_evaluation`` support package; this entry point loads the +revisioned claim bundles, wires the validators together, and keeps the import +surface the test suite and nox lanes rely on. +""" from __future__ import annotations import argparse import json -import re import sys -from collections import Counter -from collections.abc import Mapping, Sequence -from itertools import product +from collections.abc import Mapping from pathlib import Path -from statistics import median -from urllib.parse import parse_qsl, urlsplit REPO_ROOT = Path(__file__).resolve().parents[1] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) -from tools.policy.common import ( # noqa: E402 +from tools.policy.common import ( PolicyFailure, load_bounded_json_object, safe_repo_path, ) - -MANIFEST_PATH = "docs/research/dsl-language-evaluation/bundle-manifest.json" -_RETIRED_PACKAGE_PREFIX = "implementations/python/packages/" + "a" + "ces_sdl" -_HISTORICAL_PACKAGE_MOVES: tuple[tuple[str, str], ...] = ( - (_RETIRED_PACKAGE_PREFIX, "implementations/python/packages/raes"), - ("implementations/python/packages/" + "a" + "ces_cli", "implementations/python/packages/raes_cli"), - ("implementations/python/packages/" + "a" + "ces_mcp", "implementations/python/packages/raes_mcp"), +from tools.dsl_language_evaluation._analysis import _validate_analysis +from tools.dsl_language_evaluation._claims import ( + _validate_claim_binding, + _validate_claim_scope, + resolve_claim_strata, ) -_MAX_FILE_BYTES = 2 * 1024 * 1024 -_MAX_CATALOG_ITEMS = 128 -_MAX_EXECUTION_RECORDS = 20_000 - -REQUIRED_DIMENSION_IDS = { - "expressiveness", - "usability-comprehension", - "effectiveness-productivity", - "maintainability-evolution", - "ambiguity", - "diagnostic-quality", - "reviewability", - "semantic-traceability", -} -REQUIRED_PERSONA_IDS = { - "benchmark-designer", - "scenario-author", - "participant-model-author", - "backend-implementer", - "evaluator-reviewer", - "assurance-auditor", -} -REQUIRED_TASK_KINDS = { - "positive", - "negative", - "underspecified", - "ambiguous", - "round-trip", - "mutation", - "maintenance", - "independent-review", -} -EVIDENCE_STATUSES = {"untested", "partial", "demonstrated", "refuted"} -ATTEMPT_OUTCOMES = {"completed", "failed", "abandoned", "tool_failed", "missing", "withdrawn"} -OBSERVATION_OUTCOMES = {"completed", "failed", "abandoned", "tool_failed", "missing"} - -_MANIFEST_KEYS = { - "bundle_id", - "revision", - "protocol_path", - "snapshot_path", - "analysis_path", - "claim_binding", - "supplemental_bundles", -} -_BUNDLE_ENTRY_KEYS = _MANIFEST_KEYS - {"supplemental_bundles"} -_CLAIM_BINDING_KEYS = {"claim_id", "scope", "strata"} -_STRATUM_GROUP_KEYS = { - "group_id", - "role", - "partition_by", - "persona_ids", - "experience_band_ids", - "tooling_condition_ids", -} -_STRATUM_PARTITION_AXES = {"persona_id", "experience_band", "tooling_condition_id"} -_STRATUM_ROLES = {"gating", "comparison"} -_PROTOCOL_KEYS = { - "protocol_id", - "revision", - "registered_at", - "title", - "claim", - "research_question", - "evidence_status_values", - "dimensions", - "personas", - "tooling_conditions", - "artifact_stages", - "sources", - "tasks", - "variants", - "measures", - "sampling_plan", - "execution_plan", - "thresholds", - "ethics_and_privacy", - "disagreement_policy", - "validity_threats", - "analysis_plan", - "amendment_log", -} -_DIMENSION_KEYS = {"dimension_id", "label", "construct", "pass_rule", "fail_rule"} -_PERSONA_KEYS = {"persona_id", "label", "qualification", "minimum_completed_subjects"} -_CONDITION_KEYS = {"condition_id", "label", "allowed_surface", "assistance"} -_STAGE_KEYS = {"stage_id", "label", "canonical_entrypoint"} -_SOURCE_KEYS = { - "source_id", - "kind", - "title", - "authors", - "year", - "locator", - "version", - "revision", - "artifact_path", - "primary", -} -_TASK_KEYS = { - "task_id", - "title", - "kind", - "persona_ids", - "dimension_ids", - "source_refs", - "intended_semantics_ref", - "artifact_stage_ids", - "tooling_condition_ids", - "variant_ids", - "success_rule", - "failure_rule", -} -_VARIANT_KEYS = {"variant_id", "task_id", "kind", "expected_relation", "description"} -_MEASURE_KEYS = { - "measure_id", - "task_ids", - "dimension_ids", - "stage_applicability", - "unit", - "aggregation", - "direction", - "capture_rule", -} -_STAGE_APPLICABILITY_KEYS = {"task_id", "variant_ids", "artifact_stage_ids"} -_SAMPLING_KEYS = { - "target_total", - "minimum_per_persona", - "experience_bands", - "inclusion_rule", - "exclusion_rule", -} -_EXECUTION_PLAN_KEYS = { - "unit_of_analysis", - "attempts_per_subject", - "subject_task_requirements", - "task_order", - "blinding", - "stopping_rule", - "missing_data_rule", - "withdrawal_rule", -} -_SUBJECT_TASK_REQUIREMENT_KEYS = { - "requirement_id", - "minimum_assigned_attempts", - "task_kinds", -} -_THRESHOLD_KEYS = {"dimension_id", "logic", "conditions"} -_THRESHOLD_CONDITION_KEYS = {"measure_id", "operator", "target"} -_ETHICS_KEYS = { - "review_status_required", - "consent_required", - "committed_data_rule", - "prohibited_data", -} - -_HISTORICAL_REVISION_FIELD = "a" + "ces_revision" -_SNAPSHOT_KEYS = { - "snapshot_id", - "protocol_revision", - "captured_at", - "execution_status", - _HISTORICAL_REVISION_FIELD, - "public_surface", - "ethics_review", - "subjects", - "attempts", - "observations", - "reviews", - "deviations", - "withdrawals", - "disagreements", -} -_SURFACE_KEYS = {"surface_id", "kind", "artifact", "version", "parameters"} -_ETHICS_REVIEW_KEYS = { - "status", - "protocol_identifier", - "approved_population", - "approved_data_boundary", -} -_SUBJECT_KEYS = {"subject_id", "persona_id", "experience_band", "consent_status"} -_ATTEMPT_KEYS = { - "attempt_id", - "study_run_id", - "task_id", - "persona_id", - "subject_id", - "tooling_condition_id", - "variant_id", - "outcome", - "observation_ids", - "started_at", - "ended_at", -} -_OBSERVATION_KEYS = { - "observation_id", - "protocol_revision", - "study_run_id", - "task_id", - "persona_id", - "subject_id", - "tooling_condition_id", - "attempt_id", - "variant_id", - "artifact_stage", - "dimension_ids", - "measure_id", - "value", - "outcome", - "evidence_refs", -} -_REVIEW_KEYS = { - "review_id", - "attempt_id", - "reviewer_subject_id", - "task_id", - "variant_id", - "judgment", - "confidence", - "rationale_code", - "fixed_at", -} -_DEVIATION_KEYS = {"deviation_id", "scope", "severity", "disposition", "rationale"} -_WITHDRAWAL_KEYS = {"subject_id", "recorded_at", "retained_aggregate_only"} -_DISAGREEMENT_KEYS = { - "disagreement_id", - "review_ids", - "status", - "adjudication", - "originals_preserved", -} - -_ANALYSIS_KEYS = { - "analysis_id", - "protocol_revision", - "snapshot_id", - "generated_at", - "execution_status", - "measure_results", - "dimension_results", - "stratum_results", - "evidence_status", - "claim", - "plain_language_outcome", - "limitations", -} -_STRATUM_RESULT_KEYS = { - "stratum_id", - "role", - "measure_results", - "dimension_results", -} -_DIMENSION_RESULT_KEYS = { - "dimension_id", - "status", - "threshold_result", - "condition_results", - "supporting_observation_ids", -} -_MEASURE_RESULT_KEYS = { - "measure_id", - "status", - "statistic", - "numerator", - "denominator", - "opportunity_count", - "observed_count", - "missing_count", - "abandoned_count", - "tool_failed_count", - "withdrawn_count", - "value", - "supporting_observation_ids", -} -_CLAIM_KEYS = { - "claim_id", - "statement", - "threats_to_validity", - "falsification_protocol", - "objective_pass_criteria", - "objective_fail_criteria", - "allowed_evidence", - "disallowed_evidence", - "evidence_artifacts", - "scope", -} -_CLAIM_SCOPE_KEYS = { - "persona_ids", - "task_ids", - "tooling_condition_ids", - "variant_ids", - "artifact_stage_ids", - "dimension_ids", - "measure_ids", -} - -_ID_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") -_SHA_RE = re.compile(r"^[0-9a-f]{40}$") -_SENSITIVE_QUERY_KEYS = { - "access_token", - "api_key", - "apikey", - "auth", - "authorization", - "client_secret", - "key", - "password", - "secret", - "sig", - "signature", - "token", -} - - -def _failure(rule_id: str, message: str, path: str | None = None) -> PolicyFailure: - return PolicyFailure(rule_id, message, path) - - -def _resolve_repository_artifact(repo_root: Path, artifact_path: str) -> Path | None: - """Resolve an immutable evidence locator through the Python package moves.""" - - resolved_path = artifact_path - for historical_prefix, current_prefix in _HISTORICAL_PACKAGE_MOVES: - if artifact_path == historical_prefix or artifact_path.startswith(f"{historical_prefix}/"): - resolved_path = f"{current_prefix}{artifact_path[len(historical_prefix) :]}" - break - return safe_repo_path(repo_root, resolved_path) - - -def _exact_keys( - value: object, - expected: set[str], - failures: list[PolicyFailure], - *, - rule_id: str, - label: str, - path: str, -) -> bool: - if not isinstance(value, dict): - failures.append(_failure(rule_id, f"{label} must be an object", path)) - return False - actual = set(value) - if actual != expected: - failures.append( - _failure( - rule_id, - f"{label} fields must exactly match {sorted(expected)}; got {sorted(actual)}", - path, - ) - ) - return False - return True - - -def _bounded_list( - value: object, - limit: int, - failures: list[PolicyFailure], - *, - rule_id: str, - label: str, - path: str, -) -> list[object]: - if not isinstance(value, list): - failures.append(_failure(rule_id, f"{label} must be a list", path)) - return [] - if len(value) > limit: - failures.append(_failure(rule_id, f"{label} exceeds the {limit}-entry limit", path)) - return [] - return value - - -def _valid_id(value: object) -> bool: - return isinstance(value, str) and bool(_ID_RE.fullmatch(value)) - - -def _bounded_text(value: object, *, maximum: int = 6000) -> bool: - return isinstance(value, str) and bool(value.strip()) and len(value) <= maximum - - -def _string_list(value: object, *, non_empty: bool = False) -> list[str] | None: - if not isinstance(value, list) or (non_empty and not value): - return None - if not all(isinstance(item, str) for item in value): - return None - return value - - -def _record_ids( - records: Sequence[object], - field: str, - failures: list[PolicyFailure], - *, - rule_id: str, - label: str, - path: str, -) -> set[str]: - result: set[str] = set() - duplicates: set[str] = set() - for record in records: - if not isinstance(record, Mapping): - continue - value = record.get(field) - if not _valid_id(value): - failures.append(_failure(rule_id, f"{label} has invalid {field} {value!r}", path)) - continue - if value in result: - duplicates.add(value) - result.add(value) - if duplicates: - failures.append(_failure(rule_id, f"duplicate {label} ids: {sorted(duplicates)}", path)) - return result - - -def _validate_https_locator(locator: object, failures: list[PolicyFailure], source_id: object) -> None: - if not isinstance(locator, str): - failures.append(_failure("dsl-evaluation-source-locator", f"{source_id}: locator must be text")) - return - parsed = urlsplit(locator) - if parsed.scheme != "https" or not parsed.netloc: - failures.append( - _failure( - "dsl-evaluation-source-locator", - f"{source_id}: locator must be absolute HTTPS", - ) - ) - if parsed.username is not None or parsed.password is not None: - failures.append( - _failure( - "dsl-evaluation-source-secret", - f"{source_id}: locator contains URI userinfo", - ) - ) - query_keys = {key.casefold() for key, _ in parse_qsl(parsed.query, keep_blank_values=True)} - sensitive = sorted(query_keys & _SENSITIVE_QUERY_KEYS) - if sensitive: - failures.append( - _failure( - "dsl-evaluation-source-secret", - f"{source_id}: locator contains secret-bearing query keys {sensitive}", - ) - ) - - -def _protocol_records_by_id( - protocol: Mapping[str, object], - field: str, - id_field: str, -) -> dict[str, Mapping[str, object]]: - records = protocol.get(field, []) - if not isinstance(records, list): - return {} - return { - record[id_field]: record - for record in records - if isinstance(record, Mapping) and isinstance(record.get(id_field), str) - } - - -def _full_claim_scope(catalogs: Mapping[str, set[str]]) -> dict[str, set[str]]: - return { - "persona_ids": set(catalogs.get("persona_ids", set())), - "task_ids": set(catalogs.get("task_ids", set())), - "tooling_condition_ids": set(catalogs.get("condition_ids", set())), - "variant_ids": set(catalogs.get("variant_ids", set())), - "artifact_stage_ids": set(catalogs.get("stage_ids", set())), - "dimension_ids": set(catalogs.get("dimension_ids", set())), - "measure_ids": set(catalogs.get("measure_ids", set())), - } - - -def _attempt_matches_scope( - attempt: Mapping[str, object], - scope: Mapping[str, set[str]], -) -> bool: - return ( - attempt.get("task_id") in scope.get("task_ids", set()) - and attempt.get("persona_id") in scope.get("persona_ids", set()) - and attempt.get("tooling_condition_id") in scope.get("tooling_condition_ids", set()) - and attempt.get("variant_id") in scope.get("variant_ids", set()) - ) - - -def _validate_claim_scope( - protocol: Mapping[str, object], - analysis: Mapping[str, object], - catalogs: Mapping[str, set[str]], - failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/analysis-v1.json", -) -> dict[str, set[str]]: - """Validate and resolve the exact catalog slice owned by one claim.""" - - fallback = _full_claim_scope(catalogs) - claim = analysis.get("claim") - scope = claim.get("scope") if isinstance(claim, Mapping) else None - if not _exact_keys( - scope, - _CLAIM_SCOPE_KEYS, - failures, - rule_id="dsl-evaluation-claim-scope", - label="claim scope", - path=path, - ): - return fallback - - catalog_fields = { - "persona_ids": "persona_ids", - "task_ids": "task_ids", - "tooling_condition_ids": "condition_ids", - "variant_ids": "variant_ids", - "artifact_stage_ids": "stage_ids", - "dimension_ids": "dimension_ids", - "measure_ids": "measure_ids", - } - resolved: dict[str, set[str]] = {} - for field, catalog_field in catalog_fields.items(): - values = _string_list(scope[field], non_empty=True) - known = catalogs.get(catalog_field, set()) - if values is None or len(values) != len(set(values)) or not set(values).issubset(known): - failures.append( - _failure( - "dsl-evaluation-claim-scope", - f"claim scope {field} must be a non-empty unique subset of the protocol catalog", - path, - ) - ) - resolved[field] = set(values or []) & known - else: - resolved[field] = set(values) - - tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - measures = _protocol_records_by_id(protocol, "measures", "measure_id") - for task_id in resolved["task_ids"]: - task = tasks.get(task_id, {}) - joins = ( - ("persona_ids", "persona_ids"), - ("tooling_condition_ids", "tooling_condition_ids"), - ("variant_ids", "variant_ids"), - ("artifact_stage_ids", "artifact_stage_ids"), - ) - if any( - not (set(_string_list(task.get(task_field), non_empty=True) or []) & resolved[scope_field]) - for task_field, scope_field in joins - ): - failures.append( - _failure( - "dsl-evaluation-claim-scope", - f"{task_id}: claim scope must retain an eligible persona, condition, variant, and stage", - path, - ) - ) - if not any( - task_id in (_string_list(measures[measure_id].get("task_ids"), non_empty=True) or []) - for measure_id in resolved["measure_ids"] - if measure_id in measures - ): - failures.append( - _failure( - "dsl-evaluation-claim-scope", - f"{task_id}: claim scope has no applicable measure", - path, - ) - ) - - thresholds = { - threshold.get("dimension_id"): threshold - for threshold in protocol.get("thresholds", []) - if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) - } - for dimension_id in resolved["dimension_ids"]: - threshold = thresholds.get(dimension_id) - conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] - required_measures = { - condition.get("measure_id") - for condition in conditions - if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) - } - if not required_measures or not required_measures.issubset(resolved["measure_ids"]): - failures.append( - _failure( - "dsl-evaluation-claim-scope", - f"{dimension_id}: claim scope must include every threshold measure", - path, - ) - ) - return resolved - - -def _validate_claim_binding( - protocol: Mapping[str, object], - analysis: Mapping[str, object], - claim_binding: object, - catalogs: Mapping[str, set[str]], - scope: Mapping[str, set[str]], - failures: list[PolicyFailure], - *, - path: str, -) -> list[dict[str, object]]: - """Bind one stable claim to its immutable scope and reporting strata.""" - - if not _exact_keys( - claim_binding, - _CLAIM_BINDING_KEYS, - failures, - rule_id="dsl-evaluation-claim-binding", - label="claim binding", - path=path, - ): - return [] - assert isinstance(claim_binding, dict) - claim = analysis.get("claim") - claim_id = claim.get("claim_id") if isinstance(claim, Mapping) else None - if not _valid_id(claim_binding["claim_id"]) or claim_binding["claim_id"] != claim_id: - failures.append( - _failure( - "dsl-evaluation-claim-binding", - "manifest claim binding must name the analysis claim_id exactly", - path, - ) - ) - - bound_scope = claim_binding["scope"] - if _exact_keys( - bound_scope, - _CLAIM_SCOPE_KEYS, - failures, - rule_id="dsl-evaluation-claim-binding", - label="claim binding scope", - path=path, - ): - assert isinstance(bound_scope, dict) - for field in _CLAIM_SCOPE_KEYS: - values = _string_list(bound_scope[field], non_empty=True) - if values is None or len(values) != len(set(values)) or set(values) != scope.get(field, set()): - failures.append( - _failure( - "dsl-evaluation-claim-binding", - f"analysis claim scope {field} must exactly match its manifest binding", - path, - ) - ) - - groups = _bounded_list( - claim_binding["strata"], - 16, - failures, - rule_id="dsl-evaluation-claim-strata", - label="claim binding strata", - path=path, - ) - group_ids = _record_ids( - groups, - "group_id", - failures, - rule_id="dsl-evaluation-claim-strata", - label="claim stratum group", - path=path, - ) - experience_bands = set() - sampling_plan = protocol.get("sampling_plan") - if isinstance(sampling_plan, Mapping): - experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) - - expanded: list[dict[str, object]] = [] - for index, group in enumerate(groups): - if not _exact_keys( - group, - _STRATUM_GROUP_KEYS, - failures, - rule_id="dsl-evaluation-claim-strata", - label=f"claim binding strata[{index}]", - path=path, - ): - continue - assert isinstance(group, dict) - group_id = group["group_id"] - role = group["role"] - partition_by = _string_list(group["partition_by"]) - persona_ids = _string_list(group["persona_ids"], non_empty=True) - band_ids = _string_list(group["experience_band_ids"], non_empty=True) - condition_ids = _string_list(group["tooling_condition_ids"], non_empty=True) - valid = ( - _valid_id(group_id) - and role in _STRATUM_ROLES - and partition_by is not None - and len(partition_by) == len(set(partition_by)) - and set(partition_by).issubset(_STRATUM_PARTITION_AXES) - and persona_ids is not None - and len(persona_ids) == len(set(persona_ids)) - and set(persona_ids).issubset(scope.get("persona_ids", set())) - and band_ids is not None - and len(band_ids) == len(set(band_ids)) - and set(band_ids).issubset(experience_bands) - and condition_ids is not None - and len(condition_ids) == len(set(condition_ids)) - and set(condition_ids).issubset(scope.get("tooling_condition_ids", set())) - ) - if not valid: - failures.append( - _failure( - "dsl-evaluation-claim-strata", - f"claim stratum group {group_id!r} has invalid role, partition axes, or catalog filters", - path, - ) - ) - continue - - assert isinstance(group_id, str) - assert isinstance(role, str) - assert partition_by is not None - assert persona_ids is not None - assert band_ids is not None - assert condition_ids is not None - axis_values = { - "persona_id": persona_ids, - "experience_band": band_ids, - "tooling_condition_id": condition_ids, - } - split_axes = [ - axis for axis in ("persona_id", "experience_band", "tooling_condition_id") if axis in partition_by - ] - combinations = product(*(axis_values[axis] for axis in split_axes)) if split_axes else [()] - for combination in combinations: - selected = dict(zip(split_axes, combination, strict=True)) - stratum_id = "-".join([group_id, *(str(selected[axis]) for axis in split_axes)]) - stratum_personas = {str(selected["persona_id"])} if "persona_id" in selected else set(persona_ids) - stratum_bands = {str(selected["experience_band"])} if "experience_band" in selected else set(band_ids) - stratum_conditions = ( - {str(selected["tooling_condition_id"])} if "tooling_condition_id" in selected else set(condition_ids) - ) - stratum_scope = _derive_stratum_scope( - protocol, - scope, - persona_ids=stratum_personas, - experience_band_ids=stratum_bands, - tooling_condition_ids=stratum_conditions, - ) - if ( - not _valid_id(stratum_id) - or not stratum_scope["task_ids"] - or not stratum_scope["measure_ids"] - or not stratum_scope["dimension_ids"] - ): - failures.append( - _failure( - "dsl-evaluation-claim-strata", - f"claim stratum {stratum_id!r} has no complete task/measure/dimension slice", - path, - ) - ) - continue - expanded.append({"stratum_id": stratum_id, "role": role, "scope": stratum_scope}) - - expanded_ids = [str(item["stratum_id"]) for item in expanded] - if len(group_ids) != len(groups) or len(expanded_ids) != len(set(expanded_ids)) or not expanded: - failures.append( - _failure( - "dsl-evaluation-claim-strata", - "claim binding requires unique groups and expanded stratum ids", - path, - ) - ) - if not any(item["role"] == "gating" for item in expanded): - failures.append(_failure("dsl-evaluation-claim-strata", "claim binding requires a gating stratum", path)) - if len(expanded) > _MAX_CATALOG_ITEMS: - failures.append( - _failure( - "dsl-evaluation-claim-strata", - f"claim binding expands beyond {_MAX_CATALOG_ITEMS} strata", - path, - ) - ) - return expanded[:_MAX_CATALOG_ITEMS] - return expanded - - -def _derive_stratum_scope( - protocol: Mapping[str, object], - claim_scope: Mapping[str, set[str]], - *, - persona_ids: set[str], - experience_band_ids: set[str], - tooling_condition_ids: set[str], -) -> dict[str, set[str]]: - """Derive the protocol-applicable claim slice for one preregistered stratum.""" - - tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - selected_tasks = { - task_id - for task_id, task in tasks.items() - if task_id in claim_scope.get("task_ids", set()) - and set(_string_list(task.get("persona_ids"), non_empty=True) or []) & persona_ids - and set(_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) & tooling_condition_ids - } - variants = _protocol_records_by_id(protocol, "variants", "variant_id") - selected_variants = { - variant_id - for variant_id, variant in variants.items() - if variant_id in claim_scope.get("variant_ids", set()) and variant.get("task_id") in selected_tasks - } - measures = _protocol_records_by_id(protocol, "measures", "measure_id") - selected_measures = { - measure_id - for measure_id, measure in measures.items() - if measure_id in claim_scope.get("measure_ids", set()) - and set(_string_list(measure.get("task_ids"), non_empty=True) or []) & selected_tasks - } - thresholds = { - threshold.get("dimension_id"): threshold - for threshold in protocol.get("thresholds", []) - if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) - } - selected_dimensions = set() - for dimension_id in claim_scope.get("dimension_ids", set()): - threshold = thresholds.get(dimension_id) - conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] - required_measures = { - condition.get("measure_id") - for condition in conditions - if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) - } - if required_measures and required_measures.issubset(selected_measures): - selected_dimensions.add(dimension_id) - return { - "persona_ids": set(persona_ids), - "experience_band_ids": set(experience_band_ids), - "task_ids": selected_tasks, - "tooling_condition_ids": set(tooling_condition_ids), - "variant_ids": selected_variants, - "artifact_stage_ids": set(claim_scope.get("artifact_stage_ids", set())), - "dimension_ids": selected_dimensions, - "measure_ids": selected_measures, - } - - -def resolve_claim_strata( - protocol: Mapping[str, object], - analysis: Mapping[str, object], - claim_binding: Mapping[str, object], -) -> list[dict[str, object]]: - """Resolve a validated manifest binding for recomputation and fixtures.""" - - catalogs = { - "persona_ids": set(_protocol_records_by_id(protocol, "personas", "persona_id")), - "task_ids": set(_protocol_records_by_id(protocol, "tasks", "task_id")), - "condition_ids": set(_protocol_records_by_id(protocol, "tooling_conditions", "condition_id")), - "variant_ids": set(_protocol_records_by_id(protocol, "variants", "variant_id")), - "stage_ids": set(_protocol_records_by_id(protocol, "artifact_stages", "stage_id")), - "dimension_ids": set(_protocol_records_by_id(protocol, "dimensions", "dimension_id")), - "measure_ids": set(_protocol_records_by_id(protocol, "measures", "measure_id")), - } - failures: list[PolicyFailure] = [] - scope = _validate_claim_scope(protocol, analysis, catalogs, failures, path=MANIFEST_PATH) - strata = _validate_claim_binding( - protocol, - analysis, - claim_binding, - catalogs, - scope, - failures, - path=MANIFEST_PATH, - ) - if failures: - raise ValueError("; ".join(f"{failure.rule_id}: {failure.message}" for failure in failures)) - return strata - - -def _measure_stage_ids( - measure: Mapping[str, object], - task_id: str, - variant_id: str, -) -> list[str]: - """Return the single preregistered stage set for a task/variant/measure.""" - - declarations = measure.get("stage_applicability", []) - if not isinstance(declarations, list): - raise ValueError(f"{measure.get('measure_id')}: stage applicability must be a list") - matches: list[list[str]] = [] - for declaration in declarations: - if not isinstance(declaration, Mapping) or declaration.get("task_id") != task_id: - continue - variant_ids = _string_list(declaration.get("variant_ids"), non_empty=True) - stage_ids = _string_list(declaration.get("artifact_stage_ids"), non_empty=True) - if variant_ids is not None and variant_id in variant_ids and stage_ids is not None: - matches.append(stage_ids) - if len(matches) != 1: - raise ValueError(f"{measure.get('measure_id')}: expected one stage declaration for {task_id}/{variant_id}") - return matches[0] - - -def _measure_opportunities( - protocol: Mapping[str, object], - snapshot: Mapping[str, object], - scope: Mapping[str, Sequence[str] | set[str]] | None = None, -) -> tuple[ - dict[str, Mapping[str, object]], - list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], - dict[tuple[str, str, str], Mapping[str, object]], -]: - """Derive every attempt-measure-stage opportunity and frozen observation.""" - - tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - measures = _protocol_records_by_id(protocol, "measures", "measure_id") - scope_sets = {field: set(values) for field, values in scope.items()} if scope is not None else None - if scope_sets is not None: - measures = { - measure_id: measure - for measure_id, measure in measures.items() - if measure_id in scope_sets.get("measure_ids", set()) - } - attempts = snapshot.get("attempts", []) - observations = snapshot.get("observations", []) - withdrawals = snapshot.get("withdrawals", []) - subjects = snapshot.get("subjects", []) - subjects_by_id = { - subject.get("subject_id"): subject - for subject in subjects - if isinstance(subject, Mapping) and isinstance(subject.get("subject_id"), str) - } - if not isinstance(attempts, list) or not isinstance(observations, list) or not isinstance(withdrawals, list): - raise ValueError("snapshot execution records must be lists") - - withdrawn_subjects = { - withdrawal["subject_id"] - for withdrawal in withdrawals - if isinstance(withdrawal, Mapping) and isinstance(withdrawal.get("subject_id"), str) - } - observation_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} - for observation in observations: - if not isinstance(observation, Mapping): - continue - attempt_id = observation.get("attempt_id") - measure_id = observation.get("measure_id") - artifact_stage = observation.get("artifact_stage") - if not isinstance(attempt_id, str) or not isinstance(measure_id, str) or not isinstance(artifact_stage, str): - continue - key = (attempt_id, measure_id, artifact_stage) - if key in observation_by_opportunity: - raise ValueError(f"duplicate observation opportunity {attempt_id}/{measure_id}/{artifact_stage}") - observation_by_opportunity[key] = observation - - opportunities: list[tuple[Mapping[str, object], Mapping[str, object], str, bool]] = [] - expected_keys: set[tuple[str, str, str]] = set() - selected_attempt_ids: set[str] = set() - for attempt in attempts: - if not isinstance(attempt, Mapping): - continue - attempt_id = attempt.get("attempt_id") - task_id = attempt.get("task_id") - subject_id = attempt.get("subject_id") - variant_id = attempt.get("variant_id") - if not isinstance(attempt_id, str) or not isinstance(task_id, str) or not isinstance(variant_id, str): - continue - task = tasks.get(task_id) - if task is None: - continue - if scope_sets is not None and not _attempt_matches_scope(attempt, scope_sets): - continue - if scope_sets is not None and "experience_band_ids" in scope_sets: - subject = subjects_by_id.get(subject_id) - if ( - not isinstance(subject, Mapping) - or subject.get("experience_band") not in scope_sets["experience_band_ids"] - ): - continue - selected_attempt_ids.add(attempt_id) - withdrawn = (isinstance(subject_id, str) and subject_id in withdrawn_subjects) or attempt.get( - "outcome" - ) == "withdrawn" - for measure in measures.values(): - task_ids = _string_list(measure.get("task_ids"), non_empty=True) - if task_ids is None or task_id not in task_ids: - continue - measure_id = measure.get("measure_id") - if not isinstance(measure_id, str): - continue - for artifact_stage in _measure_stage_ids(measure, task_id, variant_id): - if scope_sets is not None and artifact_stage not in scope_sets.get("artifact_stage_ids", set()): - continue - opportunities.append((attempt, measure, artifact_stage, withdrawn)) - expected_keys.add((attempt_id, measure_id, artifact_stage)) - - observed_keys = set(observation_by_opportunity) - if scope_sets is not None: - observed_keys = { - key - for key in observed_keys - if key[0] in selected_attempt_ids - and key[1] in scope_sets.get("measure_ids", set()) - and key[2] in scope_sets.get("artifact_stage_ids", set()) - } - extras = sorted(observed_keys - expected_keys) - if extras: - raise ValueError(f"observations without protocol-declared opportunities: {extras[:5]}") - return measures, opportunities, observation_by_opportunity - - -def recompute_measure_results( - protocol: Mapping[str, object], - snapshot: Mapping[str, object], - *, - scope: Mapping[str, Sequence[str] | set[str]] | None = None, -) -> dict[str, dict[str, object]]: - """Recompute measures from the complete protocol-derived opportunity matrix.""" - - measures, opportunities, observation_by_opportunity = _measure_opportunities(protocol, snapshot, scope) - results: dict[str, dict[str, object]] = {} - for measure_id, measure in measures.items(): - aggregation = measure.get("aggregation") - if aggregation not in {"proportion", "median", "count"}: - raise ValueError(f"invalid measure aggregation for {measure_id!r}") - - matching = [item for item in opportunities if item[1].get("measure_id") == measure_id] - opportunity_count = len(matching) - withdrawn_count = sum(withdrawn for _, _, _, withdrawn in matching) - eligible = [(attempt, artifact_stage) for attempt, _, artifact_stage, withdrawn in matching if not withdrawn] - denominator = len(eligible) - values: list[int | float] = [] - supporting_ids: list[str] = [] - missing_count = 0 - abandoned_count = 0 - tool_failed_count = 0 - - for attempt, artifact_stage in eligible: - attempt_id = attempt.get("attempt_id") - if not isinstance(attempt_id, str): - missing_count += 1 - continue - observation = observation_by_opportunity.get((attempt_id, measure_id, artifact_stage)) - if observation is None: - missing_count += 1 - continue - observation_id = observation.get("observation_id") - if not isinstance(observation_id, str): - raise ValueError(f"{measure_id}: observations require ids") - supporting_ids.append(observation_id) - outcome = observation.get("outcome") - if outcome == "missing": - missing_count += 1 - elif outcome == "abandoned": - abandoned_count += 1 - elif outcome == "tool_failed": - tool_failed_count += 1 - value = observation.get("value") - if value is None: - continue - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"{measure_id}: observation values must be numeric or null") - if aggregation == "proportion" and value not in {0, 1}: - raise ValueError(f"{measure_id}: proportion observations must be 0 or 1") - values.append(value) - - observed_count = len(values) - numerator: int | float | None = None - value: int | float | None = None - if denominator > 0 and observed_count == denominator: - if aggregation == "proportion": - numerator = sum(values) - value = numerator / denominator - elif aggregation == "median": - value = float(median(values)) - else: - numerator = sum(values) - value = numerator - results[measure_id] = { - "statistic": aggregation, - "numerator": numerator, - "denominator": denominator, - "opportunity_count": opportunity_count, - "observed_count": observed_count, - "missing_count": missing_count, - "abandoned_count": abandoned_count, - "tool_failed_count": tool_failed_count, - "withdrawn_count": withdrawn_count, - "value": value, - "supporting_observation_ids": supporting_ids, - } - return results - - -def recompute_dimension_results( - protocol: Mapping[str, object], - measure_results: Mapping[str, Mapping[str, object]], - *, - dimension_ids: Sequence[str] | set[str] | None = None, -) -> dict[str, dict[str, object]]: - """Apply protocol-declared threshold conditions to recomputed measures.""" - - operators = { - ">=": lambda actual, target: actual >= target, - "<=": lambda actual, target: actual <= target, - "==": lambda actual, target: actual == target, - } - results: dict[str, dict[str, object]] = {} - selected_dimensions = set(dimension_ids) if dimension_ids is not None else None - for threshold in protocol.get("thresholds", []): - if not isinstance(threshold, Mapping): - continue - dimension_id = threshold.get("dimension_id") - conditions = threshold.get("conditions") - if ( - not isinstance(dimension_id, str) - or threshold.get("logic") != "all" - or not isinstance(conditions, list) - or not conditions - ): - raise ValueError(f"invalid threshold for {dimension_id!r}") - if selected_dimensions is not None and dimension_id not in selected_dimensions: - continue - resolved: list[tuple[Mapping[str, object], Mapping[str, object]]] = [] - for condition in conditions: - if not isinstance(condition, Mapping): - raise ValueError(f"{dimension_id}: threshold condition must be an object") - measure_id = condition.get("measure_id") - operator = condition.get("operator") - target = condition.get("target") - measure = measure_results.get(measure_id) if isinstance(measure_id, str) else None - if operator not in operators or isinstance(target, bool) or not isinstance(target, (int, float)): - raise ValueError(f"{dimension_id}: invalid threshold operator or target") - if measure is None: - raise ValueError(f"{dimension_id}: threshold references unknown measure {measure_id!r}") - resolved.append((condition, measure)) - if any(measure.get("value") is None for _, measure in resolved): - results[dimension_id] = { - "status": "not_evaluated", - "threshold_result": "not_evaluated", - "condition_results": [], - "supporting_observation_ids": [], - } - continue - condition_results: list[dict[str, object]] = [] - supporting_ids: list[str] = [] - for condition, measure in resolved: - actual = measure["value"] - operator = condition["operator"] - target = condition["target"] - if isinstance(actual, bool) or not isinstance(actual, (int, float)): - raise ValueError(f"{dimension_id}: measure value must be numeric") - passed = operators[operator](actual, target) - condition_results.append( - { - "measure_id": condition["measure_id"], - "operator": operator, - "target": target, - "actual": actual, - "passed": passed, - } - ) - refs = measure.get("supporting_observation_ids", []) - if not isinstance(refs, list) or not all(isinstance(item, str) for item in refs): - raise ValueError(f"{dimension_id}: measure observation refs must be text ids") - for observation_id in refs: - if observation_id not in supporting_ids: - supporting_ids.append(observation_id) - results[dimension_id] = { - "status": "evaluated", - "threshold_result": "pass" if all(item["passed"] for item in condition_results) else "fail", - "condition_results": condition_results, - "supporting_observation_ids": supporting_ids, - } - return results - - -def recompute_stratum_results( - protocol: Mapping[str, object], - snapshot: Mapping[str, object], - strata: Sequence[Mapping[str, object]], -) -> list[dict[str, object]]: - """Recompute independently persisted results for every bound claim stratum.""" - - results: list[dict[str, object]] = [] - for stratum in strata: - stratum_id = stratum.get("stratum_id") - role = stratum.get("role") - scope = stratum.get("scope") - if not isinstance(stratum_id, str) or role not in _STRATUM_ROLES or not isinstance(scope, Mapping): - raise ValueError("invalid resolved claim stratum") - measures = recompute_measure_results(protocol, snapshot, scope=scope) - measure_results = [ - { - "measure_id": measure_id, - "status": ( - "not_evaluated" - if result["denominator"] == 0 - else "incomplete" - if result["observed_count"] != result["denominator"] - else "evaluated" - ), - **result, - } - for measure_id, result in measures.items() - ] - dimensions = recompute_dimension_results( - protocol, - measures, - dimension_ids=scope.get("dimension_ids", set()), - ) - dimension_results = [{"dimension_id": dimension_id, **result} for dimension_id, result in dimensions.items()] - results.append( - { - "stratum_id": stratum_id, - "role": role, - "measure_results": measure_results, - "dimension_results": dimension_results, - } - ) - return results - - -def _validate_protocol( - repo_root: Path, - protocol: dict[str, object], - failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/protocol-v1.json", -) -> dict[str, set[str]]: - if not _exact_keys( - protocol, - _PROTOCOL_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="protocol", - path=path, - ): - return {} - - dimensions = _bounded_list( - protocol["dimensions"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="dimensions", - path=path, - ) - personas = _bounded_list( - protocol["personas"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="personas", - path=path, - ) - conditions = _bounded_list( - protocol["tooling_conditions"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="tooling_conditions", - path=path, - ) - stages = _bounded_list( - protocol["artifact_stages"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="artifact_stages", - path=path, - ) - sources = _bounded_list( - protocol["sources"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="sources", - path=path, - ) - tasks = _bounded_list( - protocol["tasks"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="tasks", - path=path, - ) - variants = _bounded_list( - protocol["variants"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="variants", - path=path, - ) - measures = _bounded_list( - protocol["measures"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="measures", - path=path, - ) - thresholds = _bounded_list( - protocol["thresholds"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="thresholds", - path=path, - ) - - dimension_ids = _record_ids( - dimensions, - "dimension_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="dimension", - path=path, - ) - persona_ids = _record_ids( - personas, - "persona_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="persona", - path=path, - ) - condition_ids = _record_ids( - conditions, - "condition_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="tooling condition", - path=path, - ) - stage_ids = _record_ids( - stages, - "stage_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="artifact stage", - path=path, - ) - source_ids = _record_ids( - sources, - "source_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="source", - path=path, - ) - task_ids = _record_ids( - tasks, - "task_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="task", - path=path, - ) - variant_ids = _record_ids( - variants, - "variant_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="variant", - path=path, - ) - measure_ids = _record_ids( - measures, - "measure_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="measure", - path=path, - ) - - if not REQUIRED_DIMENSION_IDS.issubset(dimension_ids): - failures.append( - _failure( - "dsl-evaluation-dimension-coverage", - f"missing required dimensions: {sorted(REQUIRED_DIMENSION_IDS - dimension_ids)}", - path, - ) - ) - if not REQUIRED_PERSONA_IDS.issubset(persona_ids): - failures.append( - _failure( - "dsl-evaluation-persona-coverage", - f"missing required personas: {sorted(REQUIRED_PERSONA_IDS - persona_ids)}", - path, - ) - ) - if protocol["evidence_status_values"] != [ - "untested", - "partial", - "demonstrated", - "refuted", - ]: - failures.append( - _failure( - "dsl-evaluation-status-vocabulary", - "evidence statuses must preserve ADR-021 order and vocabulary", - path, - ) - ) - - for index, record in enumerate(dimensions): - _exact_keys( - record, - _DIMENSION_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"dimensions[{index}]", - path=path, - ) - for index, record in enumerate(personas): - if not _exact_keys( - record, - _PERSONA_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"personas[{index}]", - path=path, - ): - continue - if ( - not isinstance(record["minimum_completed_subjects"], int) - or isinstance(record["minimum_completed_subjects"], bool) - or record["minimum_completed_subjects"] < 0 - ): - failures.append( - _failure( - "dsl-evaluation-sampling-plan", - f"{record['persona_id']}: minimum_completed_subjects must be non-negative", - path, - ) - ) - for index, record in enumerate(conditions): - _exact_keys( - record, - _CONDITION_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"tooling_conditions[{index}]", - path=path, - ) - for index, record in enumerate(stages): - _exact_keys( - record, - _STAGE_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"artifact_stages[{index}]", - path=path, - ) - - for index, source in enumerate(sources): - if not _exact_keys( - source, - _SOURCE_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"sources[{index}]", - path=path, - ): - continue - source_id = source["source_id"] - _validate_https_locator(source["locator"], failures, source_id) - if source["primary"] is not True: - failures.append( - _failure( - "dsl-evaluation-source-primary", - f"{source_id}: source must be primary", - path, - ) - ) - if source["kind"] == "repository-internal": - revision = source["revision"] - artifact_path = source["artifact_path"] - if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): - failures.append( - _failure( - "dsl-evaluation-source-pin", - f"{source_id}: invalid Git revision", - path, - ) - ) - elif isinstance(source["locator"], str) and revision not in source["locator"]: - failures.append( - _failure( - "dsl-evaluation-source-pin", - f"{source_id}: locator does not bind the declared Git revision", - path, - ) - ) - if not isinstance(artifact_path, str): - failures.append( - _failure( - "dsl-evaluation-source-path", - f"{source_id}: missing artifact path", - path, - ) - ) - else: - resolved = _resolve_repository_artifact(repo_root, artifact_path) - if resolved is None or not resolved.exists(): - failures.append( - _failure( - "dsl-evaluation-source-path", - f"{source_id}: unsafe or missing path", - path, - ) - ) - elif source["revision"] is not None or source["artifact_path"] is not None: - failures.append( - _failure( - "dsl-evaluation-source-shape", - f"{source_id}: publication must not claim a repository revision/path", - path, - ) - ) - - task_kinds: set[str] = set() - variants_by_task: dict[str, set[str]] = {task_id: set() for task_id in task_ids} - for index, variant in enumerate(variants): - if not _exact_keys( - variant, - _VARIANT_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"variants[{index}]", - path=path, - ): - continue - task_id = variant["task_id"] - if not isinstance(task_id, str) or task_id not in task_ids: - failures.append( - _failure( - "dsl-evaluation-variant-join", - f"{variant['variant_id']}: unknown task", - path, - ) - ) - else: - variant_id = variant["variant_id"] - if isinstance(variant_id, str): - variants_by_task[task_id].add(variant_id) - for index, task in enumerate(tasks): - if not _exact_keys( - task, - _TASK_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"tasks[{index}]", - path=path, - ): - continue - task_id = task["task_id"] - if not isinstance(task_id, str): - failures.append(_failure("dsl-evaluation-protocol-id", "task id must be text", path)) - continue - task_kind = task["kind"] - if isinstance(task_kind, str): - task_kinds.add(task_kind) - else: - failures.append( - _failure( - "dsl-evaluation-task-kind-coverage", - f"{task_id}: task kind must be text", - path, - ) - ) - joins = ( - ("persona_ids", persona_ids), - ("dimension_ids", dimension_ids), - ("source_refs", source_ids), - ("artifact_stage_ids", stage_ids), - ("tooling_condition_ids", condition_ids), - ("variant_ids", variant_ids), - ) - for field, allowed in joins: - values = task[field] - value_ids = _string_list(values, non_empty=True) - if value_ids is None or not set(value_ids).issubset(allowed): - failures.append( - _failure( - "dsl-evaluation-task-join", - f"{task_id}: invalid or empty {field}", - path, - ) - ) - task_variant_ids = task["variant_ids"] - valid_task_variant_ids = _string_list(task_variant_ids) - if valid_task_variant_ids is not None and set(valid_task_variant_ids) != variants_by_task.get(task_id, set()): - failures.append( - _failure( - "dsl-evaluation-task-variant-coverage", - f"{task_id}: task and variant catalogs disagree", - path, - ) - ) - if not REQUIRED_TASK_KINDS.issubset(task_kinds): - failures.append( - _failure( - "dsl-evaluation-task-kind-coverage", - f"missing required task kinds: {sorted(REQUIRED_TASK_KINDS - task_kinds)}", - path, - ) - ) - - tasks_by_id = _protocol_records_by_id(protocol, "tasks", "task_id") - for index, measure in enumerate(measures): - if not _exact_keys( - measure, - _MEASURE_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"measures[{index}]", - path=path, - ): - continue - measure_dimensions = _string_list(measure["dimension_ids"], non_empty=True) - if measure_dimensions is None or not set(measure_dimensions).issubset(dimension_ids): - failures.append( - _failure( - "dsl-evaluation-measure-join", - f"{measure['measure_id']}: invalid dimension ids", - path, - ) - ) - measure_tasks = _string_list(measure["task_ids"], non_empty=True) - if measure_tasks is None or not set(measure_tasks).issubset(task_ids): - failures.append( - _failure( - "dsl-evaluation-measure-join", - f"{measure['measure_id']}: invalid task ids", - path, - ) - ) - elif not all( - set(measure_dimensions or []) & set(task["dimension_ids"]) - for task in tasks - if isinstance(task, Mapping) - and task.get("task_id") in measure_tasks - and _string_list(task.get("dimension_ids"), non_empty=True) is not None - ): - failures.append( - _failure( - "dsl-evaluation-measure-join", - f"{measure['measure_id']}: task applicability must share a measured dimension", - path, - ) - ) - applicability = _bounded_list( - measure["stage_applicability"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-measure-stage-applicability", - label=f"measures[{index}].stage_applicability", - path=path, - ) - actual_stage_pairs: set[tuple[str, str]] = set() - duplicate_stage_pairs: set[tuple[str, str]] = set() - for applicability_index, declaration in enumerate(applicability): - if not _exact_keys( - declaration, - _STAGE_APPLICABILITY_KEYS, - failures, - rule_id="dsl-evaluation-measure-stage-applicability", - label=(f"measures[{index}].stage_applicability[{applicability_index}]"), - path=path, - ): - continue - task_id = declaration["task_id"] - task = tasks_by_id.get(task_id) if isinstance(task_id, str) else None - declaration_variants = _string_list(declaration["variant_ids"], non_empty=True) - declaration_stages = _string_list(declaration["artifact_stage_ids"], non_empty=True) - task_variants = ( - set(_string_list(task.get("variant_ids"), non_empty=True) or []) if task is not None else set() - ) - task_stages = ( - set(_string_list(task.get("artifact_stage_ids"), non_empty=True) or []) if task is not None else set() - ) - if ( - task is None - or measure_tasks is None - or task_id not in measure_tasks - or declaration_variants is None - or not set(declaration_variants).issubset(task_variants) - or declaration_stages is None - or not set(declaration_stages).issubset(task_stages) - ): - failures.append( - _failure( - "dsl-evaluation-measure-stage-applicability", - f"{measure['measure_id']}: invalid task, variant, or stage applicability", - path, - ) - ) - continue - for variant_id in declaration_variants: - pair = (task_id, variant_id) - if pair in actual_stage_pairs: - duplicate_stage_pairs.add(pair) - actual_stage_pairs.add(pair) - expected_stage_pairs = { - (task_id, variant_id) - for task_id in (measure_tasks or []) - for variant_id in (_string_list(tasks_by_id.get(task_id, {}).get("variant_ids"), non_empty=True) or []) - } - if actual_stage_pairs != expected_stage_pairs or duplicate_stage_pairs: - failures.append( - _failure( - "dsl-evaluation-measure-stage-coverage", - f"{measure['measure_id']}: every task/variant requires one stage declaration", - path, - ) - ) - threshold_ids = _record_ids( - thresholds, - "dimension_id", - failures, - rule_id="dsl-evaluation-threshold-id", - label="threshold", - path=path, - ) - for index, threshold in enumerate(thresholds): - if not _exact_keys( - threshold, - _THRESHOLD_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"thresholds[{index}]", - path=path, - ): - continue - conditions = _bounded_list( - threshold["conditions"], - 16, - failures, - rule_id="dsl-evaluation-threshold-shape", - label=f"thresholds[{index}].conditions", - path=path, - ) - if threshold["logic"] != "all" or not conditions: - failures.append( - _failure( - "dsl-evaluation-threshold-shape", - f"{threshold['dimension_id']}: threshold requires non-empty all conditions", - path, - ) - ) - for condition_index, condition in enumerate(conditions): - if not _exact_keys( - condition, - _THRESHOLD_CONDITION_KEYS, - failures, - rule_id="dsl-evaluation-threshold-shape", - label=f"thresholds[{index}].conditions[{condition_index}]", - path=path, - ): - continue - if ( - not isinstance(condition["measure_id"], str) - or condition["measure_id"] not in measure_ids - or not isinstance(condition["operator"], str) - or condition["operator"] not in {">=", "<=", "=="} - or isinstance(condition["target"], bool) - or not isinstance(condition["target"], (int, float)) - ): - failures.append( - _failure( - "dsl-evaluation-threshold-join", - f"{threshold['dimension_id']}: invalid measure, operator, or target", - path, - ) - ) - if threshold_ids != dimension_ids: - failures.append( - _failure( - "dsl-evaluation-threshold-coverage", - "every dimension requires exactly one preregistered threshold", - path, - ) - ) - - if _exact_keys( - protocol["sampling_plan"], - _SAMPLING_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="sampling_plan", - path=path, - ): - sampling = protocol["sampling_plan"] - minimum = sampling["minimum_per_persona"] - target = sampling["target_total"] - if not isinstance(minimum, int) or minimum < 1 or not isinstance(target, int): - failures.append(_failure("dsl-evaluation-sampling-plan", "invalid sample sizes", path)) - elif target < sum( - item.get("minimum_completed_subjects", 0) - for item in personas - if isinstance(item, Mapping) and isinstance(item.get("minimum_completed_subjects"), int) - ): - failures.append( - _failure( - "dsl-evaluation-sampling-plan", - "target_total cannot cover the required per-persona minimum", - path, - ) - ) - if _exact_keys( - protocol["execution_plan"], - _EXECUTION_PLAN_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="execution_plan", - path=path, - ): - execution_plan = protocol["execution_plan"] - requirements = _bounded_list( - execution_plan["subject_task_requirements"], - 16, - failures, - rule_id="dsl-evaluation-subject-workload-plan", - label="execution_plan.subject_task_requirements", - path=path, - ) - requirement_ids = _record_ids( - requirements, - "requirement_id", - failures, - rule_id="dsl-evaluation-subject-workload-plan", - label="subject task requirement", - path=path, - ) - declared_kinds: set[str] = set() - duplicate_kinds: set[str] = set() - for index, requirement in enumerate(requirements): - if not _exact_keys( - requirement, - _SUBJECT_TASK_REQUIREMENT_KEYS, - failures, - rule_id="dsl-evaluation-subject-workload-plan", - label=f"execution_plan.subject_task_requirements[{index}]", - path=path, - ): - continue - task_kind_values = _string_list(requirement["task_kinds"], non_empty=True) - minimum = requirement["minimum_assigned_attempts"] - if ( - task_kind_values is None - or not set(task_kind_values).issubset(REQUIRED_TASK_KINDS) - or isinstance(minimum, bool) - or not isinstance(minimum, int) - or minimum < 1 - ): - failures.append( - _failure( - "dsl-evaluation-subject-workload-plan", - f"{requirement['requirement_id']}: invalid task kinds or minimum", - path, - ) - ) - continue - overlap = declared_kinds & set(task_kind_values) - duplicate_kinds.update(overlap) - declared_kinds.update(task_kind_values) - for persona_id in persona_ids: - if not any( - task.get("kind") in task_kind_values - and persona_id in (_string_list(task.get("persona_ids"), non_empty=True) or []) - for task in tasks_by_id.values() - ): - failures.append( - _failure( - "dsl-evaluation-subject-workload-plan", - f"{requirement['requirement_id']}: no eligible task for {persona_id}", - path, - ) - ) - if not requirement_ids or declared_kinds != REQUIRED_TASK_KINDS or duplicate_kinds: - failures.append( - _failure( - "dsl-evaluation-subject-workload-plan", - "subject task requirements must partition every required task kind", - path, - ) - ) - _exact_keys( - protocol["ethics_and_privacy"], - _ETHICS_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="ethics_and_privacy", - path=path, - ) - if not protocol["validity_threats"] or not protocol["analysis_plan"]: - failures.append( - _failure( - "dsl-evaluation-preregistration", - "validity threats and the analysis plan must be preregistered", - path, - ) - ) - if not isinstance(protocol["amendment_log"], list): - failures.append(_failure("dsl-evaluation-protocol-shape", "amendment_log must be a list", path)) - return { - "dimension_ids": dimension_ids, - "persona_ids": persona_ids, - "condition_ids": condition_ids, - "stage_ids": stage_ids, - "task_ids": task_ids, - "variant_ids": variant_ids, - "measure_ids": measure_ids, - } - - -def _validate_snapshot( - repo_root: Path, - protocol: Mapping[str, object], - snapshot: dict[str, object], - catalogs: Mapping[str, set[str]], - scope: Mapping[str, set[str]], - failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/execution-snapshot-v1.json", -) -> set[str]: - if not _exact_keys( - snapshot, - _SNAPSHOT_KEYS, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label="snapshot", - path=path, - ): - return set() - if snapshot["protocol_revision"] != protocol.get("revision"): - failures.append(_failure("dsl-evaluation-snapshot-join", "protocol revision mismatch", path)) - revision = snapshot[_HISTORICAL_REVISION_FIELD] - if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): - failures.append( - _failure( - "dsl-evaluation-snapshot-pin", - "RAES revision must be full Git SHA", - path, - ) - ) - if snapshot["execution_status"] not in {"not_started", "in_progress", "complete"}: - failures.append(_failure("dsl-evaluation-snapshot-status", "invalid execution status", path)) - - surfaces = _bounded_list( - snapshot["public_surface"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label="public_surface", - path=path, - ) - _record_ids( - surfaces, - "surface_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="public surface", - path=path, - ) - for index, surface in enumerate(surfaces): - if not _exact_keys( - surface, - _SURFACE_KEYS, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label=f"public_surface[{index}]", - path=path, - ): - continue - artifact = surface["artifact"] - resolved = _resolve_repository_artifact(repo_root, artifact) if isinstance(artifact, str) else None - if resolved is None or not resolved.exists(): - failures.append( - _failure( - "dsl-evaluation-public-surface-path", - f"{surface['surface_id']}: unsafe or missing artifact", - path, - ) - ) - _exact_keys( - snapshot["ethics_review"], - _ETHICS_REVIEW_KEYS, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label="ethics_review", - path=path, - ) - - record_fields = { - "subjects": _SUBJECT_KEYS, - "attempts": _ATTEMPT_KEYS, - "observations": _OBSERVATION_KEYS, - "reviews": _REVIEW_KEYS, - "deviations": _DEVIATION_KEYS, - "withdrawals": _WITHDRAWAL_KEYS, - "disagreements": _DISAGREEMENT_KEYS, - } - records: dict[str, list[object]] = {} - for field, keys in record_fields.items(): - records[field] = _bounded_list( - snapshot[field], - _MAX_EXECUTION_RECORDS, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label=field, - path=path, - ) - for index, record in enumerate(records[field]): - _exact_keys( - record, - keys, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label=f"{field}[{index}]", - path=path, - ) - if snapshot["execution_status"] == "not_started": - populated = sorted(field for field, value in records.items() if value) - if populated: - failures.append( - _failure( - "dsl-evaluation-not-started-observations", - f"not-started snapshot contains execution records: {populated}", - path, - ) - ) - ethics = snapshot["ethics_review"] - if isinstance(ethics, Mapping) and ethics.get("status") not in { - "pending", - "not_required", - }: - failures.append( - _failure( - "dsl-evaluation-ethics-state", - "not-started snapshot must remain pending or explicitly not-required", - path, - ) - ) - return set() - - subjects = records["subjects"] - attempts = records["attempts"] - observations = records["observations"] - reviews = records["reviews"] - withdrawals = records["withdrawals"] - tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - variants = _protocol_records_by_id(protocol, "variants", "variant_id") - measures = _protocol_records_by_id(protocol, "measures", "measure_id") - - _record_ids( - subjects, - "subject_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="subject", - path=path, - ) - attempt_ids = _record_ids( - attempts, - "attempt_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="attempt", - path=path, - ) - observation_ids = _record_ids( - observations, - "observation_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="observation", - path=path, - ) - review_ids = _record_ids( - reviews, - "review_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="review", - path=path, - ) - withdrawal_subject_ids = _record_ids( - withdrawals, - "subject_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="withdrawal", - path=path, - ) - - subjects_by_id = { - subject["subject_id"]: subject - for subject in subjects - if isinstance(subject, Mapping) and set(subject) == _SUBJECT_KEYS and isinstance(subject.get("subject_id"), str) - } - sampling_plan = protocol.get("sampling_plan") - experience_bands = set() - if isinstance(sampling_plan, Mapping): - experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) - for subject_id, subject in subjects_by_id.items(): - if subject.get("persona_id") not in catalogs.get("persona_ids", set()): - failures.append(_failure("dsl-evaluation-subject-join", f"{subject_id}: unknown persona", path)) - if subject.get("consent_status") not in {"consented", "withdrawn"}: - failures.append(_failure("dsl-evaluation-consent-status", f"{subject_id}: invalid consent status", path)) - if ( - not _bounded_text(subject.get("experience_band"), maximum=200) - or subject.get("experience_band") not in experience_bands - ): - failures.append(_failure("dsl-evaluation-subject-shape", f"{subject_id}: invalid experience band", path)) - declared_withdrawn_subjects = { - subject_id for subject_id, subject in subjects_by_id.items() if subject.get("consent_status") == "withdrawn" - } - if withdrawal_subject_ids != declared_withdrawn_subjects: - failures.append( - _failure( - "dsl-evaluation-withdrawal-join", - "withdrawal records must exactly match subjects with withdrawn consent", - path, - ) - ) - for withdrawal in withdrawals: - if not isinstance(withdrawal, Mapping) or set(withdrawal) != _WITHDRAWAL_KEYS: - continue - if ( - not _bounded_text(withdrawal.get("recorded_at"), maximum=100) - or withdrawal.get("retained_aggregate_only") is not True - ): - failures.append( - _failure( - "dsl-evaluation-withdrawal-shape", - f"{withdrawal.get('subject_id')}: withdrawal must retain aggregate counts only", - path, - ) - ) - - attempts_by_id: dict[str, Mapping[str, object]] = {} - for attempt in attempts: - if not isinstance(attempt, Mapping) or set(attempt) != _ATTEMPT_KEYS: - continue - attempt_id = attempt.get("attempt_id") - if not isinstance(attempt_id, str): - continue - attempts_by_id[attempt_id] = attempt - task_id = attempt.get("task_id") - task = tasks.get(task_id) if isinstance(task_id, str) else None - subject_id = attempt.get("subject_id") - subject = subjects_by_id.get(subject_id) if isinstance(subject_id, str) else None - outcome = attempt.get("outcome") - if outcome not in ATTEMPT_OUTCOMES: - failures.append(_failure("dsl-evaluation-attempt-outcome", f"{attempt_id}: invalid outcome", path)) - if task is None or subject is None: - failures.append(_failure("dsl-evaluation-attempt-join", f"{attempt_id}: unknown task or subject", path)) - continue - task_personas = _string_list(task.get("persona_ids"), non_empty=True) or [] - task_conditions = _string_list(task.get("tooling_condition_ids"), non_empty=True) or [] - task_variants = _string_list(task.get("variant_ids"), non_empty=True) or [] - if ( - attempt.get("persona_id") != subject.get("persona_id") - or attempt.get("persona_id") not in task_personas - or attempt.get("tooling_condition_id") not in task_conditions - or attempt.get("variant_id") not in task_variants - ): - failures.append( - _failure( - "dsl-evaluation-attempt-join", - f"{attempt_id}: subject, persona, task, condition, or variant mismatch", - path, - ) - ) - variant = variants.get(attempt.get("variant_id")) - if variant is None or variant.get("task_id") != task_id: - failures.append( - _failure("dsl-evaluation-attempt-join", f"{attempt_id}: variant belongs to another task", path) - ) - withdrawn = subject_id in withdrawal_subject_ids - if withdrawn != (outcome == "withdrawn"): - failures.append( - _failure( - "dsl-evaluation-withdrawal-join", - f"{attempt_id}: withdrawn subject and attempt outcome disagree", - path, - ) - ) - if not _valid_id(attempt.get("study_run_id")): - failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: invalid study run id", path)) - if not _bounded_text(attempt.get("started_at"), maximum=100) or not _bounded_text( - attempt.get("ended_at"), maximum=100 - ): - failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: timestamps must be text", path)) - if _string_list(attempt.get("observation_ids")) is None: - failures.append( - _failure("dsl-evaluation-attempt-observation-join", f"{attempt_id}: invalid observation ids", path) - ) - - observations_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} - child_observations: dict[str, set[str]] = {attempt_id: set() for attempt_id in attempt_ids} - for observation in observations: - if not isinstance(observation, Mapping) or set(observation) != _OBSERVATION_KEYS: - continue - observation_id = observation.get("observation_id") - attempt_id = observation.get("attempt_id") - measure_id = observation.get("measure_id") - artifact_stage = observation.get("artifact_stage") - if ( - not isinstance(observation_id, str) - or not isinstance(attempt_id, str) - or not isinstance(measure_id, str) - or not isinstance(artifact_stage, str) - ): - failures.append( - _failure("dsl-evaluation-observation-identity", "observation identity fields must be text", path) - ) - continue - opportunity = (attempt_id, measure_id, artifact_stage) - observation_dimensions = _string_list(observation.get("dimension_ids"), non_empty=True) - if opportunity in observations_by_opportunity: - failures.append( - _failure( - "dsl-evaluation-observation-identity", - f"duplicate attempt-measure-stage observation at {observation_id}", - path, - ) - ) - observations_by_opportunity[opportunity] = observation - child_observations.setdefault(attempt_id, set()).add(observation_id) - attempt = attempts_by_id.get(attempt_id) - measure = measures.get(measure_id) - task = tasks.get(observation.get("task_id")) - if attempt is None or measure is None or task is None: - failures.append( - _failure( - "dsl-evaluation-observation-join", - f"{observation_id}: unknown parent attempt, task, or measure", - path, - ) - ) - continue - parent_fields = ( - "study_run_id", - "task_id", - "persona_id", - "subject_id", - "tooling_condition_id", - "variant_id", - "outcome", - ) - if observation.get("protocol_revision") != protocol.get("revision") or any( - observation.get(field) != attempt.get(field) for field in parent_fields - ): - failures.append( - _failure( - "dsl-evaluation-observation-parent-join", - f"{observation_id}: observation does not match its parent attempt", - path, - ) - ) - task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) or [] - measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] - task_dimensions = set(_string_list(task.get("dimension_ids"), non_empty=True) or []) - measure_dimensions = set(_string_list(measure.get("dimension_ids"), non_empty=True) or []) - expected_dimensions = task_dimensions & measure_dimensions - parent_task_id = attempt.get("task_id") - parent_variant_id = attempt.get("variant_id") - try: - applicable_stages = ( - _measure_stage_ids(measure, parent_task_id, parent_variant_id) - if isinstance(parent_task_id, str) and isinstance(parent_variant_id, str) - else [] - ) - except ValueError: - applicable_stages = [] - if ( - artifact_stage not in task_stages - or artifact_stage not in applicable_stages - or observation.get("task_id") not in measure_tasks - or observation_dimensions is None - or set(observation_dimensions) != expected_dimensions - ): - failures.append( - _failure( - "dsl-evaluation-observation-task-join", - f"{observation_id}: stage, measure, or dimensions are not declared for the task", - path, - ) - ) - outcome = observation.get("outcome") - value = observation.get("value") - if outcome not in OBSERVATION_OUTCOMES: - failures.append(_failure("dsl-evaluation-observation-outcome", f"{observation_id}: invalid outcome", path)) - if measure_id == "task-completion": - expected_value = 1 if outcome == "completed" else 0 - valid_value = value == expected_value and not isinstance(value, bool) - elif outcome in {"completed", "failed"}: - valid_value = not isinstance(value, bool) and isinstance(value, (int, float)) - else: - valid_value = value is None - if not valid_value: - failures.append( - _failure( - "dsl-evaluation-observation-value", - f"{observation_id}: value does not represent its attempt outcome", - path, - ) - ) - refs = _string_list(observation.get("evidence_refs")) - if refs is None or any(safe_repo_path(repo_root, ref) is None for ref in refs): - failures.append( - _failure( - "dsl-evaluation-observation-evidence", - f"{observation_id}: evidence refs must be repository-confined paths", - path, - ) - ) - - expected_opportunities: set[tuple[str, str, str]] = set() - withdrawn_opportunities: set[tuple[str, str, str]] = set() - for attempt_id, attempt in attempts_by_id.items(): - task_id = attempt.get("task_id") - variant_id = attempt.get("variant_id") - for measure_id, measure in measures.items(): - measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] - if task_id not in measure_tasks or not isinstance(task_id, str) or not isinstance(variant_id, str): - continue - try: - applicable_stages = _measure_stage_ids(measure, task_id, variant_id) - except ValueError: - continue - for artifact_stage in applicable_stages: - opportunity = (attempt_id, measure_id, artifact_stage) - if attempt.get("outcome") == "withdrawn": - withdrawn_opportunities.add(opportunity) - else: - expected_opportunities.add(opportunity) - actual_opportunities = set(observations_by_opportunity) - if actual_opportunities != expected_opportunities or actual_opportunities & withdrawn_opportunities: - failures.append( - _failure( - "dsl-evaluation-opportunity-coverage", - "observations must exactly cover every non-withdrawn protocol-declared attempt-measure-stage opportunity", - path, - ) - ) - for attempt_id, attempt in attempts_by_id.items(): - stored_ids = _string_list(attempt.get("observation_ids")) - if stored_ids is not None and set(stored_ids) != child_observations.get(attempt_id, set()): - failures.append( - _failure( - "dsl-evaluation-attempt-observation-join", - f"{attempt_id}: observation ids do not match child records", - path, - ) - ) - - reviews_by_attempt: Counter[str] = Counter() - for review in reviews: - if not isinstance(review, Mapping) or set(review) != _REVIEW_KEYS: - continue - review_id = review.get("review_id") - attempt_id = review.get("attempt_id") - attempt = attempts_by_id.get(attempt_id) if isinstance(attempt_id, str) else None - reviewer_id = review.get("reviewer_subject_id") - reviewer = subjects_by_id.get(reviewer_id) if isinstance(reviewer_id, str) else None - if attempt is None or reviewer is None: - failures.append(_failure("dsl-evaluation-review-join", f"{review_id}: unknown attempt or reviewer", path)) - continue - task = tasks.get(attempt.get("task_id")) - task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) if task else None - task_personas = _string_list(task.get("persona_ids"), non_empty=True) if task else None - if ( - review.get("task_id") != attempt.get("task_id") - or review.get("variant_id") != attempt.get("variant_id") - or reviewer_id == attempt.get("subject_id") - or reviewer.get("consent_status") != "consented" - or task_stages is None - or "review-judgment" not in task_stages - or task_personas is None - or reviewer.get("persona_id") not in task_personas - ): - failures.append( - _failure( - "dsl-evaluation-review-join", - f"{review_id}: review does not match an eligible independent reviewer and parent task", - path, - ) - ) - if ( - not _bounded_text(review.get("judgment"), maximum=500) - or isinstance(review.get("confidence"), bool) - or not isinstance(review.get("confidence"), (int, float)) - or not 0 <= review["confidence"] <= 1 - or not _valid_id(review.get("rationale_code")) - or not _bounded_text(review.get("fixed_at"), maximum=100) - ): - failures.append(_failure("dsl-evaluation-review-shape", f"{review_id}: invalid fixed judgment", path)) - reviews_by_attempt[attempt_id] += 1 - - for disagreement in records["disagreements"]: - if not isinstance(disagreement, Mapping) or set(disagreement) != _DISAGREEMENT_KEYS: - continue - disagreement_review_ids = _string_list(disagreement["review_ids"], non_empty=True) - linked_reviews = [ - review - for review in reviews - if isinstance(review, Mapping) - and disagreement_review_ids is not None - and review.get("review_id") in disagreement_review_ids - ] - linked_attempts = {review.get("attempt_id") for review in linked_reviews} - if ( - disagreement_review_ids is None - or len(disagreement_review_ids) < 2 - or not set(disagreement_review_ids).issubset(review_ids) - or len(linked_attempts) != 1 - ): - failures.append( - _failure( - "dsl-evaluation-disagreement-join", - f"{disagreement['disagreement_id']}: reviews must share one parent attempt", - path, - ) - ) - if disagreement["originals_preserved"] is not True: - failures.append( - _failure( - "dsl-evaluation-disagreement-preservation", - f"{disagreement['disagreement_id']}: originals must be preserved", - path, - ) - ) - - if snapshot["execution_status"] == "complete": - nonwithdrawn_subject_ids = { - subject_id - for subject_id, subject in subjects_by_id.items() - if subject.get("consent_status") == "consented" - and subject.get("persona_id") in scope.get("persona_ids", set()) - } - active_subject_ids = { - subject_id - for subject_id in nonwithdrawn_subject_ids - if any( - attempt.get("subject_id") == subject_id - and attempt.get("outcome") != "withdrawn" - and _attempt_matches_scope(attempt, scope) - for attempt in attempts_by_id.values() - ) - } - persona_minimums = { - item["persona_id"]: item["minimum_completed_subjects"] - for item in protocol.get("personas", []) - if isinstance(item, Mapping) - and isinstance(item.get("persona_id"), str) - and isinstance(item.get("minimum_completed_subjects"), int) - and item["persona_id"] in scope.get("persona_ids", set()) - } - persona_counts = Counter(subjects_by_id[subject_id]["persona_id"] for subject_id in active_subject_ids) - missing_personas = sorted( - persona_id for persona_id, minimum in persona_minimums.items() if persona_counts[persona_id] < minimum - ) - expected_task_shapes = { - (task["task_id"], condition_id, variant_id) - for task in tasks.values() - if task["task_id"] in scope.get("task_ids", set()) - for condition_id in (_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) - if condition_id in scope.get("tooling_condition_ids", set()) - for variant_id in (_string_list(task.get("variant_ids"), non_empty=True) or []) - if variant_id in scope.get("variant_ids", set()) - } - actual_task_shapes = { - (attempt.get("task_id"), attempt.get("tooling_condition_id"), attempt.get("variant_id")) - for attempt in attempts_by_id.values() - if attempt.get("outcome") != "withdrawn" and _attempt_matches_scope(attempt, scope) - } - review_required_attempts = { - attempt_id - for attempt_id, attempt in attempts_by_id.items() - if _attempt_matches_scope(attempt, scope) - if "review-judgment" - in (_string_list(tasks.get(attempt.get("task_id"), {}).get("artifact_stage_ids"), non_empty=True) or []) - and attempt.get("outcome") != "withdrawn" - } - sampling_plan = protocol.get("sampling_plan", {}) - target_total = sampling_plan.get("target_total") if isinstance(sampling_plan, Mapping) else None - ethics = snapshot["ethics_review"] - ethics_approved = isinstance(ethics, Mapping) and ethics.get("status") == "approved" - execution_plan = protocol.get("execution_plan", {}) - subject_task_requirements = ( - execution_plan.get("subject_task_requirements", []) if isinstance(execution_plan, Mapping) else [] - ) - missing_subject_workloads: list[tuple[str, str]] = [] - if isinstance(subject_task_requirements, list): - for subject_id in nonwithdrawn_subject_ids: - subject_attempts = [ - attempt - for attempt in attempts_by_id.values() - if attempt.get("subject_id") == subject_id - and attempt.get("outcome") != "withdrawn" - and _attempt_matches_scope(attempt, scope) - ] - for requirement in subject_task_requirements: - if not isinstance(requirement, Mapping): - continue - task_kind_values = _string_list(requirement.get("task_kinds"), non_empty=True) - minimum = requirement.get("minimum_assigned_attempts") - requirement_id = requirement.get("requirement_id") - if ( - task_kind_values is None - or not isinstance(minimum, int) - or isinstance(minimum, bool) - or not isinstance(requirement_id, str) - ): - continue - assigned = sum( - tasks.get(attempt.get("task_id"), {}).get("kind") in task_kind_values - for attempt in subject_attempts - ) - if assigned < minimum: - missing_subject_workloads.append((subject_id, requirement_id)) - if missing_subject_workloads: - failures.append( - _failure( - "dsl-evaluation-subject-workload", - "complete execution does not satisfy every active subject's assigned task groups", - path, - ) - ) - if ( - not ethics_approved - or missing_personas - or not isinstance(target_total, int) - or len(active_subject_ids) < target_total - or not expected_task_shapes.issubset(actual_task_shapes) - or any(reviews_by_attempt[attempt_id] == 0 for attempt_id in review_required_attempts) - or missing_subject_workloads - ): - failures.append( - _failure( - "dsl-evaluation-completion-coverage", - "complete execution lacks approved ethics, subject workload/minima, task/condition/variant coverage, or required independent reviews", - path, - ) - ) - return observation_ids - - -def _validate_analysis( - repo_root: Path, - protocol: Mapping[str, object], - snapshot: Mapping[str, object], - analysis: dict[str, object], - catalogs: Mapping[str, set[str]], - scope: Mapping[str, set[str]], - strata: Sequence[Mapping[str, object]], - observation_ids: set[str], - failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/analysis-v1.json", -) -> None: - if not _exact_keys( - analysis, - _ANALYSIS_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label="analysis", - path=path, - ): - return - if analysis["protocol_revision"] != protocol.get("revision"): - failures.append(_failure("dsl-evaluation-analysis-join", "protocol revision mismatch", path)) - if analysis["snapshot_id"] != snapshot.get("snapshot_id"): - failures.append(_failure("dsl-evaluation-analysis-join", "snapshot id mismatch", path)) - if analysis["execution_status"] != snapshot.get("execution_status"): - failures.append(_failure("dsl-evaluation-analysis-join", "execution status mismatch", path)) - if analysis["evidence_status"] not in EVIDENCE_STATUSES: - failures.append(_failure("dsl-evaluation-evidence-status", "invalid evidence status", path)) - - measure_results = _bounded_list( - analysis["measure_results"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label="measure_results", - path=path, - ) - measure_result_ids = _record_ids( - measure_results, - "measure_id", - failures, - rule_id="dsl-evaluation-analysis-id", - label="measure result", - path=path, - ) - if measure_result_ids != scope.get("measure_ids", set()): - failures.append( - _failure( - "dsl-evaluation-analysis-measure-coverage", - "analysis must contain one result for every claim-scoped measure", - path, - ) - ) - try: - recomputed_measures = recompute_measure_results(protocol, snapshot, scope=scope) - except ValueError as exc: - failures.append(_failure("dsl-evaluation-observation-value", str(exc), path)) - recomputed_measures = {} - for index, result in enumerate(measure_results): - if not _exact_keys( - result, - _MEASURE_RESULT_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label=f"measure_results[{index}]", - path=path, - ): - continue - expected = recomputed_measures.get(result["measure_id"]) - if expected and expected["denominator"] == 0: - expected_status = "not_evaluated" - elif expected and expected["observed_count"] != expected["denominator"]: - expected_status = "incomplete" - else: - expected_status = "evaluated" - if expected is None or result != { - "measure_id": result["measure_id"], - "status": expected_status, - **expected, - }: - failures.append( - _failure( - "dsl-evaluation-analysis-measure-drift", - f"{result['measure_id']}: stored aggregate does not match frozen observations", - path, - ) - ) - - try: - recomputed_dimensions = recompute_dimension_results( - protocol, - recomputed_measures, - dimension_ids=scope.get("dimension_ids", set()), - ) - except ValueError as exc: - failures.append(_failure("dsl-evaluation-threshold-evaluation", str(exc), path)) - recomputed_dimensions = {} - - results = _bounded_list( - analysis["dimension_results"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label="dimension_results", - path=path, - ) - result_ids = _record_ids( - results, - "dimension_id", - failures, - rule_id="dsl-evaluation-analysis-id", - label="dimension result", - path=path, - ) - if result_ids != scope.get("dimension_ids", set()): - failures.append( - _failure( - "dsl-evaluation-analysis-dimension-coverage", - "analysis must contain one result for every claim-scoped dimension", - path, - ) - ) - for index, result in enumerate(results): - if not _exact_keys( - result, - _DIMENSION_RESULT_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label=f"dimension_results[{index}]", - path=path, - ): - continue - refs = _string_list(result["supporting_observation_ids"]) - if refs is None or not set(refs).issubset(observation_ids): - failures.append( - _failure( - "dsl-evaluation-analysis-observation-join", - f"{result['dimension_id']}: unknown supporting observations", - path, - ) - ) - expected_dimension = recomputed_dimensions.get(result["dimension_id"]) - if expected_dimension is None or result != { - "dimension_id": result["dimension_id"], - **expected_dimension, - }: - failures.append( - _failure( - "dsl-evaluation-analysis-dimension-drift", - f"{result['dimension_id']}: stored threshold result does not match recomputed measures", - path, - ) - ) - - stored_strata = _bounded_list( - analysis["stratum_results"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label="stratum_results", - path=path, - ) - expected_strata: list[dict[str, object]] = [] - if snapshot.get("execution_status") == "not_started": - if stored_strata: - failures.append( - _failure( - "dsl-evaluation-stratum-drift", - "not-started analysis cannot persist derived stratum results", - path, - ) - ) - else: - try: - expected_strata = recompute_stratum_results(protocol, snapshot, strata) - except ValueError as exc: - failures.append(_failure("dsl-evaluation-stratum-drift", str(exc), path)) - expected_by_id = {item["stratum_id"]: item for item in expected_strata} - stored_ids = _record_ids( - stored_strata, - "stratum_id", - failures, - rule_id="dsl-evaluation-analysis-id", - label="stratum result", - path=path, - ) - if stored_ids != set(expected_by_id): - failures.append( - _failure( - "dsl-evaluation-stratum-coverage", - "analysis must persist one independently recomputed result for every bound stratum", - path, - ) - ) - for index, result in enumerate(stored_strata): - if not _exact_keys( - result, - _STRATUM_RESULT_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label=f"stratum_results[{index}]", - path=path, - ): - continue - assert isinstance(result, dict) - if result != expected_by_id.get(result["stratum_id"]): - failures.append( - _failure( - "dsl-evaluation-stratum-drift", - f"{result['stratum_id']}: stored stratum result does not match frozen observations", - path, - ) - ) - claim = analysis["claim"] - if _exact_keys( - claim, - _CLAIM_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label="claim", - path=path, - ): - evidence_artifacts = claim["evidence_artifacts"] - if not isinstance(evidence_artifacts, list) or len(evidence_artifacts) != 3: - failures.append( - _failure( - "dsl-evaluation-claim-evidence", - "claim must name protocol, snapshot, and analysis artifacts", - path, - ) - ) - else: - for artifact in evidence_artifacts: - resolved = safe_repo_path(repo_root, artifact) if isinstance(artifact, str) else None - if resolved is None or not resolved.is_file(): - failures.append( - _failure( - "dsl-evaluation-claim-evidence-path", - f"unsafe or missing claim evidence artifact {artifact!r}", - path, - ) - ) - status = analysis["evidence_status"] - if snapshot.get("execution_status") == "not_started": - if status != "untested": - failures.append( - _failure( - "dsl-evaluation-evidence-status", - "not-started execution must remain untested", - path, - ) - ) - for result in results: - if not isinstance(result, Mapping): - continue - expected = { - "status": "not_evaluated", - "threshold_result": "not_evaluated", - "condition_results": [], - "supporting_observation_ids": [], - } - if any(result.get(field) != value for field, value in expected.items()): - failures.append( - _failure( - "dsl-evaluation-not-started-analysis", - f"{result.get('dimension_id')}: not-started result contains derived evidence", - path, - ) - ) - gating_strata = [item for item in expected_strata if item.get("role") == "gating"] - gating_dimensions = [ - item.get("dimension_results", []) for item in gating_strata if isinstance(item.get("dimension_results"), list) - ] - all_pass = ( - bool(gating_strata) - and len(gating_dimensions) == len(gating_strata) - and all( - dimension_results - and all( - isinstance(result, Mapping) - and result.get("status") == "evaluated" - and result.get("threshold_result") == "pass" - for result in dimension_results - ) - for dimension_results in gating_dimensions - ) - ) - any_fail = any( - isinstance(result, Mapping) and result.get("status") == "evaluated" and result.get("threshold_result") == "fail" - for dimension_results in gating_dimensions - for result in dimension_results - ) - unresolved = any( - isinstance(item, Mapping) and item.get("status") == "unresolved" for item in snapshot.get("disagreements", []) - ) - invalidating_deviation = any( - isinstance(item, Mapping) and item.get("severity") == "invalidating" for item in snapshot.get("deviations", []) - ) - execution_complete = snapshot.get("execution_status") == "complete" - qualifies_demonstrated = execution_complete and all_pass and not unresolved and not invalidating_deviation - qualifies_refuted = execution_complete and any_fail - execution_records_present = any( - isinstance(snapshot.get(field), list) and bool(snapshot[field]) - for field in ("subjects", "attempts", "observations", "reviews", "deviations", "withdrawals") - ) - if status == "demonstrated" and not qualifies_demonstrated: - failures.append( - _failure( - "dsl-evaluation-evidence-status", - "demonstrated requires every bound gating stratum to pass without unresolved critical disagreement or invalidating deviation", - path, - ) - ) - elif status == "refuted" and not qualifies_refuted: - failures.append( - _failure( - "dsl-evaluation-evidence-status", - "refuted requires a complete execution with at least one gating-stratum dimension failure", - path, - ) - ) - elif status == "partial" and ( - snapshot.get("execution_status") == "not_started" - or not execution_records_present - or qualifies_demonstrated - or qualifies_refuted - ): - failures.append( - _failure( - "dsl-evaluation-evidence-status", - "partial requires relevant execution evidence that is not yet a demonstrated or refuted result", - path, - ) - ) - elif status == "untested" and snapshot.get("execution_status") != "not_started": - failures.append( - _failure( - "dsl-evaluation-evidence-status", - "untested is reserved for a not-started execution without evidence records", - path, - ) - ) +from tools.dsl_language_evaluation._keys import ( + _BUNDLE_ENTRY_KEYS, + _MANIFEST_KEYS, + _MAX_FILE_BYTES, + MANIFEST_PATH, + REQUIRED_DIMENSION_IDS, + REQUIRED_PERSONA_IDS, + REQUIRED_TASK_KINDS, +) +from tools.dsl_language_evaluation._measures import ( + recompute_dimension_results, + recompute_measure_results, + recompute_stratum_results, +) +from tools.dsl_language_evaluation._protocol import _validate_protocol +from tools.dsl_language_evaluation._shape import _failure, _valid_id +from tools.dsl_language_evaluation._snapshot import _validate_snapshot + +__all__ = [ + "MANIFEST_PATH", + "REQUIRED_DIMENSION_IDS", + "REQUIRED_PERSONA_IDS", + "REQUIRED_TASK_KINDS", + "evaluate", + "load_bundle", + "load_bundles", + "main", + "recompute_dimension_results", + "recompute_measure_results", + "recompute_stratum_results", + "resolve_claim_strata", + "validate_bundle", +] def validate_bundle( diff --git a/tools/dsl_language_evaluation/__init__.py b/tools/dsl_language_evaluation/__init__.py new file mode 100644 index 000000000..e9dee57eb --- /dev/null +++ b/tools/dsl_language_evaluation/__init__.py @@ -0,0 +1,10 @@ +"""Split support package for the DSL language-evaluation checker.""" + +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_PYTHON_PACKAGES = _REPO_ROOT / "implementations" / "python" / "packages" +for _import_root in (_REPO_ROOT, _PYTHON_PACKAGES): + if str(_import_root) not in sys.path: + sys.path.insert(0, str(_import_root)) diff --git a/tools/dsl_language_evaluation/_analysis.py b/tools/dsl_language_evaluation/_analysis.py new file mode 100644 index 000000000..5966f9111 --- /dev/null +++ b/tools/dsl_language_evaluation/_analysis.py @@ -0,0 +1,375 @@ +"""Analysis validation and evidence-status honesty for the DSL evaluation.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path + +from tools.dsl_language_evaluation._keys import ( + _ANALYSIS_KEYS, + _CLAIM_KEYS, + _DIMENSION_RESULT_KEYS, + _MAX_CATALOG_ITEMS, + _MEASURE_RESULT_KEYS, + _STRATUM_RESULT_KEYS, + EVIDENCE_STATUSES, +) +from tools.dsl_language_evaluation._measures import ( + recompute_dimension_results, + recompute_measure_results, + recompute_stratum_results, +) +from tools.dsl_language_evaluation._shape import ( + _bounded_list, + _exact_keys, + _failure, + _record_ids, + _string_list, +) +from tools.policy.common import PolicyFailure, safe_repo_path + + +def _validate_analysis( + repo_root: Path, + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + analysis: dict[str, object], + catalogs: Mapping[str, set[str]], + scope: Mapping[str, set[str]], + strata: Sequence[Mapping[str, object]], + observation_ids: set[str], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/analysis-v1.json", +) -> None: + if not _exact_keys( + analysis, + _ANALYSIS_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label="analysis", + path=path, + ): + return + if analysis["protocol_revision"] != protocol.get("revision"): + failures.append(_failure("dsl-evaluation-analysis-join", "protocol revision mismatch", path)) + if analysis["snapshot_id"] != snapshot.get("snapshot_id"): + failures.append(_failure("dsl-evaluation-analysis-join", "snapshot id mismatch", path)) + if analysis["execution_status"] != snapshot.get("execution_status"): + failures.append(_failure("dsl-evaluation-analysis-join", "execution status mismatch", path)) + if analysis["evidence_status"] not in EVIDENCE_STATUSES: + failures.append(_failure("dsl-evaluation-evidence-status", "invalid evidence status", path)) + + measure_results = _bounded_list( + analysis["measure_results"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label="measure_results", + path=path, + ) + measure_result_ids = _record_ids( + measure_results, + "measure_id", + failures, + rule_id="dsl-evaluation-analysis-id", + label="measure result", + path=path, + ) + if measure_result_ids != scope.get("measure_ids", set()): + failures.append( + _failure( + "dsl-evaluation-analysis-measure-coverage", + "analysis must contain one result for every claim-scoped measure", + path, + ) + ) + try: + recomputed_measures = recompute_measure_results(protocol, snapshot, scope=scope) + except ValueError as exc: + failures.append(_failure("dsl-evaluation-observation-value", str(exc), path)) + recomputed_measures = {} + for index, result in enumerate(measure_results): + if not _exact_keys( + result, + _MEASURE_RESULT_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label=f"measure_results[{index}]", + path=path, + ): + continue + expected = recomputed_measures.get(result["measure_id"]) + if expected and expected["denominator"] == 0: + expected_status = "not_evaluated" + elif expected and expected["observed_count"] != expected["denominator"]: + expected_status = "incomplete" + else: + expected_status = "evaluated" + if expected is None or result != { + "measure_id": result["measure_id"], + "status": expected_status, + **expected, + }: + failures.append( + _failure( + "dsl-evaluation-analysis-measure-drift", + f"{result['measure_id']}: stored aggregate does not match frozen observations", + path, + ) + ) + + try: + recomputed_dimensions = recompute_dimension_results( + protocol, + recomputed_measures, + dimension_ids=scope.get("dimension_ids", set()), + ) + except ValueError as exc: + failures.append(_failure("dsl-evaluation-threshold-evaluation", str(exc), path)) + recomputed_dimensions = {} + + results = _bounded_list( + analysis["dimension_results"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label="dimension_results", + path=path, + ) + result_ids = _record_ids( + results, + "dimension_id", + failures, + rule_id="dsl-evaluation-analysis-id", + label="dimension result", + path=path, + ) + if result_ids != scope.get("dimension_ids", set()): + failures.append( + _failure( + "dsl-evaluation-analysis-dimension-coverage", + "analysis must contain one result for every claim-scoped dimension", + path, + ) + ) + for index, result in enumerate(results): + if not _exact_keys( + result, + _DIMENSION_RESULT_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label=f"dimension_results[{index}]", + path=path, + ): + continue + refs = _string_list(result["supporting_observation_ids"]) + if refs is None or not set(refs).issubset(observation_ids): + failures.append( + _failure( + "dsl-evaluation-analysis-observation-join", + f"{result['dimension_id']}: unknown supporting observations", + path, + ) + ) + expected_dimension = recomputed_dimensions.get(result["dimension_id"]) + if expected_dimension is None or result != { + "dimension_id": result["dimension_id"], + **expected_dimension, + }: + failures.append( + _failure( + "dsl-evaluation-analysis-dimension-drift", + f"{result['dimension_id']}: stored threshold result does not match recomputed measures", + path, + ) + ) + + stored_strata = _bounded_list( + analysis["stratum_results"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label="stratum_results", + path=path, + ) + expected_strata: list[dict[str, object]] = [] + if snapshot.get("execution_status") == "not_started": + if stored_strata: + failures.append( + _failure( + "dsl-evaluation-stratum-drift", + "not-started analysis cannot persist derived stratum results", + path, + ) + ) + else: + try: + expected_strata = recompute_stratum_results(protocol, snapshot, strata) + except ValueError as exc: + failures.append(_failure("dsl-evaluation-stratum-drift", str(exc), path)) + expected_by_id = {item["stratum_id"]: item for item in expected_strata} + stored_ids = _record_ids( + stored_strata, + "stratum_id", + failures, + rule_id="dsl-evaluation-analysis-id", + label="stratum result", + path=path, + ) + if stored_ids != set(expected_by_id): + failures.append( + _failure( + "dsl-evaluation-stratum-coverage", + "analysis must persist one independently recomputed result for every bound stratum", + path, + ) + ) + for index, result in enumerate(stored_strata): + if not _exact_keys( + result, + _STRATUM_RESULT_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label=f"stratum_results[{index}]", + path=path, + ): + continue + assert isinstance(result, dict) + if result != expected_by_id.get(result["stratum_id"]): + failures.append( + _failure( + "dsl-evaluation-stratum-drift", + f"{result['stratum_id']}: stored stratum result does not match frozen observations", + path, + ) + ) + claim = analysis["claim"] + if _exact_keys( + claim, + _CLAIM_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label="claim", + path=path, + ): + evidence_artifacts = claim["evidence_artifacts"] + if not isinstance(evidence_artifacts, list) or len(evidence_artifacts) != 3: + failures.append( + _failure( + "dsl-evaluation-claim-evidence", + "claim must name protocol, snapshot, and analysis artifacts", + path, + ) + ) + else: + for artifact in evidence_artifacts: + resolved = safe_repo_path(repo_root, artifact) if isinstance(artifact, str) else None + if resolved is None or not resolved.is_file(): + failures.append( + _failure( + "dsl-evaluation-claim-evidence-path", + f"unsafe or missing claim evidence artifact {artifact!r}", + path, + ) + ) + status = analysis["evidence_status"] + if snapshot.get("execution_status") == "not_started": + if status != "untested": + failures.append( + _failure( + "dsl-evaluation-evidence-status", + "not-started execution must remain untested", + path, + ) + ) + for result in results: + if not isinstance(result, Mapping): + continue + expected = { + "status": "not_evaluated", + "threshold_result": "not_evaluated", + "condition_results": [], + "supporting_observation_ids": [], + } + if any(result.get(field) != value for field, value in expected.items()): + failures.append( + _failure( + "dsl-evaluation-not-started-analysis", + f"{result.get('dimension_id')}: not-started result contains derived evidence", + path, + ) + ) + gating_strata = [item for item in expected_strata if item.get("role") == "gating"] + gating_dimensions = [ + item.get("dimension_results", []) for item in gating_strata if isinstance(item.get("dimension_results"), list) + ] + all_pass = ( + bool(gating_strata) + and len(gating_dimensions) == len(gating_strata) + and all( + dimension_results + and all( + isinstance(result, Mapping) + and result.get("status") == "evaluated" + and result.get("threshold_result") == "pass" + for result in dimension_results + ) + for dimension_results in gating_dimensions + ) + ) + any_fail = any( + isinstance(result, Mapping) and result.get("status") == "evaluated" and result.get("threshold_result") == "fail" + for dimension_results in gating_dimensions + for result in dimension_results + ) + unresolved = any( + isinstance(item, Mapping) and item.get("status") == "unresolved" for item in snapshot.get("disagreements", []) + ) + invalidating_deviation = any( + isinstance(item, Mapping) and item.get("severity") == "invalidating" for item in snapshot.get("deviations", []) + ) + execution_complete = snapshot.get("execution_status") == "complete" + qualifies_demonstrated = execution_complete and all_pass and not unresolved and not invalidating_deviation + qualifies_refuted = execution_complete and any_fail + execution_records_present = any( + isinstance(snapshot.get(field), list) and bool(snapshot[field]) + for field in ("subjects", "attempts", "observations", "reviews", "deviations", "withdrawals") + ) + if status == "demonstrated" and not qualifies_demonstrated: + failures.append( + _failure( + "dsl-evaluation-evidence-status", + "demonstrated requires every bound gating stratum to pass without unresolved critical disagreement or invalidating deviation", + path, + ) + ) + elif status == "refuted" and not qualifies_refuted: + failures.append( + _failure( + "dsl-evaluation-evidence-status", + "refuted requires a complete execution with at least one gating-stratum dimension failure", + path, + ) + ) + elif status == "partial" and ( + snapshot.get("execution_status") == "not_started" + or not execution_records_present + or qualifies_demonstrated + or qualifies_refuted + ): + failures.append( + _failure( + "dsl-evaluation-evidence-status", + "partial requires relevant execution evidence that is not yet a demonstrated or refuted result", + path, + ) + ) + elif status == "untested" and snapshot.get("execution_status") != "not_started": + failures.append( + _failure( + "dsl-evaluation-evidence-status", + "untested is reserved for a not-started execution without evidence records", + path, + ) + ) diff --git a/tools/dsl_language_evaluation/_claims.py b/tools/dsl_language_evaluation/_claims.py new file mode 100644 index 000000000..dbfcfcc18 --- /dev/null +++ b/tools/dsl_language_evaluation/_claims.py @@ -0,0 +1,435 @@ +"""Claim scope and stratum binding validation.""" + +from __future__ import annotations + +from collections.abc import Mapping +from itertools import product + +from tools.dsl_language_evaluation._keys import ( + _CLAIM_BINDING_KEYS, + _CLAIM_SCOPE_KEYS, + _MAX_CATALOG_ITEMS, + _STRATUM_GROUP_KEYS, + _STRATUM_PARTITION_AXES, + _STRATUM_ROLES, + MANIFEST_PATH, +) +from tools.dsl_language_evaluation._shape import ( + _bounded_list, + _exact_keys, + _failure, + _protocol_records_by_id, + _record_ids, + _string_list, + _valid_id, +) +from tools.policy.common import PolicyFailure + + +def _full_claim_scope(catalogs: Mapping[str, set[str]]) -> dict[str, set[str]]: + return { + "persona_ids": set(catalogs.get("persona_ids", set())), + "task_ids": set(catalogs.get("task_ids", set())), + "tooling_condition_ids": set(catalogs.get("condition_ids", set())), + "variant_ids": set(catalogs.get("variant_ids", set())), + "artifact_stage_ids": set(catalogs.get("stage_ids", set())), + "dimension_ids": set(catalogs.get("dimension_ids", set())), + "measure_ids": set(catalogs.get("measure_ids", set())), + } + + +def _attempt_matches_scope( + attempt: Mapping[str, object], + scope: Mapping[str, set[str]], +) -> bool: + return ( + attempt.get("task_id") in scope.get("task_ids", set()) + and attempt.get("persona_id") in scope.get("persona_ids", set()) + and attempt.get("tooling_condition_id") in scope.get("tooling_condition_ids", set()) + and attempt.get("variant_id") in scope.get("variant_ids", set()) + ) + + +def _validate_claim_scope( + protocol: Mapping[str, object], + analysis: Mapping[str, object], + catalogs: Mapping[str, set[str]], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/analysis-v1.json", +) -> dict[str, set[str]]: + """Validate and resolve the exact catalog slice owned by one claim.""" + + fallback = _full_claim_scope(catalogs) + claim = analysis.get("claim") + scope = claim.get("scope") if isinstance(claim, Mapping) else None + if not _exact_keys( + scope, + _CLAIM_SCOPE_KEYS, + failures, + rule_id="dsl-evaluation-claim-scope", + label="claim scope", + path=path, + ): + return fallback + + catalog_fields = { + "persona_ids": "persona_ids", + "task_ids": "task_ids", + "tooling_condition_ids": "condition_ids", + "variant_ids": "variant_ids", + "artifact_stage_ids": "stage_ids", + "dimension_ids": "dimension_ids", + "measure_ids": "measure_ids", + } + resolved: dict[str, set[str]] = {} + for field, catalog_field in catalog_fields.items(): + values = _string_list(scope[field], non_empty=True) + known = catalogs.get(catalog_field, set()) + if values is None or len(values) != len(set(values)) or not set(values).issubset(known): + failures.append( + _failure( + "dsl-evaluation-claim-scope", + f"claim scope {field} must be a non-empty unique subset of the protocol catalog", + path, + ) + ) + resolved[field] = set(values or []) & known + else: + resolved[field] = set(values) + + tasks = _protocol_records_by_id(protocol, "tasks", "task_id") + measures = _protocol_records_by_id(protocol, "measures", "measure_id") + for task_id in resolved["task_ids"]: + task = tasks.get(task_id, {}) + joins = ( + ("persona_ids", "persona_ids"), + ("tooling_condition_ids", "tooling_condition_ids"), + ("variant_ids", "variant_ids"), + ("artifact_stage_ids", "artifact_stage_ids"), + ) + if any( + not (set(_string_list(task.get(task_field), non_empty=True) or []) & resolved[scope_field]) + for task_field, scope_field in joins + ): + failures.append( + _failure( + "dsl-evaluation-claim-scope", + f"{task_id}: claim scope must retain an eligible persona, condition, variant, and stage", + path, + ) + ) + if not any( + task_id in (_string_list(measures[measure_id].get("task_ids"), non_empty=True) or []) + for measure_id in resolved["measure_ids"] + if measure_id in measures + ): + failures.append( + _failure( + "dsl-evaluation-claim-scope", + f"{task_id}: claim scope has no applicable measure", + path, + ) + ) + + thresholds = { + threshold.get("dimension_id"): threshold + for threshold in protocol.get("thresholds", []) + if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) + } + for dimension_id in resolved["dimension_ids"]: + threshold = thresholds.get(dimension_id) + conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] + required_measures = { + condition.get("measure_id") + for condition in conditions + if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) + } + if not required_measures or not required_measures.issubset(resolved["measure_ids"]): + failures.append( + _failure( + "dsl-evaluation-claim-scope", + f"{dimension_id}: claim scope must include every threshold measure", + path, + ) + ) + return resolved + + +def _validate_claim_binding( + protocol: Mapping[str, object], + analysis: Mapping[str, object], + claim_binding: object, + catalogs: Mapping[str, set[str]], + scope: Mapping[str, set[str]], + failures: list[PolicyFailure], + *, + path: str, +) -> list[dict[str, object]]: + """Bind one stable claim to its immutable scope and reporting strata.""" + + if not _exact_keys( + claim_binding, + _CLAIM_BINDING_KEYS, + failures, + rule_id="dsl-evaluation-claim-binding", + label="claim binding", + path=path, + ): + return [] + assert isinstance(claim_binding, dict) + claim = analysis.get("claim") + claim_id = claim.get("claim_id") if isinstance(claim, Mapping) else None + if not _valid_id(claim_binding["claim_id"]) or claim_binding["claim_id"] != claim_id: + failures.append( + _failure( + "dsl-evaluation-claim-binding", + "manifest claim binding must name the analysis claim_id exactly", + path, + ) + ) + + bound_scope = claim_binding["scope"] + if _exact_keys( + bound_scope, + _CLAIM_SCOPE_KEYS, + failures, + rule_id="dsl-evaluation-claim-binding", + label="claim binding scope", + path=path, + ): + assert isinstance(bound_scope, dict) + for field in _CLAIM_SCOPE_KEYS: + values = _string_list(bound_scope[field], non_empty=True) + if values is None or len(values) != len(set(values)) or set(values) != scope.get(field, set()): + failures.append( + _failure( + "dsl-evaluation-claim-binding", + f"analysis claim scope {field} must exactly match its manifest binding", + path, + ) + ) + + groups = _bounded_list( + claim_binding["strata"], + 16, + failures, + rule_id="dsl-evaluation-claim-strata", + label="claim binding strata", + path=path, + ) + group_ids = _record_ids( + groups, + "group_id", + failures, + rule_id="dsl-evaluation-claim-strata", + label="claim stratum group", + path=path, + ) + experience_bands = set() + sampling_plan = protocol.get("sampling_plan") + if isinstance(sampling_plan, Mapping): + experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) + + expanded: list[dict[str, object]] = [] + for index, group in enumerate(groups): + if not _exact_keys( + group, + _STRATUM_GROUP_KEYS, + failures, + rule_id="dsl-evaluation-claim-strata", + label=f"claim binding strata[{index}]", + path=path, + ): + continue + assert isinstance(group, dict) + group_id = group["group_id"] + role = group["role"] + partition_by = _string_list(group["partition_by"]) + persona_ids = _string_list(group["persona_ids"], non_empty=True) + band_ids = _string_list(group["experience_band_ids"], non_empty=True) + condition_ids = _string_list(group["tooling_condition_ids"], non_empty=True) + valid = ( + _valid_id(group_id) + and role in _STRATUM_ROLES + and partition_by is not None + and len(partition_by) == len(set(partition_by)) + and set(partition_by).issubset(_STRATUM_PARTITION_AXES) + and persona_ids is not None + and len(persona_ids) == len(set(persona_ids)) + and set(persona_ids).issubset(scope.get("persona_ids", set())) + and band_ids is not None + and len(band_ids) == len(set(band_ids)) + and set(band_ids).issubset(experience_bands) + and condition_ids is not None + and len(condition_ids) == len(set(condition_ids)) + and set(condition_ids).issubset(scope.get("tooling_condition_ids", set())) + ) + if not valid: + failures.append( + _failure( + "dsl-evaluation-claim-strata", + f"claim stratum group {group_id!r} has invalid role, partition axes, or catalog filters", + path, + ) + ) + continue + + assert isinstance(group_id, str) + assert isinstance(role, str) + assert partition_by is not None + assert persona_ids is not None + assert band_ids is not None + assert condition_ids is not None + axis_values = { + "persona_id": persona_ids, + "experience_band": band_ids, + "tooling_condition_id": condition_ids, + } + split_axes = [ + axis for axis in ("persona_id", "experience_band", "tooling_condition_id") if axis in partition_by + ] + combinations = product(*(axis_values[axis] for axis in split_axes)) if split_axes else [()] + for combination in combinations: + selected = dict(zip(split_axes, combination, strict=True)) + stratum_id = "-".join([group_id, *(str(selected[axis]) for axis in split_axes)]) + stratum_personas = {str(selected["persona_id"])} if "persona_id" in selected else set(persona_ids) + stratum_bands = {str(selected["experience_band"])} if "experience_band" in selected else set(band_ids) + stratum_conditions = ( + {str(selected["tooling_condition_id"])} if "tooling_condition_id" in selected else set(condition_ids) + ) + stratum_scope = _derive_stratum_scope( + protocol, + scope, + persona_ids=stratum_personas, + experience_band_ids=stratum_bands, + tooling_condition_ids=stratum_conditions, + ) + if ( + not _valid_id(stratum_id) + or not stratum_scope["task_ids"] + or not stratum_scope["measure_ids"] + or not stratum_scope["dimension_ids"] + ): + failures.append( + _failure( + "dsl-evaluation-claim-strata", + f"claim stratum {stratum_id!r} has no complete task/measure/dimension slice", + path, + ) + ) + continue + expanded.append({"stratum_id": stratum_id, "role": role, "scope": stratum_scope}) + + expanded_ids = [str(item["stratum_id"]) for item in expanded] + if len(group_ids) != len(groups) or len(expanded_ids) != len(set(expanded_ids)) or not expanded: + failures.append( + _failure( + "dsl-evaluation-claim-strata", + "claim binding requires unique groups and expanded stratum ids", + path, + ) + ) + if not any(item["role"] == "gating" for item in expanded): + failures.append(_failure("dsl-evaluation-claim-strata", "claim binding requires a gating stratum", path)) + if len(expanded) > _MAX_CATALOG_ITEMS: + failures.append( + _failure( + "dsl-evaluation-claim-strata", + f"claim binding expands beyond {_MAX_CATALOG_ITEMS} strata", + path, + ) + ) + return expanded[:_MAX_CATALOG_ITEMS] + return expanded + + +def _derive_stratum_scope( + protocol: Mapping[str, object], + claim_scope: Mapping[str, set[str]], + *, + persona_ids: set[str], + experience_band_ids: set[str], + tooling_condition_ids: set[str], +) -> dict[str, set[str]]: + """Derive the protocol-applicable claim slice for one preregistered stratum.""" + + tasks = _protocol_records_by_id(protocol, "tasks", "task_id") + selected_tasks = { + task_id + for task_id, task in tasks.items() + if task_id in claim_scope.get("task_ids", set()) + and set(_string_list(task.get("persona_ids"), non_empty=True) or []) & persona_ids + and set(_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) & tooling_condition_ids + } + variants = _protocol_records_by_id(protocol, "variants", "variant_id") + selected_variants = { + variant_id + for variant_id, variant in variants.items() + if variant_id in claim_scope.get("variant_ids", set()) and variant.get("task_id") in selected_tasks + } + measures = _protocol_records_by_id(protocol, "measures", "measure_id") + selected_measures = { + measure_id + for measure_id, measure in measures.items() + if measure_id in claim_scope.get("measure_ids", set()) + and set(_string_list(measure.get("task_ids"), non_empty=True) or []) & selected_tasks + } + thresholds = { + threshold.get("dimension_id"): threshold + for threshold in protocol.get("thresholds", []) + if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) + } + selected_dimensions = set() + for dimension_id in claim_scope.get("dimension_ids", set()): + threshold = thresholds.get(dimension_id) + conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] + required_measures = { + condition.get("measure_id") + for condition in conditions + if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) + } + if required_measures and required_measures.issubset(selected_measures): + selected_dimensions.add(dimension_id) + return { + "persona_ids": set(persona_ids), + "experience_band_ids": set(experience_band_ids), + "task_ids": selected_tasks, + "tooling_condition_ids": set(tooling_condition_ids), + "variant_ids": selected_variants, + "artifact_stage_ids": set(claim_scope.get("artifact_stage_ids", set())), + "dimension_ids": selected_dimensions, + "measure_ids": selected_measures, + } + + +def resolve_claim_strata( + protocol: Mapping[str, object], + analysis: Mapping[str, object], + claim_binding: Mapping[str, object], +) -> list[dict[str, object]]: + """Resolve a validated manifest binding for recomputation and fixtures.""" + + catalogs = { + "persona_ids": set(_protocol_records_by_id(protocol, "personas", "persona_id")), + "task_ids": set(_protocol_records_by_id(protocol, "tasks", "task_id")), + "condition_ids": set(_protocol_records_by_id(protocol, "tooling_conditions", "condition_id")), + "variant_ids": set(_protocol_records_by_id(protocol, "variants", "variant_id")), + "stage_ids": set(_protocol_records_by_id(protocol, "artifact_stages", "stage_id")), + "dimension_ids": set(_protocol_records_by_id(protocol, "dimensions", "dimension_id")), + "measure_ids": set(_protocol_records_by_id(protocol, "measures", "measure_id")), + } + failures: list[PolicyFailure] = [] + scope = _validate_claim_scope(protocol, analysis, catalogs, failures, path=MANIFEST_PATH) + strata = _validate_claim_binding( + protocol, + analysis, + claim_binding, + catalogs, + scope, + failures, + path=MANIFEST_PATH, + ) + if failures: + raise ValueError("; ".join(f"{failure.rule_id}: {failure.message}" for failure in failures)) + return strata diff --git a/tools/dsl_language_evaluation/_keys.py b/tools/dsl_language_evaluation/_keys.py new file mode 100644 index 000000000..30d4ae466 --- /dev/null +++ b/tools/dsl_language_evaluation/_keys.py @@ -0,0 +1,324 @@ +"""Bundle paths, closed key sets, and bounded limits for the DSL evaluation.""" + +from __future__ import annotations + +import re + +MANIFEST_PATH = "docs/research/dsl-language-evaluation/bundle-manifest.json" +_RETIRED_PACKAGE_PREFIX = "implementations/python/packages/" + "a" + "ces_sdl" +_HISTORICAL_PACKAGE_MOVES: tuple[tuple[str, str], ...] = ( + (_RETIRED_PACKAGE_PREFIX, "implementations/python/packages/raes"), + ("implementations/python/packages/" + "a" + "ces_cli", "implementations/python/packages/raes_cli"), + ("implementations/python/packages/" + "a" + "ces_mcp", "implementations/python/packages/raes_mcp"), +) +_MAX_FILE_BYTES = 2 * 1024 * 1024 +_MAX_CATALOG_ITEMS = 128 +_MAX_EXECUTION_RECORDS = 20_000 + +REQUIRED_DIMENSION_IDS = { + "expressiveness", + "usability-comprehension", + "effectiveness-productivity", + "maintainability-evolution", + "ambiguity", + "diagnostic-quality", + "reviewability", + "semantic-traceability", +} +REQUIRED_PERSONA_IDS = { + "benchmark-designer", + "scenario-author", + "participant-model-author", + "backend-implementer", + "evaluator-reviewer", + "assurance-auditor", +} +REQUIRED_TASK_KINDS = { + "positive", + "negative", + "underspecified", + "ambiguous", + "round-trip", + "mutation", + "maintenance", + "independent-review", +} +EVIDENCE_STATUSES = {"untested", "partial", "demonstrated", "refuted"} +ATTEMPT_OUTCOMES = {"completed", "failed", "abandoned", "tool_failed", "missing", "withdrawn"} +OBSERVATION_OUTCOMES = {"completed", "failed", "abandoned", "tool_failed", "missing"} + +_MANIFEST_KEYS = { + "bundle_id", + "revision", + "protocol_path", + "snapshot_path", + "analysis_path", + "claim_binding", + "supplemental_bundles", +} +_BUNDLE_ENTRY_KEYS = _MANIFEST_KEYS - {"supplemental_bundles"} +_CLAIM_BINDING_KEYS = {"claim_id", "scope", "strata"} +_STRATUM_GROUP_KEYS = { + "group_id", + "role", + "partition_by", + "persona_ids", + "experience_band_ids", + "tooling_condition_ids", +} +_STRATUM_PARTITION_AXES = {"persona_id", "experience_band", "tooling_condition_id"} +_STRATUM_ROLES = {"gating", "comparison"} +_PROTOCOL_KEYS = { + "protocol_id", + "revision", + "registered_at", + "title", + "claim", + "research_question", + "evidence_status_values", + "dimensions", + "personas", + "tooling_conditions", + "artifact_stages", + "sources", + "tasks", + "variants", + "measures", + "sampling_plan", + "execution_plan", + "thresholds", + "ethics_and_privacy", + "disagreement_policy", + "validity_threats", + "analysis_plan", + "amendment_log", +} +_DIMENSION_KEYS = {"dimension_id", "label", "construct", "pass_rule", "fail_rule"} +_PERSONA_KEYS = {"persona_id", "label", "qualification", "minimum_completed_subjects"} +_CONDITION_KEYS = {"condition_id", "label", "allowed_surface", "assistance"} +_STAGE_KEYS = {"stage_id", "label", "canonical_entrypoint"} +_SOURCE_KEYS = { + "source_id", + "kind", + "title", + "authors", + "year", + "locator", + "version", + "revision", + "artifact_path", + "primary", +} +_TASK_KEYS = { + "task_id", + "title", + "kind", + "persona_ids", + "dimension_ids", + "source_refs", + "intended_semantics_ref", + "artifact_stage_ids", + "tooling_condition_ids", + "variant_ids", + "success_rule", + "failure_rule", +} +_VARIANT_KEYS = {"variant_id", "task_id", "kind", "expected_relation", "description"} +_MEASURE_KEYS = { + "measure_id", + "task_ids", + "dimension_ids", + "stage_applicability", + "unit", + "aggregation", + "direction", + "capture_rule", +} +_STAGE_APPLICABILITY_KEYS = {"task_id", "variant_ids", "artifact_stage_ids"} +_SAMPLING_KEYS = { + "target_total", + "minimum_per_persona", + "experience_bands", + "inclusion_rule", + "exclusion_rule", +} +_EXECUTION_PLAN_KEYS = { + "unit_of_analysis", + "attempts_per_subject", + "subject_task_requirements", + "task_order", + "blinding", + "stopping_rule", + "missing_data_rule", + "withdrawal_rule", +} +_SUBJECT_TASK_REQUIREMENT_KEYS = { + "requirement_id", + "minimum_assigned_attempts", + "task_kinds", +} +_THRESHOLD_KEYS = {"dimension_id", "logic", "conditions"} +_THRESHOLD_CONDITION_KEYS = {"measure_id", "operator", "target"} +_ETHICS_KEYS = { + "review_status_required", + "consent_required", + "committed_data_rule", + "prohibited_data", +} + +_HISTORICAL_REVISION_FIELD = "a" + "ces_revision" +_SNAPSHOT_KEYS = { + "snapshot_id", + "protocol_revision", + "captured_at", + "execution_status", + _HISTORICAL_REVISION_FIELD, + "public_surface", + "ethics_review", + "subjects", + "attempts", + "observations", + "reviews", + "deviations", + "withdrawals", + "disagreements", +} +_SURFACE_KEYS = {"surface_id", "kind", "artifact", "version", "parameters"} +_ETHICS_REVIEW_KEYS = { + "status", + "protocol_identifier", + "approved_population", + "approved_data_boundary", +} +_SUBJECT_KEYS = {"subject_id", "persona_id", "experience_band", "consent_status"} +_ATTEMPT_KEYS = { + "attempt_id", + "study_run_id", + "task_id", + "persona_id", + "subject_id", + "tooling_condition_id", + "variant_id", + "outcome", + "observation_ids", + "started_at", + "ended_at", +} +_OBSERVATION_KEYS = { + "observation_id", + "protocol_revision", + "study_run_id", + "task_id", + "persona_id", + "subject_id", + "tooling_condition_id", + "attempt_id", + "variant_id", + "artifact_stage", + "dimension_ids", + "measure_id", + "value", + "outcome", + "evidence_refs", +} +_REVIEW_KEYS = { + "review_id", + "attempt_id", + "reviewer_subject_id", + "task_id", + "variant_id", + "judgment", + "confidence", + "rationale_code", + "fixed_at", +} +_DEVIATION_KEYS = {"deviation_id", "scope", "severity", "disposition", "rationale"} +_WITHDRAWAL_KEYS = {"subject_id", "recorded_at", "retained_aggregate_only"} +_DISAGREEMENT_KEYS = { + "disagreement_id", + "review_ids", + "status", + "adjudication", + "originals_preserved", +} + +_ANALYSIS_KEYS = { + "analysis_id", + "protocol_revision", + "snapshot_id", + "generated_at", + "execution_status", + "measure_results", + "dimension_results", + "stratum_results", + "evidence_status", + "claim", + "plain_language_outcome", + "limitations", +} +_STRATUM_RESULT_KEYS = { + "stratum_id", + "role", + "measure_results", + "dimension_results", +} +_DIMENSION_RESULT_KEYS = { + "dimension_id", + "status", + "threshold_result", + "condition_results", + "supporting_observation_ids", +} +_MEASURE_RESULT_KEYS = { + "measure_id", + "status", + "statistic", + "numerator", + "denominator", + "opportunity_count", + "observed_count", + "missing_count", + "abandoned_count", + "tool_failed_count", + "withdrawn_count", + "value", + "supporting_observation_ids", +} +_CLAIM_KEYS = { + "claim_id", + "statement", + "threats_to_validity", + "falsification_protocol", + "objective_pass_criteria", + "objective_fail_criteria", + "allowed_evidence", + "disallowed_evidence", + "evidence_artifacts", + "scope", +} +_CLAIM_SCOPE_KEYS = { + "persona_ids", + "task_ids", + "tooling_condition_ids", + "variant_ids", + "artifact_stage_ids", + "dimension_ids", + "measure_ids", +} + +_ID_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") +_SHA_RE = re.compile(r"^[0-9a-f]{40}$") +_SENSITIVE_QUERY_KEYS = { + "access_token", + "api_key", + "apikey", + "auth", + "authorization", + "client_secret", + "key", + "password", + "secret", + "sig", + "signature", + "token", +} diff --git a/tools/dsl_language_evaluation/_measures.py b/tools/dsl_language_evaluation/_measures.py new file mode 100644 index 000000000..e95d3c7a6 --- /dev/null +++ b/tools/dsl_language_evaluation/_measures.py @@ -0,0 +1,352 @@ +"""Measure opportunity derivation and result recomputation.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from statistics import median + +from tools.dsl_language_evaluation._claims import _attempt_matches_scope +from tools.dsl_language_evaluation._keys import _STRATUM_ROLES +from tools.dsl_language_evaluation._shape import _protocol_records_by_id, _string_list + + +def _measure_stage_ids( + measure: Mapping[str, object], + task_id: str, + variant_id: str, +) -> list[str]: + """Return the single preregistered stage set for a task/variant/measure.""" + + declarations = measure.get("stage_applicability", []) + if not isinstance(declarations, list): + raise ValueError(f"{measure.get('measure_id')}: stage applicability must be a list") + matches: list[list[str]] = [] + for declaration in declarations: + if not isinstance(declaration, Mapping) or declaration.get("task_id") != task_id: + continue + variant_ids = _string_list(declaration.get("variant_ids"), non_empty=True) + stage_ids = _string_list(declaration.get("artifact_stage_ids"), non_empty=True) + if variant_ids is not None and variant_id in variant_ids and stage_ids is not None: + matches.append(stage_ids) + if len(matches) != 1: + raise ValueError(f"{measure.get('measure_id')}: expected one stage declaration for {task_id}/{variant_id}") + return matches[0] + + +def _measure_opportunities( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + scope: Mapping[str, Sequence[str] | set[str]] | None = None, +) -> tuple[ + dict[str, Mapping[str, object]], + list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], + dict[tuple[str, str, str], Mapping[str, object]], +]: + """Derive every attempt-measure-stage opportunity and frozen observation.""" + + tasks = _protocol_records_by_id(protocol, "tasks", "task_id") + measures = _protocol_records_by_id(protocol, "measures", "measure_id") + scope_sets = {field: set(values) for field, values in scope.items()} if scope is not None else None + if scope_sets is not None: + measures = { + measure_id: measure + for measure_id, measure in measures.items() + if measure_id in scope_sets.get("measure_ids", set()) + } + attempts = snapshot.get("attempts", []) + observations = snapshot.get("observations", []) + withdrawals = snapshot.get("withdrawals", []) + subjects = snapshot.get("subjects", []) + subjects_by_id = { + subject.get("subject_id"): subject + for subject in subjects + if isinstance(subject, Mapping) and isinstance(subject.get("subject_id"), str) + } + if not isinstance(attempts, list) or not isinstance(observations, list) or not isinstance(withdrawals, list): + raise ValueError("snapshot execution records must be lists") + + withdrawn_subjects = { + withdrawal["subject_id"] + for withdrawal in withdrawals + if isinstance(withdrawal, Mapping) and isinstance(withdrawal.get("subject_id"), str) + } + observation_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} + for observation in observations: + if not isinstance(observation, Mapping): + continue + attempt_id = observation.get("attempt_id") + measure_id = observation.get("measure_id") + artifact_stage = observation.get("artifact_stage") + if not isinstance(attempt_id, str) or not isinstance(measure_id, str) or not isinstance(artifact_stage, str): + continue + key = (attempt_id, measure_id, artifact_stage) + if key in observation_by_opportunity: + raise ValueError(f"duplicate observation opportunity {attempt_id}/{measure_id}/{artifact_stage}") + observation_by_opportunity[key] = observation + + opportunities: list[tuple[Mapping[str, object], Mapping[str, object], str, bool]] = [] + expected_keys: set[tuple[str, str, str]] = set() + selected_attempt_ids: set[str] = set() + for attempt in attempts: + if not isinstance(attempt, Mapping): + continue + attempt_id = attempt.get("attempt_id") + task_id = attempt.get("task_id") + subject_id = attempt.get("subject_id") + variant_id = attempt.get("variant_id") + if not isinstance(attempt_id, str) or not isinstance(task_id, str) or not isinstance(variant_id, str): + continue + task = tasks.get(task_id) + if task is None: + continue + if scope_sets is not None and not _attempt_matches_scope(attempt, scope_sets): + continue + if scope_sets is not None and "experience_band_ids" in scope_sets: + subject = subjects_by_id.get(subject_id) + if ( + not isinstance(subject, Mapping) + or subject.get("experience_band") not in scope_sets["experience_band_ids"] + ): + continue + selected_attempt_ids.add(attempt_id) + withdrawn = (isinstance(subject_id, str) and subject_id in withdrawn_subjects) or attempt.get( + "outcome" + ) == "withdrawn" + for measure in measures.values(): + task_ids = _string_list(measure.get("task_ids"), non_empty=True) + if task_ids is None or task_id not in task_ids: + continue + measure_id = measure.get("measure_id") + if not isinstance(measure_id, str): + continue + for artifact_stage in _measure_stage_ids(measure, task_id, variant_id): + if scope_sets is not None and artifact_stage not in scope_sets.get("artifact_stage_ids", set()): + continue + opportunities.append((attempt, measure, artifact_stage, withdrawn)) + expected_keys.add((attempt_id, measure_id, artifact_stage)) + + observed_keys = set(observation_by_opportunity) + if scope_sets is not None: + observed_keys = { + key + for key in observed_keys + if key[0] in selected_attempt_ids + and key[1] in scope_sets.get("measure_ids", set()) + and key[2] in scope_sets.get("artifact_stage_ids", set()) + } + extras = sorted(observed_keys - expected_keys) + if extras: + raise ValueError(f"observations without protocol-declared opportunities: {extras[:5]}") + return measures, opportunities, observation_by_opportunity + + +def recompute_measure_results( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + *, + scope: Mapping[str, Sequence[str] | set[str]] | None = None, +) -> dict[str, dict[str, object]]: + """Recompute measures from the complete protocol-derived opportunity matrix.""" + + measures, opportunities, observation_by_opportunity = _measure_opportunities(protocol, snapshot, scope) + results: dict[str, dict[str, object]] = {} + for measure_id, measure in measures.items(): + aggregation = measure.get("aggregation") + if aggregation not in {"proportion", "median", "count"}: + raise ValueError(f"invalid measure aggregation for {measure_id!r}") + + matching = [item for item in opportunities if item[1].get("measure_id") == measure_id] + opportunity_count = len(matching) + withdrawn_count = sum(withdrawn for _, _, _, withdrawn in matching) + eligible = [(attempt, artifact_stage) for attempt, _, artifact_stage, withdrawn in matching if not withdrawn] + denominator = len(eligible) + values: list[int | float] = [] + supporting_ids: list[str] = [] + missing_count = 0 + abandoned_count = 0 + tool_failed_count = 0 + + for attempt, artifact_stage in eligible: + attempt_id = attempt.get("attempt_id") + if not isinstance(attempt_id, str): + missing_count += 1 + continue + observation = observation_by_opportunity.get((attempt_id, measure_id, artifact_stage)) + if observation is None: + missing_count += 1 + continue + observation_id = observation.get("observation_id") + if not isinstance(observation_id, str): + raise ValueError(f"{measure_id}: observations require ids") + supporting_ids.append(observation_id) + outcome = observation.get("outcome") + if outcome == "missing": + missing_count += 1 + elif outcome == "abandoned": + abandoned_count += 1 + elif outcome == "tool_failed": + tool_failed_count += 1 + value = observation.get("value") + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{measure_id}: observation values must be numeric or null") + if aggregation == "proportion" and value not in {0, 1}: + raise ValueError(f"{measure_id}: proportion observations must be 0 or 1") + values.append(value) + + observed_count = len(values) + numerator: int | float | None = None + value: int | float | None = None + if denominator > 0 and observed_count == denominator: + if aggregation == "proportion": + numerator = sum(values) + value = numerator / denominator + elif aggregation == "median": + value = float(median(values)) + else: + numerator = sum(values) + value = numerator + results[measure_id] = { + "statistic": aggregation, + "numerator": numerator, + "denominator": denominator, + "opportunity_count": opportunity_count, + "observed_count": observed_count, + "missing_count": missing_count, + "abandoned_count": abandoned_count, + "tool_failed_count": tool_failed_count, + "withdrawn_count": withdrawn_count, + "value": value, + "supporting_observation_ids": supporting_ids, + } + return results + + +def recompute_dimension_results( + protocol: Mapping[str, object], + measure_results: Mapping[str, Mapping[str, object]], + *, + dimension_ids: Sequence[str] | set[str] | None = None, +) -> dict[str, dict[str, object]]: + """Apply protocol-declared threshold conditions to recomputed measures.""" + + operators = { + ">=": lambda actual, target: actual >= target, + "<=": lambda actual, target: actual <= target, + "==": lambda actual, target: actual == target, + } + results: dict[str, dict[str, object]] = {} + selected_dimensions = set(dimension_ids) if dimension_ids is not None else None + for threshold in protocol.get("thresholds", []): + if not isinstance(threshold, Mapping): + continue + dimension_id = threshold.get("dimension_id") + conditions = threshold.get("conditions") + if ( + not isinstance(dimension_id, str) + or threshold.get("logic") != "all" + or not isinstance(conditions, list) + or not conditions + ): + raise ValueError(f"invalid threshold for {dimension_id!r}") + if selected_dimensions is not None and dimension_id not in selected_dimensions: + continue + resolved: list[tuple[Mapping[str, object], Mapping[str, object]]] = [] + for condition in conditions: + if not isinstance(condition, Mapping): + raise ValueError(f"{dimension_id}: threshold condition must be an object") + measure_id = condition.get("measure_id") + operator = condition.get("operator") + target = condition.get("target") + measure = measure_results.get(measure_id) if isinstance(measure_id, str) else None + if operator not in operators or isinstance(target, bool) or not isinstance(target, (int, float)): + raise ValueError(f"{dimension_id}: invalid threshold operator or target") + if measure is None: + raise ValueError(f"{dimension_id}: threshold references unknown measure {measure_id!r}") + resolved.append((condition, measure)) + if any(measure.get("value") is None for _, measure in resolved): + results[dimension_id] = { + "status": "not_evaluated", + "threshold_result": "not_evaluated", + "condition_results": [], + "supporting_observation_ids": [], + } + continue + condition_results: list[dict[str, object]] = [] + supporting_ids: list[str] = [] + for condition, measure in resolved: + actual = measure["value"] + operator = condition["operator"] + target = condition["target"] + if isinstance(actual, bool) or not isinstance(actual, (int, float)): + raise ValueError(f"{dimension_id}: measure value must be numeric") + passed = operators[operator](actual, target) + condition_results.append( + { + "measure_id": condition["measure_id"], + "operator": operator, + "target": target, + "actual": actual, + "passed": passed, + } + ) + refs = measure.get("supporting_observation_ids", []) + if not isinstance(refs, list) or not all(isinstance(item, str) for item in refs): + raise ValueError(f"{dimension_id}: measure observation refs must be text ids") + for observation_id in refs: + if observation_id not in supporting_ids: + supporting_ids.append(observation_id) + results[dimension_id] = { + "status": "evaluated", + "threshold_result": "pass" if all(item["passed"] for item in condition_results) else "fail", + "condition_results": condition_results, + "supporting_observation_ids": supporting_ids, + } + return results + + +def recompute_stratum_results( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + strata: Sequence[Mapping[str, object]], +) -> list[dict[str, object]]: + """Recompute independently persisted results for every bound claim stratum.""" + + results: list[dict[str, object]] = [] + for stratum in strata: + stratum_id = stratum.get("stratum_id") + role = stratum.get("role") + scope = stratum.get("scope") + if not isinstance(stratum_id, str) or role not in _STRATUM_ROLES or not isinstance(scope, Mapping): + raise ValueError("invalid resolved claim stratum") + measures = recompute_measure_results(protocol, snapshot, scope=scope) + measure_results = [ + { + "measure_id": measure_id, + "status": ( + "not_evaluated" + if result["denominator"] == 0 + else "incomplete" + if result["observed_count"] != result["denominator"] + else "evaluated" + ), + **result, + } + for measure_id, result in measures.items() + ] + dimensions = recompute_dimension_results( + protocol, + measures, + dimension_ids=scope.get("dimension_ids", set()), + ) + dimension_results = [{"dimension_id": dimension_id, **result} for dimension_id, result in dimensions.items()] + results.append( + { + "stratum_id": stratum_id, + "role": role, + "measure_results": measure_results, + "dimension_results": dimension_results, + } + ) + return results diff --git a/tools/dsl_language_evaluation/_protocol.py b/tools/dsl_language_evaluation/_protocol.py new file mode 100644 index 000000000..69e5d81a1 --- /dev/null +++ b/tools/dsl_language_evaluation/_protocol.py @@ -0,0 +1,444 @@ +"""Preregistered-protocol validation for the DSL evaluation bundle.""" + +from __future__ import annotations + +from pathlib import Path + +from tools.dsl_language_evaluation._keys import ( + _CONDITION_KEYS, + _DIMENSION_KEYS, + _MAX_CATALOG_ITEMS, + _PERSONA_KEYS, + _PROTOCOL_KEYS, + _SHA_RE, + _SOURCE_KEYS, + _STAGE_KEYS, + _TASK_KEYS, + _VARIANT_KEYS, + REQUIRED_DIMENSION_IDS, + REQUIRED_PERSONA_IDS, + REQUIRED_TASK_KINDS, +) +from tools.dsl_language_evaluation._protocol_plans import _plan_failures, _preregistration_failures +from tools.dsl_language_evaluation._shape import ( + _bounded_list, + _exact_keys, + _failure, + _record_ids, + _resolve_repository_artifact, + _string_list, + _validate_https_locator, +) +from tools.policy.common import PolicyFailure + + +def _validate_protocol( + repo_root: Path, + protocol: dict[str, object], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/protocol-v1.json", +) -> dict[str, set[str]]: + if not _exact_keys( + protocol, + _PROTOCOL_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="protocol", + path=path, + ): + return {} + + dimensions = _bounded_list( + protocol["dimensions"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="dimensions", + path=path, + ) + personas = _bounded_list( + protocol["personas"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="personas", + path=path, + ) + conditions = _bounded_list( + protocol["tooling_conditions"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="tooling_conditions", + path=path, + ) + stages = _bounded_list( + protocol["artifact_stages"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="artifact_stages", + path=path, + ) + sources = _bounded_list( + protocol["sources"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="sources", + path=path, + ) + tasks = _bounded_list( + protocol["tasks"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="tasks", + path=path, + ) + variants = _bounded_list( + protocol["variants"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="variants", + path=path, + ) + measures = _bounded_list( + protocol["measures"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="measures", + path=path, + ) + thresholds = _bounded_list( + protocol["thresholds"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="thresholds", + path=path, + ) + + dimension_ids = _record_ids( + dimensions, + "dimension_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="dimension", + path=path, + ) + persona_ids = _record_ids( + personas, + "persona_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="persona", + path=path, + ) + condition_ids = _record_ids( + conditions, + "condition_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="tooling condition", + path=path, + ) + stage_ids = _record_ids( + stages, + "stage_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="artifact stage", + path=path, + ) + source_ids = _record_ids( + sources, + "source_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="source", + path=path, + ) + task_ids = _record_ids( + tasks, + "task_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="task", + path=path, + ) + variant_ids = _record_ids( + variants, + "variant_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="variant", + path=path, + ) + measure_ids = _record_ids( + measures, + "measure_id", + failures, + rule_id="dsl-evaluation-protocol-id", + label="measure", + path=path, + ) + + if not REQUIRED_DIMENSION_IDS.issubset(dimension_ids): + failures.append( + _failure( + "dsl-evaluation-dimension-coverage", + f"missing required dimensions: {sorted(REQUIRED_DIMENSION_IDS - dimension_ids)}", + path, + ) + ) + if not REQUIRED_PERSONA_IDS.issubset(persona_ids): + failures.append( + _failure( + "dsl-evaluation-persona-coverage", + f"missing required personas: {sorted(REQUIRED_PERSONA_IDS - persona_ids)}", + path, + ) + ) + if protocol["evidence_status_values"] != [ + "untested", + "partial", + "demonstrated", + "refuted", + ]: + failures.append( + _failure( + "dsl-evaluation-status-vocabulary", + "evidence statuses must preserve ADR-021 order and vocabulary", + path, + ) + ) + + for index, record in enumerate(dimensions): + _exact_keys( + record, + _DIMENSION_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"dimensions[{index}]", + path=path, + ) + for index, record in enumerate(personas): + if not _exact_keys( + record, + _PERSONA_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"personas[{index}]", + path=path, + ): + continue + if ( + not isinstance(record["minimum_completed_subjects"], int) + or isinstance(record["minimum_completed_subjects"], bool) + or record["minimum_completed_subjects"] < 0 + ): + failures.append( + _failure( + "dsl-evaluation-sampling-plan", + f"{record['persona_id']}: minimum_completed_subjects must be non-negative", + path, + ) + ) + for index, record in enumerate(conditions): + _exact_keys( + record, + _CONDITION_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"tooling_conditions[{index}]", + path=path, + ) + for index, record in enumerate(stages): + _exact_keys( + record, + _STAGE_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"artifact_stages[{index}]", + path=path, + ) + + for index, source in enumerate(sources): + if not _exact_keys( + source, + _SOURCE_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"sources[{index}]", + path=path, + ): + continue + source_id = source["source_id"] + _validate_https_locator(source["locator"], failures, source_id) + if source["primary"] is not True: + failures.append( + _failure( + "dsl-evaluation-source-primary", + f"{source_id}: source must be primary", + path, + ) + ) + if source["kind"] == "repository-internal": + revision = source["revision"] + artifact_path = source["artifact_path"] + if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): + failures.append( + _failure( + "dsl-evaluation-source-pin", + f"{source_id}: invalid Git revision", + path, + ) + ) + elif isinstance(source["locator"], str) and revision not in source["locator"]: + failures.append( + _failure( + "dsl-evaluation-source-pin", + f"{source_id}: locator does not bind the declared Git revision", + path, + ) + ) + if not isinstance(artifact_path, str): + failures.append( + _failure( + "dsl-evaluation-source-path", + f"{source_id}: missing artifact path", + path, + ) + ) + else: + resolved = _resolve_repository_artifact(repo_root, artifact_path) + if resolved is None or not resolved.exists(): + failures.append( + _failure( + "dsl-evaluation-source-path", + f"{source_id}: unsafe or missing path", + path, + ) + ) + elif source["revision"] is not None or source["artifact_path"] is not None: + failures.append( + _failure( + "dsl-evaluation-source-shape", + f"{source_id}: publication must not claim a repository revision/path", + path, + ) + ) + + task_kinds: set[str] = set() + variants_by_task: dict[str, set[str]] = {task_id: set() for task_id in task_ids} + for index, variant in enumerate(variants): + if not _exact_keys( + variant, + _VARIANT_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"variants[{index}]", + path=path, + ): + continue + task_id = variant["task_id"] + if not isinstance(task_id, str) or task_id not in task_ids: + failures.append( + _failure( + "dsl-evaluation-variant-join", + f"{variant['variant_id']}: unknown task", + path, + ) + ) + else: + variant_id = variant["variant_id"] + if isinstance(variant_id, str): + variants_by_task[task_id].add(variant_id) + for index, task in enumerate(tasks): + if not _exact_keys( + task, + _TASK_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"tasks[{index}]", + path=path, + ): + continue + task_id = task["task_id"] + if not isinstance(task_id, str): + failures.append(_failure("dsl-evaluation-protocol-id", "task id must be text", path)) + continue + task_kind = task["kind"] + if isinstance(task_kind, str): + task_kinds.add(task_kind) + else: + failures.append( + _failure( + "dsl-evaluation-task-kind-coverage", + f"{task_id}: task kind must be text", + path, + ) + ) + joins = ( + ("persona_ids", persona_ids), + ("dimension_ids", dimension_ids), + ("source_refs", source_ids), + ("artifact_stage_ids", stage_ids), + ("tooling_condition_ids", condition_ids), + ("variant_ids", variant_ids), + ) + for field, allowed in joins: + values = task[field] + value_ids = _string_list(values, non_empty=True) + if value_ids is None or not set(value_ids).issubset(allowed): + failures.append( + _failure( + "dsl-evaluation-task-join", + f"{task_id}: invalid or empty {field}", + path, + ) + ) + task_variant_ids = task["variant_ids"] + valid_task_variant_ids = _string_list(task_variant_ids) + if valid_task_variant_ids is not None and set(valid_task_variant_ids) != variants_by_task.get(task_id, set()): + failures.append( + _failure( + "dsl-evaluation-task-variant-coverage", + f"{task_id}: task and variant catalogs disagree", + path, + ) + ) + if not REQUIRED_TASK_KINDS.issubset(task_kinds): + failures.append( + _failure( + "dsl-evaluation-task-kind-coverage", + f"missing required task kinds: {sorted(REQUIRED_TASK_KINDS - task_kinds)}", + path, + ) + ) + + catalogs_ids = { + "dimension_ids": dimension_ids, + "persona_ids": persona_ids, + "condition_ids": condition_ids, + "stage_ids": stage_ids, + "source_ids": source_ids, + "task_ids": task_ids, + "variant_ids": variant_ids, + "measure_ids": measure_ids, + } + _plan_failures(protocol, catalogs_ids, personas, tasks, measures, thresholds, failures, path) + _preregistration_failures(protocol, failures, path) + return { + "dimension_ids": dimension_ids, + "persona_ids": persona_ids, + "condition_ids": condition_ids, + "stage_ids": stage_ids, + "task_ids": task_ids, + "variant_ids": variant_ids, + "measure_ids": measure_ids, + } diff --git a/tools/dsl_language_evaluation/_protocol_plans.py b/tools/dsl_language_evaluation/_protocol_plans.py new file mode 100644 index 000000000..9903e2841 --- /dev/null +++ b/tools/dsl_language_evaluation/_protocol_plans.py @@ -0,0 +1,343 @@ +"""Measure, threshold, sampling, and execution-plan validation for the protocol.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from tools.dsl_language_evaluation._keys import ( + _ETHICS_KEYS, + _EXECUTION_PLAN_KEYS, + _MAX_CATALOG_ITEMS, + _MEASURE_KEYS, + _SAMPLING_KEYS, + _STAGE_APPLICABILITY_KEYS, + _SUBJECT_TASK_REQUIREMENT_KEYS, + _THRESHOLD_CONDITION_KEYS, + _THRESHOLD_KEYS, + REQUIRED_TASK_KINDS, +) +from tools.dsl_language_evaluation._shape import ( + _bounded_list, + _exact_keys, + _failure, + _protocol_records_by_id, + _record_ids, + _string_list, +) +from tools.policy.common import PolicyFailure + + +def _preregistration_failures(protocol: dict[str, object], failures: list[PolicyFailure], path: str) -> None: + _exact_keys( + protocol["ethics_and_privacy"], + _ETHICS_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="ethics_and_privacy", + path=path, + ) + if not protocol["validity_threats"] or not protocol["analysis_plan"]: + failures.append( + _failure( + "dsl-evaluation-preregistration", + "validity threats and the analysis plan must be preregistered", + path, + ) + ) + if not isinstance(protocol["amendment_log"], list): + failures.append(_failure("dsl-evaluation-protocol-shape", "amendment_log must be a list", path)) + + +def _plan_failures( + protocol: dict[str, object], + catalogs_ids: Mapping[str, set[str]], + personas: list[object], + tasks: list[object], + measures: list[object], + thresholds: list[object], + failures: list[PolicyFailure], + path: str, +) -> None: + dimension_ids = catalogs_ids["dimension_ids"] + persona_ids = catalogs_ids["persona_ids"] + task_ids = catalogs_ids["task_ids"] + measure_ids = catalogs_ids["measure_ids"] + tasks_by_id = _protocol_records_by_id(protocol, "tasks", "task_id") + for index, measure in enumerate(measures): + if not _exact_keys( + measure, + _MEASURE_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"measures[{index}]", + path=path, + ): + continue + measure_dimensions = _string_list(measure["dimension_ids"], non_empty=True) + if measure_dimensions is None or not set(measure_dimensions).issubset(dimension_ids): + failures.append( + _failure( + "dsl-evaluation-measure-join", + f"{measure['measure_id']}: invalid dimension ids", + path, + ) + ) + measure_tasks = _string_list(measure["task_ids"], non_empty=True) + if measure_tasks is None or not set(measure_tasks).issubset(task_ids): + failures.append( + _failure( + "dsl-evaluation-measure-join", + f"{measure['measure_id']}: invalid task ids", + path, + ) + ) + elif not all( + set(measure_dimensions or []) & set(task["dimension_ids"]) + for task in tasks + if isinstance(task, Mapping) + and task.get("task_id") in measure_tasks + and _string_list(task.get("dimension_ids"), non_empty=True) is not None + ): + failures.append( + _failure( + "dsl-evaluation-measure-join", + f"{measure['measure_id']}: task applicability must share a measured dimension", + path, + ) + ) + applicability = _bounded_list( + measure["stage_applicability"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-measure-stage-applicability", + label=f"measures[{index}].stage_applicability", + path=path, + ) + actual_stage_pairs: set[tuple[str, str]] = set() + duplicate_stage_pairs: set[tuple[str, str]] = set() + for applicability_index, declaration in enumerate(applicability): + if not _exact_keys( + declaration, + _STAGE_APPLICABILITY_KEYS, + failures, + rule_id="dsl-evaluation-measure-stage-applicability", + label=(f"measures[{index}].stage_applicability[{applicability_index}]"), + path=path, + ): + continue + task_id = declaration["task_id"] + task = tasks_by_id.get(task_id) if isinstance(task_id, str) else None + declaration_variants = _string_list(declaration["variant_ids"], non_empty=True) + declaration_stages = _string_list(declaration["artifact_stage_ids"], non_empty=True) + task_variants = ( + set(_string_list(task.get("variant_ids"), non_empty=True) or []) if task is not None else set() + ) + task_stages = ( + set(_string_list(task.get("artifact_stage_ids"), non_empty=True) or []) if task is not None else set() + ) + if ( + task is None + or measure_tasks is None + or task_id not in measure_tasks + or declaration_variants is None + or not set(declaration_variants).issubset(task_variants) + or declaration_stages is None + or not set(declaration_stages).issubset(task_stages) + ): + failures.append( + _failure( + "dsl-evaluation-measure-stage-applicability", + f"{measure['measure_id']}: invalid task, variant, or stage applicability", + path, + ) + ) + continue + for variant_id in declaration_variants: + pair = (task_id, variant_id) + if pair in actual_stage_pairs: + duplicate_stage_pairs.add(pair) + actual_stage_pairs.add(pair) + expected_stage_pairs = { + (task_id, variant_id) + for task_id in (measure_tasks or []) + for variant_id in (_string_list(tasks_by_id.get(task_id, {}).get("variant_ids"), non_empty=True) or []) + } + if actual_stage_pairs != expected_stage_pairs or duplicate_stage_pairs: + failures.append( + _failure( + "dsl-evaluation-measure-stage-coverage", + f"{measure['measure_id']}: every task/variant requires one stage declaration", + path, + ) + ) + threshold_ids = _record_ids( + thresholds, + "dimension_id", + failures, + rule_id="dsl-evaluation-threshold-id", + label="threshold", + path=path, + ) + for index, threshold in enumerate(thresholds): + if not _exact_keys( + threshold, + _THRESHOLD_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"thresholds[{index}]", + path=path, + ): + continue + conditions = _bounded_list( + threshold["conditions"], + 16, + failures, + rule_id="dsl-evaluation-threshold-shape", + label=f"thresholds[{index}].conditions", + path=path, + ) + if threshold["logic"] != "all" or not conditions: + failures.append( + _failure( + "dsl-evaluation-threshold-shape", + f"{threshold['dimension_id']}: threshold requires non-empty all conditions", + path, + ) + ) + for condition_index, condition in enumerate(conditions): + if not _exact_keys( + condition, + _THRESHOLD_CONDITION_KEYS, + failures, + rule_id="dsl-evaluation-threshold-shape", + label=f"thresholds[{index}].conditions[{condition_index}]", + path=path, + ): + continue + if ( + not isinstance(condition["measure_id"], str) + or condition["measure_id"] not in measure_ids + or not isinstance(condition["operator"], str) + or condition["operator"] not in {">=", "<=", "=="} + or isinstance(condition["target"], bool) + or not isinstance(condition["target"], (int, float)) + ): + failures.append( + _failure( + "dsl-evaluation-threshold-join", + f"{threshold['dimension_id']}: invalid measure, operator, or target", + path, + ) + ) + if threshold_ids != dimension_ids: + failures.append( + _failure( + "dsl-evaluation-threshold-coverage", + "every dimension requires exactly one preregistered threshold", + path, + ) + ) + + if _exact_keys( + protocol["sampling_plan"], + _SAMPLING_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="sampling_plan", + path=path, + ): + sampling = protocol["sampling_plan"] + minimum = sampling["minimum_per_persona"] + target = sampling["target_total"] + if not isinstance(minimum, int) or minimum < 1 or not isinstance(target, int): + failures.append(_failure("dsl-evaluation-sampling-plan", "invalid sample sizes", path)) + elif target < sum( + item.get("minimum_completed_subjects", 0) + for item in personas + if isinstance(item, Mapping) and isinstance(item.get("minimum_completed_subjects"), int) + ): + failures.append( + _failure( + "dsl-evaluation-sampling-plan", + "target_total cannot cover the required per-persona minimum", + path, + ) + ) + if _exact_keys( + protocol["execution_plan"], + _EXECUTION_PLAN_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="execution_plan", + path=path, + ): + execution_plan = protocol["execution_plan"] + requirements = _bounded_list( + execution_plan["subject_task_requirements"], + 16, + failures, + rule_id="dsl-evaluation-subject-workload-plan", + label="execution_plan.subject_task_requirements", + path=path, + ) + requirement_ids = _record_ids( + requirements, + "requirement_id", + failures, + rule_id="dsl-evaluation-subject-workload-plan", + label="subject task requirement", + path=path, + ) + declared_kinds: set[str] = set() + duplicate_kinds: set[str] = set() + for index, requirement in enumerate(requirements): + if not _exact_keys( + requirement, + _SUBJECT_TASK_REQUIREMENT_KEYS, + failures, + rule_id="dsl-evaluation-subject-workload-plan", + label=f"execution_plan.subject_task_requirements[{index}]", + path=path, + ): + continue + task_kind_values = _string_list(requirement["task_kinds"], non_empty=True) + minimum = requirement["minimum_assigned_attempts"] + if ( + task_kind_values is None + or not set(task_kind_values).issubset(REQUIRED_TASK_KINDS) + or isinstance(minimum, bool) + or not isinstance(minimum, int) + or minimum < 1 + ): + failures.append( + _failure( + "dsl-evaluation-subject-workload-plan", + f"{requirement['requirement_id']}: invalid task kinds or minimum", + path, + ) + ) + continue + overlap = declared_kinds & set(task_kind_values) + duplicate_kinds.update(overlap) + declared_kinds.update(task_kind_values) + for persona_id in persona_ids: + if not any( + task.get("kind") in task_kind_values + and persona_id in (_string_list(task.get("persona_ids"), non_empty=True) or []) + for task in tasks_by_id.values() + ): + failures.append( + _failure( + "dsl-evaluation-subject-workload-plan", + f"{requirement['requirement_id']}: no eligible task for {persona_id}", + path, + ) + ) + if not requirement_ids or declared_kinds != REQUIRED_TASK_KINDS or duplicate_kinds: + failures.append( + _failure( + "dsl-evaluation-subject-workload-plan", + "subject task requirements must partition every required task kind", + path, + ) + ) diff --git a/tools/dsl_language_evaluation/_shape.py b/tools/dsl_language_evaluation/_shape.py new file mode 100644 index 000000000..cfc74bf74 --- /dev/null +++ b/tools/dsl_language_evaluation/_shape.py @@ -0,0 +1,159 @@ +"""Shape, id, and locator primitives for the DSL evaluation checker.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path +from urllib.parse import parse_qsl, urlsplit + +from tools.dsl_language_evaluation._keys import ( + _HISTORICAL_PACKAGE_MOVES, + _ID_RE, + _SENSITIVE_QUERY_KEYS, +) +from tools.policy.common import PolicyFailure, safe_repo_path + + +def _failure(rule_id: str, message: str, path: str | None = None) -> PolicyFailure: + return PolicyFailure(rule_id, message, path) + + +def _resolve_repository_artifact(repo_root: Path, artifact_path: str) -> Path | None: + """Resolve an immutable evidence locator through the Python package moves.""" + + resolved_path = artifact_path + for historical_prefix, current_prefix in _HISTORICAL_PACKAGE_MOVES: + if artifact_path == historical_prefix or artifact_path.startswith(f"{historical_prefix}/"): + resolved_path = f"{current_prefix}{artifact_path[len(historical_prefix) :]}" + break + return safe_repo_path(repo_root, resolved_path) + + +def _exact_keys( + value: object, + expected: set[str], + failures: list[PolicyFailure], + *, + rule_id: str, + label: str, + path: str, +) -> bool: + if not isinstance(value, dict): + failures.append(_failure(rule_id, f"{label} must be an object", path)) + return False + actual = set(value) + if actual != expected: + failures.append( + _failure( + rule_id, + f"{label} fields must exactly match {sorted(expected)}; got {sorted(actual)}", + path, + ) + ) + return False + return True + + +def _bounded_list( + value: object, + limit: int, + failures: list[PolicyFailure], + *, + rule_id: str, + label: str, + path: str, +) -> list[object]: + if not isinstance(value, list): + failures.append(_failure(rule_id, f"{label} must be a list", path)) + return [] + if len(value) > limit: + failures.append(_failure(rule_id, f"{label} exceeds the {limit}-entry limit", path)) + return [] + return value + + +def _valid_id(value: object) -> bool: + return isinstance(value, str) and bool(_ID_RE.fullmatch(value)) + + +def _bounded_text(value: object, *, maximum: int = 6000) -> bool: + return isinstance(value, str) and bool(value.strip()) and len(value) <= maximum + + +def _string_list(value: object, *, non_empty: bool = False) -> list[str] | None: + if not isinstance(value, list) or (non_empty and not value): + return None + if not all(isinstance(item, str) for item in value): + return None + return value + + +def _record_ids( + records: Sequence[object], + field: str, + failures: list[PolicyFailure], + *, + rule_id: str, + label: str, + path: str, +) -> set[str]: + result: set[str] = set() + duplicates: set[str] = set() + for record in records: + if not isinstance(record, Mapping): + continue + value = record.get(field) + if not _valid_id(value): + failures.append(_failure(rule_id, f"{label} has invalid {field} {value!r}", path)) + continue + if value in result: + duplicates.add(value) + result.add(value) + if duplicates: + failures.append(_failure(rule_id, f"duplicate {label} ids: {sorted(duplicates)}", path)) + return result + + +def _validate_https_locator(locator: object, failures: list[PolicyFailure], source_id: object) -> None: + if not isinstance(locator, str): + failures.append(_failure("dsl-evaluation-source-locator", f"{source_id}: locator must be text")) + return + parsed = urlsplit(locator) + if parsed.scheme != "https" or not parsed.netloc: + failures.append( + _failure( + "dsl-evaluation-source-locator", + f"{source_id}: locator must be absolute HTTPS", + ) + ) + if parsed.username is not None or parsed.password is not None: + failures.append( + _failure( + "dsl-evaluation-source-secret", + f"{source_id}: locator contains URI userinfo", + ) + ) + query_keys = {key.casefold() for key, _ in parse_qsl(parsed.query, keep_blank_values=True)} + sensitive = sorted(query_keys & _SENSITIVE_QUERY_KEYS) + if sensitive: + failures.append( + _failure( + "dsl-evaluation-source-secret", + f"{source_id}: locator contains secret-bearing query keys {sensitive}", + ) + ) + + +def _protocol_records_by_id( + protocol: Mapping[str, object], + field: str, + id_field: str, +) -> dict[str, Mapping[str, object]]: + records = protocol.get(field, []) + if not isinstance(records, list): + return {} + return { + record[id_field]: record + for record in records + if isinstance(record, Mapping) and isinstance(record.get(id_field), str) + } diff --git a/tools/dsl_language_evaluation/_snapshot.py b/tools/dsl_language_evaluation/_snapshot.py new file mode 100644 index 000000000..1584a855f --- /dev/null +++ b/tools/dsl_language_evaluation/_snapshot.py @@ -0,0 +1,464 @@ +"""Execution-snapshot validation for the DSL evaluation bundle.""" + +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path + +from tools.dsl_language_evaluation._keys import ( + _ATTEMPT_KEYS, + _DEVIATION_KEYS, + _DISAGREEMENT_KEYS, + _ETHICS_REVIEW_KEYS, + _HISTORICAL_REVISION_FIELD, + _MAX_CATALOG_ITEMS, + _MAX_EXECUTION_RECORDS, + _OBSERVATION_KEYS, + _REVIEW_KEYS, + _SHA_RE, + _SNAPSHOT_KEYS, + _SUBJECT_KEYS, + _SURFACE_KEYS, + _WITHDRAWAL_KEYS, + ATTEMPT_OUTCOMES, + OBSERVATION_OUTCOMES, +) +from tools.dsl_language_evaluation._measures import _measure_stage_ids +from tools.dsl_language_evaluation._shape import ( + _bounded_list, + _bounded_text, + _exact_keys, + _failure, + _protocol_records_by_id, + _record_ids, + _resolve_repository_artifact, + _string_list, + _valid_id, +) +from tools.dsl_language_evaluation._snapshot_completion import ( + _review_and_completion_failures, + _SnapshotJoins, + _subject_failures, +) +from tools.policy.common import PolicyFailure, safe_repo_path + + +def _validate_snapshot( + repo_root: Path, + protocol: Mapping[str, object], + snapshot: dict[str, object], + catalogs: Mapping[str, set[str]], + scope: Mapping[str, set[str]], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/execution-snapshot-v1.json", +) -> set[str]: + if not _exact_keys( + snapshot, + _SNAPSHOT_KEYS, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label="snapshot", + path=path, + ): + return set() + if snapshot["protocol_revision"] != protocol.get("revision"): + failures.append(_failure("dsl-evaluation-snapshot-join", "protocol revision mismatch", path)) + revision = snapshot[_HISTORICAL_REVISION_FIELD] + if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): + failures.append( + _failure( + "dsl-evaluation-snapshot-pin", + "RAES revision must be full Git SHA", + path, + ) + ) + if snapshot["execution_status"] not in {"not_started", "in_progress", "complete"}: + failures.append(_failure("dsl-evaluation-snapshot-status", "invalid execution status", path)) + + surfaces = _bounded_list( + snapshot["public_surface"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label="public_surface", + path=path, + ) + _record_ids( + surfaces, + "surface_id", + failures, + rule_id="dsl-evaluation-snapshot-id", + label="public surface", + path=path, + ) + for index, surface in enumerate(surfaces): + if not _exact_keys( + surface, + _SURFACE_KEYS, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label=f"public_surface[{index}]", + path=path, + ): + continue + artifact = surface["artifact"] + resolved = _resolve_repository_artifact(repo_root, artifact) if isinstance(artifact, str) else None + if resolved is None or not resolved.exists(): + failures.append( + _failure( + "dsl-evaluation-public-surface-path", + f"{surface['surface_id']}: unsafe or missing artifact", + path, + ) + ) + _exact_keys( + snapshot["ethics_review"], + _ETHICS_REVIEW_KEYS, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label="ethics_review", + path=path, + ) + + record_fields = { + "subjects": _SUBJECT_KEYS, + "attempts": _ATTEMPT_KEYS, + "observations": _OBSERVATION_KEYS, + "reviews": _REVIEW_KEYS, + "deviations": _DEVIATION_KEYS, + "withdrawals": _WITHDRAWAL_KEYS, + "disagreements": _DISAGREEMENT_KEYS, + } + records: dict[str, list[object]] = {} + for field, keys in record_fields.items(): + records[field] = _bounded_list( + snapshot[field], + _MAX_EXECUTION_RECORDS, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label=field, + path=path, + ) + for index, record in enumerate(records[field]): + _exact_keys( + record, + keys, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label=f"{field}[{index}]", + path=path, + ) + if snapshot["execution_status"] == "not_started": + populated = sorted(field for field, value in records.items() if value) + if populated: + failures.append( + _failure( + "dsl-evaluation-not-started-observations", + f"not-started snapshot contains execution records: {populated}", + path, + ) + ) + ethics = snapshot["ethics_review"] + if isinstance(ethics, Mapping) and ethics.get("status") not in { + "pending", + "not_required", + }: + failures.append( + _failure( + "dsl-evaluation-ethics-state", + "not-started snapshot must remain pending or explicitly not-required", + path, + ) + ) + return set() + + subjects = records["subjects"] + attempts = records["attempts"] + observations = records["observations"] + reviews = records["reviews"] + withdrawals = records["withdrawals"] + tasks = _protocol_records_by_id(protocol, "tasks", "task_id") + variants = _protocol_records_by_id(protocol, "variants", "variant_id") + measures = _protocol_records_by_id(protocol, "measures", "measure_id") + + _record_ids( + subjects, + "subject_id", + failures, + rule_id="dsl-evaluation-snapshot-id", + label="subject", + path=path, + ) + attempt_ids = _record_ids( + attempts, + "attempt_id", + failures, + rule_id="dsl-evaluation-snapshot-id", + label="attempt", + path=path, + ) + observation_ids = _record_ids( + observations, + "observation_id", + failures, + rule_id="dsl-evaluation-snapshot-id", + label="observation", + path=path, + ) + review_ids = _record_ids( + reviews, + "review_id", + failures, + rule_id="dsl-evaluation-snapshot-id", + label="review", + path=path, + ) + withdrawal_subject_ids = _record_ids( + withdrawals, + "subject_id", + failures, + rule_id="dsl-evaluation-snapshot-id", + label="withdrawal", + path=path, + ) + + subjects_by_id, withdrawal_subject_ids = _subject_failures( + protocol, catalogs, subjects, withdrawals, withdrawal_subject_ids, failures, path + ) + + attempts_by_id: dict[str, Mapping[str, object]] = {} + for attempt in attempts: + if not isinstance(attempt, Mapping) or set(attempt) != _ATTEMPT_KEYS: + continue + attempt_id = attempt.get("attempt_id") + if not isinstance(attempt_id, str): + continue + attempts_by_id[attempt_id] = attempt + task_id = attempt.get("task_id") + task = tasks.get(task_id) if isinstance(task_id, str) else None + subject_id = attempt.get("subject_id") + subject = subjects_by_id.get(subject_id) if isinstance(subject_id, str) else None + outcome = attempt.get("outcome") + if outcome not in ATTEMPT_OUTCOMES: + failures.append(_failure("dsl-evaluation-attempt-outcome", f"{attempt_id}: invalid outcome", path)) + if task is None or subject is None: + failures.append(_failure("dsl-evaluation-attempt-join", f"{attempt_id}: unknown task or subject", path)) + continue + task_personas = _string_list(task.get("persona_ids"), non_empty=True) or [] + task_conditions = _string_list(task.get("tooling_condition_ids"), non_empty=True) or [] + task_variants = _string_list(task.get("variant_ids"), non_empty=True) or [] + if ( + attempt.get("persona_id") != subject.get("persona_id") + or attempt.get("persona_id") not in task_personas + or attempt.get("tooling_condition_id") not in task_conditions + or attempt.get("variant_id") not in task_variants + ): + failures.append( + _failure( + "dsl-evaluation-attempt-join", + f"{attempt_id}: subject, persona, task, condition, or variant mismatch", + path, + ) + ) + variant = variants.get(attempt.get("variant_id")) + if variant is None or variant.get("task_id") != task_id: + failures.append( + _failure("dsl-evaluation-attempt-join", f"{attempt_id}: variant belongs to another task", path) + ) + withdrawn = subject_id in withdrawal_subject_ids + if withdrawn != (outcome == "withdrawn"): + failures.append( + _failure( + "dsl-evaluation-withdrawal-join", + f"{attempt_id}: withdrawn subject and attempt outcome disagree", + path, + ) + ) + if not _valid_id(attempt.get("study_run_id")): + failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: invalid study run id", path)) + if not _bounded_text(attempt.get("started_at"), maximum=100) or not _bounded_text( + attempt.get("ended_at"), maximum=100 + ): + failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: timestamps must be text", path)) + if _string_list(attempt.get("observation_ids")) is None: + failures.append( + _failure("dsl-evaluation-attempt-observation-join", f"{attempt_id}: invalid observation ids", path) + ) + + observations_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} + child_observations: dict[str, set[str]] = {attempt_id: set() for attempt_id in attempt_ids} + for observation in observations: + if not isinstance(observation, Mapping) or set(observation) != _OBSERVATION_KEYS: + continue + observation_id = observation.get("observation_id") + attempt_id = observation.get("attempt_id") + measure_id = observation.get("measure_id") + artifact_stage = observation.get("artifact_stage") + if ( + not isinstance(observation_id, str) + or not isinstance(attempt_id, str) + or not isinstance(measure_id, str) + or not isinstance(artifact_stage, str) + ): + failures.append( + _failure("dsl-evaluation-observation-identity", "observation identity fields must be text", path) + ) + continue + opportunity = (attempt_id, measure_id, artifact_stage) + observation_dimensions = _string_list(observation.get("dimension_ids"), non_empty=True) + if opportunity in observations_by_opportunity: + failures.append( + _failure( + "dsl-evaluation-observation-identity", + f"duplicate attempt-measure-stage observation at {observation_id}", + path, + ) + ) + observations_by_opportunity[opportunity] = observation + child_observations.setdefault(attempt_id, set()).add(observation_id) + attempt = attempts_by_id.get(attempt_id) + measure = measures.get(measure_id) + task = tasks.get(observation.get("task_id")) + if attempt is None or measure is None or task is None: + failures.append( + _failure( + "dsl-evaluation-observation-join", + f"{observation_id}: unknown parent attempt, task, or measure", + path, + ) + ) + continue + parent_fields = ( + "study_run_id", + "task_id", + "persona_id", + "subject_id", + "tooling_condition_id", + "variant_id", + "outcome", + ) + if observation.get("protocol_revision") != protocol.get("revision") or any( + observation.get(field) != attempt.get(field) for field in parent_fields + ): + failures.append( + _failure( + "dsl-evaluation-observation-parent-join", + f"{observation_id}: observation does not match its parent attempt", + path, + ) + ) + task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) or [] + measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] + task_dimensions = set(_string_list(task.get("dimension_ids"), non_empty=True) or []) + measure_dimensions = set(_string_list(measure.get("dimension_ids"), non_empty=True) or []) + expected_dimensions = task_dimensions & measure_dimensions + parent_task_id = attempt.get("task_id") + parent_variant_id = attempt.get("variant_id") + try: + applicable_stages = ( + _measure_stage_ids(measure, parent_task_id, parent_variant_id) + if isinstance(parent_task_id, str) and isinstance(parent_variant_id, str) + else [] + ) + except ValueError: + applicable_stages = [] + if ( + artifact_stage not in task_stages + or artifact_stage not in applicable_stages + or observation.get("task_id") not in measure_tasks + or observation_dimensions is None + or set(observation_dimensions) != expected_dimensions + ): + failures.append( + _failure( + "dsl-evaluation-observation-task-join", + f"{observation_id}: stage, measure, or dimensions are not declared for the task", + path, + ) + ) + outcome = observation.get("outcome") + value = observation.get("value") + if outcome not in OBSERVATION_OUTCOMES: + failures.append(_failure("dsl-evaluation-observation-outcome", f"{observation_id}: invalid outcome", path)) + if measure_id == "task-completion": + expected_value = 1 if outcome == "completed" else 0 + valid_value = value == expected_value and not isinstance(value, bool) + elif outcome in {"completed", "failed"}: + valid_value = not isinstance(value, bool) and isinstance(value, (int, float)) + else: + valid_value = value is None + if not valid_value: + failures.append( + _failure( + "dsl-evaluation-observation-value", + f"{observation_id}: value does not represent its attempt outcome", + path, + ) + ) + refs = _string_list(observation.get("evidence_refs")) + if refs is None or any(safe_repo_path(repo_root, ref) is None for ref in refs): + failures.append( + _failure( + "dsl-evaluation-observation-evidence", + f"{observation_id}: evidence refs must be repository-confined paths", + path, + ) + ) + + expected_opportunities: set[tuple[str, str, str]] = set() + withdrawn_opportunities: set[tuple[str, str, str]] = set() + for attempt_id, attempt in attempts_by_id.items(): + task_id = attempt.get("task_id") + variant_id = attempt.get("variant_id") + for measure_id, measure in measures.items(): + measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] + if task_id not in measure_tasks or not isinstance(task_id, str) or not isinstance(variant_id, str): + continue + try: + applicable_stages = _measure_stage_ids(measure, task_id, variant_id) + except ValueError: + continue + for artifact_stage in applicable_stages: + opportunity = (attempt_id, measure_id, artifact_stage) + if attempt.get("outcome") == "withdrawn": + withdrawn_opportunities.add(opportunity) + else: + expected_opportunities.add(opportunity) + actual_opportunities = set(observations_by_opportunity) + if actual_opportunities != expected_opportunities or actual_opportunities & withdrawn_opportunities: + failures.append( + _failure( + "dsl-evaluation-opportunity-coverage", + "observations must exactly cover every non-withdrawn protocol-declared attempt-measure-stage opportunity", + path, + ) + ) + for attempt_id, attempt in attempts_by_id.items(): + stored_ids = _string_list(attempt.get("observation_ids")) + if stored_ids is not None and set(stored_ids) != child_observations.get(attempt_id, set()): + failures.append( + _failure( + "dsl-evaluation-attempt-observation-join", + f"{attempt_id}: observation ids do not match child records", + path, + ) + ) + + _review_and_completion_failures( + protocol, + snapshot, + scope, + _SnapshotJoins( + subjects_by_id=subjects_by_id, + attempts_by_id=attempts_by_id, + tasks=tasks, + review_ids=review_ids, + reviews=reviews, + disagreements=records["disagreements"], + withdrawal_subject_ids=withdrawal_subject_ids, + ), + failures, + path, + ) + return observation_ids diff --git a/tools/dsl_language_evaluation/_snapshot_completion.py b/tools/dsl_language_evaluation/_snapshot_completion.py new file mode 100644 index 000000000..76c4a2864 --- /dev/null +++ b/tools/dsl_language_evaluation/_snapshot_completion.py @@ -0,0 +1,296 @@ +"""Review joins and completion-coverage validation for the execution snapshot.""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass + +from tools.dsl_language_evaluation._claims import _attempt_matches_scope +from tools.dsl_language_evaluation._keys import ( + _DISAGREEMENT_KEYS, + _REVIEW_KEYS, + _SUBJECT_KEYS, + _WITHDRAWAL_KEYS, +) +from tools.dsl_language_evaluation._shape import ( + _bounded_text, + _failure, + _string_list, + _valid_id, +) +from tools.policy.common import PolicyFailure + + +@dataclass(frozen=True) +class _SnapshotJoins: + """Joined execution-record indexes shared by review and completion checks.""" + + subjects_by_id: Mapping[str, Mapping[str, object]] + attempts_by_id: Mapping[str, Mapping[str, object]] + tasks: Mapping[str, Mapping[str, object]] + review_ids: set[str] + reviews: list[object] + disagreements: list[object] + withdrawal_subject_ids: set[str] + + +def _subject_failures( + protocol: Mapping[str, object], + catalogs: Mapping[str, set[str]], + subjects: list[object], + withdrawals: list[object], + withdrawal_subject_ids: set[str], + failures: list[PolicyFailure], + path: str, +) -> tuple[dict[str, Mapping[str, object]], set[str]]: + subjects_by_id = { + subject["subject_id"]: subject + for subject in subjects + if isinstance(subject, Mapping) and set(subject) == _SUBJECT_KEYS and isinstance(subject.get("subject_id"), str) + } + sampling_plan = protocol.get("sampling_plan") + experience_bands = set() + if isinstance(sampling_plan, Mapping): + experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) + for subject_id, subject in subjects_by_id.items(): + if subject.get("persona_id") not in catalogs.get("persona_ids", set()): + failures.append(_failure("dsl-evaluation-subject-join", f"{subject_id}: unknown persona", path)) + if subject.get("consent_status") not in {"consented", "withdrawn"}: + failures.append(_failure("dsl-evaluation-consent-status", f"{subject_id}: invalid consent status", path)) + if ( + not _bounded_text(subject.get("experience_band"), maximum=200) + or subject.get("experience_band") not in experience_bands + ): + failures.append(_failure("dsl-evaluation-subject-shape", f"{subject_id}: invalid experience band", path)) + declared_withdrawn_subjects = { + subject_id for subject_id, subject in subjects_by_id.items() if subject.get("consent_status") == "withdrawn" + } + if withdrawal_subject_ids != declared_withdrawn_subjects: + failures.append( + _failure( + "dsl-evaluation-withdrawal-join", + "withdrawal records must exactly match subjects with withdrawn consent", + path, + ) + ) + for withdrawal in withdrawals: + if not isinstance(withdrawal, Mapping) or set(withdrawal) != _WITHDRAWAL_KEYS: + continue + if ( + not _bounded_text(withdrawal.get("recorded_at"), maximum=100) + or withdrawal.get("retained_aggregate_only") is not True + ): + failures.append( + _failure( + "dsl-evaluation-withdrawal-shape", + f"{withdrawal.get('subject_id')}: withdrawal must retain aggregate counts only", + path, + ) + ) + return subjects_by_id, withdrawal_subject_ids + + +def _review_and_completion_failures( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + scope: Mapping[str, set[str]], + joins: _SnapshotJoins, + failures: list[PolicyFailure], + path: str, +) -> None: + subjects_by_id = joins.subjects_by_id + attempts_by_id = joins.attempts_by_id + tasks = joins.tasks + review_ids = joins.review_ids + reviews = joins.reviews + records = {"disagreements": joins.disagreements} + reviews_by_attempt: Counter[str] = Counter() + for review in reviews: + if not isinstance(review, Mapping) or set(review) != _REVIEW_KEYS: + continue + review_id = review.get("review_id") + attempt_id = review.get("attempt_id") + attempt = attempts_by_id.get(attempt_id) if isinstance(attempt_id, str) else None + reviewer_id = review.get("reviewer_subject_id") + reviewer = subjects_by_id.get(reviewer_id) if isinstance(reviewer_id, str) else None + if attempt is None or reviewer is None: + failures.append(_failure("dsl-evaluation-review-join", f"{review_id}: unknown attempt or reviewer", path)) + continue + task = tasks.get(attempt.get("task_id")) + task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) if task else None + task_personas = _string_list(task.get("persona_ids"), non_empty=True) if task else None + if ( + review.get("task_id") != attempt.get("task_id") + or review.get("variant_id") != attempt.get("variant_id") + or reviewer_id == attempt.get("subject_id") + or reviewer.get("consent_status") != "consented" + or task_stages is None + or "review-judgment" not in task_stages + or task_personas is None + or reviewer.get("persona_id") not in task_personas + ): + failures.append( + _failure( + "dsl-evaluation-review-join", + f"{review_id}: review does not match an eligible independent reviewer and parent task", + path, + ) + ) + if ( + not _bounded_text(review.get("judgment"), maximum=500) + or isinstance(review.get("confidence"), bool) + or not isinstance(review.get("confidence"), (int, float)) + or not 0 <= review["confidence"] <= 1 + or not _valid_id(review.get("rationale_code")) + or not _bounded_text(review.get("fixed_at"), maximum=100) + ): + failures.append(_failure("dsl-evaluation-review-shape", f"{review_id}: invalid fixed judgment", path)) + reviews_by_attempt[attempt_id] += 1 + + for disagreement in records["disagreements"]: + if not isinstance(disagreement, Mapping) or set(disagreement) != _DISAGREEMENT_KEYS: + continue + disagreement_review_ids = _string_list(disagreement["review_ids"], non_empty=True) + linked_reviews = [ + review + for review in reviews + if isinstance(review, Mapping) + and disagreement_review_ids is not None + and review.get("review_id") in disagreement_review_ids + ] + linked_attempts = {review.get("attempt_id") for review in linked_reviews} + if ( + disagreement_review_ids is None + or len(disagreement_review_ids) < 2 + or not set(disagreement_review_ids).issubset(review_ids) + or len(linked_attempts) != 1 + ): + failures.append( + _failure( + "dsl-evaluation-disagreement-join", + f"{disagreement['disagreement_id']}: reviews must share one parent attempt", + path, + ) + ) + if disagreement["originals_preserved"] is not True: + failures.append( + _failure( + "dsl-evaluation-disagreement-preservation", + f"{disagreement['disagreement_id']}: originals must be preserved", + path, + ) + ) + + if snapshot["execution_status"] == "complete": + nonwithdrawn_subject_ids = { + subject_id + for subject_id, subject in subjects_by_id.items() + if subject.get("consent_status") == "consented" + and subject.get("persona_id") in scope.get("persona_ids", set()) + } + active_subject_ids = { + subject_id + for subject_id in nonwithdrawn_subject_ids + if any( + attempt.get("subject_id") == subject_id + and attempt.get("outcome") != "withdrawn" + and _attempt_matches_scope(attempt, scope) + for attempt in attempts_by_id.values() + ) + } + persona_minimums = { + item["persona_id"]: item["minimum_completed_subjects"] + for item in protocol.get("personas", []) + if isinstance(item, Mapping) + and isinstance(item.get("persona_id"), str) + and isinstance(item.get("minimum_completed_subjects"), int) + and item["persona_id"] in scope.get("persona_ids", set()) + } + persona_counts = Counter(subjects_by_id[subject_id]["persona_id"] for subject_id in active_subject_ids) + missing_personas = sorted( + persona_id for persona_id, minimum in persona_minimums.items() if persona_counts[persona_id] < minimum + ) + expected_task_shapes = { + (task["task_id"], condition_id, variant_id) + for task in tasks.values() + if task["task_id"] in scope.get("task_ids", set()) + for condition_id in (_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) + if condition_id in scope.get("tooling_condition_ids", set()) + for variant_id in (_string_list(task.get("variant_ids"), non_empty=True) or []) + if variant_id in scope.get("variant_ids", set()) + } + actual_task_shapes = { + (attempt.get("task_id"), attempt.get("tooling_condition_id"), attempt.get("variant_id")) + for attempt in attempts_by_id.values() + if attempt.get("outcome") != "withdrawn" and _attempt_matches_scope(attempt, scope) + } + review_required_attempts = { + attempt_id + for attempt_id, attempt in attempts_by_id.items() + if _attempt_matches_scope(attempt, scope) + if "review-judgment" + in (_string_list(tasks.get(attempt.get("task_id"), {}).get("artifact_stage_ids"), non_empty=True) or []) + and attempt.get("outcome") != "withdrawn" + } + sampling_plan = protocol.get("sampling_plan", {}) + target_total = sampling_plan.get("target_total") if isinstance(sampling_plan, Mapping) else None + ethics = snapshot["ethics_review"] + ethics_approved = isinstance(ethics, Mapping) and ethics.get("status") == "approved" + execution_plan = protocol.get("execution_plan", {}) + subject_task_requirements = ( + execution_plan.get("subject_task_requirements", []) if isinstance(execution_plan, Mapping) else [] + ) + missing_subject_workloads: list[tuple[str, str]] = [] + if isinstance(subject_task_requirements, list): + for subject_id in nonwithdrawn_subject_ids: + subject_attempts = [ + attempt + for attempt in attempts_by_id.values() + if attempt.get("subject_id") == subject_id + and attempt.get("outcome") != "withdrawn" + and _attempt_matches_scope(attempt, scope) + ] + for requirement in subject_task_requirements: + if not isinstance(requirement, Mapping): + continue + task_kind_values = _string_list(requirement.get("task_kinds"), non_empty=True) + minimum = requirement.get("minimum_assigned_attempts") + requirement_id = requirement.get("requirement_id") + if ( + task_kind_values is None + or not isinstance(minimum, int) + or isinstance(minimum, bool) + or not isinstance(requirement_id, str) + ): + continue + assigned = sum( + tasks.get(attempt.get("task_id"), {}).get("kind") in task_kind_values + for attempt in subject_attempts + ) + if assigned < minimum: + missing_subject_workloads.append((subject_id, requirement_id)) + if missing_subject_workloads: + failures.append( + _failure( + "dsl-evaluation-subject-workload", + "complete execution does not satisfy every active subject's assigned task groups", + path, + ) + ) + if ( + not ethics_approved + or missing_personas + or not isinstance(target_total, int) + or len(active_subject_ids) < target_total + or not expected_task_shapes.issubset(actual_task_shapes) + or any(reviews_by_attempt[attempt_id] == 0 for attempt_id in review_required_attempts) + or missing_subject_workloads + ): + failures.append( + _failure( + "dsl-evaluation-completion-coverage", + "complete execution lacks approved ethics, subject workload/minima, task/condition/variant coverage, or required independent reviews", + path, + ) + ) From 6e43eb6cac37c466e86b9c0916702e6375d8bf93 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:02:54 -0700 Subject: [PATCH 2/4] refactor(tools): decompose DSL evaluation validators below Sonar thresholds Split every moved mega-function into single-purpose helpers so the relocated code meets the raes-strict new-code gate: cognitive complexity <=15, <=100 lines, <=7 parameters, <=3 returns per function, and <=500 lines per module. - _measures/_claims/_protocol/_protocol_plans: helper-per-concern rewrites of the recompute, scope, catalog, and plan validators - _snapshot: not-started, record-id, attempt, observation, and opportunity-coverage sections extracted; the execution-record joins now live in _snapshot_execution.py to stay under the module cap - _snapshot_completion: review, disagreement, and completion coverage split into scoped helpers - _analysis: section helpers plus an _AnalysisContext bundle; drops the unused catalogs parameter Verified byte-identical behavior: the checker exits 0 with empty text and JSON output on the real bundles, all 40 suite tests pass, and a 73-scenario differential harness (mutating every record type across protocol, snapshot, and analysis) produces identical failure lists on the old and new implementations. Co-Authored-By: Claude Fable 5 --- tools/check_dsl_language_evaluation.py | 22 +- tools/dsl_language_evaluation/_analysis.py | 304 +++++++---- tools/dsl_language_evaluation/_claims.py | 323 ++++++----- tools/dsl_language_evaluation/_keys.py | 7 +- tools/dsl_language_evaluation/_measures.py | 456 +++++++++------- tools/dsl_language_evaluation/_protocol.py | 501 +++++++++-------- .../_protocol_plans.py | 503 +++++++++++------- tools/dsl_language_evaluation/_snapshot.py | 424 +++++---------- .../_snapshot_completion.py | 473 +++++++++------- .../_snapshot_execution.py | 371 +++++++++++++ 10 files changed, 2001 insertions(+), 1383 deletions(-) create mode 100644 tools/dsl_language_evaluation/_snapshot_execution.py diff --git a/tools/check_dsl_language_evaluation.py b/tools/check_dsl_language_evaluation.py index df0548423..02a2c268c 100644 --- a/tools/check_dsl_language_evaluation.py +++ b/tools/check_dsl_language_evaluation.py @@ -26,7 +26,7 @@ load_bounded_json_object, safe_repo_path, ) -from tools.dsl_language_evaluation._analysis import _validate_analysis +from tools.dsl_language_evaluation._analysis import _AnalysisContext, _validate_analysis from tools.dsl_language_evaluation._claims import ( _validate_claim_binding, _validate_claim_scope, @@ -93,7 +93,7 @@ def validate_bundle( if isinstance(entry[0].get("claim_binding"), Mapping) and entry[0]["claim_binding"].get("claim_id") == claim_id ) - except (OSError, ValueError, json.JSONDecodeError, StopIteration): + except (OSError, ValueError, StopIteration): claim_binding = None catalogs = _validate_protocol( repo_root, @@ -116,7 +116,6 @@ def validate_bundle( protocol, analysis, claim_binding, - catalogs, scope, failures, path=analysis_path @@ -137,14 +136,15 @@ def validate_bundle( ), ) _validate_analysis( - repo_root, - protocol, - snapshot, + _AnalysisContext( + repo_root=repo_root, + protocol=protocol, + snapshot=snapshot, + scope=scope, + strata=strata, + observation_ids=observation_ids, + ), analysis, - catalogs, - scope, - strata, - observation_ids, failures, path=analysis_path if isinstance(analysis_path, str) @@ -207,7 +207,7 @@ def load_bundles( def evaluate(repo_root: Path = REPO_ROOT) -> list[PolicyFailure]: try: bundles = load_bundles(repo_root) - except (OSError, ValueError, json.JSONDecodeError) as exc: + except (OSError, ValueError) as exc: return [_failure("dsl-evaluation-bundle-invalid", str(exc), MANIFEST_PATH)] failures: list[PolicyFailure] = [] for entry, protocol, snapshot, analysis in bundles: diff --git a/tools/dsl_language_evaluation/_analysis.py b/tools/dsl_language_evaluation/_analysis.py index 5966f9111..067c3a1bb 100644 --- a/tools/dsl_language_evaluation/_analysis.py +++ b/tools/dsl_language_evaluation/_analysis.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Mapping, Sequence +from dataclasses import dataclass from pathlib import Path from tools.dsl_language_evaluation._keys import ( @@ -29,28 +30,25 @@ from tools.policy.common import PolicyFailure, safe_repo_path -def _validate_analysis( - repo_root: Path, +@dataclass(frozen=True) +class _AnalysisContext: + """Frozen upstream artifacts and joins the analysis is validated against.""" + + repo_root: Path + protocol: Mapping[str, object] + snapshot: Mapping[str, object] + scope: Mapping[str, set[str]] + strata: Sequence[Mapping[str, object]] + observation_ids: set[str] + + +def _analysis_header_failures( protocol: Mapping[str, object], snapshot: Mapping[str, object], - analysis: dict[str, object], - catalogs: Mapping[str, set[str]], - scope: Mapping[str, set[str]], - strata: Sequence[Mapping[str, object]], - observation_ids: set[str], + analysis: Mapping[str, object], failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/analysis-v1.json", + path: str, ) -> None: - if not _exact_keys( - analysis, - _ANALYSIS_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label="analysis", - path=path, - ): - return if analysis["protocol_revision"] != protocol.get("revision"): failures.append(_failure("dsl-evaluation-analysis-join", "protocol revision mismatch", path)) if analysis["snapshot_id"] != snapshot.get("snapshot_id"): @@ -60,6 +58,15 @@ def _validate_analysis( if analysis["evidence_status"] not in EVIDENCE_STATUSES: failures.append(_failure("dsl-evaluation-evidence-status", "invalid evidence status", path)) + +def _measure_result_failures( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + analysis: Mapping[str, object], + scope: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> dict[str, dict[str, object]]: measure_results = _bounded_list( analysis["measure_results"], _MAX_CATALOG_ITEMS, @@ -118,12 +125,21 @@ def _validate_analysis( path, ) ) + return recomputed_measures + +def _dimension_result_failures( + context: _AnalysisContext, + analysis: Mapping[str, object], + recomputed_measures: dict[str, dict[str, object]], + failures: list[PolicyFailure], + path: str, +) -> list[object]: try: recomputed_dimensions = recompute_dimension_results( - protocol, + context.protocol, recomputed_measures, - dimension_ids=scope.get("dimension_ids", set()), + dimension_ids=context.scope.get("dimension_ids", set()), ) except ValueError as exc: failures.append(_failure("dsl-evaluation-threshold-evaluation", str(exc), path)) @@ -145,7 +161,7 @@ def _validate_analysis( label="dimension result", path=path, ) - if result_ids != scope.get("dimension_ids", set()): + if result_ids != context.scope.get("dimension_ids", set()): failures.append( _failure( "dsl-evaluation-analysis-dimension-coverage", @@ -164,7 +180,7 @@ def _validate_analysis( ): continue refs = _string_list(result["supporting_observation_ids"]) - if refs is None or not set(refs).issubset(observation_ids): + if refs is None or not set(refs).issubset(context.observation_ids): failures.append( _failure( "dsl-evaluation-analysis-observation-join", @@ -184,7 +200,65 @@ def _validate_analysis( path, ) ) + return results + + +def _recomputed_stratum_failures( + context: _AnalysisContext, + stored_strata: list[object], + failures: list[PolicyFailure], + path: str, +) -> list[dict[str, object]]: + expected_strata: list[dict[str, object]] = [] + try: + expected_strata = recompute_stratum_results(context.protocol, context.snapshot, context.strata) + except ValueError as exc: + failures.append(_failure("dsl-evaluation-stratum-drift", str(exc), path)) + expected_by_id = {item["stratum_id"]: item for item in expected_strata} + stored_ids = _record_ids( + stored_strata, + "stratum_id", + failures, + rule_id="dsl-evaluation-analysis-id", + label="stratum result", + path=path, + ) + if stored_ids != set(expected_by_id): + failures.append( + _failure( + "dsl-evaluation-stratum-coverage", + "analysis must persist one independently recomputed result for every bound stratum", + path, + ) + ) + for index, result in enumerate(stored_strata): + if not _exact_keys( + result, + _STRATUM_RESULT_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label=f"stratum_results[{index}]", + path=path, + ): + continue + assert isinstance(result, dict) + if result != expected_by_id.get(result["stratum_id"]): + failures.append( + _failure( + "dsl-evaluation-stratum-drift", + f"{result['stratum_id']}: stored stratum result does not match frozen observations", + path, + ) + ) + return expected_strata + +def _stratum_result_failures( + context: _AnalysisContext, + analysis: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> list[dict[str, object]]: stored_strata = _bounded_list( analysis["stratum_results"], _MAX_CATALOG_ITEMS, @@ -193,8 +267,7 @@ def _validate_analysis( label="stratum_results", path=path, ) - expected_strata: list[dict[str, object]] = [] - if snapshot.get("execution_status") == "not_started": + if context.snapshot.get("execution_status") == "not_started": if stored_strata: failures.append( _failure( @@ -203,49 +276,18 @@ def _validate_analysis( path, ) ) - else: - try: - expected_strata = recompute_stratum_results(protocol, snapshot, strata) - except ValueError as exc: - failures.append(_failure("dsl-evaluation-stratum-drift", str(exc), path)) - expected_by_id = {item["stratum_id"]: item for item in expected_strata} - stored_ids = _record_ids( - stored_strata, - "stratum_id", - failures, - rule_id="dsl-evaluation-analysis-id", - label="stratum result", - path=path, - ) - if stored_ids != set(expected_by_id): - failures.append( - _failure( - "dsl-evaluation-stratum-coverage", - "analysis must persist one independently recomputed result for every bound stratum", - path, - ) - ) - for index, result in enumerate(stored_strata): - if not _exact_keys( - result, - _STRATUM_RESULT_KEYS, - failures, - rule_id="dsl-evaluation-analysis-shape", - label=f"stratum_results[{index}]", - path=path, - ): - continue - assert isinstance(result, dict) - if result != expected_by_id.get(result["stratum_id"]): - failures.append( - _failure( - "dsl-evaluation-stratum-drift", - f"{result['stratum_id']}: stored stratum result does not match frozen observations", - path, - ) - ) + return [] + return _recomputed_stratum_failures(context, stored_strata, failures, path) + + +def _claim_evidence_failures( + repo_root: Path, + analysis: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: claim = analysis["claim"] - if _exact_keys( + if not _exact_keys( claim, _CLAIM_KEYS, failures, @@ -253,53 +295,66 @@ def _validate_analysis( label="claim", path=path, ): - evidence_artifacts = claim["evidence_artifacts"] - if not isinstance(evidence_artifacts, list) or len(evidence_artifacts) != 3: + return + evidence_artifacts = claim["evidence_artifacts"] + if not isinstance(evidence_artifacts, list) or len(evidence_artifacts) != 3: + failures.append( + _failure( + "dsl-evaluation-claim-evidence", + "claim must name protocol, snapshot, and analysis artifacts", + path, + ) + ) + return + for artifact in evidence_artifacts: + resolved = safe_repo_path(repo_root, artifact) if isinstance(artifact, str) else None + if resolved is None or not resolved.is_file(): failures.append( _failure( - "dsl-evaluation-claim-evidence", - "claim must name protocol, snapshot, and analysis artifacts", + "dsl-evaluation-claim-evidence-path", + f"unsafe or missing claim evidence artifact {artifact!r}", path, ) ) - else: - for artifact in evidence_artifacts: - resolved = safe_repo_path(repo_root, artifact) if isinstance(artifact, str) else None - if resolved is None or not resolved.is_file(): - failures.append( - _failure( - "dsl-evaluation-claim-evidence-path", - f"unsafe or missing claim evidence artifact {artifact!r}", - path, - ) - ) - status = analysis["evidence_status"] - if snapshot.get("execution_status") == "not_started": - if status != "untested": + + +def _not_started_analysis_failures( + status: object, + results: list[object], + failures: list[PolicyFailure], + path: str, +) -> None: + if status != "untested": + failures.append( + _failure( + "dsl-evaluation-evidence-status", + "not-started execution must remain untested", + path, + ) + ) + for result in results: + if not isinstance(result, Mapping): + continue + expected = { + "status": "not_evaluated", + "threshold_result": "not_evaluated", + "condition_results": [], + "supporting_observation_ids": [], + } + if any(result.get(field) != value for field, value in expected.items()): failures.append( _failure( - "dsl-evaluation-evidence-status", - "not-started execution must remain untested", + "dsl-evaluation-not-started-analysis", + f"{result.get('dimension_id')}: not-started result contains derived evidence", path, ) ) - for result in results: - if not isinstance(result, Mapping): - continue - expected = { - "status": "not_evaluated", - "threshold_result": "not_evaluated", - "condition_results": [], - "supporting_observation_ids": [], - } - if any(result.get(field) != value for field, value in expected.items()): - failures.append( - _failure( - "dsl-evaluation-not-started-analysis", - f"{result.get('dimension_id')}: not-started result contains derived evidence", - path, - ) - ) + + +def _gating_qualification( + snapshot: Mapping[str, object], + expected_strata: list[dict[str, object]], +) -> tuple[bool, bool]: gating_strata = [item for item in expected_strata if item.get("role") == "gating"] gating_dimensions = [ item.get("dimension_results", []) for item in gating_strata if isinstance(item.get("dimension_results"), list) @@ -332,6 +387,17 @@ def _validate_analysis( execution_complete = snapshot.get("execution_status") == "complete" qualifies_demonstrated = execution_complete and all_pass and not unresolved and not invalidating_deviation qualifies_refuted = execution_complete and any_fail + return qualifies_demonstrated, qualifies_refuted + + +def _evidence_status_failures( + snapshot: Mapping[str, object], + status: object, + expected_strata: list[dict[str, object]], + failures: list[PolicyFailure], + path: str, +) -> None: + qualifies_demonstrated, qualifies_refuted = _gating_qualification(snapshot, expected_strata) execution_records_present = any( isinstance(snapshot.get(field), list) and bool(snapshot[field]) for field in ("subjects", "attempts", "observations", "reviews", "deviations", "withdrawals") @@ -340,7 +406,8 @@ def _validate_analysis( failures.append( _failure( "dsl-evaluation-evidence-status", - "demonstrated requires every bound gating stratum to pass without unresolved critical disagreement or invalidating deviation", + "demonstrated requires every bound gating stratum to pass " + "without unresolved critical disagreement or invalidating deviation", path, ) ) @@ -373,3 +440,32 @@ def _validate_analysis( path, ) ) + + +def _validate_analysis( + context: _AnalysisContext, + analysis: dict[str, object], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/analysis-v1.json", +) -> None: + if not _exact_keys( + analysis, + _ANALYSIS_KEYS, + failures, + rule_id="dsl-evaluation-analysis-shape", + label="analysis", + path=path, + ): + return + _analysis_header_failures(context.protocol, context.snapshot, analysis, failures, path) + recomputed_measures = _measure_result_failures( + context.protocol, context.snapshot, analysis, context.scope, failures, path + ) + results = _dimension_result_failures(context, analysis, recomputed_measures, failures, path) + expected_strata = _stratum_result_failures(context, analysis, failures, path) + _claim_evidence_failures(context.repo_root, analysis, failures, path) + status = analysis["evidence_status"] + if context.snapshot.get("execution_status") == "not_started": + _not_started_analysis_failures(status, results, failures, path) + _evidence_status_failures(context.snapshot, status, expected_strata, failures, path) diff --git a/tools/dsl_language_evaluation/_claims.py b/tools/dsl_language_evaluation/_claims.py index dbfcfcc18..dc90dbaf2 100644 --- a/tools/dsl_language_evaluation/_claims.py +++ b/tools/dsl_language_evaluation/_claims.py @@ -73,6 +73,18 @@ def _validate_claim_scope( ): return fallback + resolved = _resolved_scope_fields(scope, catalogs, failures, path) + _scope_task_failures(protocol, resolved, failures, path) + _scope_threshold_failures(protocol, resolved, failures, path) + return resolved + + +def _resolved_scope_fields( + scope: Mapping[str, object], + catalogs: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> dict[str, set[str]]: catalog_fields = { "persona_ids": "persona_ids", "task_ids": "task_ids", @@ -97,17 +109,25 @@ def _validate_claim_scope( resolved[field] = set(values or []) & known else: resolved[field] = set(values) + return resolved + +def _scope_task_failures( + protocol: Mapping[str, object], + resolved: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> None: tasks = _protocol_records_by_id(protocol, "tasks", "task_id") measures = _protocol_records_by_id(protocol, "measures", "measure_id") + joins = ( + ("persona_ids", "persona_ids"), + ("tooling_condition_ids", "tooling_condition_ids"), + ("variant_ids", "variant_ids"), + ("artifact_stage_ids", "artifact_stage_ids"), + ) for task_id in resolved["task_ids"]: task = tasks.get(task_id, {}) - joins = ( - ("persona_ids", "persona_ids"), - ("tooling_condition_ids", "tooling_condition_ids"), - ("variant_ids", "variant_ids"), - ("artifact_stage_ids", "artifact_stage_ids"), - ) if any( not (set(_string_list(task.get(task_field), non_empty=True) or []) & resolved[scope_field]) for task_field, scope_field in joins @@ -132,19 +152,16 @@ def _validate_claim_scope( ) ) - thresholds = { - threshold.get("dimension_id"): threshold - for threshold in protocol.get("thresholds", []) - if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) - } + +def _scope_threshold_failures( + protocol: Mapping[str, object], + resolved: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> None: + thresholds = _thresholds_by_dimension(protocol) for dimension_id in resolved["dimension_ids"]: - threshold = thresholds.get(dimension_id) - conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] - required_measures = { - condition.get("measure_id") - for condition in conditions - if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) - } + required_measures = _threshold_measures(thresholds.get(dimension_id)) if not required_measures or not required_measures.issubset(resolved["measure_ids"]): failures.append( _failure( @@ -153,14 +170,29 @@ def _validate_claim_scope( path, ) ) - return resolved + + +def _thresholds_by_dimension(protocol: Mapping[str, object]) -> dict[object, Mapping[str, object]]: + return { + threshold.get("dimension_id"): threshold + for threshold in protocol.get("thresholds", []) + if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) + } + + +def _threshold_measures(threshold: Mapping[str, object] | None) -> set[object]: + conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] + return { + condition.get("measure_id") + for condition in conditions + if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) + } def _validate_claim_binding( protocol: Mapping[str, object], analysis: Mapping[str, object], claim_binding: object, - catalogs: Mapping[str, set[str]], scope: Mapping[str, set[str]], failures: list[PolicyFailure], *, @@ -178,37 +210,8 @@ def _validate_claim_binding( ): return [] assert isinstance(claim_binding, dict) - claim = analysis.get("claim") - claim_id = claim.get("claim_id") if isinstance(claim, Mapping) else None - if not _valid_id(claim_binding["claim_id"]) or claim_binding["claim_id"] != claim_id: - failures.append( - _failure( - "dsl-evaluation-claim-binding", - "manifest claim binding must name the analysis claim_id exactly", - path, - ) - ) - - bound_scope = claim_binding["scope"] - if _exact_keys( - bound_scope, - _CLAIM_SCOPE_KEYS, - failures, - rule_id="dsl-evaluation-claim-binding", - label="claim binding scope", - path=path, - ): - assert isinstance(bound_scope, dict) - for field in _CLAIM_SCOPE_KEYS: - values = _string_list(bound_scope[field], non_empty=True) - if values is None or len(values) != len(set(values)) or set(values) != scope.get(field, set()): - failures.append( - _failure( - "dsl-evaluation-claim-binding", - f"analysis claim scope {field} must exactly match its manifest binding", - path, - ) - ) + _claim_identity_failures(analysis, claim_binding, failures, path) + _bound_scope_failures(claim_binding, scope, failures, path) groups = _bounded_list( claim_binding["strata"], @@ -243,83 +246,17 @@ def _validate_claim_binding( ): continue assert isinstance(group, dict) - group_id = group["group_id"] - role = group["role"] - partition_by = _string_list(group["partition_by"]) - persona_ids = _string_list(group["persona_ids"], non_empty=True) - band_ids = _string_list(group["experience_band_ids"], non_empty=True) - condition_ids = _string_list(group["tooling_condition_ids"], non_empty=True) - valid = ( - _valid_id(group_id) - and role in _STRATUM_ROLES - and partition_by is not None - and len(partition_by) == len(set(partition_by)) - and set(partition_by).issubset(_STRATUM_PARTITION_AXES) - and persona_ids is not None - and len(persona_ids) == len(set(persona_ids)) - and set(persona_ids).issubset(scope.get("persona_ids", set())) - and band_ids is not None - and len(band_ids) == len(set(band_ids)) - and set(band_ids).issubset(experience_bands) - and condition_ids is not None - and len(condition_ids) == len(set(condition_ids)) - and set(condition_ids).issubset(scope.get("tooling_condition_ids", set())) - ) - if not valid: + filters = _validated_group_filters(group, scope, experience_bands) + if filters is None: failures.append( _failure( "dsl-evaluation-claim-strata", - f"claim stratum group {group_id!r} has invalid role, partition axes, or catalog filters", + f"claim stratum group {group['group_id']!r} has invalid role, partition axes, or catalog filters", path, ) ) continue - - assert isinstance(group_id, str) - assert isinstance(role, str) - assert partition_by is not None - assert persona_ids is not None - assert band_ids is not None - assert condition_ids is not None - axis_values = { - "persona_id": persona_ids, - "experience_band": band_ids, - "tooling_condition_id": condition_ids, - } - split_axes = [ - axis for axis in ("persona_id", "experience_band", "tooling_condition_id") if axis in partition_by - ] - combinations = product(*(axis_values[axis] for axis in split_axes)) if split_axes else [()] - for combination in combinations: - selected = dict(zip(split_axes, combination, strict=True)) - stratum_id = "-".join([group_id, *(str(selected[axis]) for axis in split_axes)]) - stratum_personas = {str(selected["persona_id"])} if "persona_id" in selected else set(persona_ids) - stratum_bands = {str(selected["experience_band"])} if "experience_band" in selected else set(band_ids) - stratum_conditions = ( - {str(selected["tooling_condition_id"])} if "tooling_condition_id" in selected else set(condition_ids) - ) - stratum_scope = _derive_stratum_scope( - protocol, - scope, - persona_ids=stratum_personas, - experience_band_ids=stratum_bands, - tooling_condition_ids=stratum_conditions, - ) - if ( - not _valid_id(stratum_id) - or not stratum_scope["task_ids"] - or not stratum_scope["measure_ids"] - or not stratum_scope["dimension_ids"] - ): - failures.append( - _failure( - "dsl-evaluation-claim-strata", - f"claim stratum {stratum_id!r} has no complete task/measure/dimension slice", - path, - ) - ) - continue - expanded.append({"stratum_id": stratum_id, "role": role, "scope": stratum_scope}) + expanded.extend(_expanded_group_strata(protocol, scope, group, filters, failures, path)) expanded_ids = [str(item["stratum_id"]) for item in expanded] if len(group_ids) != len(groups) or len(expanded_ids) != len(set(expanded_ids)) or not expanded: @@ -344,6 +281,135 @@ def _validate_claim_binding( return expanded +def _claim_identity_failures( + analysis: Mapping[str, object], + claim_binding: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: + claim = analysis.get("claim") + claim_id = claim.get("claim_id") if isinstance(claim, Mapping) else None + if not _valid_id(claim_binding["claim_id"]) or claim_binding["claim_id"] != claim_id: + failures.append( + _failure( + "dsl-evaluation-claim-binding", + "manifest claim binding must name the analysis claim_id exactly", + path, + ) + ) + + +def _bound_scope_failures( + claim_binding: Mapping[str, object], + scope: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> None: + bound_scope = claim_binding["scope"] + if not _exact_keys( + bound_scope, + _CLAIM_SCOPE_KEYS, + failures, + rule_id="dsl-evaluation-claim-binding", + label="claim binding scope", + path=path, + ): + return + assert isinstance(bound_scope, dict) + for field in _CLAIM_SCOPE_KEYS: + values = _string_list(bound_scope[field], non_empty=True) + if values is None or len(values) != len(set(values)) or set(values) != scope.get(field, set()): + failures.append( + _failure( + "dsl-evaluation-claim-binding", + f"analysis claim scope {field} must exactly match its manifest binding", + path, + ) + ) + + +def _unique_subset(values: list[str] | None, allowed: set[str]) -> bool: + return values is not None and len(values) == len(set(values)) and set(values).issubset(allowed) + + +def _validated_group_filters( + group: Mapping[str, object], + scope: Mapping[str, set[str]], + experience_bands: set[str], +) -> tuple[str, str, list[str], list[str], list[str], list[str]] | None: + """Return (group_id, role, partition_by, personas, bands, conditions) or None.""" + + group_id = group["group_id"] + role = group["role"] + partition_by = _string_list(group["partition_by"]) + persona_ids = _string_list(group["persona_ids"], non_empty=True) + band_ids = _string_list(group["experience_band_ids"], non_empty=True) + condition_ids = _string_list(group["tooling_condition_ids"], non_empty=True) + valid = ( + _valid_id(group_id) + and isinstance(group_id, str) + and isinstance(role, str) + and role in _STRATUM_ROLES + and _unique_subset(partition_by, _STRATUM_PARTITION_AXES) + and _unique_subset(persona_ids, scope.get("persona_ids", set())) + and _unique_subset(band_ids, experience_bands) + and _unique_subset(condition_ids, scope.get("tooling_condition_ids", set())) + ) + if not valid: + return None + assert partition_by is not None and persona_ids is not None + assert band_ids is not None and condition_ids is not None + return group_id, role, partition_by, persona_ids, band_ids, condition_ids + + +def _expanded_group_strata( + protocol: Mapping[str, object], + scope: Mapping[str, set[str]], + group: Mapping[str, object], + filters: tuple[str, str, list[str], list[str], list[str], list[str]], + failures: list[PolicyFailure], + path: str, +) -> list[dict[str, object]]: + del group + group_id, role, partition_by, persona_ids, band_ids, condition_ids = filters + axis_values = { + "persona_id": persona_ids, + "experience_band": band_ids, + "tooling_condition_id": condition_ids, + } + split_axes = [axis for axis in ("persona_id", "experience_band", "tooling_condition_id") if axis in partition_by] + combinations = product(*(axis_values[axis] for axis in split_axes)) if split_axes else [()] + expanded: list[dict[str, object]] = [] + for combination in combinations: + selected = dict(zip(split_axes, combination, strict=True)) + stratum_id = "-".join([group_id, *(str(selected[axis]) for axis in split_axes)]) + stratum_scope = _derive_stratum_scope( + protocol, + scope, + persona_ids={str(selected["persona_id"])} if "persona_id" in selected else set(persona_ids), + experience_band_ids={str(selected["experience_band"])} if "experience_band" in selected else set(band_ids), + tooling_condition_ids=( + {str(selected["tooling_condition_id"])} if "tooling_condition_id" in selected else set(condition_ids) + ), + ) + if ( + not _valid_id(stratum_id) + or not stratum_scope["task_ids"] + or not stratum_scope["measure_ids"] + or not stratum_scope["dimension_ids"] + ): + failures.append( + _failure( + "dsl-evaluation-claim-strata", + f"claim stratum {stratum_id!r} has no complete task/measure/dimension slice", + path, + ) + ) + continue + expanded.append({"stratum_id": stratum_id, "role": role, "scope": stratum_scope}) + return expanded + + def _derive_stratum_scope( protocol: Mapping[str, object], claim_scope: Mapping[str, set[str]], @@ -375,20 +441,10 @@ def _derive_stratum_scope( if measure_id in claim_scope.get("measure_ids", set()) and set(_string_list(measure.get("task_ids"), non_empty=True) or []) & selected_tasks } - thresholds = { - threshold.get("dimension_id"): threshold - for threshold in protocol.get("thresholds", []) - if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) - } + thresholds = _thresholds_by_dimension(protocol) selected_dimensions = set() for dimension_id in claim_scope.get("dimension_ids", set()): - threshold = thresholds.get(dimension_id) - conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] - required_measures = { - condition.get("measure_id") - for condition in conditions - if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) - } + required_measures = _threshold_measures(thresholds.get(dimension_id)) if required_measures and required_measures.issubset(selected_measures): selected_dimensions.add(dimension_id) return { @@ -425,7 +481,6 @@ def resolve_claim_strata( protocol, analysis, claim_binding, - catalogs, scope, failures, path=MANIFEST_PATH, diff --git a/tools/dsl_language_evaluation/_keys.py b/tools/dsl_language_evaluation/_keys.py index 30d4ae466..00a043671 100644 --- a/tools/dsl_language_evaluation/_keys.py +++ b/tools/dsl_language_evaluation/_keys.py @@ -4,12 +4,13 @@ import re +_PACKAGES_PREFIX = "implementations/python/packages/" MANIFEST_PATH = "docs/research/dsl-language-evaluation/bundle-manifest.json" -_RETIRED_PACKAGE_PREFIX = "implementations/python/packages/" + "a" + "ces_sdl" +_RETIRED_PACKAGE_PREFIX = _PACKAGES_PREFIX + "a" + "ces_sdl" _HISTORICAL_PACKAGE_MOVES: tuple[tuple[str, str], ...] = ( (_RETIRED_PACKAGE_PREFIX, "implementations/python/packages/raes"), - ("implementations/python/packages/" + "a" + "ces_cli", "implementations/python/packages/raes_cli"), - ("implementations/python/packages/" + "a" + "ces_mcp", "implementations/python/packages/raes_mcp"), + (_PACKAGES_PREFIX + "a" + "ces_cli", "implementations/python/packages/raes_cli"), + (_PACKAGES_PREFIX + "a" + "ces_mcp", "implementations/python/packages/raes_mcp"), ) _MAX_FILE_BYTES = 2 * 1024 * 1024 _MAX_CATALOG_ITEMS = 128 diff --git a/tools/dsl_language_evaluation/_measures.py b/tools/dsl_language_evaluation/_measures.py index e95d3c7a6..cd80c4e0a 100644 --- a/tools/dsl_language_evaluation/_measures.py +++ b/tools/dsl_language_evaluation/_measures.py @@ -9,6 +9,13 @@ from tools.dsl_language_evaluation._keys import _STRATUM_ROLES from tools.dsl_language_evaluation._shape import _protocol_records_by_id, _string_list +_AGGREGATIONS = {"proportion", "median", "count"} +_THRESHOLD_OPERATORS = { + ">=": lambda actual, target: actual >= target, + "<=": lambda actual, target: actual <= target, + "==": lambda actual, target: actual == target, +} + def _measure_stage_ids( measure: Mapping[str, object], @@ -33,6 +40,100 @@ def _measure_stage_ids( return matches[0] +def _scoped_measures( + protocol: Mapping[str, object], + scope_sets: Mapping[str, set[str]] | None, +) -> dict[str, Mapping[str, object]]: + measures = _protocol_records_by_id(protocol, "measures", "measure_id") + if scope_sets is None: + return measures + return { + measure_id: measure + for measure_id, measure in measures.items() + if measure_id in scope_sets.get("measure_ids", set()) + } + + +def _observation_index(observations: list[object]) -> dict[tuple[str, str, str], Mapping[str, object]]: + index: dict[tuple[str, str, str], Mapping[str, object]] = {} + for observation in observations: + if not isinstance(observation, Mapping): + continue + attempt_id = observation.get("attempt_id") + measure_id = observation.get("measure_id") + artifact_stage = observation.get("artifact_stage") + if not isinstance(attempt_id, str) or not isinstance(measure_id, str) or not isinstance(artifact_stage, str): + continue + key = (attempt_id, measure_id, artifact_stage) + if key in index: + raise ValueError(f"duplicate observation opportunity {attempt_id}/{measure_id}/{artifact_stage}") + index[key] = observation + return index + + +def _attempt_in_scope( + attempt: Mapping[str, object], + scope_sets: Mapping[str, set[str]] | None, + subjects_by_id: Mapping[object, Mapping[str, object]], +) -> bool: + if scope_sets is None: + return True + if not _attempt_matches_scope(attempt, scope_sets): + return False + if "experience_band_ids" in scope_sets: + subject = subjects_by_id.get(attempt.get("subject_id")) + if not isinstance(subject, Mapping) or subject.get("experience_band") not in scope_sets["experience_band_ids"]: + return False + return True + + +def _attempt_opportunities( + attempt: Mapping[str, object], + measures: Mapping[str, Mapping[str, object]], + scope_sets: Mapping[str, set[str]] | None, + *, + withdrawn: bool, +) -> tuple[list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], set[tuple[str, str, str]]]: + attempt_id = str(attempt.get("attempt_id")) + task_id = str(attempt.get("task_id")) + variant_id = str(attempt.get("variant_id")) + opportunities: list[tuple[Mapping[str, object], Mapping[str, object], str, bool]] = [] + expected_keys: set[tuple[str, str, str]] = set() + for measure in measures.values(): + task_ids = _string_list(measure.get("task_ids"), non_empty=True) + if task_ids is None or task_id not in task_ids: + continue + measure_id = measure.get("measure_id") + if not isinstance(measure_id, str): + continue + for artifact_stage in _measure_stage_ids(measure, task_id, variant_id): + if scope_sets is not None and artifact_stage not in scope_sets.get("artifact_stage_ids", set()): + continue + opportunities.append((attempt, measure, artifact_stage, withdrawn)) + expected_keys.add((attempt_id, measure_id, artifact_stage)) + return opportunities, expected_keys + + +def _check_observed_keys( + observation_by_opportunity: Mapping[tuple[str, str, str], Mapping[str, object]], + expected_keys: set[tuple[str, str, str]], + selected_attempt_ids: set[str], + scope_sets: Mapping[str, set[str]] | None, +) -> None: + observed_keys = set(observation_by_opportunity) + if scope_sets is not None: + observed_keys = { + key + for key in observed_keys + if key[0] in selected_attempt_ids + and key[1] in scope_sets.get("measure_ids", set()) + and key[2] in scope_sets.get("artifact_stage_ids", set()) + } + extras = sorted(observed_keys - expected_keys) + if extras: + raise ValueError(f"observations without protocol-declared opportunities: {extras[:5]}") + + def _measure_opportunities( protocol: Mapping[str, object], snapshot: Mapping[str, object], @@ -45,14 +146,8 @@ def _measure_opportunities( """Derive every attempt-measure-stage opportunity and frozen observation.""" tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - measures = _protocol_records_by_id(protocol, "measures", "measure_id") scope_sets = {field: set(values) for field, values in scope.items()} if scope is not None else None - if scope_sets is not None: - measures = { - measure_id: measure - for measure_id, measure in measures.items() - if measure_id in scope_sets.get("measure_ids", set()) - } + measures = _scoped_measures(protocol, scope_sets) attempts = snapshot.get("attempts", []) observations = snapshot.get("observations", []) withdrawals = snapshot.get("withdrawals", []) @@ -70,19 +165,7 @@ def _measure_opportunities( for withdrawal in withdrawals if isinstance(withdrawal, Mapping) and isinstance(withdrawal.get("subject_id"), str) } - observation_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} - for observation in observations: - if not isinstance(observation, Mapping): - continue - attempt_id = observation.get("attempt_id") - measure_id = observation.get("measure_id") - artifact_stage = observation.get("artifact_stage") - if not isinstance(attempt_id, str) or not isinstance(measure_id, str) or not isinstance(artifact_stage, str): - continue - key = (attempt_id, measure_id, artifact_stage) - if key in observation_by_opportunity: - raise ValueError(f"duplicate observation opportunity {attempt_id}/{measure_id}/{artifact_stage}") - observation_by_opportunity[key] = observation + observation_by_opportunity = _observation_index(observations) opportunities: list[tuple[Mapping[str, object], Mapping[str, object], str, bool]] = [] expected_keys: set[tuple[str, str, str]] = set() @@ -93,53 +176,108 @@ def _measure_opportunities( attempt_id = attempt.get("attempt_id") task_id = attempt.get("task_id") subject_id = attempt.get("subject_id") - variant_id = attempt.get("variant_id") - if not isinstance(attempt_id, str) or not isinstance(task_id, str) or not isinstance(variant_id, str): - continue - task = tasks.get(task_id) - if task is None: + if ( + not isinstance(attempt_id, str) + or not isinstance(task_id, str) + or not isinstance(attempt.get("variant_id"), str) + ): continue - if scope_sets is not None and not _attempt_matches_scope(attempt, scope_sets): + if tasks.get(task_id) is None or not _attempt_in_scope(attempt, scope_sets, subjects_by_id): continue - if scope_sets is not None and "experience_band_ids" in scope_sets: - subject = subjects_by_id.get(subject_id) - if ( - not isinstance(subject, Mapping) - or subject.get("experience_band") not in scope_sets["experience_band_ids"] - ): - continue selected_attempt_ids.add(attempt_id) withdrawn = (isinstance(subject_id, str) and subject_id in withdrawn_subjects) or attempt.get( "outcome" ) == "withdrawn" - for measure in measures.values(): - task_ids = _string_list(measure.get("task_ids"), non_empty=True) - if task_ids is None or task_id not in task_ids: - continue - measure_id = measure.get("measure_id") - if not isinstance(measure_id, str): - continue - for artifact_stage in _measure_stage_ids(measure, task_id, variant_id): - if scope_sets is not None and artifact_stage not in scope_sets.get("artifact_stage_ids", set()): - continue - opportunities.append((attempt, measure, artifact_stage, withdrawn)) - expected_keys.add((attempt_id, measure_id, artifact_stage)) + attempt_opportunities, attempt_keys = _attempt_opportunities(attempt, measures, scope_sets, withdrawn=withdrawn) + opportunities.extend(attempt_opportunities) + expected_keys.update(attempt_keys) - observed_keys = set(observation_by_opportunity) - if scope_sets is not None: - observed_keys = { - key - for key in observed_keys - if key[0] in selected_attempt_ids - and key[1] in scope_sets.get("measure_ids", set()) - and key[2] in scope_sets.get("artifact_stage_ids", set()) - } - extras = sorted(observed_keys - expected_keys) - if extras: - raise ValueError(f"observations without protocol-declared opportunities: {extras[:5]}") + _check_observed_keys(observation_by_opportunity, expected_keys, selected_attempt_ids, scope_sets) return measures, opportunities, observation_by_opportunity +def _observation_tallies( + measure_id: str, + aggregation: object, + eligible: list[tuple[Mapping[str, object], str]], + observation_by_opportunity: Mapping[tuple[str, str, str], Mapping[str, object]], +) -> tuple[list[int | float], list[str], dict[str, int]]: + values: list[int | float] = [] + supporting_ids: list[str] = [] + counts = {"missing": 0, "abandoned": 0, "tool_failed": 0} + for attempt, artifact_stage in eligible: + attempt_id = attempt.get("attempt_id") + if not isinstance(attempt_id, str): + counts["missing"] += 1 + continue + observation = observation_by_opportunity.get((attempt_id, measure_id, artifact_stage)) + if observation is None: + counts["missing"] += 1 + continue + observation_id = observation.get("observation_id") + if not isinstance(observation_id, str): + raise ValueError(f"{measure_id}: observations require ids") + supporting_ids.append(observation_id) + outcome = observation.get("outcome") + if outcome in counts: + counts[outcome] += 1 + value = observation.get("value") + if value is None: + continue + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{measure_id}: observation values must be numeric or null") + if aggregation == "proportion" and value not in {0, 1}: + raise ValueError(f"{measure_id}: proportion observations must be 0 or 1") + values.append(value) + return values, supporting_ids, counts + + +def _aggregated_value( + aggregation: object, + values: list[int | float], + denominator: int, +) -> tuple[int | float | None, int | float | None]: + if denominator == 0 or len(values) != denominator: + return None, None + if aggregation == "proportion": + numerator = sum(values) + return numerator, numerator / denominator + if aggregation == "median": + return None, float(median(values)) + numerator = sum(values) + return numerator, numerator + + +def _measure_result( + measure_id: str, + measure: Mapping[str, object], + opportunities: list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], + observation_by_opportunity: Mapping[tuple[str, str, str], Mapping[str, object]], +) -> dict[str, object]: + aggregation = measure.get("aggregation") + if aggregation not in _AGGREGATIONS: + raise ValueError(f"invalid measure aggregation for {measure_id!r}") + matching = [item for item in opportunities if item[1].get("measure_id") == measure_id] + withdrawn_count = sum(withdrawn for _, _, _, withdrawn in matching) + eligible = [(attempt, artifact_stage) for attempt, _, artifact_stage, withdrawn in matching if not withdrawn] + denominator = len(eligible) + values, supporting_ids, counts = _observation_tallies(measure_id, aggregation, eligible, observation_by_opportunity) + numerator, value = _aggregated_value(aggregation, values, denominator) + return { + "statistic": aggregation, + "numerator": numerator, + "denominator": denominator, + "opportunity_count": len(matching), + "observed_count": len(values), + "missing_count": counts["missing"], + "abandoned_count": counts["abandoned"], + "tool_failed_count": counts["tool_failed"], + "withdrawn_count": withdrawn_count, + "value": value, + "supporting_observation_ids": supporting_ids, + } + + def recompute_measure_results( protocol: Mapping[str, object], snapshot: Mapping[str, object], @@ -149,78 +287,83 @@ def recompute_measure_results( """Recompute measures from the complete protocol-derived opportunity matrix.""" measures, opportunities, observation_by_opportunity = _measure_opportunities(protocol, snapshot, scope) - results: dict[str, dict[str, object]] = {} - for measure_id, measure in measures.items(): - aggregation = measure.get("aggregation") - if aggregation not in {"proportion", "median", "count"}: - raise ValueError(f"invalid measure aggregation for {measure_id!r}") - - matching = [item for item in opportunities if item[1].get("measure_id") == measure_id] - opportunity_count = len(matching) - withdrawn_count = sum(withdrawn for _, _, _, withdrawn in matching) - eligible = [(attempt, artifact_stage) for attempt, _, artifact_stage, withdrawn in matching if not withdrawn] - denominator = len(eligible) - values: list[int | float] = [] - supporting_ids: list[str] = [] - missing_count = 0 - abandoned_count = 0 - tool_failed_count = 0 - - for attempt, artifact_stage in eligible: - attempt_id = attempt.get("attempt_id") - if not isinstance(attempt_id, str): - missing_count += 1 - continue - observation = observation_by_opportunity.get((attempt_id, measure_id, artifact_stage)) - if observation is None: - missing_count += 1 - continue - observation_id = observation.get("observation_id") - if not isinstance(observation_id, str): - raise ValueError(f"{measure_id}: observations require ids") - supporting_ids.append(observation_id) - outcome = observation.get("outcome") - if outcome == "missing": - missing_count += 1 - elif outcome == "abandoned": - abandoned_count += 1 - elif outcome == "tool_failed": - tool_failed_count += 1 - value = observation.get("value") - if value is None: - continue - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"{measure_id}: observation values must be numeric or null") - if aggregation == "proportion" and value not in {0, 1}: - raise ValueError(f"{measure_id}: proportion observations must be 0 or 1") - values.append(value) - - observed_count = len(values) - numerator: int | float | None = None - value: int | float | None = None - if denominator > 0 and observed_count == denominator: - if aggregation == "proportion": - numerator = sum(values) - value = numerator / denominator - elif aggregation == "median": - value = float(median(values)) - else: - numerator = sum(values) - value = numerator - results[measure_id] = { - "statistic": aggregation, - "numerator": numerator, - "denominator": denominator, - "opportunity_count": opportunity_count, - "observed_count": observed_count, - "missing_count": missing_count, - "abandoned_count": abandoned_count, - "tool_failed_count": tool_failed_count, - "withdrawn_count": withdrawn_count, - "value": value, - "supporting_observation_ids": supporting_ids, + return { + measure_id: _measure_result(measure_id, measure, opportunities, observation_by_opportunity) + for measure_id, measure in measures.items() + } + + +def _resolved_conditions( + dimension_id: str, + conditions: list[object], + measure_results: Mapping[str, Mapping[str, object]], +) -> list[tuple[Mapping[str, object], Mapping[str, object]]]: + resolved: list[tuple[Mapping[str, object], Mapping[str, object]]] = [] + for condition in conditions: + if not isinstance(condition, Mapping): + raise ValueError(f"{dimension_id}: threshold condition must be an object") + measure_id = condition.get("measure_id") + operator = condition.get("operator") + target = condition.get("target") + measure = measure_results.get(measure_id) if isinstance(measure_id, str) else None + if operator not in _THRESHOLD_OPERATORS or isinstance(target, bool) or not isinstance(target, (int, float)): + raise ValueError(f"{dimension_id}: invalid threshold operator or target") + if measure is None: + raise ValueError(f"{dimension_id}: threshold references unknown measure {measure_id!r}") + resolved.append((condition, measure)) + return resolved + + +def _condition_outcomes( + dimension_id: str, + resolved: list[tuple[Mapping[str, object], Mapping[str, object]]], +) -> tuple[list[dict[str, object]], list[str]]: + condition_results: list[dict[str, object]] = [] + supporting_ids: list[str] = [] + for condition, measure in resolved: + actual = measure["value"] + operator = condition["operator"] + target = condition["target"] + if isinstance(actual, bool) or not isinstance(actual, (int, float)): + raise ValueError(f"{dimension_id}: measure value must be numeric") + condition_results.append( + { + "measure_id": condition["measure_id"], + "operator": operator, + "target": target, + "actual": actual, + "passed": _THRESHOLD_OPERATORS[operator](actual, target), + } + ) + refs = measure.get("supporting_observation_ids", []) + if not isinstance(refs, list) or not all(isinstance(item, str) for item in refs): + raise ValueError(f"{dimension_id}: measure observation refs must be text ids") + for observation_id in refs: + if observation_id not in supporting_ids: + supporting_ids.append(observation_id) + return condition_results, supporting_ids + + +def _dimension_result( + dimension_id: str, + conditions: list[object], + measure_results: Mapping[str, Mapping[str, object]], +) -> dict[str, object]: + resolved = _resolved_conditions(dimension_id, conditions, measure_results) + if any(measure.get("value") is None for _, measure in resolved): + return { + "status": "not_evaluated", + "threshold_result": "not_evaluated", + "condition_results": [], + "supporting_observation_ids": [], } - return results + condition_results, supporting_ids = _condition_outcomes(dimension_id, resolved) + return { + "status": "evaluated", + "threshold_result": "pass" if all(item["passed"] for item in condition_results) else "fail", + "condition_results": condition_results, + "supporting_observation_ids": supporting_ids, + } def recompute_dimension_results( @@ -231,11 +374,6 @@ def recompute_dimension_results( ) -> dict[str, dict[str, object]]: """Apply protocol-declared threshold conditions to recomputed measures.""" - operators = { - ">=": lambda actual, target: actual >= target, - "<=": lambda actual, target: actual <= target, - "==": lambda actual, target: actual == target, - } results: dict[str, dict[str, object]] = {} selected_dimensions = set(dimension_ids) if dimension_ids is not None else None for threshold in protocol.get("thresholds", []): @@ -252,57 +390,7 @@ def recompute_dimension_results( raise ValueError(f"invalid threshold for {dimension_id!r}") if selected_dimensions is not None and dimension_id not in selected_dimensions: continue - resolved: list[tuple[Mapping[str, object], Mapping[str, object]]] = [] - for condition in conditions: - if not isinstance(condition, Mapping): - raise ValueError(f"{dimension_id}: threshold condition must be an object") - measure_id = condition.get("measure_id") - operator = condition.get("operator") - target = condition.get("target") - measure = measure_results.get(measure_id) if isinstance(measure_id, str) else None - if operator not in operators or isinstance(target, bool) or not isinstance(target, (int, float)): - raise ValueError(f"{dimension_id}: invalid threshold operator or target") - if measure is None: - raise ValueError(f"{dimension_id}: threshold references unknown measure {measure_id!r}") - resolved.append((condition, measure)) - if any(measure.get("value") is None for _, measure in resolved): - results[dimension_id] = { - "status": "not_evaluated", - "threshold_result": "not_evaluated", - "condition_results": [], - "supporting_observation_ids": [], - } - continue - condition_results: list[dict[str, object]] = [] - supporting_ids: list[str] = [] - for condition, measure in resolved: - actual = measure["value"] - operator = condition["operator"] - target = condition["target"] - if isinstance(actual, bool) or not isinstance(actual, (int, float)): - raise ValueError(f"{dimension_id}: measure value must be numeric") - passed = operators[operator](actual, target) - condition_results.append( - { - "measure_id": condition["measure_id"], - "operator": operator, - "target": target, - "actual": actual, - "passed": passed, - } - ) - refs = measure.get("supporting_observation_ids", []) - if not isinstance(refs, list) or not all(isinstance(item, str) for item in refs): - raise ValueError(f"{dimension_id}: measure observation refs must be text ids") - for observation_id in refs: - if observation_id not in supporting_ids: - supporting_ids.append(observation_id) - results[dimension_id] = { - "status": "evaluated", - "threshold_result": "pass" if all(item["passed"] for item in condition_results) else "fail", - "condition_results": condition_results, - "supporting_observation_ids": supporting_ids, - } + results[dimension_id] = _dimension_result(dimension_id, conditions, measure_results) return results diff --git a/tools/dsl_language_evaluation/_protocol.py b/tools/dsl_language_evaluation/_protocol.py index 69e5d81a1..6c01f84d9 100644 --- a/tools/dsl_language_evaluation/_protocol.py +++ b/tools/dsl_language_evaluation/_protocol.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Mapping from pathlib import Path from tools.dsl_language_evaluation._keys import ( @@ -31,162 +32,73 @@ ) from tools.policy.common import PolicyFailure +_CATALOG_FIELDS = ( + ("dimensions", "dimension_id", "dimension"), + ("personas", "persona_id", "persona"), + ("tooling_conditions", "condition_id", "tooling condition"), + ("artifact_stages", "stage_id", "artifact stage"), + ("sources", "source_id", "source"), + ("tasks", "task_id", "task"), + ("variants", "variant_id", "variant"), + ("measures", "measure_id", "measure"), +) -def _validate_protocol( - repo_root: Path, + +def _protocol_catalog_records( protocol: dict[str, object], failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/protocol-v1.json", + path: str, +) -> dict[str, list[object]]: + records: dict[str, list[object]] = {} + for field, _id_field, _label in (*_CATALOG_FIELDS, ("thresholds", "", "")): + records[field] = _bounded_list( + protocol[field], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=field, + path=path, + ) + return records + + +def _protocol_catalog_ids( + records: Mapping[str, list[object]], + failures: list[PolicyFailure], + path: str, ) -> dict[str, set[str]]: - if not _exact_keys( - protocol, - _PROTOCOL_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="protocol", - path=path, - ): - return {} + ids: dict[str, set[str]] = {} + for field, id_field, label in _CATALOG_FIELDS: + ids[_ID_KEYS[field]] = _record_ids( + records[field], + id_field, + failures, + rule_id="dsl-evaluation-protocol-id", + label=label, + path=path, + ) + return ids - dimensions = _bounded_list( - protocol["dimensions"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="dimensions", - path=path, - ) - personas = _bounded_list( - protocol["personas"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="personas", - path=path, - ) - conditions = _bounded_list( - protocol["tooling_conditions"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="tooling_conditions", - path=path, - ) - stages = _bounded_list( - protocol["artifact_stages"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="artifact_stages", - path=path, - ) - sources = _bounded_list( - protocol["sources"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="sources", - path=path, - ) - tasks = _bounded_list( - protocol["tasks"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="tasks", - path=path, - ) - variants = _bounded_list( - protocol["variants"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="variants", - path=path, - ) - measures = _bounded_list( - protocol["measures"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="measures", - path=path, - ) - thresholds = _bounded_list( - protocol["thresholds"], - _MAX_CATALOG_ITEMS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label="thresholds", - path=path, - ) - dimension_ids = _record_ids( - dimensions, - "dimension_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="dimension", - path=path, - ) - persona_ids = _record_ids( - personas, - "persona_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="persona", - path=path, - ) - condition_ids = _record_ids( - conditions, - "condition_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="tooling condition", - path=path, - ) - stage_ids = _record_ids( - stages, - "stage_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="artifact stage", - path=path, - ) - source_ids = _record_ids( - sources, - "source_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="source", - path=path, - ) - task_ids = _record_ids( - tasks, - "task_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="task", - path=path, - ) - variant_ids = _record_ids( - variants, - "variant_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="variant", - path=path, - ) - measure_ids = _record_ids( - measures, - "measure_id", - failures, - rule_id="dsl-evaluation-protocol-id", - label="measure", - path=path, - ) +_ID_KEYS = { + "dimensions": "dimension_ids", + "personas": "persona_ids", + "tooling_conditions": "condition_ids", + "artifact_stages": "stage_ids", + "sources": "source_ids", + "tasks": "task_ids", + "variants": "variant_ids", + "measures": "measure_ids", +} + +def _required_coverage_failures( + protocol: dict[str, object], + dimension_ids: set[str], + persona_ids: set[str], + failures: list[PolicyFailure], + path: str, +) -> None: if not REQUIRED_DIMENSION_IDS.issubset(dimension_ids): failures.append( _failure( @@ -217,7 +129,13 @@ def _validate_protocol( ) ) - for index, record in enumerate(dimensions): + +def _catalog_shape_failures( + records: Mapping[str, list[object]], + failures: list[PolicyFailure], + path: str, +) -> None: + for index, record in enumerate(records["dimensions"]): _exact_keys( record, _DIMENSION_KEYS, @@ -226,7 +144,7 @@ def _validate_protocol( label=f"dimensions[{index}]", path=path, ) - for index, record in enumerate(personas): + for index, record in enumerate(records["personas"]): if not _exact_keys( record, _PERSONA_KEYS, @@ -248,7 +166,7 @@ def _validate_protocol( path, ) ) - for index, record in enumerate(conditions): + for index, record in enumerate(records["tooling_conditions"]): _exact_keys( record, _CONDITION_KEYS, @@ -257,7 +175,7 @@ def _validate_protocol( label=f"tooling_conditions[{index}]", path=path, ) - for index, record in enumerate(stages): + for index, record in enumerate(records["artifact_stages"]): _exact_keys( record, _STAGE_KEYS, @@ -267,73 +185,86 @@ def _validate_protocol( path=path, ) - for index, source in enumerate(sources): - if not _exact_keys( - source, - _SOURCE_KEYS, - failures, - rule_id="dsl-evaluation-protocol-shape", - label=f"sources[{index}]", - path=path, - ): - continue - source_id = source["source_id"] - _validate_https_locator(source["locator"], failures, source_id) - if source["primary"] is not True: - failures.append( - _failure( - "dsl-evaluation-source-primary", - f"{source_id}: source must be primary", - path, - ) + +def _repository_source_failures( + repo_root: Path, + source: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: + source_id = source["source_id"] + revision = source["revision"] + artifact_path = source["artifact_path"] + if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): + failures.append( + _failure( + "dsl-evaluation-source-pin", + f"{source_id}: invalid Git revision", + path, ) - if source["kind"] == "repository-internal": - revision = source["revision"] - artifact_path = source["artifact_path"] - if not isinstance(revision, str) or not _SHA_RE.fullmatch(revision): - failures.append( - _failure( - "dsl-evaluation-source-pin", - f"{source_id}: invalid Git revision", - path, - ) - ) - elif isinstance(source["locator"], str) and revision not in source["locator"]: - failures.append( - _failure( - "dsl-evaluation-source-pin", - f"{source_id}: locator does not bind the declared Git revision", - path, - ) - ) - if not isinstance(artifact_path, str): - failures.append( - _failure( - "dsl-evaluation-source-path", - f"{source_id}: missing artifact path", - path, - ) - ) - else: - resolved = _resolve_repository_artifact(repo_root, artifact_path) - if resolved is None or not resolved.exists(): - failures.append( - _failure( - "dsl-evaluation-source-path", - f"{source_id}: unsafe or missing path", - path, - ) - ) - elif source["revision"] is not None or source["artifact_path"] is not None: + ) + elif isinstance(source["locator"], str) and revision not in source["locator"]: + failures.append( + _failure( + "dsl-evaluation-source-pin", + f"{source_id}: locator does not bind the declared Git revision", + path, + ) + ) + if not isinstance(artifact_path, str): + failures.append( + _failure( + "dsl-evaluation-source-path", + f"{source_id}: missing artifact path", + path, + ) + ) + else: + resolved = _resolve_repository_artifact(repo_root, artifact_path) + if resolved is None or not resolved.exists(): failures.append( _failure( - "dsl-evaluation-source-shape", - f"{source_id}: publication must not claim a repository revision/path", + "dsl-evaluation-source-path", + f"{source_id}: unsafe or missing path", path, ) ) - task_kinds: set[str] = set() + +def _source_entry_failures( + repo_root: Path, + source: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: + source_id = source["source_id"] + _validate_https_locator(source["locator"], failures, source_id) + if source["primary"] is not True: + failures.append( + _failure( + "dsl-evaluation-source-primary", + f"{source_id}: source must be primary", + path, + ) + ) + if source["kind"] == "repository-internal": + _repository_source_failures(repo_root, source, failures, path) + elif source["revision"] is not None or source["artifact_path"] is not None: + failures.append( + _failure( + "dsl-evaluation-source-shape", + f"{source_id}: publication must not claim a repository revision/path", + path, + ) + ) + + +def _variants_by_task( + variants: list[object], + task_ids: set[str], + failures: list[PolicyFailure], + path: str, +) -> dict[str, set[str]]: variants_by_task: dict[str, set[str]] = {task_id: set() for task_id in task_ids} for index, variant in enumerate(variants): if not _exact_keys( @@ -358,7 +289,55 @@ def _validate_protocol( variant_id = variant["variant_id"] if isinstance(variant_id, str): variants_by_task[task_id].add(variant_id) - for index, task in enumerate(tasks): + return variants_by_task + + +def _task_entry_failures( + task: Mapping[str, object], + task_id: str, + ids: Mapping[str, set[str]], + variants_by_task: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> None: + joins = ( + ("persona_ids", ids["persona_ids"]), + ("dimension_ids", ids["dimension_ids"]), + ("source_refs", ids["source_ids"]), + ("artifact_stage_ids", ids["stage_ids"]), + ("tooling_condition_ids", ids["condition_ids"]), + ("variant_ids", ids["variant_ids"]), + ) + for field, allowed in joins: + value_ids = _string_list(task[field], non_empty=True) + if value_ids is None or not set(value_ids).issubset(allowed): + failures.append( + _failure( + "dsl-evaluation-task-join", + f"{task_id}: invalid or empty {field}", + path, + ) + ) + valid_task_variant_ids = _string_list(task["variant_ids"]) + if valid_task_variant_ids is not None and set(valid_task_variant_ids) != variants_by_task.get(task_id, set()): + failures.append( + _failure( + "dsl-evaluation-task-variant-coverage", + f"{task_id}: task and variant catalogs disagree", + path, + ) + ) + + +def _task_failures( + records: Mapping[str, list[object]], + ids: Mapping[str, set[str]], + variants_by_task: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> None: + task_kinds: set[str] = set() + for index, task in enumerate(records["tasks"]): if not _exact_keys( task, _TASK_KEYS, @@ -383,35 +362,7 @@ def _validate_protocol( path, ) ) - joins = ( - ("persona_ids", persona_ids), - ("dimension_ids", dimension_ids), - ("source_refs", source_ids), - ("artifact_stage_ids", stage_ids), - ("tooling_condition_ids", condition_ids), - ("variant_ids", variant_ids), - ) - for field, allowed in joins: - values = task[field] - value_ids = _string_list(values, non_empty=True) - if value_ids is None or not set(value_ids).issubset(allowed): - failures.append( - _failure( - "dsl-evaluation-task-join", - f"{task_id}: invalid or empty {field}", - path, - ) - ) - task_variant_ids = task["variant_ids"] - valid_task_variant_ids = _string_list(task_variant_ids) - if valid_task_variant_ids is not None and set(valid_task_variant_ids) != variants_by_task.get(task_id, set()): - failures.append( - _failure( - "dsl-evaluation-task-variant-coverage", - f"{task_id}: task and variant catalogs disagree", - path, - ) - ) + _task_entry_failures(task, task_id, ids, variants_by_task, failures, path) if not REQUIRED_TASK_KINDS.issubset(task_kinds): failures.append( _failure( @@ -421,24 +372,52 @@ def _validate_protocol( ) ) - catalogs_ids = { - "dimension_ids": dimension_ids, - "persona_ids": persona_ids, - "condition_ids": condition_ids, - "stage_ids": stage_ids, - "source_ids": source_ids, - "task_ids": task_ids, - "variant_ids": variant_ids, - "measure_ids": measure_ids, - } - _plan_failures(protocol, catalogs_ids, personas, tasks, measures, thresholds, failures, path) + +def _validate_protocol( + repo_root: Path, + protocol: dict[str, object], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/protocol-v1.json", +) -> dict[str, set[str]]: + if not _exact_keys( + protocol, + _PROTOCOL_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label="protocol", + path=path, + ): + return {} + + records = _protocol_catalog_records(protocol, failures, path) + ids = _protocol_catalog_ids(records, failures, path) + + _required_coverage_failures(protocol, ids["dimension_ids"], ids["persona_ids"], failures, path) + _catalog_shape_failures(records, failures, path) + + for index, source in enumerate(records["sources"]): + if _exact_keys( + source, + _SOURCE_KEYS, + failures, + rule_id="dsl-evaluation-protocol-shape", + label=f"sources[{index}]", + path=path, + ): + _source_entry_failures(repo_root, source, failures, path) + + variants_by_task = _variants_by_task(records["variants"], ids["task_ids"], failures, path) + _task_failures(records, ids, variants_by_task, failures, path) + + _plan_failures(protocol, ids, records, failures, path) _preregistration_failures(protocol, failures, path) return { - "dimension_ids": dimension_ids, - "persona_ids": persona_ids, - "condition_ids": condition_ids, - "stage_ids": stage_ids, - "task_ids": task_ids, - "variant_ids": variant_ids, - "measure_ids": measure_ids, + "dimension_ids": ids["dimension_ids"], + "persona_ids": ids["persona_ids"], + "condition_ids": ids["condition_ids"], + "stage_ids": ids["stage_ids"], + "task_ids": ids["task_ids"], + "variant_ids": ids["variant_ids"], + "measure_ids": ids["measure_ids"], } diff --git a/tools/dsl_language_evaluation/_protocol_plans.py b/tools/dsl_language_evaluation/_protocol_plans.py index 9903e2841..a6fa0073f 100644 --- a/tools/dsl_language_evaluation/_protocol_plans.py +++ b/tools/dsl_language_evaluation/_protocol_plans.py @@ -48,22 +48,145 @@ def _preregistration_failures(protocol: dict[str, object], failures: list[Policy failures.append(_failure("dsl-evaluation-protocol-shape", "amendment_log must be a list", path)) -def _plan_failures( +def _measure_join_failures( + measure: Mapping[str, object], + tasks: list[object], + catalogs_ids: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> list[str] | None: + """Validate one measure's dimension/task joins; return its task ids.""" + + measure_dimensions = _string_list(measure["dimension_ids"], non_empty=True) + if measure_dimensions is None or not set(measure_dimensions).issubset(catalogs_ids["dimension_ids"]): + failures.append( + _failure( + "dsl-evaluation-measure-join", + f"{measure['measure_id']}: invalid dimension ids", + path, + ) + ) + measure_tasks = _string_list(measure["task_ids"], non_empty=True) + if measure_tasks is None or not set(measure_tasks).issubset(catalogs_ids["task_ids"]): + failures.append( + _failure( + "dsl-evaluation-measure-join", + f"{measure['measure_id']}: invalid task ids", + path, + ) + ) + elif not all( + set(measure_dimensions or []) & set(task["dimension_ids"]) + for task in tasks + if isinstance(task, Mapping) + and task.get("task_id") in measure_tasks + and _string_list(task.get("dimension_ids"), non_empty=True) is not None + ): + failures.append( + _failure( + "dsl-evaluation-measure-join", + f"{measure['measure_id']}: task applicability must share a measured dimension", + path, + ) + ) + return measure_tasks + + +def _declaration_stage_pairs( + measure: Mapping[str, object], + declaration: Mapping[str, object], + tasks_by_id: Mapping[str, Mapping[str, object]], + measure_tasks: list[str] | None, + failures: list[PolicyFailure], + path: str, +) -> list[tuple[str, str]] | None: + """Validate one stage-applicability declaration; return its task/variant pairs.""" + + task_id = declaration["task_id"] + task = tasks_by_id.get(task_id) if isinstance(task_id, str) else None + declaration_variants = _string_list(declaration["variant_ids"], non_empty=True) + declaration_stages = _string_list(declaration["artifact_stage_ids"], non_empty=True) + task_variants = set(_string_list(task.get("variant_ids"), non_empty=True) or []) if task is not None else set() + task_stages = set(_string_list(task.get("artifact_stage_ids"), non_empty=True) or []) if task is not None else set() + if ( + task is None + or measure_tasks is None + or task_id not in measure_tasks + or declaration_variants is None + or not set(declaration_variants).issubset(task_variants) + or declaration_stages is None + or not set(declaration_stages).issubset(task_stages) + ): + failures.append( + _failure( + "dsl-evaluation-measure-stage-applicability", + f"{measure['measure_id']}: invalid task, variant, or stage applicability", + path, + ) + ) + return None + return [(task_id, variant_id) for variant_id in declaration_variants] + + +def _stage_applicability_failures( + measure: Mapping[str, object], + index: int, + tasks_by_id: Mapping[str, Mapping[str, object]], + measure_tasks: list[str] | None, + failures: list[PolicyFailure], + path: str, +) -> None: + applicability = _bounded_list( + measure["stage_applicability"], + _MAX_CATALOG_ITEMS, + failures, + rule_id="dsl-evaluation-measure-stage-applicability", + label=f"measures[{index}].stage_applicability", + path=path, + ) + actual_stage_pairs: set[tuple[str, str]] = set() + duplicate_stage_pairs: set[tuple[str, str]] = set() + for applicability_index, declaration in enumerate(applicability): + if not _exact_keys( + declaration, + _STAGE_APPLICABILITY_KEYS, + failures, + rule_id="dsl-evaluation-measure-stage-applicability", + label=(f"measures[{index}].stage_applicability[{applicability_index}]"), + path=path, + ): + continue + pairs = _declaration_stage_pairs(measure, declaration, tasks_by_id, measure_tasks, failures, path) + if pairs is None: + continue + for pair in pairs: + if pair in actual_stage_pairs: + duplicate_stage_pairs.add(pair) + actual_stage_pairs.add(pair) + expected_stage_pairs = { + (task_id, variant_id) + for task_id in (measure_tasks or []) + for variant_id in (_string_list(tasks_by_id.get(task_id, {}).get("variant_ids"), non_empty=True) or []) + } + if actual_stage_pairs != expected_stage_pairs or duplicate_stage_pairs: + failures.append( + _failure( + "dsl-evaluation-measure-stage-coverage", + f"{measure['measure_id']}: every task/variant requires one stage declaration", + path, + ) + ) + + +def _measure_failures( protocol: dict[str, object], catalogs_ids: Mapping[str, set[str]], - personas: list[object], - tasks: list[object], - measures: list[object], - thresholds: list[object], + records: Mapping[str, list[object]], failures: list[PolicyFailure], path: str, ) -> None: - dimension_ids = catalogs_ids["dimension_ids"] - persona_ids = catalogs_ids["persona_ids"] - task_ids = catalogs_ids["task_ids"] - measure_ids = catalogs_ids["measure_ids"] tasks_by_id = _protocol_records_by_id(protocol, "tasks", "task_id") - for index, measure in enumerate(measures): + for index, measure in enumerate(records["measures"]): if not _exact_keys( measure, _MEASURE_KEYS, @@ -73,103 +196,51 @@ def _plan_failures( path=path, ): continue - measure_dimensions = _string_list(measure["dimension_ids"], non_empty=True) - if measure_dimensions is None or not set(measure_dimensions).issubset(dimension_ids): - failures.append( - _failure( - "dsl-evaluation-measure-join", - f"{measure['measure_id']}: invalid dimension ids", - path, - ) - ) - measure_tasks = _string_list(measure["task_ids"], non_empty=True) - if measure_tasks is None or not set(measure_tasks).issubset(task_ids): - failures.append( - _failure( - "dsl-evaluation-measure-join", - f"{measure['measure_id']}: invalid task ids", - path, - ) - ) - elif not all( - set(measure_dimensions or []) & set(task["dimension_ids"]) - for task in tasks - if isinstance(task, Mapping) - and task.get("task_id") in measure_tasks - and _string_list(task.get("dimension_ids"), non_empty=True) is not None - ): - failures.append( - _failure( - "dsl-evaluation-measure-join", - f"{measure['measure_id']}: task applicability must share a measured dimension", - path, - ) - ) - applicability = _bounded_list( - measure["stage_applicability"], - _MAX_CATALOG_ITEMS, + measure_tasks = _measure_join_failures(measure, records["tasks"], catalogs_ids, failures, path) + _stage_applicability_failures(measure, index, tasks_by_id, measure_tasks, failures, path) + + +def _threshold_condition_failures( + threshold: Mapping[str, object], + index: int, + conditions: list[object], + measure_ids: set[str], + failures: list[PolicyFailure], + path: str, +) -> None: + for condition_index, condition in enumerate(conditions): + if not _exact_keys( + condition, + _THRESHOLD_CONDITION_KEYS, failures, - rule_id="dsl-evaluation-measure-stage-applicability", - label=f"measures[{index}].stage_applicability", + rule_id="dsl-evaluation-threshold-shape", + label=f"thresholds[{index}].conditions[{condition_index}]", path=path, - ) - actual_stage_pairs: set[tuple[str, str]] = set() - duplicate_stage_pairs: set[tuple[str, str]] = set() - for applicability_index, declaration in enumerate(applicability): - if not _exact_keys( - declaration, - _STAGE_APPLICABILITY_KEYS, - failures, - rule_id="dsl-evaluation-measure-stage-applicability", - label=(f"measures[{index}].stage_applicability[{applicability_index}]"), - path=path, - ): - continue - task_id = declaration["task_id"] - task = tasks_by_id.get(task_id) if isinstance(task_id, str) else None - declaration_variants = _string_list(declaration["variant_ids"], non_empty=True) - declaration_stages = _string_list(declaration["artifact_stage_ids"], non_empty=True) - task_variants = ( - set(_string_list(task.get("variant_ids"), non_empty=True) or []) if task is not None else set() - ) - task_stages = ( - set(_string_list(task.get("artifact_stage_ids"), non_empty=True) or []) if task is not None else set() - ) - if ( - task is None - or measure_tasks is None - or task_id not in measure_tasks - or declaration_variants is None - or not set(declaration_variants).issubset(task_variants) - or declaration_stages is None - or not set(declaration_stages).issubset(task_stages) - ): - failures.append( - _failure( - "dsl-evaluation-measure-stage-applicability", - f"{measure['measure_id']}: invalid task, variant, or stage applicability", - path, - ) - ) - continue - for variant_id in declaration_variants: - pair = (task_id, variant_id) - if pair in actual_stage_pairs: - duplicate_stage_pairs.add(pair) - actual_stage_pairs.add(pair) - expected_stage_pairs = { - (task_id, variant_id) - for task_id in (measure_tasks or []) - for variant_id in (_string_list(tasks_by_id.get(task_id, {}).get("variant_ids"), non_empty=True) or []) - } - if actual_stage_pairs != expected_stage_pairs or duplicate_stage_pairs: + ): + continue + if ( + not isinstance(condition["measure_id"], str) + or condition["measure_id"] not in measure_ids + or not isinstance(condition["operator"], str) + or condition["operator"] not in {">=", "<=", "=="} + or isinstance(condition["target"], bool) + or not isinstance(condition["target"], (int, float)) + ): failures.append( _failure( - "dsl-evaluation-measure-stage-coverage", - f"{measure['measure_id']}: every task/variant requires one stage declaration", + "dsl-evaluation-threshold-join", + f"{threshold['dimension_id']}: invalid measure, operator, or target", path, ) ) + + +def _threshold_failures( + catalogs_ids: Mapping[str, set[str]], + thresholds: list[object], + failures: list[PolicyFailure], + path: str, +) -> None: threshold_ids = _record_ids( thresholds, "dimension_id", @@ -204,32 +275,8 @@ def _plan_failures( path, ) ) - for condition_index, condition in enumerate(conditions): - if not _exact_keys( - condition, - _THRESHOLD_CONDITION_KEYS, - failures, - rule_id="dsl-evaluation-threshold-shape", - label=f"thresholds[{index}].conditions[{condition_index}]", - path=path, - ): - continue - if ( - not isinstance(condition["measure_id"], str) - or condition["measure_id"] not in measure_ids - or not isinstance(condition["operator"], str) - or condition["operator"] not in {">=", "<=", "=="} - or isinstance(condition["target"], bool) - or not isinstance(condition["target"], (int, float)) - ): - failures.append( - _failure( - "dsl-evaluation-threshold-join", - f"{threshold['dimension_id']}: invalid measure, operator, or target", - path, - ) - ) - if threshold_ids != dimension_ids: + _threshold_condition_failures(threshold, index, conditions, catalogs_ids["measure_ids"], failures, path) + if threshold_ids != catalogs_ids["dimension_ids"]: failures.append( _failure( "dsl-evaluation-threshold-coverage", @@ -238,7 +285,14 @@ def _plan_failures( ) ) - if _exact_keys( + +def _sampling_plan_failures( + protocol: dict[str, object], + personas: list[object], + failures: list[PolicyFailure], + path: str, +) -> None: + if not _exact_keys( protocol["sampling_plan"], _SAMPLING_KEYS, failures, @@ -246,24 +300,75 @@ def _plan_failures( label="sampling_plan", path=path, ): - sampling = protocol["sampling_plan"] - minimum = sampling["minimum_per_persona"] - target = sampling["target_total"] - if not isinstance(minimum, int) or minimum < 1 or not isinstance(target, int): - failures.append(_failure("dsl-evaluation-sampling-plan", "invalid sample sizes", path)) - elif target < sum( - item.get("minimum_completed_subjects", 0) - for item in personas - if isinstance(item, Mapping) and isinstance(item.get("minimum_completed_subjects"), int) + return + sampling = protocol["sampling_plan"] + minimum = sampling["minimum_per_persona"] + target = sampling["target_total"] + if not isinstance(minimum, int) or minimum < 1 or not isinstance(target, int): + failures.append(_failure("dsl-evaluation-sampling-plan", "invalid sample sizes", path)) + elif target < sum( + item.get("minimum_completed_subjects", 0) + for item in personas + if isinstance(item, Mapping) and isinstance(item.get("minimum_completed_subjects"), int) + ): + failures.append( + _failure( + "dsl-evaluation-sampling-plan", + "target_total cannot cover the required per-persona minimum", + path, + ) + ) + + +def _subject_requirement_failures( + requirement: Mapping[str, object], + persona_ids: set[str], + tasks_by_id: Mapping[str, Mapping[str, object]], + failures: list[PolicyFailure], + path: str, +) -> list[str] | None: + """Validate one workload requirement; return its task kinds when valid.""" + + task_kind_values = _string_list(requirement["task_kinds"], non_empty=True) + minimum = requirement["minimum_assigned_attempts"] + if ( + task_kind_values is None + or not set(task_kind_values).issubset(REQUIRED_TASK_KINDS) + or isinstance(minimum, bool) + or not isinstance(minimum, int) + or minimum < 1 + ): + failures.append( + _failure( + "dsl-evaluation-subject-workload-plan", + f"{requirement['requirement_id']}: invalid task kinds or minimum", + path, + ) + ) + return None + for persona_id in persona_ids: + if not any( + task.get("kind") in task_kind_values + and persona_id in (_string_list(task.get("persona_ids"), non_empty=True) or []) + for task in tasks_by_id.values() ): failures.append( _failure( - "dsl-evaluation-sampling-plan", - "target_total cannot cover the required per-persona minimum", + "dsl-evaluation-subject-workload-plan", + f"{requirement['requirement_id']}: no eligible task for {persona_id}", path, ) ) - if _exact_keys( + return task_kind_values + + +def _execution_plan_failures( + protocol: dict[str, object], + persona_ids: set[str], + failures: list[PolicyFailure], + path: str, +) -> None: + if not _exact_keys( protocol["execution_plan"], _EXECUTION_PLAN_KEYS, failures, @@ -271,73 +376,61 @@ def _plan_failures( label="execution_plan", path=path, ): - execution_plan = protocol["execution_plan"] - requirements = _bounded_list( - execution_plan["subject_task_requirements"], - 16, - failures, - rule_id="dsl-evaluation-subject-workload-plan", - label="execution_plan.subject_task_requirements", - path=path, - ) - requirement_ids = _record_ids( - requirements, - "requirement_id", + return + tasks_by_id = _protocol_records_by_id(protocol, "tasks", "task_id") + execution_plan = protocol["execution_plan"] + requirements = _bounded_list( + execution_plan["subject_task_requirements"], + 16, + failures, + rule_id="dsl-evaluation-subject-workload-plan", + label="execution_plan.subject_task_requirements", + path=path, + ) + requirement_ids = _record_ids( + requirements, + "requirement_id", + failures, + rule_id="dsl-evaluation-subject-workload-plan", + label="subject task requirement", + path=path, + ) + declared_kinds: set[str] = set() + duplicate_kinds: set[str] = set() + for index, requirement in enumerate(requirements): + if not _exact_keys( + requirement, + _SUBJECT_TASK_REQUIREMENT_KEYS, failures, rule_id="dsl-evaluation-subject-workload-plan", - label="subject task requirement", + label=f"execution_plan.subject_task_requirements[{index}]", path=path, - ) - declared_kinds: set[str] = set() - duplicate_kinds: set[str] = set() - for index, requirement in enumerate(requirements): - if not _exact_keys( - requirement, - _SUBJECT_TASK_REQUIREMENT_KEYS, - failures, - rule_id="dsl-evaluation-subject-workload-plan", - label=f"execution_plan.subject_task_requirements[{index}]", - path=path, - ): - continue - task_kind_values = _string_list(requirement["task_kinds"], non_empty=True) - minimum = requirement["minimum_assigned_attempts"] - if ( - task_kind_values is None - or not set(task_kind_values).issubset(REQUIRED_TASK_KINDS) - or isinstance(minimum, bool) - or not isinstance(minimum, int) - or minimum < 1 - ): - failures.append( - _failure( - "dsl-evaluation-subject-workload-plan", - f"{requirement['requirement_id']}: invalid task kinds or minimum", - path, - ) - ) - continue - overlap = declared_kinds & set(task_kind_values) - duplicate_kinds.update(overlap) - declared_kinds.update(task_kind_values) - for persona_id in persona_ids: - if not any( - task.get("kind") in task_kind_values - and persona_id in (_string_list(task.get("persona_ids"), non_empty=True) or []) - for task in tasks_by_id.values() - ): - failures.append( - _failure( - "dsl-evaluation-subject-workload-plan", - f"{requirement['requirement_id']}: no eligible task for {persona_id}", - path, - ) - ) - if not requirement_ids or declared_kinds != REQUIRED_TASK_KINDS or duplicate_kinds: - failures.append( - _failure( - "dsl-evaluation-subject-workload-plan", - "subject task requirements must partition every required task kind", - path, - ) + ): + continue + task_kind_values = _subject_requirement_failures(requirement, persona_ids, tasks_by_id, failures, path) + if task_kind_values is None: + continue + overlap = declared_kinds & set(task_kind_values) + duplicate_kinds.update(overlap) + declared_kinds.update(task_kind_values) + if not requirement_ids or declared_kinds != REQUIRED_TASK_KINDS or duplicate_kinds: + failures.append( + _failure( + "dsl-evaluation-subject-workload-plan", + "subject task requirements must partition every required task kind", + path, ) + ) + + +def _plan_failures( + protocol: dict[str, object], + catalogs_ids: Mapping[str, set[str]], + records: Mapping[str, list[object]], + failures: list[PolicyFailure], + path: str, +) -> None: + _measure_failures(protocol, catalogs_ids, records, failures, path) + _threshold_failures(catalogs_ids, records["thresholds"], failures, path) + _sampling_plan_failures(protocol, records["personas"], failures, path) + _execution_plan_failures(protocol, catalogs_ids["persona_ids"], failures, path) diff --git a/tools/dsl_language_evaluation/_snapshot.py b/tools/dsl_language_evaluation/_snapshot.py index 1584a855f..652d31bee 100644 --- a/tools/dsl_language_evaluation/_snapshot.py +++ b/tools/dsl_language_evaluation/_snapshot.py @@ -20,48 +20,35 @@ _SUBJECT_KEYS, _SURFACE_KEYS, _WITHDRAWAL_KEYS, - ATTEMPT_OUTCOMES, - OBSERVATION_OUTCOMES, ) -from tools.dsl_language_evaluation._measures import _measure_stage_ids from tools.dsl_language_evaluation._shape import ( _bounded_list, - _bounded_text, _exact_keys, _failure, _protocol_records_by_id, _record_ids, _resolve_repository_artifact, - _string_list, - _valid_id, ) from tools.dsl_language_evaluation._snapshot_completion import ( _review_and_completion_failures, _SnapshotJoins, _subject_failures, ) -from tools.policy.common import PolicyFailure, safe_repo_path +from tools.dsl_language_evaluation._snapshot_execution import ( + _attempt_failures, + _ExecutionJoins, + _observation_failures, + _opportunity_coverage_failures, +) +from tools.policy.common import PolicyFailure -def _validate_snapshot( - repo_root: Path, +def _snapshot_header_failures( protocol: Mapping[str, object], - snapshot: dict[str, object], - catalogs: Mapping[str, set[str]], - scope: Mapping[str, set[str]], + snapshot: Mapping[str, object], failures: list[PolicyFailure], - *, - path: str = "docs/research/dsl-language-evaluation/execution-snapshot-v1.json", -) -> set[str]: - if not _exact_keys( - snapshot, - _SNAPSHOT_KEYS, - failures, - rule_id="dsl-evaluation-snapshot-shape", - label="snapshot", - path=path, - ): - return set() + path: str, +) -> None: if snapshot["protocol_revision"] != protocol.get("revision"): failures.append(_failure("dsl-evaluation-snapshot-join", "protocol revision mismatch", path)) revision = snapshot[_HISTORICAL_REVISION_FIELD] @@ -76,6 +63,13 @@ def _validate_snapshot( if snapshot["execution_status"] not in {"not_started", "in_progress", "complete"}: failures.append(_failure("dsl-evaluation-snapshot-status", "invalid execution status", path)) + +def _public_surface_failures( + repo_root: Path, + snapshot: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: surfaces = _bounded_list( snapshot["public_surface"], _MAX_CATALOG_ITEMS, @@ -121,6 +115,12 @@ def _validate_snapshot( path=path, ) + +def _snapshot_records( + snapshot: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> dict[str, list[object]]: record_fields = { "subjects": _SUBJECT_KEYS, "attempts": _ATTEMPT_KEYS, @@ -149,28 +149,87 @@ def _validate_snapshot( label=f"{field}[{index}]", path=path, ) - if snapshot["execution_status"] == "not_started": - populated = sorted(field for field, value in records.items() if value) - if populated: - failures.append( - _failure( - "dsl-evaluation-not-started-observations", - f"not-started snapshot contains execution records: {populated}", - path, - ) + return records + + +def _not_started_failures( + snapshot: Mapping[str, object], + records: Mapping[str, list[object]], + failures: list[PolicyFailure], + path: str, +) -> None: + populated = sorted(field for field, value in records.items() if value) + if populated: + failures.append( + _failure( + "dsl-evaluation-not-started-observations", + f"not-started snapshot contains execution records: {populated}", + path, ) - ethics = snapshot["ethics_review"] - if isinstance(ethics, Mapping) and ethics.get("status") not in { - "pending", - "not_required", - }: - failures.append( - _failure( - "dsl-evaluation-ethics-state", - "not-started snapshot must remain pending or explicitly not-required", - path, - ) + ) + ethics = snapshot["ethics_review"] + if isinstance(ethics, Mapping) and ethics.get("status") not in { + "pending", + "not_required", + }: + failures.append( + _failure( + "dsl-evaluation-ethics-state", + "not-started snapshot must remain pending or explicitly not-required", + path, ) + ) + + +def _execution_record_ids( + records: Mapping[str, list[object]], + failures: list[PolicyFailure], + path: str, +) -> dict[str, set[str]]: + spec = ( + ("subjects", "subject_id", "subject", "subject_ids"), + ("attempts", "attempt_id", "attempt", "attempt_ids"), + ("observations", "observation_id", "observation", "observation_ids"), + ("reviews", "review_id", "review", "review_ids"), + ("withdrawals", "subject_id", "withdrawal", "withdrawal_subject_ids"), + ) + ids: dict[str, set[str]] = {} + for field, id_field, label, key in spec: + ids[key] = _record_ids( + records[field], + id_field, + failures, + rule_id="dsl-evaluation-snapshot-id", + label=label, + path=path, + ) + return ids + + +def _validate_snapshot( + repo_root: Path, + protocol: Mapping[str, object], + snapshot: dict[str, object], + catalogs: Mapping[str, set[str]], + scope: Mapping[str, set[str]], + failures: list[PolicyFailure], + *, + path: str = "docs/research/dsl-language-evaluation/execution-snapshot-v1.json", +) -> set[str]: + if not _exact_keys( + snapshot, + _SNAPSHOT_KEYS, + failures, + rule_id="dsl-evaluation-snapshot-shape", + label="snapshot", + path=path, + ): + return set() + _snapshot_header_failures(protocol, snapshot, failures, path) + _public_surface_failures(repo_root, snapshot, failures, path) + records = _snapshot_records(snapshot, failures, path) + if snapshot["execution_status"] == "not_started": + _not_started_failures(snapshot, records, failures, path) return set() subjects = records["subjects"] @@ -181,269 +240,28 @@ def _validate_snapshot( tasks = _protocol_records_by_id(protocol, "tasks", "task_id") variants = _protocol_records_by_id(protocol, "variants", "variant_id") measures = _protocol_records_by_id(protocol, "measures", "measure_id") - - _record_ids( - subjects, - "subject_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="subject", - path=path, - ) - attempt_ids = _record_ids( - attempts, - "attempt_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="attempt", - path=path, - ) - observation_ids = _record_ids( - observations, - "observation_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="observation", - path=path, - ) - review_ids = _record_ids( - reviews, - "review_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="review", - path=path, - ) - withdrawal_subject_ids = _record_ids( - withdrawals, - "subject_id", - failures, - rule_id="dsl-evaluation-snapshot-id", - label="withdrawal", - path=path, - ) + ids = _execution_record_ids(records, failures, path) + attempt_ids = ids["attempt_ids"] + observation_ids = ids["observation_ids"] + review_ids = ids["review_ids"] + withdrawal_subject_ids = ids["withdrawal_subject_ids"] subjects_by_id, withdrawal_subject_ids = _subject_failures( protocol, catalogs, subjects, withdrawals, withdrawal_subject_ids, failures, path ) - attempts_by_id: dict[str, Mapping[str, object]] = {} - for attempt in attempts: - if not isinstance(attempt, Mapping) or set(attempt) != _ATTEMPT_KEYS: - continue - attempt_id = attempt.get("attempt_id") - if not isinstance(attempt_id, str): - continue - attempts_by_id[attempt_id] = attempt - task_id = attempt.get("task_id") - task = tasks.get(task_id) if isinstance(task_id, str) else None - subject_id = attempt.get("subject_id") - subject = subjects_by_id.get(subject_id) if isinstance(subject_id, str) else None - outcome = attempt.get("outcome") - if outcome not in ATTEMPT_OUTCOMES: - failures.append(_failure("dsl-evaluation-attempt-outcome", f"{attempt_id}: invalid outcome", path)) - if task is None or subject is None: - failures.append(_failure("dsl-evaluation-attempt-join", f"{attempt_id}: unknown task or subject", path)) - continue - task_personas = _string_list(task.get("persona_ids"), non_empty=True) or [] - task_conditions = _string_list(task.get("tooling_condition_ids"), non_empty=True) or [] - task_variants = _string_list(task.get("variant_ids"), non_empty=True) or [] - if ( - attempt.get("persona_id") != subject.get("persona_id") - or attempt.get("persona_id") not in task_personas - or attempt.get("tooling_condition_id") not in task_conditions - or attempt.get("variant_id") not in task_variants - ): - failures.append( - _failure( - "dsl-evaluation-attempt-join", - f"{attempt_id}: subject, persona, task, condition, or variant mismatch", - path, - ) - ) - variant = variants.get(attempt.get("variant_id")) - if variant is None or variant.get("task_id") != task_id: - failures.append( - _failure("dsl-evaluation-attempt-join", f"{attempt_id}: variant belongs to another task", path) - ) - withdrawn = subject_id in withdrawal_subject_ids - if withdrawn != (outcome == "withdrawn"): - failures.append( - _failure( - "dsl-evaluation-withdrawal-join", - f"{attempt_id}: withdrawn subject and attempt outcome disagree", - path, - ) - ) - if not _valid_id(attempt.get("study_run_id")): - failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: invalid study run id", path)) - if not _bounded_text(attempt.get("started_at"), maximum=100) or not _bounded_text( - attempt.get("ended_at"), maximum=100 - ): - failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: timestamps must be text", path)) - if _string_list(attempt.get("observation_ids")) is None: - failures.append( - _failure("dsl-evaluation-attempt-observation-join", f"{attempt_id}: invalid observation ids", path) - ) - - observations_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} - child_observations: dict[str, set[str]] = {attempt_id: set() for attempt_id in attempt_ids} - for observation in observations: - if not isinstance(observation, Mapping) or set(observation) != _OBSERVATION_KEYS: - continue - observation_id = observation.get("observation_id") - attempt_id = observation.get("attempt_id") - measure_id = observation.get("measure_id") - artifact_stage = observation.get("artifact_stage") - if ( - not isinstance(observation_id, str) - or not isinstance(attempt_id, str) - or not isinstance(measure_id, str) - or not isinstance(artifact_stage, str) - ): - failures.append( - _failure("dsl-evaluation-observation-identity", "observation identity fields must be text", path) - ) - continue - opportunity = (attempt_id, measure_id, artifact_stage) - observation_dimensions = _string_list(observation.get("dimension_ids"), non_empty=True) - if opportunity in observations_by_opportunity: - failures.append( - _failure( - "dsl-evaluation-observation-identity", - f"duplicate attempt-measure-stage observation at {observation_id}", - path, - ) - ) - observations_by_opportunity[opportunity] = observation - child_observations.setdefault(attempt_id, set()).add(observation_id) - attempt = attempts_by_id.get(attempt_id) - measure = measures.get(measure_id) - task = tasks.get(observation.get("task_id")) - if attempt is None or measure is None or task is None: - failures.append( - _failure( - "dsl-evaluation-observation-join", - f"{observation_id}: unknown parent attempt, task, or measure", - path, - ) - ) - continue - parent_fields = ( - "study_run_id", - "task_id", - "persona_id", - "subject_id", - "tooling_condition_id", - "variant_id", - "outcome", - ) - if observation.get("protocol_revision") != protocol.get("revision") or any( - observation.get(field) != attempt.get(field) for field in parent_fields - ): - failures.append( - _failure( - "dsl-evaluation-observation-parent-join", - f"{observation_id}: observation does not match its parent attempt", - path, - ) - ) - task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) or [] - measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] - task_dimensions = set(_string_list(task.get("dimension_ids"), non_empty=True) or []) - measure_dimensions = set(_string_list(measure.get("dimension_ids"), non_empty=True) or []) - expected_dimensions = task_dimensions & measure_dimensions - parent_task_id = attempt.get("task_id") - parent_variant_id = attempt.get("variant_id") - try: - applicable_stages = ( - _measure_stage_ids(measure, parent_task_id, parent_variant_id) - if isinstance(parent_task_id, str) and isinstance(parent_variant_id, str) - else [] - ) - except ValueError: - applicable_stages = [] - if ( - artifact_stage not in task_stages - or artifact_stage not in applicable_stages - or observation.get("task_id") not in measure_tasks - or observation_dimensions is None - or set(observation_dimensions) != expected_dimensions - ): - failures.append( - _failure( - "dsl-evaluation-observation-task-join", - f"{observation_id}: stage, measure, or dimensions are not declared for the task", - path, - ) - ) - outcome = observation.get("outcome") - value = observation.get("value") - if outcome not in OBSERVATION_OUTCOMES: - failures.append(_failure("dsl-evaluation-observation-outcome", f"{observation_id}: invalid outcome", path)) - if measure_id == "task-completion": - expected_value = 1 if outcome == "completed" else 0 - valid_value = value == expected_value and not isinstance(value, bool) - elif outcome in {"completed", "failed"}: - valid_value = not isinstance(value, bool) and isinstance(value, (int, float)) - else: - valid_value = value is None - if not valid_value: - failures.append( - _failure( - "dsl-evaluation-observation-value", - f"{observation_id}: value does not represent its attempt outcome", - path, - ) - ) - refs = _string_list(observation.get("evidence_refs")) - if refs is None or any(safe_repo_path(repo_root, ref) is None for ref in refs): - failures.append( - _failure( - "dsl-evaluation-observation-evidence", - f"{observation_id}: evidence refs must be repository-confined paths", - path, - ) - ) - - expected_opportunities: set[tuple[str, str, str]] = set() - withdrawn_opportunities: set[tuple[str, str, str]] = set() - for attempt_id, attempt in attempts_by_id.items(): - task_id = attempt.get("task_id") - variant_id = attempt.get("variant_id") - for measure_id, measure in measures.items(): - measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] - if task_id not in measure_tasks or not isinstance(task_id, str) or not isinstance(variant_id, str): - continue - try: - applicable_stages = _measure_stage_ids(measure, task_id, variant_id) - except ValueError: - continue - for artifact_stage in applicable_stages: - opportunity = (attempt_id, measure_id, artifact_stage) - if attempt.get("outcome") == "withdrawn": - withdrawn_opportunities.add(opportunity) - else: - expected_opportunities.add(opportunity) - actual_opportunities = set(observations_by_opportunity) - if actual_opportunities != expected_opportunities or actual_opportunities & withdrawn_opportunities: - failures.append( - _failure( - "dsl-evaluation-opportunity-coverage", - "observations must exactly cover every non-withdrawn protocol-declared attempt-measure-stage opportunity", - path, - ) - ) - for attempt_id, attempt in attempts_by_id.items(): - stored_ids = _string_list(attempt.get("observation_ids")) - if stored_ids is not None and set(stored_ids) != child_observations.get(attempt_id, set()): - failures.append( - _failure( - "dsl-evaluation-attempt-observation-join", - f"{attempt_id}: observation ids do not match child records", - path, - ) - ) + joins = _ExecutionJoins( + attempts_by_id=_attempt_failures( + attempts, tasks, variants, subjects_by_id, withdrawal_subject_ids, failures, path + ), + attempt_ids=attempt_ids, + tasks=tasks, + measures=measures, + ) + observations_by_opportunity, child_observations = _observation_failures( + repo_root, protocol, observations, joins, failures, path + ) + _opportunity_coverage_failures(joins, observations_by_opportunity, child_observations, failures, path) _review_and_completion_failures( protocol, @@ -451,7 +269,7 @@ def _validate_snapshot( scope, _SnapshotJoins( subjects_by_id=subjects_by_id, - attempts_by_id=attempts_by_id, + attempts_by_id=joins.attempts_by_id, tasks=tasks, review_ids=review_ids, reviews=reviews, diff --git a/tools/dsl_language_evaluation/_snapshot_completion.py b/tools/dsl_language_evaluation/_snapshot_completion.py index 76c4a2864..9aafc4781 100644 --- a/tools/dsl_language_evaluation/_snapshot_completion.py +++ b/tools/dsl_language_evaluation/_snapshot_completion.py @@ -35,6 +35,46 @@ class _SnapshotJoins: withdrawal_subject_ids: set[str] +def _subject_record_failures( + subjects_by_id: Mapping[str, Mapping[str, object]], + catalogs: Mapping[str, set[str]], + experience_bands: set[str], + failures: list[PolicyFailure], + path: str, +) -> None: + for subject_id, subject in subjects_by_id.items(): + if subject.get("persona_id") not in catalogs.get("persona_ids", set()): + failures.append(_failure("dsl-evaluation-subject-join", f"{subject_id}: unknown persona", path)) + if subject.get("consent_status") not in {"consented", "withdrawn"}: + failures.append(_failure("dsl-evaluation-consent-status", f"{subject_id}: invalid consent status", path)) + if ( + not _bounded_text(subject.get("experience_band"), maximum=200) + or subject.get("experience_band") not in experience_bands + ): + failures.append(_failure("dsl-evaluation-subject-shape", f"{subject_id}: invalid experience band", path)) + + +def _withdrawal_shape_failures( + withdrawals: list[object], + failures: list[PolicyFailure], + path: str, +) -> None: + for withdrawal in withdrawals: + if not isinstance(withdrawal, Mapping) or set(withdrawal) != _WITHDRAWAL_KEYS: + continue + if ( + not _bounded_text(withdrawal.get("recorded_at"), maximum=100) + or withdrawal.get("retained_aggregate_only") is not True + ): + failures.append( + _failure( + "dsl-evaluation-withdrawal-shape", + f"{withdrawal.get('subject_id')}: withdrawal must retain aggregate counts only", + path, + ) + ) + + def _subject_failures( protocol: Mapping[str, object], catalogs: Mapping[str, set[str]], @@ -53,16 +93,7 @@ def _subject_failures( experience_bands = set() if isinstance(sampling_plan, Mapping): experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) - for subject_id, subject in subjects_by_id.items(): - if subject.get("persona_id") not in catalogs.get("persona_ids", set()): - failures.append(_failure("dsl-evaluation-subject-join", f"{subject_id}: unknown persona", path)) - if subject.get("consent_status") not in {"consented", "withdrawn"}: - failures.append(_failure("dsl-evaluation-consent-status", f"{subject_id}: invalid consent status", path)) - if ( - not _bounded_text(subject.get("experience_band"), maximum=200) - or subject.get("experience_band") not in experience_bands - ): - failures.append(_failure("dsl-evaluation-subject-shape", f"{subject_id}: invalid experience band", path)) + _subject_record_failures(subjects_by_id, catalogs, experience_bands, failures, path) declared_withdrawn_subjects = { subject_id for subject_id, subject in subjects_by_id.items() if subject.get("consent_status") == "withdrawn" } @@ -74,87 +105,95 @@ def _subject_failures( path, ) ) - for withdrawal in withdrawals: - if not isinstance(withdrawal, Mapping) or set(withdrawal) != _WITHDRAWAL_KEYS: - continue - if ( - not _bounded_text(withdrawal.get("recorded_at"), maximum=100) - or withdrawal.get("retained_aggregate_only") is not True - ): - failures.append( - _failure( - "dsl-evaluation-withdrawal-shape", - f"{withdrawal.get('subject_id')}: withdrawal must retain aggregate counts only", - path, - ) - ) + _withdrawal_shape_failures(withdrawals, failures, path) return subjects_by_id, withdrawal_subject_ids -def _review_and_completion_failures( - protocol: Mapping[str, object], - snapshot: Mapping[str, object], - scope: Mapping[str, set[str]], - joins: _SnapshotJoins, +def _review_join_failures( + review: Mapping[str, object], + attempt: Mapping[str, object], + reviewer: Mapping[str, object], + tasks: Mapping[str, Mapping[str, object]], + failures: list[PolicyFailure], + path: str, +) -> None: + review_id = review.get("review_id") + reviewer_id = review.get("reviewer_subject_id") + task = tasks.get(attempt.get("task_id")) + task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) if task else None + task_personas = _string_list(task.get("persona_ids"), non_empty=True) if task else None + if ( + review.get("task_id") != attempt.get("task_id") + or review.get("variant_id") != attempt.get("variant_id") + or reviewer_id == attempt.get("subject_id") + or reviewer.get("consent_status") != "consented" + or task_stages is None + or "review-judgment" not in task_stages + or task_personas is None + or reviewer.get("persona_id") not in task_personas + ): + failures.append( + _failure( + "dsl-evaluation-review-join", + f"{review_id}: review does not match an eligible independent reviewer and parent task", + path, + ) + ) + + +def _review_shape_failures( + review: Mapping[str, object], failures: list[PolicyFailure], path: str, ) -> None: - subjects_by_id = joins.subjects_by_id - attempts_by_id = joins.attempts_by_id - tasks = joins.tasks - review_ids = joins.review_ids - reviews = joins.reviews - records = {"disagreements": joins.disagreements} + if ( + not _bounded_text(review.get("judgment"), maximum=500) + or isinstance(review.get("confidence"), bool) + or not isinstance(review.get("confidence"), (int, float)) + or not 0 <= review["confidence"] <= 1 + or not _valid_id(review.get("rationale_code")) + or not _bounded_text(review.get("fixed_at"), maximum=100) + ): + failures.append( + _failure("dsl-evaluation-review-shape", f"{review.get('review_id')}: invalid fixed judgment", path) + ) + + +def _review_failures( + joins: _SnapshotJoins, + failures: list[PolicyFailure], + path: str, +) -> Counter[str]: reviews_by_attempt: Counter[str] = Counter() - for review in reviews: + for review in joins.reviews: if not isinstance(review, Mapping) or set(review) != _REVIEW_KEYS: continue review_id = review.get("review_id") attempt_id = review.get("attempt_id") - attempt = attempts_by_id.get(attempt_id) if isinstance(attempt_id, str) else None + attempt = joins.attempts_by_id.get(attempt_id) if isinstance(attempt_id, str) else None reviewer_id = review.get("reviewer_subject_id") - reviewer = subjects_by_id.get(reviewer_id) if isinstance(reviewer_id, str) else None + reviewer = joins.subjects_by_id.get(reviewer_id) if isinstance(reviewer_id, str) else None if attempt is None or reviewer is None: failures.append(_failure("dsl-evaluation-review-join", f"{review_id}: unknown attempt or reviewer", path)) continue - task = tasks.get(attempt.get("task_id")) - task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) if task else None - task_personas = _string_list(task.get("persona_ids"), non_empty=True) if task else None - if ( - review.get("task_id") != attempt.get("task_id") - or review.get("variant_id") != attempt.get("variant_id") - or reviewer_id == attempt.get("subject_id") - or reviewer.get("consent_status") != "consented" - or task_stages is None - or "review-judgment" not in task_stages - or task_personas is None - or reviewer.get("persona_id") not in task_personas - ): - failures.append( - _failure( - "dsl-evaluation-review-join", - f"{review_id}: review does not match an eligible independent reviewer and parent task", - path, - ) - ) - if ( - not _bounded_text(review.get("judgment"), maximum=500) - or isinstance(review.get("confidence"), bool) - or not isinstance(review.get("confidence"), (int, float)) - or not 0 <= review["confidence"] <= 1 - or not _valid_id(review.get("rationale_code")) - or not _bounded_text(review.get("fixed_at"), maximum=100) - ): - failures.append(_failure("dsl-evaluation-review-shape", f"{review_id}: invalid fixed judgment", path)) + _review_join_failures(review, attempt, reviewer, joins.tasks, failures, path) + _review_shape_failures(review, failures, path) reviews_by_attempt[attempt_id] += 1 + return reviews_by_attempt + - for disagreement in records["disagreements"]: +def _disagreement_failures( + joins: _SnapshotJoins, + failures: list[PolicyFailure], + path: str, +) -> None: + for disagreement in joins.disagreements: if not isinstance(disagreement, Mapping) or set(disagreement) != _DISAGREEMENT_KEYS: continue disagreement_review_ids = _string_list(disagreement["review_ids"], non_empty=True) linked_reviews = [ review - for review in reviews + for review in joins.reviews if isinstance(review, Mapping) and disagreement_review_ids is not None and review.get("review_id") in disagreement_review_ids @@ -163,7 +202,7 @@ def _review_and_completion_failures( if ( disagreement_review_ids is None or len(disagreement_review_ids) < 2 - or not set(disagreement_review_ids).issubset(review_ids) + or not set(disagreement_review_ids).issubset(joins.review_ids) or len(linked_attempts) != 1 ): failures.append( @@ -182,115 +221,193 @@ def _review_and_completion_failures( ) ) - if snapshot["execution_status"] == "complete": - nonwithdrawn_subject_ids = { - subject_id - for subject_id, subject in subjects_by_id.items() - if subject.get("consent_status") == "consented" - and subject.get("persona_id") in scope.get("persona_ids", set()) - } - active_subject_ids = { - subject_id - for subject_id in nonwithdrawn_subject_ids - if any( - attempt.get("subject_id") == subject_id - and attempt.get("outcome") != "withdrawn" - and _attempt_matches_scope(attempt, scope) - for attempt in attempts_by_id.values() - ) - } - persona_minimums = { - item["persona_id"]: item["minimum_completed_subjects"] - for item in protocol.get("personas", []) - if isinstance(item, Mapping) - and isinstance(item.get("persona_id"), str) - and isinstance(item.get("minimum_completed_subjects"), int) - and item["persona_id"] in scope.get("persona_ids", set()) - } - persona_counts = Counter(subjects_by_id[subject_id]["persona_id"] for subject_id in active_subject_ids) - missing_personas = sorted( - persona_id for persona_id, minimum in persona_minimums.items() if persona_counts[persona_id] < minimum - ) - expected_task_shapes = { - (task["task_id"], condition_id, variant_id) - for task in tasks.values() - if task["task_id"] in scope.get("task_ids", set()) - for condition_id in (_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) - if condition_id in scope.get("tooling_condition_ids", set()) - for variant_id in (_string_list(task.get("variant_ids"), non_empty=True) or []) - if variant_id in scope.get("variant_ids", set()) - } - actual_task_shapes = { - (attempt.get("task_id"), attempt.get("tooling_condition_id"), attempt.get("variant_id")) - for attempt in attempts_by_id.values() - if attempt.get("outcome") != "withdrawn" and _attempt_matches_scope(attempt, scope) - } - review_required_attempts = { - attempt_id - for attempt_id, attempt in attempts_by_id.items() - if _attempt_matches_scope(attempt, scope) - if "review-judgment" - in (_string_list(tasks.get(attempt.get("task_id"), {}).get("artifact_stage_ids"), non_empty=True) or []) + +def _scoped_subject_ids( + subjects_by_id: Mapping[str, Mapping[str, object]], + attempts_by_id: Mapping[str, Mapping[str, object]], + scope: Mapping[str, set[str]], +) -> tuple[set[str], set[str]]: + nonwithdrawn_subject_ids = { + subject_id + for subject_id, subject in subjects_by_id.items() + if subject.get("consent_status") == "consented" and subject.get("persona_id") in scope.get("persona_ids", set()) + } + active_subject_ids = { + subject_id + for subject_id in nonwithdrawn_subject_ids + if any( + attempt.get("subject_id") == subject_id and attempt.get("outcome") != "withdrawn" - } - sampling_plan = protocol.get("sampling_plan", {}) - target_total = sampling_plan.get("target_total") if isinstance(sampling_plan, Mapping) else None - ethics = snapshot["ethics_review"] - ethics_approved = isinstance(ethics, Mapping) and ethics.get("status") == "approved" - execution_plan = protocol.get("execution_plan", {}) - subject_task_requirements = ( - execution_plan.get("subject_task_requirements", []) if isinstance(execution_plan, Mapping) else [] + and _attempt_matches_scope(attempt, scope) + for attempt in attempts_by_id.values() ) - missing_subject_workloads: list[tuple[str, str]] = [] - if isinstance(subject_task_requirements, list): - for subject_id in nonwithdrawn_subject_ids: - subject_attempts = [ - attempt - for attempt in attempts_by_id.values() - if attempt.get("subject_id") == subject_id - and attempt.get("outcome") != "withdrawn" - and _attempt_matches_scope(attempt, scope) - ] - for requirement in subject_task_requirements: - if not isinstance(requirement, Mapping): - continue - task_kind_values = _string_list(requirement.get("task_kinds"), non_empty=True) - minimum = requirement.get("minimum_assigned_attempts") - requirement_id = requirement.get("requirement_id") - if ( - task_kind_values is None - or not isinstance(minimum, int) - or isinstance(minimum, bool) - or not isinstance(requirement_id, str) - ): - continue - assigned = sum( - tasks.get(attempt.get("task_id"), {}).get("kind") in task_kind_values - for attempt in subject_attempts - ) - if assigned < minimum: - missing_subject_workloads.append((subject_id, requirement_id)) - if missing_subject_workloads: - failures.append( - _failure( - "dsl-evaluation-subject-workload", - "complete execution does not satisfy every active subject's assigned task groups", - path, - ) + } + return nonwithdrawn_subject_ids, active_subject_ids + + +def _missing_personas( + protocol: Mapping[str, object], + scope: Mapping[str, set[str]], + subjects_by_id: Mapping[str, Mapping[str, object]], + active_subject_ids: set[str], +) -> list[str]: + persona_minimums = { + item["persona_id"]: item["minimum_completed_subjects"] + for item in protocol.get("personas", []) + if isinstance(item, Mapping) + and isinstance(item.get("persona_id"), str) + and isinstance(item.get("minimum_completed_subjects"), int) + and item["persona_id"] in scope.get("persona_ids", set()) + } + persona_counts = Counter(subjects_by_id[subject_id]["persona_id"] for subject_id in active_subject_ids) + return sorted( + persona_id for persona_id, minimum in persona_minimums.items() if persona_counts[persona_id] < minimum + ) + + +def _task_shape_coverage( + tasks: Mapping[str, Mapping[str, object]], + attempts_by_id: Mapping[str, Mapping[str, object]], + scope: Mapping[str, set[str]], +) -> tuple[set[tuple[object, object, object]], set[tuple[object, object, object]]]: + expected_task_shapes = { + (task["task_id"], condition_id, variant_id) + for task in tasks.values() + if task["task_id"] in scope.get("task_ids", set()) + for condition_id in (_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) + if condition_id in scope.get("tooling_condition_ids", set()) + for variant_id in (_string_list(task.get("variant_ids"), non_empty=True) or []) + if variant_id in scope.get("variant_ids", set()) + } + actual_task_shapes = { + (attempt.get("task_id"), attempt.get("tooling_condition_id"), attempt.get("variant_id")) + for attempt in attempts_by_id.values() + if attempt.get("outcome") != "withdrawn" and _attempt_matches_scope(attempt, scope) + } + return expected_task_shapes, actual_task_shapes + + +def _review_required_attempts( + attempts_by_id: Mapping[str, Mapping[str, object]], + tasks: Mapping[str, Mapping[str, object]], + scope: Mapping[str, set[str]], +) -> set[str]: + return { + attempt_id + for attempt_id, attempt in attempts_by_id.items() + if _attempt_matches_scope(attempt, scope) + if "review-judgment" + in (_string_list(tasks.get(attempt.get("task_id"), {}).get("artifact_stage_ids"), non_empty=True) or []) + and attempt.get("outcome") != "withdrawn" + } + + +def _requirement_gap( + requirement: object, + subject_attempts: list[Mapping[str, object]], + tasks: Mapping[str, Mapping[str, object]], +) -> str | None: + if not isinstance(requirement, Mapping): + return None + task_kind_values = _string_list(requirement.get("task_kinds"), non_empty=True) + minimum = requirement.get("minimum_assigned_attempts") + requirement_id = requirement.get("requirement_id") + if ( + task_kind_values is None + or not isinstance(minimum, int) + or isinstance(minimum, bool) + or not isinstance(requirement_id, str) + ): + return None + assigned = sum( + tasks.get(attempt.get("task_id"), {}).get("kind") in task_kind_values for attempt in subject_attempts + ) + return requirement_id if assigned < minimum else None + + +def _missing_subject_workloads( + protocol: Mapping[str, object], + joins: _SnapshotJoins, + scope: Mapping[str, set[str]], + nonwithdrawn_subject_ids: set[str], +) -> list[tuple[str, str]]: + execution_plan = protocol.get("execution_plan", {}) + subject_task_requirements = ( + execution_plan.get("subject_task_requirements", []) if isinstance(execution_plan, Mapping) else [] + ) + if not isinstance(subject_task_requirements, list): + return [] + missing_subject_workloads: list[tuple[str, str]] = [] + for subject_id in nonwithdrawn_subject_ids: + subject_attempts = [ + attempt + for attempt in joins.attempts_by_id.values() + if attempt.get("subject_id") == subject_id + and attempt.get("outcome") != "withdrawn" + and _attempt_matches_scope(attempt, scope) + ] + for requirement in subject_task_requirements: + gap = _requirement_gap(requirement, subject_attempts, joins.tasks) + if gap is not None: + missing_subject_workloads.append((subject_id, gap)) + return missing_subject_workloads + + +def _completion_failures( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + scope: Mapping[str, set[str]], + joins: _SnapshotJoins, + reviews_by_attempt: Counter[str], + failures: list[PolicyFailure], + path: str, +) -> None: + nonwithdrawn_subject_ids, active_subject_ids = _scoped_subject_ids( + joins.subjects_by_id, joins.attempts_by_id, scope + ) + missing_personas = _missing_personas(protocol, scope, joins.subjects_by_id, active_subject_ids) + expected_task_shapes, actual_task_shapes = _task_shape_coverage(joins.tasks, joins.attempts_by_id, scope) + review_required_attempts = _review_required_attempts(joins.attempts_by_id, joins.tasks, scope) + sampling_plan = protocol.get("sampling_plan", {}) + target_total = sampling_plan.get("target_total") if isinstance(sampling_plan, Mapping) else None + ethics = snapshot["ethics_review"] + ethics_approved = isinstance(ethics, Mapping) and ethics.get("status") == "approved" + missing_subject_workloads = _missing_subject_workloads(protocol, joins, scope, nonwithdrawn_subject_ids) + if missing_subject_workloads: + failures.append( + _failure( + "dsl-evaluation-subject-workload", + "complete execution does not satisfy every active subject's assigned task groups", + path, ) - if ( - not ethics_approved - or missing_personas - or not isinstance(target_total, int) - or len(active_subject_ids) < target_total - or not expected_task_shapes.issubset(actual_task_shapes) - or any(reviews_by_attempt[attempt_id] == 0 for attempt_id in review_required_attempts) - or missing_subject_workloads - ): - failures.append( - _failure( - "dsl-evaluation-completion-coverage", - "complete execution lacks approved ethics, subject workload/minima, task/condition/variant coverage, or required independent reviews", - path, - ) + ) + if ( + not ethics_approved + or missing_personas + or not isinstance(target_total, int) + or len(active_subject_ids) < target_total + or not expected_task_shapes.issubset(actual_task_shapes) + or any(reviews_by_attempt[attempt_id] == 0 for attempt_id in review_required_attempts) + or missing_subject_workloads + ): + failures.append( + _failure( + "dsl-evaluation-completion-coverage", + "complete execution lacks approved ethics, subject workload/minima, task/condition/variant coverage, or required independent reviews", + path, ) + ) + + +def _review_and_completion_failures( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + scope: Mapping[str, set[str]], + joins: _SnapshotJoins, + failures: list[PolicyFailure], + path: str, +) -> None: + reviews_by_attempt = _review_failures(joins, failures, path) + _disagreement_failures(joins, failures, path) + if snapshot["execution_status"] == "complete": + _completion_failures(protocol, snapshot, scope, joins, reviews_by_attempt, failures, path) diff --git a/tools/dsl_language_evaluation/_snapshot_execution.py b/tools/dsl_language_evaluation/_snapshot_execution.py new file mode 100644 index 000000000..37bac0a33 --- /dev/null +++ b/tools/dsl_language_evaluation/_snapshot_execution.py @@ -0,0 +1,371 @@ +"""Attempt, observation, and opportunity-coverage joins for the snapshot.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +from tools.dsl_language_evaluation._keys import ( + _ATTEMPT_KEYS, + _OBSERVATION_KEYS, + ATTEMPT_OUTCOMES, + OBSERVATION_OUTCOMES, +) +from tools.dsl_language_evaluation._measures import _measure_stage_ids +from tools.dsl_language_evaluation._shape import ( + _bounded_text, + _failure, + _string_list, + _valid_id, +) +from tools.policy.common import PolicyFailure, safe_repo_path + + +@dataclass(frozen=True) +class _ExecutionJoins: + attempts_by_id: Mapping[str, Mapping[str, object]] + attempt_ids: set[str] + tasks: Mapping[str, Mapping[str, object]] + measures: Mapping[str, Mapping[str, object]] + + +def _attempt_parents( + attempt: Mapping[str, object], + tasks: Mapping[str, Mapping[str, object]], + subjects_by_id: Mapping[str, Mapping[str, object]], +) -> tuple[Mapping[str, object] | None, Mapping[str, object] | None]: + task_id = attempt.get("task_id") + subject_id = attempt.get("subject_id") + task = tasks.get(task_id) if isinstance(task_id, str) else None + subject = subjects_by_id.get(subject_id) if isinstance(subject_id, str) else None + return task, subject + + +def _attempt_join_failures( + attempt: Mapping[str, object], + attempt_id: str, + task: Mapping[str, object], + subject: Mapping[str, object], + variants: Mapping[str, Mapping[str, object]], + failures: list[PolicyFailure], + path: str, +) -> None: + task_personas = _string_list(task.get("persona_ids"), non_empty=True) or [] + task_conditions = _string_list(task.get("tooling_condition_ids"), non_empty=True) or [] + task_variants = _string_list(task.get("variant_ids"), non_empty=True) or [] + if ( + attempt.get("persona_id") != subject.get("persona_id") + or attempt.get("persona_id") not in task_personas + or attempt.get("tooling_condition_id") not in task_conditions + or attempt.get("variant_id") not in task_variants + ): + failures.append( + _failure( + "dsl-evaluation-attempt-join", + f"{attempt_id}: subject, persona, task, condition, or variant mismatch", + path, + ) + ) + variant = variants.get(attempt.get("variant_id")) + if variant is None or variant.get("task_id") != attempt.get("task_id"): + failures.append(_failure("dsl-evaluation-attempt-join", f"{attempt_id}: variant belongs to another task", path)) + + +def _attempt_identity_failures( + attempt: Mapping[str, object], + attempt_id: str, + failures: list[PolicyFailure], + path: str, +) -> None: + if not _valid_id(attempt.get("study_run_id")): + failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: invalid study run id", path)) + if not _bounded_text(attempt.get("started_at"), maximum=100) or not _bounded_text( + attempt.get("ended_at"), maximum=100 + ): + failures.append(_failure("dsl-evaluation-attempt-identity", f"{attempt_id}: timestamps must be text", path)) + if _string_list(attempt.get("observation_ids")) is None: + failures.append( + _failure("dsl-evaluation-attempt-observation-join", f"{attempt_id}: invalid observation ids", path) + ) + + +def _attempt_failures( + attempts: list[object], + tasks: Mapping[str, Mapping[str, object]], + variants: Mapping[str, Mapping[str, object]], + subjects_by_id: Mapping[str, Mapping[str, object]], + withdrawal_subject_ids: set[str], + failures: list[PolicyFailure], + path: str, +) -> dict[str, Mapping[str, object]]: + attempts_by_id: dict[str, Mapping[str, object]] = {} + for attempt in attempts: + if not isinstance(attempt, Mapping) or set(attempt) != _ATTEMPT_KEYS: + continue + attempt_id = attempt.get("attempt_id") + if not isinstance(attempt_id, str): + continue + attempts_by_id[attempt_id] = attempt + task, subject = _attempt_parents(attempt, tasks, subjects_by_id) + outcome = attempt.get("outcome") + if outcome not in ATTEMPT_OUTCOMES: + failures.append(_failure("dsl-evaluation-attempt-outcome", f"{attempt_id}: invalid outcome", path)) + if task is None or subject is None: + failures.append(_failure("dsl-evaluation-attempt-join", f"{attempt_id}: unknown task or subject", path)) + continue + _attempt_join_failures(attempt, attempt_id, task, subject, variants, failures, path) + if (attempt.get("subject_id") in withdrawal_subject_ids) != (outcome == "withdrawn"): + failures.append( + _failure( + "dsl-evaluation-withdrawal-join", + f"{attempt_id}: withdrawn subject and attempt outcome disagree", + path, + ) + ) + _attempt_identity_failures(attempt, attempt_id, failures, path) + return attempts_by_id + + +def _observation_identity( + observation: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> tuple[str, str, str, str] | None: + observation_id = observation.get("observation_id") + attempt_id = observation.get("attempt_id") + measure_id = observation.get("measure_id") + artifact_stage = observation.get("artifact_stage") + if ( + not isinstance(observation_id, str) + or not isinstance(attempt_id, str) + or not isinstance(measure_id, str) + or not isinstance(artifact_stage, str) + ): + failures.append( + _failure("dsl-evaluation-observation-identity", "observation identity fields must be text", path) + ) + return None + return observation_id, attempt_id, measure_id, artifact_stage + + +def _observation_parent_failures( + protocol: Mapping[str, object], + observation: Mapping[str, object], + observation_id: str, + attempt: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: + parent_fields = ( + "study_run_id", + "task_id", + "persona_id", + "subject_id", + "tooling_condition_id", + "variant_id", + "outcome", + ) + if observation.get("protocol_revision") != protocol.get("revision") or any( + observation.get(field) != attempt.get(field) for field in parent_fields + ): + failures.append( + _failure( + "dsl-evaluation-observation-parent-join", + f"{observation_id}: observation does not match its parent attempt", + path, + ) + ) + + +def _observation_stage_failures( + observation: Mapping[str, object], + observation_id: str, + attempt: Mapping[str, object], + task: Mapping[str, object], + measure: Mapping[str, object], + failures: list[PolicyFailure], + path: str, +) -> None: + observation_dimensions = _string_list(observation.get("dimension_ids"), non_empty=True) + task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) or [] + measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] + task_dimensions = set(_string_list(task.get("dimension_ids"), non_empty=True) or []) + measure_dimensions = set(_string_list(measure.get("dimension_ids"), non_empty=True) or []) + expected_dimensions = task_dimensions & measure_dimensions + parent_task_id = attempt.get("task_id") + parent_variant_id = attempt.get("variant_id") + try: + applicable_stages = ( + _measure_stage_ids(measure, parent_task_id, parent_variant_id) + if isinstance(parent_task_id, str) and isinstance(parent_variant_id, str) + else [] + ) + except ValueError: + applicable_stages = [] + if ( + observation.get("artifact_stage") not in task_stages + or observation.get("artifact_stage") not in applicable_stages + or observation.get("task_id") not in measure_tasks + or observation_dimensions is None + or set(observation_dimensions) != expected_dimensions + ): + failures.append( + _failure( + "dsl-evaluation-observation-task-join", + f"{observation_id}: stage, measure, or dimensions are not declared for the task", + path, + ) + ) + + +def _observation_value_failures( + observation: Mapping[str, object], + observation_id: str, + measure_id: str, + failures: list[PolicyFailure], + path: str, +) -> None: + outcome = observation.get("outcome") + value = observation.get("value") + if outcome not in OBSERVATION_OUTCOMES: + failures.append(_failure("dsl-evaluation-observation-outcome", f"{observation_id}: invalid outcome", path)) + if measure_id == "task-completion": + expected_value = 1 if outcome == "completed" else 0 + valid_value = value == expected_value and not isinstance(value, bool) + elif outcome in {"completed", "failed"}: + valid_value = not isinstance(value, bool) and isinstance(value, (int, float)) + else: + valid_value = value is None + if not valid_value: + failures.append( + _failure( + "dsl-evaluation-observation-value", + f"{observation_id}: value does not represent its attempt outcome", + path, + ) + ) + + +def _observation_evidence_failures( + repo_root: Path, + observation: Mapping[str, object], + observation_id: str, + failures: list[PolicyFailure], + path: str, +) -> None: + refs = _string_list(observation.get("evidence_refs")) + if refs is None or any(safe_repo_path(repo_root, ref) is None for ref in refs): + failures.append( + _failure( + "dsl-evaluation-observation-evidence", + f"{observation_id}: evidence refs must be repository-confined paths", + path, + ) + ) + + +def _observation_failures( + repo_root: Path, + protocol: Mapping[str, object], + observations: list[object], + joins: _ExecutionJoins, + failures: list[PolicyFailure], + path: str, +) -> tuple[dict[tuple[str, str, str], Mapping[str, object]], dict[str, set[str]]]: + observations_by_opportunity: dict[tuple[str, str, str], Mapping[str, object]] = {} + child_observations: dict[str, set[str]] = {attempt_id: set() for attempt_id in joins.attempt_ids} + for observation in observations: + if not isinstance(observation, Mapping) or set(observation) != _OBSERVATION_KEYS: + continue + identity = _observation_identity(observation, failures, path) + if identity is None: + continue + observation_id, attempt_id, measure_id, artifact_stage = identity + opportunity = (attempt_id, measure_id, artifact_stage) + if opportunity in observations_by_opportunity: + failures.append( + _failure( + "dsl-evaluation-observation-identity", + f"duplicate attempt-measure-stage observation at {observation_id}", + path, + ) + ) + observations_by_opportunity[opportunity] = observation + child_observations.setdefault(attempt_id, set()).add(observation_id) + attempt = joins.attempts_by_id.get(attempt_id) + measure = joins.measures.get(measure_id) + task = joins.tasks.get(observation.get("task_id")) + if attempt is None or measure is None or task is None: + failures.append( + _failure( + "dsl-evaluation-observation-join", + f"{observation_id}: unknown parent attempt, task, or measure", + path, + ) + ) + continue + _observation_parent_failures(protocol, observation, observation_id, attempt, failures, path) + _observation_stage_failures(observation, observation_id, attempt, task, measure, failures, path) + _observation_value_failures(observation, observation_id, measure_id, failures, path) + _observation_evidence_failures(repo_root, observation, observation_id, failures, path) + return observations_by_opportunity, child_observations + + +def _opportunity_stages( + measure: Mapping[str, object], + task_id: object, + variant_id: object, +) -> list[str]: + measure_tasks = _string_list(measure.get("task_ids"), non_empty=True) or [] + if task_id not in measure_tasks or not isinstance(task_id, str) or not isinstance(variant_id, str): + return [] + try: + return _measure_stage_ids(measure, task_id, variant_id) + except ValueError: + return [] + + +def _expected_opportunities( + attempts_by_id: Mapping[str, Mapping[str, object]], + measures: Mapping[str, Mapping[str, object]], +) -> tuple[set[tuple[str, str, str]], set[tuple[str, str, str]]]: + expected: set[tuple[str, str, str]] = set() + withdrawn: set[tuple[str, str, str]] = set() + for attempt_id, attempt in attempts_by_id.items(): + task_id = attempt.get("task_id") + variant_id = attempt.get("variant_id") + target = withdrawn if attempt.get("outcome") == "withdrawn" else expected + for measure_id, measure in measures.items(): + for artifact_stage in _opportunity_stages(measure, task_id, variant_id): + target.add((attempt_id, measure_id, artifact_stage)) + return expected, withdrawn + + +def _opportunity_coverage_failures( + joins: _ExecutionJoins, + observations_by_opportunity: Mapping[tuple[str, str, str], Mapping[str, object]], + child_observations: Mapping[str, set[str]], + failures: list[PolicyFailure], + path: str, +) -> None: + expected_opportunities, withdrawn_opportunities = _expected_opportunities(joins.attempts_by_id, joins.measures) + actual_opportunities = set(observations_by_opportunity) + if actual_opportunities != expected_opportunities or actual_opportunities & withdrawn_opportunities: + failures.append( + _failure( + "dsl-evaluation-opportunity-coverage", + "observations must exactly cover every non-withdrawn protocol-declared attempt-measure-stage opportunity", + path, + ) + ) + for attempt_id, attempt in joins.attempts_by_id.items(): + stored_ids = _string_list(attempt.get("observation_ids")) + if stored_ids is not None and set(stored_ids) != child_observations.get(attempt_id, set()): + failures.append( + _failure( + "dsl-evaluation-attempt-observation-join", + f"{attempt_id}: observation ids do not match child records", + path, + ) + ) From 6b60812737d7853461dbb31dcf3862194f0c6039 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:50:03 -0700 Subject: [PATCH 3/4] refactor(tools): meet the cyclomatic gate across the DSL evaluation package The raes-strict gate also caps plain cyclomatic complexity at 10, which the first decomposition round exceeded in thirteen helpers. Flatten wide or-chains into any()/all() tuples over pre-normalized operands (behavior preserved: every operand is pure, and each normalized fallback is reachable only when its is-None guard already decides the outcome), extract the remaining hot loops into named helpers, cap returns at three, wrap the two over-long diagnostic strings, and hoist the duplicated analysis-path literal into a constant. The claim-stratum expansion cluster moves to _claim_strata.py to keep _claims.py under the 500-line module cap. Same evidence as the previous round: checker exits 0 with byte-identical text and JSON output, all 40 suite tests pass, the 73-scenario differential harness matches exactly, and nox -s tests passes. Co-Authored-By: Claude Fable 5 --- tools/check_dsl_language_evaluation.py | 88 ++---- tools/dsl_language_evaluation/_analysis.py | 61 ++-- .../dsl_language_evaluation/_claim_strata.py | 277 ++++++++++++++++++ tools/dsl_language_evaluation/_claims.py | 200 +------------ tools/dsl_language_evaluation/_measures.py | 126 ++++---- .../_protocol_plans.py | 30 +- .../_snapshot_completion.py | 124 ++++---- .../_snapshot_execution.py | 20 +- 8 files changed, 535 insertions(+), 391 deletions(-) create mode 100644 tools/dsl_language_evaluation/_claim_strata.py diff --git a/tools/check_dsl_language_evaluation.py b/tools/check_dsl_language_evaluation.py index 02a2c268c..d46c25f3a 100644 --- a/tools/check_dsl_language_evaluation.py +++ b/tools/check_dsl_language_evaluation.py @@ -67,6 +67,28 @@ ] +_ANALYSIS_ARTIFACT_PATH = "docs/research/dsl-language-evaluation/analysis-v1.json" + + +def _artifact_path(paths: Mapping[str, object], key: str, default: str) -> str: + value = paths.get(key) + return value if isinstance(value, str) else default + + +def _manifest_claim_binding(repo_root: Path, analysis: dict[str, object]) -> object: + claim = analysis.get("claim") + claim_id = claim.get("claim_id") if isinstance(claim, Mapping) else None + try: + return next( + entry[0]["claim_binding"] + for entry in load_bundles(repo_root) + if isinstance(entry[0].get("claim_binding"), Mapping) + and entry[0]["claim_binding"].get("claim_id") == claim_id + ) + except (OSError, ValueError, StopIteration): + return None + + def validate_bundle( repo_root: Path, protocol: dict[str, object], @@ -79,62 +101,18 @@ def validate_bundle( failures: list[PolicyFailure] = [] paths = artifact_paths or {} - protocol_path = paths.get("protocol_path") - snapshot_path = paths.get("snapshot_path") - analysis_path = paths.get("analysis_path") claim_binding = paths.get("claim_binding") if claim_binding is None: - claim = analysis.get("claim") - claim_id = claim.get("claim_id") if isinstance(claim, Mapping) else None - try: - claim_binding = next( - entry[0]["claim_binding"] - for entry in load_bundles(repo_root) - if isinstance(entry[0].get("claim_binding"), Mapping) - and entry[0]["claim_binding"].get("claim_id") == claim_id - ) - except (OSError, ValueError, StopIteration): - claim_binding = None - catalogs = _validate_protocol( - repo_root, - protocol, - failures, - path=protocol_path - if isinstance(protocol_path, str) - else "docs/research/dsl-language-evaluation/protocol-v1.json", - ) - scope = _validate_claim_scope( - protocol, - analysis, - catalogs, - failures, - path=analysis_path - if isinstance(analysis_path, str) - else "docs/research/dsl-language-evaluation/analysis-v1.json", - ) - strata = _validate_claim_binding( - protocol, - analysis, - claim_binding, - scope, - failures, - path=analysis_path - if isinstance(analysis_path, str) - else "docs/research/dsl-language-evaluation/analysis-v1.json", - ) - observation_ids = _validate_snapshot( - repo_root, - protocol, - snapshot, - catalogs, - scope, - failures, - path=( - snapshot_path - if isinstance(snapshot_path, str) - else "docs/research/dsl-language-evaluation/execution-snapshot-v1.json" - ), + claim_binding = _manifest_claim_binding(repo_root, analysis) + protocol_path = _artifact_path(paths, "protocol_path", "docs/research/dsl-language-evaluation/protocol-v1.json") + snapshot_path = _artifact_path( + paths, "snapshot_path", "docs/research/dsl-language-evaluation/execution-snapshot-v1.json" ) + analysis_path = _artifact_path(paths, "analysis_path", _ANALYSIS_ARTIFACT_PATH) + catalogs = _validate_protocol(repo_root, protocol, failures, path=protocol_path) + scope = _validate_claim_scope(protocol, analysis, catalogs, failures, path=analysis_path) + strata = _validate_claim_binding(protocol, analysis, claim_binding, scope, failures, path=analysis_path) + observation_ids = _validate_snapshot(repo_root, protocol, snapshot, catalogs, scope, failures, path=snapshot_path) _validate_analysis( _AnalysisContext( repo_root=repo_root, @@ -146,9 +124,7 @@ def validate_bundle( ), analysis, failures, - path=analysis_path - if isinstance(analysis_path, str) - else "docs/research/dsl-language-evaluation/analysis-v1.json", + path=analysis_path, ) return failures diff --git a/tools/dsl_language_evaluation/_analysis.py b/tools/dsl_language_evaluation/_analysis.py index 067c3a1bb..93cba6db8 100644 --- a/tools/dsl_language_evaluation/_analysis.py +++ b/tools/dsl_language_evaluation/_analysis.py @@ -351,39 +351,50 @@ def _not_started_analysis_failures( ) -def _gating_qualification( - snapshot: Mapping[str, object], - expected_strata: list[dict[str, object]], -) -> tuple[bool, bool]: - gating_strata = [item for item in expected_strata if item.get("role") == "gating"] - gating_dimensions = [ - item.get("dimension_results", []) for item in gating_strata if isinstance(item.get("dimension_results"), list) - ] - all_pass = ( +def _dimension_outcome(result: object, threshold_result: str) -> bool: + return ( + isinstance(result, Mapping) + and result.get("status") == "evaluated" + and result.get("threshold_result") == threshold_result + ) + + +def _all_gating_pass(gating_strata: list[dict[str, object]], gating_dimensions: list[object]) -> bool: + return ( bool(gating_strata) and len(gating_dimensions) == len(gating_strata) and all( - dimension_results - and all( - isinstance(result, Mapping) - and result.get("status") == "evaluated" - and result.get("threshold_result") == "pass" - for result in dimension_results - ) + dimension_results and all(_dimension_outcome(result, "pass") for result in dimension_results) for dimension_results in gating_dimensions ) ) + + +def _any_record_matches(records: object, field: str, value: str) -> bool: + return any(isinstance(item, Mapping) and item.get(field) == value for item in records) + + +def _gating_dimension_lists( + expected_strata: list[dict[str, object]], +) -> tuple[list[dict[str, object]], list[object]]: + gating_strata = [item for item in expected_strata if item.get("role") == "gating"] + gating_dimensions = [ + item.get("dimension_results", []) for item in gating_strata if isinstance(item.get("dimension_results"), list) + ] + return gating_strata, gating_dimensions + + +def _gating_qualification( + snapshot: Mapping[str, object], + expected_strata: list[dict[str, object]], +) -> tuple[bool, bool]: + gating_strata, gating_dimensions = _gating_dimension_lists(expected_strata) + all_pass = _all_gating_pass(gating_strata, gating_dimensions) any_fail = any( - isinstance(result, Mapping) and result.get("status") == "evaluated" and result.get("threshold_result") == "fail" - for dimension_results in gating_dimensions - for result in dimension_results - ) - unresolved = any( - isinstance(item, Mapping) and item.get("status") == "unresolved" for item in snapshot.get("disagreements", []) - ) - invalidating_deviation = any( - isinstance(item, Mapping) and item.get("severity") == "invalidating" for item in snapshot.get("deviations", []) + _dimension_outcome(result, "fail") for dimension_results in gating_dimensions for result in dimension_results ) + unresolved = _any_record_matches(snapshot.get("disagreements", []), "status", "unresolved") + invalidating_deviation = _any_record_matches(snapshot.get("deviations", []), "severity", "invalidating") execution_complete = snapshot.get("execution_status") == "complete" qualifies_demonstrated = execution_complete and all_pass and not unresolved and not invalidating_deviation qualifies_refuted = execution_complete and any_fail diff --git a/tools/dsl_language_evaluation/_claim_strata.py b/tools/dsl_language_evaluation/_claim_strata.py new file mode 100644 index 000000000..1b27b3bb6 --- /dev/null +++ b/tools/dsl_language_evaluation/_claim_strata.py @@ -0,0 +1,277 @@ +"""Claim-binding stratum expansion and per-stratum scope derivation.""" + +from __future__ import annotations + +from collections.abc import Mapping +from itertools import product + +from tools.dsl_language_evaluation._keys import ( + _STRATUM_GROUP_KEYS, + _STRATUM_PARTITION_AXES, + _STRATUM_ROLES, +) +from tools.dsl_language_evaluation._shape import ( + _exact_keys, + _failure, + _protocol_records_by_id, + _string_list, + _valid_id, +) +from tools.policy.common import PolicyFailure + + +def _thresholds_by_dimension(protocol: Mapping[str, object]) -> dict[object, Mapping[str, object]]: + return { + threshold.get("dimension_id"): threshold + for threshold in protocol.get("thresholds", []) + if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) + } + + +def _threshold_measures(threshold: Mapping[str, object] | None) -> set[object]: + conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] + return { + condition.get("measure_id") + for condition in conditions + if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) + } + + +def _unique_subset(values: list[str] | None, allowed: set[str]) -> bool: + return values is not None and len(values) == len(set(values)) and set(values).issubset(allowed) + + +def _validated_group_filters( + group: Mapping[str, object], + scope: Mapping[str, set[str]], + experience_bands: set[str], +) -> tuple[str, str, list[str], list[str], list[str], list[str]] | None: + """Return (group_id, role, partition_by, personas, bands, conditions) or None.""" + + group_id = group["group_id"] + role = group["role"] + partition_by = _string_list(group["partition_by"]) + persona_ids = _string_list(group["persona_ids"], non_empty=True) + band_ids = _string_list(group["experience_band_ids"], non_empty=True) + condition_ids = _string_list(group["tooling_condition_ids"], non_empty=True) + identity_valid = _valid_id(group_id) and isinstance(group_id, str) + role_valid = isinstance(role, str) and role in _STRATUM_ROLES + valid = all( + ( + identity_valid, + role_valid, + _unique_subset(partition_by, _STRATUM_PARTITION_AXES), + _unique_subset(persona_ids, scope.get("persona_ids", set())), + _unique_subset(band_ids, experience_bands), + _unique_subset(condition_ids, scope.get("tooling_condition_ids", set())), + ) + ) + if not valid: + return None + assert partition_by is not None + assert persona_ids is not None + assert band_ids is not None + assert condition_ids is not None + return group_id, role, partition_by, persona_ids, band_ids, condition_ids + + +def _expanded_strata( + protocol: Mapping[str, object], + scope: Mapping[str, set[str]], + groups: list[object], + failures: list[PolicyFailure], + path: str, +) -> list[dict[str, object]]: + experience_bands = set() + sampling_plan = protocol.get("sampling_plan") + if isinstance(sampling_plan, Mapping): + experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) + expanded: list[dict[str, object]] = [] + for index, group in enumerate(groups): + if not _exact_keys( + group, + _STRATUM_GROUP_KEYS, + failures, + rule_id="dsl-evaluation-claim-strata", + label=f"claim binding strata[{index}]", + path=path, + ): + continue + assert isinstance(group, dict) + filters = _validated_group_filters(group, scope, experience_bands) + if filters is None: + failures.append( + _failure( + "dsl-evaluation-claim-strata", + f"claim stratum group {group['group_id']!r} has invalid role, partition axes, or catalog filters", + path, + ) + ) + continue + expanded.extend(_expanded_group_strata(protocol, scope, group, filters, failures, path)) + return expanded + + +def _expanded_strata_failures( + groups: list[object], + group_ids: set[str], + expanded: list[dict[str, object]], + failures: list[PolicyFailure], + path: str, +) -> None: + expanded_ids = [str(item["stratum_id"]) for item in expanded] + if len(group_ids) != len(groups) or len(expanded_ids) != len(set(expanded_ids)) or not expanded: + failures.append( + _failure( + "dsl-evaluation-claim-strata", + "claim binding requires unique groups and expanded stratum ids", + path, + ) + ) + if not any(item["role"] == "gating" for item in expanded): + failures.append(_failure("dsl-evaluation-claim-strata", "claim binding requires a gating stratum", path)) + + +def _expanded_group_strata( + protocol: Mapping[str, object], + scope: Mapping[str, set[str]], + group: Mapping[str, object], + filters: tuple[str, str, list[str], list[str], list[str], list[str]], + failures: list[PolicyFailure], + path: str, +) -> list[dict[str, object]]: + del group + group_id, role, partition_by, persona_ids, band_ids, condition_ids = filters + axis_values = { + "persona_id": persona_ids, + "experience_band": band_ids, + "tooling_condition_id": condition_ids, + } + split_axes = [axis for axis in ("persona_id", "experience_band", "tooling_condition_id") if axis in partition_by] + combinations = product(*(axis_values[axis] for axis in split_axes)) if split_axes else [()] + expanded: list[dict[str, object]] = [] + for combination in combinations: + selected = dict(zip(split_axes, combination, strict=True)) + stratum_id = "-".join([group_id, *(str(selected[axis]) for axis in split_axes)]) + stratum_scope = _combination_scope(protocol, scope, filters, selected) + if _incomplete_stratum(stratum_id, stratum_scope): + failures.append( + _failure( + "dsl-evaluation-claim-strata", + f"claim stratum {stratum_id!r} has no complete task/measure/dimension slice", + path, + ) + ) + continue + expanded.append({"stratum_id": stratum_id, "role": role, "scope": stratum_scope}) + return expanded + + +def _combination_scope( + protocol: Mapping[str, object], + scope: Mapping[str, set[str]], + filters: tuple[str, str, list[str], list[str], list[str], list[str]], + selected: Mapping[str, object], +) -> dict[str, set[str]]: + _, _, _, persona_ids, band_ids, condition_ids = filters + return _derive_stratum_scope( + protocol, + scope, + persona_ids={str(selected["persona_id"])} if "persona_id" in selected else set(persona_ids), + experience_band_ids={str(selected["experience_band"])} if "experience_band" in selected else set(band_ids), + tooling_condition_ids=( + {str(selected["tooling_condition_id"])} if "tooling_condition_id" in selected else set(condition_ids) + ), + ) + + +def _incomplete_stratum(stratum_id: str, stratum_scope: Mapping[str, set[str]]) -> bool: + return ( + not _valid_id(stratum_id) + or not stratum_scope["task_ids"] + or not stratum_scope["measure_ids"] + or not stratum_scope["dimension_ids"] + ) + + +def _derive_stratum_scope( + protocol: Mapping[str, object], + claim_scope: Mapping[str, set[str]], + *, + persona_ids: set[str], + experience_band_ids: set[str], + tooling_condition_ids: set[str], +) -> dict[str, set[str]]: + """Derive the protocol-applicable claim slice for one preregistered stratum.""" + + selected_tasks = _stratum_tasks(protocol, claim_scope, persona_ids, tooling_condition_ids) + selected_variants = _stratum_variants(protocol, claim_scope, selected_tasks) + selected_measures = _stratum_measures(protocol, claim_scope, selected_tasks) + selected_dimensions = _stratum_dimensions(protocol, claim_scope, selected_measures) + return { + "persona_ids": set(persona_ids), + "experience_band_ids": set(experience_band_ids), + "task_ids": selected_tasks, + "tooling_condition_ids": set(tooling_condition_ids), + "variant_ids": selected_variants, + "artifact_stage_ids": set(claim_scope.get("artifact_stage_ids", set())), + "dimension_ids": selected_dimensions, + "measure_ids": selected_measures, + } + + +def _stratum_tasks( + protocol: Mapping[str, object], + claim_scope: Mapping[str, set[str]], + persona_ids: set[str], + tooling_condition_ids: set[str], +) -> set[str]: + tasks = _protocol_records_by_id(protocol, "tasks", "task_id") + return { + task_id + for task_id, task in tasks.items() + if task_id in claim_scope.get("task_ids", set()) + and set(_string_list(task.get("persona_ids"), non_empty=True) or []) & persona_ids + and set(_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) & tooling_condition_ids + } + + +def _stratum_variants( + protocol: Mapping[str, object], + claim_scope: Mapping[str, set[str]], + selected_tasks: set[str], +) -> set[str]: + variants = _protocol_records_by_id(protocol, "variants", "variant_id") + return { + variant_id + for variant_id, variant in variants.items() + if variant_id in claim_scope.get("variant_ids", set()) and variant.get("task_id") in selected_tasks + } + + +def _stratum_measures( + protocol: Mapping[str, object], + claim_scope: Mapping[str, set[str]], + selected_tasks: set[str], +) -> set[str]: + measures = _protocol_records_by_id(protocol, "measures", "measure_id") + return { + measure_id + for measure_id, measure in measures.items() + if measure_id in claim_scope.get("measure_ids", set()) + and set(_string_list(measure.get("task_ids"), non_empty=True) or []) & selected_tasks + } + + +def _stratum_dimensions( + protocol: Mapping[str, object], + claim_scope: Mapping[str, set[str]], + selected_measures: set[str], +) -> set[str]: + thresholds = _thresholds_by_dimension(protocol) + selected_dimensions = set() + for dimension_id in claim_scope.get("dimension_ids", set()): + required_measures = _threshold_measures(thresholds.get(dimension_id)) + if required_measures and required_measures.issubset(selected_measures): + selected_dimensions.add(dimension_id) + return selected_dimensions diff --git a/tools/dsl_language_evaluation/_claims.py b/tools/dsl_language_evaluation/_claims.py index dc90dbaf2..525afae72 100644 --- a/tools/dsl_language_evaluation/_claims.py +++ b/tools/dsl_language_evaluation/_claims.py @@ -3,15 +3,17 @@ from __future__ import annotations from collections.abc import Mapping -from itertools import product +from tools.dsl_language_evaluation._claim_strata import ( + _expanded_strata, + _expanded_strata_failures, + _threshold_measures, + _thresholds_by_dimension, +) from tools.dsl_language_evaluation._keys import ( _CLAIM_BINDING_KEYS, _CLAIM_SCOPE_KEYS, _MAX_CATALOG_ITEMS, - _STRATUM_GROUP_KEYS, - _STRATUM_PARTITION_AXES, - _STRATUM_ROLES, MANIFEST_PATH, ) from tools.dsl_language_evaluation._shape import ( @@ -172,23 +174,6 @@ def _scope_threshold_failures( ) -def _thresholds_by_dimension(protocol: Mapping[str, object]) -> dict[object, Mapping[str, object]]: - return { - threshold.get("dimension_id"): threshold - for threshold in protocol.get("thresholds", []) - if isinstance(threshold, Mapping) and isinstance(threshold.get("dimension_id"), str) - } - - -def _threshold_measures(threshold: Mapping[str, object] | None) -> set[object]: - conditions = threshold.get("conditions", []) if isinstance(threshold, Mapping) else [] - return { - condition.get("measure_id") - for condition in conditions - if isinstance(condition, Mapping) and isinstance(condition.get("measure_id"), str) - } - - def _validate_claim_binding( protocol: Mapping[str, object], analysis: Mapping[str, object], @@ -229,46 +214,8 @@ def _validate_claim_binding( label="claim stratum group", path=path, ) - experience_bands = set() - sampling_plan = protocol.get("sampling_plan") - if isinstance(sampling_plan, Mapping): - experience_bands = set(_string_list(sampling_plan.get("experience_bands"), non_empty=True) or []) - - expanded: list[dict[str, object]] = [] - for index, group in enumerate(groups): - if not _exact_keys( - group, - _STRATUM_GROUP_KEYS, - failures, - rule_id="dsl-evaluation-claim-strata", - label=f"claim binding strata[{index}]", - path=path, - ): - continue - assert isinstance(group, dict) - filters = _validated_group_filters(group, scope, experience_bands) - if filters is None: - failures.append( - _failure( - "dsl-evaluation-claim-strata", - f"claim stratum group {group['group_id']!r} has invalid role, partition axes, or catalog filters", - path, - ) - ) - continue - expanded.extend(_expanded_group_strata(protocol, scope, group, filters, failures, path)) - - expanded_ids = [str(item["stratum_id"]) for item in expanded] - if len(group_ids) != len(groups) or len(expanded_ids) != len(set(expanded_ids)) or not expanded: - failures.append( - _failure( - "dsl-evaluation-claim-strata", - "claim binding requires unique groups and expanded stratum ids", - path, - ) - ) - if not any(item["role"] == "gating" for item in expanded): - failures.append(_failure("dsl-evaluation-claim-strata", "claim binding requires a gating stratum", path)) + expanded = _expanded_strata(protocol, scope, groups, failures, path) + _expanded_strata_failures(groups, group_ids, expanded, failures, path) if len(expanded) > _MAX_CATALOG_ITEMS: failures.append( _failure( @@ -328,137 +275,6 @@ def _bound_scope_failures( ) -def _unique_subset(values: list[str] | None, allowed: set[str]) -> bool: - return values is not None and len(values) == len(set(values)) and set(values).issubset(allowed) - - -def _validated_group_filters( - group: Mapping[str, object], - scope: Mapping[str, set[str]], - experience_bands: set[str], -) -> tuple[str, str, list[str], list[str], list[str], list[str]] | None: - """Return (group_id, role, partition_by, personas, bands, conditions) or None.""" - - group_id = group["group_id"] - role = group["role"] - partition_by = _string_list(group["partition_by"]) - persona_ids = _string_list(group["persona_ids"], non_empty=True) - band_ids = _string_list(group["experience_band_ids"], non_empty=True) - condition_ids = _string_list(group["tooling_condition_ids"], non_empty=True) - valid = ( - _valid_id(group_id) - and isinstance(group_id, str) - and isinstance(role, str) - and role in _STRATUM_ROLES - and _unique_subset(partition_by, _STRATUM_PARTITION_AXES) - and _unique_subset(persona_ids, scope.get("persona_ids", set())) - and _unique_subset(band_ids, experience_bands) - and _unique_subset(condition_ids, scope.get("tooling_condition_ids", set())) - ) - if not valid: - return None - assert partition_by is not None and persona_ids is not None - assert band_ids is not None and condition_ids is not None - return group_id, role, partition_by, persona_ids, band_ids, condition_ids - - -def _expanded_group_strata( - protocol: Mapping[str, object], - scope: Mapping[str, set[str]], - group: Mapping[str, object], - filters: tuple[str, str, list[str], list[str], list[str], list[str]], - failures: list[PolicyFailure], - path: str, -) -> list[dict[str, object]]: - del group - group_id, role, partition_by, persona_ids, band_ids, condition_ids = filters - axis_values = { - "persona_id": persona_ids, - "experience_band": band_ids, - "tooling_condition_id": condition_ids, - } - split_axes = [axis for axis in ("persona_id", "experience_band", "tooling_condition_id") if axis in partition_by] - combinations = product(*(axis_values[axis] for axis in split_axes)) if split_axes else [()] - expanded: list[dict[str, object]] = [] - for combination in combinations: - selected = dict(zip(split_axes, combination, strict=True)) - stratum_id = "-".join([group_id, *(str(selected[axis]) for axis in split_axes)]) - stratum_scope = _derive_stratum_scope( - protocol, - scope, - persona_ids={str(selected["persona_id"])} if "persona_id" in selected else set(persona_ids), - experience_band_ids={str(selected["experience_band"])} if "experience_band" in selected else set(band_ids), - tooling_condition_ids=( - {str(selected["tooling_condition_id"])} if "tooling_condition_id" in selected else set(condition_ids) - ), - ) - if ( - not _valid_id(stratum_id) - or not stratum_scope["task_ids"] - or not stratum_scope["measure_ids"] - or not stratum_scope["dimension_ids"] - ): - failures.append( - _failure( - "dsl-evaluation-claim-strata", - f"claim stratum {stratum_id!r} has no complete task/measure/dimension slice", - path, - ) - ) - continue - expanded.append({"stratum_id": stratum_id, "role": role, "scope": stratum_scope}) - return expanded - - -def _derive_stratum_scope( - protocol: Mapping[str, object], - claim_scope: Mapping[str, set[str]], - *, - persona_ids: set[str], - experience_band_ids: set[str], - tooling_condition_ids: set[str], -) -> dict[str, set[str]]: - """Derive the protocol-applicable claim slice for one preregistered stratum.""" - - tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - selected_tasks = { - task_id - for task_id, task in tasks.items() - if task_id in claim_scope.get("task_ids", set()) - and set(_string_list(task.get("persona_ids"), non_empty=True) or []) & persona_ids - and set(_string_list(task.get("tooling_condition_ids"), non_empty=True) or []) & tooling_condition_ids - } - variants = _protocol_records_by_id(protocol, "variants", "variant_id") - selected_variants = { - variant_id - for variant_id, variant in variants.items() - if variant_id in claim_scope.get("variant_ids", set()) and variant.get("task_id") in selected_tasks - } - measures = _protocol_records_by_id(protocol, "measures", "measure_id") - selected_measures = { - measure_id - for measure_id, measure in measures.items() - if measure_id in claim_scope.get("measure_ids", set()) - and set(_string_list(measure.get("task_ids"), non_empty=True) or []) & selected_tasks - } - thresholds = _thresholds_by_dimension(protocol) - selected_dimensions = set() - for dimension_id in claim_scope.get("dimension_ids", set()): - required_measures = _threshold_measures(thresholds.get(dimension_id)) - if required_measures and required_measures.issubset(selected_measures): - selected_dimensions.add(dimension_id) - return { - "persona_ids": set(persona_ids), - "experience_band_ids": set(experience_band_ids), - "task_ids": selected_tasks, - "tooling_condition_ids": set(tooling_condition_ids), - "variant_ids": selected_variants, - "artifact_stage_ids": set(claim_scope.get("artifact_stage_ids", set())), - "dimension_ids": selected_dimensions, - "measure_ids": selected_measures, - } - - def resolve_claim_strata( protocol: Mapping[str, object], analysis: Mapping[str, object], diff --git a/tools/dsl_language_evaluation/_measures.py b/tools/dsl_language_evaluation/_measures.py index cd80c4e0a..aa4a4034d 100644 --- a/tools/dsl_language_evaluation/_measures.py +++ b/tools/dsl_language_evaluation/_measures.py @@ -78,13 +78,11 @@ def _attempt_in_scope( ) -> bool: if scope_sets is None: return True - if not _attempt_matches_scope(attempt, scope_sets): - return False - if "experience_band_ids" in scope_sets: + in_scope = _attempt_matches_scope(attempt, scope_sets) + if in_scope and "experience_band_ids" in scope_sets: subject = subjects_by_id.get(attempt.get("subject_id")) - if not isinstance(subject, Mapping) or subject.get("experience_band") not in scope_sets["experience_band_ids"]: - return False - return True + in_scope = isinstance(subject, Mapping) and subject.get("experience_band") in scope_sets["experience_band_ids"] + return in_scope def _attempt_opportunities( @@ -134,39 +132,41 @@ def _check_observed_keys( raise ValueError(f"observations without protocol-declared opportunities: {extras[:5]}") -def _measure_opportunities( - protocol: Mapping[str, object], - snapshot: Mapping[str, object], - scope: Mapping[str, Sequence[str] | set[str]] | None = None, -) -> tuple[ - dict[str, Mapping[str, object]], - list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], - dict[tuple[str, str, str], Mapping[str, object]], -]: - """Derive every attempt-measure-stage opportunity and frozen observation.""" - - tasks = _protocol_records_by_id(protocol, "tasks", "task_id") - scope_sets = {field: set(values) for field, values in scope.items()} if scope is not None else None - measures = _scoped_measures(protocol, scope_sets) - attempts = snapshot.get("attempts", []) - observations = snapshot.get("observations", []) - withdrawals = snapshot.get("withdrawals", []) - subjects = snapshot.get("subjects", []) - subjects_by_id = { +def _subjects_by_id(subjects: object) -> dict[object, Mapping[str, object]]: + return { subject.get("subject_id"): subject for subject in subjects if isinstance(subject, Mapping) and isinstance(subject.get("subject_id"), str) } - if not isinstance(attempts, list) or not isinstance(observations, list) or not isinstance(withdrawals, list): - raise ValueError("snapshot execution records must be lists") - withdrawn_subjects = { + +def _withdrawn_subjects(withdrawals: list[object]) -> set[str]: + return { withdrawal["subject_id"] for withdrawal in withdrawals if isinstance(withdrawal, Mapping) and isinstance(withdrawal.get("subject_id"), str) } - observation_by_opportunity = _observation_index(observations) + +def _attempt_withdrawn(attempt: Mapping[str, object], withdrawn_subjects: set[str]) -> bool: + subject_id = attempt.get("subject_id") + if isinstance(subject_id, str) and subject_id in withdrawn_subjects: + return True + return attempt.get("outcome") == "withdrawn" + + +def _selected_opportunities( + attempts: list[object], + tasks: Mapping[str, Mapping[str, object]], + measures: Mapping[str, Mapping[str, object]], + scope_sets: Mapping[str, set[str]] | None, + subjects_by_id: Mapping[object, Mapping[str, object]], + withdrawn_subjects: set[str], +) -> tuple[ + list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], + set[tuple[str, str, str]], + set[str], +]: opportunities: list[tuple[Mapping[str, object], Mapping[str, object], str, bool]] = [] expected_keys: set[tuple[str, str, str]] = set() selected_attempt_ids: set[str] = set() @@ -175,7 +175,6 @@ def _measure_opportunities( continue attempt_id = attempt.get("attempt_id") task_id = attempt.get("task_id") - subject_id = attempt.get("subject_id") if ( not isinstance(attempt_id, str) or not isinstance(task_id, str) @@ -185,17 +184,53 @@ def _measure_opportunities( if tasks.get(task_id) is None or not _attempt_in_scope(attempt, scope_sets, subjects_by_id): continue selected_attempt_ids.add(attempt_id) - withdrawn = (isinstance(subject_id, str) and subject_id in withdrawn_subjects) or attempt.get( - "outcome" - ) == "withdrawn" + withdrawn = _attempt_withdrawn(attempt, withdrawn_subjects) attempt_opportunities, attempt_keys = _attempt_opportunities(attempt, measures, scope_sets, withdrawn=withdrawn) opportunities.extend(attempt_opportunities) expected_keys.update(attempt_keys) + return opportunities, expected_keys, selected_attempt_ids + + +def _measure_opportunities( + protocol: Mapping[str, object], + snapshot: Mapping[str, object], + scope: Mapping[str, Sequence[str] | set[str]] | None = None, +) -> tuple[ + dict[str, Mapping[str, object]], + list[tuple[Mapping[str, object], Mapping[str, object], str, bool]], + dict[tuple[str, str, str], Mapping[str, object]], +]: + """Derive every attempt-measure-stage opportunity and frozen observation.""" + + tasks = _protocol_records_by_id(protocol, "tasks", "task_id") + scope_sets = {field: set(values) for field, values in scope.items()} if scope is not None else None + measures = _scoped_measures(protocol, scope_sets) + attempts = snapshot.get("attempts", []) + observations = snapshot.get("observations", []) + withdrawals = snapshot.get("withdrawals", []) + subjects_by_id = _subjects_by_id(snapshot.get("subjects", [])) + if not isinstance(attempts, list) or not isinstance(observations, list) or not isinstance(withdrawals, list): + raise ValueError("snapshot execution records must be lists") + withdrawn_subjects = _withdrawn_subjects(withdrawals) + observation_by_opportunity = _observation_index(observations) + opportunities, expected_keys, selected_attempt_ids = _selected_opportunities( + attempts, tasks, measures, scope_sets, subjects_by_id, withdrawn_subjects + ) _check_observed_keys(observation_by_opportunity, expected_keys, selected_attempt_ids, scope_sets) return measures, opportunities, observation_by_opportunity +def _validated_value(measure_id: str, aggregation: object, value: object) -> int | float | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{measure_id}: observation values must be numeric or null") + if aggregation == "proportion" and value not in {0, 1}: + raise ValueError(f"{measure_id}: proportion observations must be 0 or 1") + return value + + def _observation_tallies( measure_id: str, aggregation: object, @@ -207,10 +242,11 @@ def _observation_tallies( counts = {"missing": 0, "abandoned": 0, "tool_failed": 0} for attempt, artifact_stage in eligible: attempt_id = attempt.get("attempt_id") - if not isinstance(attempt_id, str): - counts["missing"] += 1 - continue - observation = observation_by_opportunity.get((attempt_id, measure_id, artifact_stage)) + observation = ( + observation_by_opportunity.get((attempt_id, measure_id, artifact_stage)) + if isinstance(attempt_id, str) + else None + ) if observation is None: counts["missing"] += 1 continue @@ -221,14 +257,9 @@ def _observation_tallies( outcome = observation.get("outcome") if outcome in counts: counts[outcome] += 1 - value = observation.get("value") - if value is None: - continue - if isinstance(value, bool) or not isinstance(value, (int, float)): - raise ValueError(f"{measure_id}: observation values must be numeric or null") - if aggregation == "proportion" and value not in {0, 1}: - raise ValueError(f"{measure_id}: proportion observations must be 0 or 1") - values.append(value) + value = _validated_value(measure_id, aggregation, observation.get("value")) + if value is not None: + values.append(value) return values, supporting_ids, counts @@ -239,13 +270,10 @@ def _aggregated_value( ) -> tuple[int | float | None, int | float | None]: if denominator == 0 or len(values) != denominator: return None, None - if aggregation == "proportion": - numerator = sum(values) - return numerator, numerator / denominator if aggregation == "median": return None, float(median(values)) numerator = sum(values) - return numerator, numerator + return numerator, numerator / denominator if aggregation == "proportion" else numerator def _measure_result( diff --git a/tools/dsl_language_evaluation/_protocol_plans.py b/tools/dsl_language_evaluation/_protocol_plans.py index a6fa0073f..4c41f306b 100644 --- a/tools/dsl_language_evaluation/_protocol_plans.py +++ b/tools/dsl_language_evaluation/_protocol_plans.py @@ -106,17 +106,18 @@ def _declaration_stage_pairs( task = tasks_by_id.get(task_id) if isinstance(task_id, str) else None declaration_variants = _string_list(declaration["variant_ids"], non_empty=True) declaration_stages = _string_list(declaration["artifact_stage_ids"], non_empty=True) - task_variants = set(_string_list(task.get("variant_ids"), non_empty=True) or []) if task is not None else set() - task_stages = set(_string_list(task.get("artifact_stage_ids"), non_empty=True) or []) if task is not None else set() - if ( - task is None - or measure_tasks is None - or task_id not in measure_tasks - or declaration_variants is None - or not set(declaration_variants).issubset(task_variants) - or declaration_stages is None - or not set(declaration_stages).issubset(task_stages) - ): + invalid = any( + ( + task is None, + measure_tasks is None, + task_id not in (measure_tasks or []), + declaration_variants is None, + not set(declaration_variants or []).issubset(_task_id_set(task, "variant_ids")), + declaration_stages is None, + not set(declaration_stages or []).issubset(_task_id_set(task, "artifact_stage_ids")), + ) + ) + if invalid: failures.append( _failure( "dsl-evaluation-measure-stage-applicability", @@ -125,9 +126,16 @@ def _declaration_stage_pairs( ) ) return None + assert declaration_variants is not None return [(task_id, variant_id) for variant_id in declaration_variants] +def _task_id_set(task: Mapping[str, object] | None, field: str) -> set[str]: + if task is None: + return set() + return set(_string_list(task.get(field), non_empty=True) or []) + + def _stage_applicability_failures( measure: Mapping[str, object], index: int, diff --git a/tools/dsl_language_evaluation/_snapshot_completion.py b/tools/dsl_language_evaluation/_snapshot_completion.py index 9aafc4781..00aa4e79b 100644 --- a/tools/dsl_language_evaluation/_snapshot_completion.py +++ b/tools/dsl_language_evaluation/_snapshot_completion.py @@ -122,16 +122,19 @@ def _review_join_failures( task = tasks.get(attempt.get("task_id")) task_stages = _string_list(task.get("artifact_stage_ids"), non_empty=True) if task else None task_personas = _string_list(task.get("persona_ids"), non_empty=True) if task else None - if ( - review.get("task_id") != attempt.get("task_id") - or review.get("variant_id") != attempt.get("variant_id") - or reviewer_id == attempt.get("subject_id") - or reviewer.get("consent_status") != "consented" - or task_stages is None - or "review-judgment" not in task_stages - or task_personas is None - or reviewer.get("persona_id") not in task_personas - ): + ineligible = any( + ( + review.get("task_id") != attempt.get("task_id"), + review.get("variant_id") != attempt.get("variant_id"), + reviewer_id == attempt.get("subject_id"), + reviewer.get("consent_status") != "consented", + task_stages is None, + "review-judgment" not in (task_stages or []), + task_personas is None, + reviewer.get("persona_id") not in (task_personas or []), + ) + ) + if ineligible: failures.append( _failure( "dsl-evaluation-review-join", @@ -182,6 +185,51 @@ def _review_failures( return reviews_by_attempt +def _linked_attempt_ids(reviews: list[object], disagreement_review_ids: list[str] | None) -> set[object]: + if disagreement_review_ids is None: + return set() + return { + review.get("attempt_id") + for review in reviews + if isinstance(review, Mapping) and review.get("review_id") in disagreement_review_ids + } + + +def _disagreement_record_failures( + disagreement: Mapping[str, object], + joins: _SnapshotJoins, + failures: list[PolicyFailure], + path: str, +) -> None: + disagreement_review_ids = _string_list(disagreement["review_ids"], non_empty=True) + linked_attempts = _linked_attempt_ids(joins.reviews, disagreement_review_ids) + listed_ids = disagreement_review_ids or [] + broken = any( + ( + disagreement_review_ids is None, + len(listed_ids) < 2, + not set(listed_ids).issubset(joins.review_ids), + len(linked_attempts) != 1, + ) + ) + if broken: + failures.append( + _failure( + "dsl-evaluation-disagreement-join", + f"{disagreement['disagreement_id']}: reviews must share one parent attempt", + path, + ) + ) + if disagreement["originals_preserved"] is not True: + failures.append( + _failure( + "dsl-evaluation-disagreement-preservation", + f"{disagreement['disagreement_id']}: originals must be preserved", + path, + ) + ) + + def _disagreement_failures( joins: _SnapshotJoins, failures: list[PolicyFailure], @@ -190,36 +238,7 @@ def _disagreement_failures( for disagreement in joins.disagreements: if not isinstance(disagreement, Mapping) or set(disagreement) != _DISAGREEMENT_KEYS: continue - disagreement_review_ids = _string_list(disagreement["review_ids"], non_empty=True) - linked_reviews = [ - review - for review in joins.reviews - if isinstance(review, Mapping) - and disagreement_review_ids is not None - and review.get("review_id") in disagreement_review_ids - ] - linked_attempts = {review.get("attempt_id") for review in linked_reviews} - if ( - disagreement_review_ids is None - or len(disagreement_review_ids) < 2 - or not set(disagreement_review_ids).issubset(joins.review_ids) - or len(linked_attempts) != 1 - ): - failures.append( - _failure( - "dsl-evaluation-disagreement-join", - f"{disagreement['disagreement_id']}: reviews must share one parent attempt", - path, - ) - ) - if disagreement["originals_preserved"] is not True: - failures.append( - _failure( - "dsl-evaluation-disagreement-preservation", - f"{disagreement['disagreement_id']}: originals must be preserved", - path, - ) - ) + _disagreement_record_failures(disagreement, joins, failures, path) def _scoped_subject_ids( @@ -381,19 +400,24 @@ def _completion_failures( path, ) ) - if ( - not ethics_approved - or missing_personas - or not isinstance(target_total, int) - or len(active_subject_ids) < target_total - or not expected_task_shapes.issubset(actual_task_shapes) - or any(reviews_by_attempt[attempt_id] == 0 for attempt_id in review_required_attempts) - or missing_subject_workloads - ): + subject_floor = target_total if isinstance(target_total, int) else 0 + incomplete = any( + ( + not ethics_approved, + bool(missing_personas), + not isinstance(target_total, int), + len(active_subject_ids) < subject_floor, + not expected_task_shapes.issubset(actual_task_shapes), + any(reviews_by_attempt[attempt_id] == 0 for attempt_id in review_required_attempts), + bool(missing_subject_workloads), + ) + ) + if incomplete: failures.append( _failure( "dsl-evaluation-completion-coverage", - "complete execution lacks approved ethics, subject workload/minima, task/condition/variant coverage, or required independent reviews", + "complete execution lacks approved ethics, subject workload/minima, " + "task/condition/variant coverage, or required independent reviews", path, ) ) diff --git a/tools/dsl_language_evaluation/_snapshot_execution.py b/tools/dsl_language_evaluation/_snapshot_execution.py index 37bac0a33..82e643ee2 100644 --- a/tools/dsl_language_evaluation/_snapshot_execution.py +++ b/tools/dsl_language_evaluation/_snapshot_execution.py @@ -203,13 +203,16 @@ def _observation_stage_failures( ) except ValueError: applicable_stages = [] - if ( - observation.get("artifact_stage") not in task_stages - or observation.get("artifact_stage") not in applicable_stages - or observation.get("task_id") not in measure_tasks - or observation_dimensions is None - or set(observation_dimensions) != expected_dimensions - ): + undeclared = any( + ( + observation.get("artifact_stage") not in task_stages, + observation.get("artifact_stage") not in applicable_stages, + observation.get("task_id") not in measure_tasks, + observation_dimensions is None, + set(observation_dimensions or []) != expected_dimensions, + ) + ) + if undeclared: failures.append( _failure( "dsl-evaluation-observation-task-join", @@ -355,7 +358,8 @@ def _opportunity_coverage_failures( failures.append( _failure( "dsl-evaluation-opportunity-coverage", - "observations must exactly cover every non-withdrawn protocol-declared attempt-measure-stage opportunity", + "observations must exactly cover every non-withdrawn " + "protocol-declared attempt-measure-stage opportunity", path, ) ) From 1a1fd72376a1db5b0eed9c31c2709939d0df3244 Mon Sep 17 00:00:00 2001 From: Yernat Yestekov <2068106+doublewhy@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:06:54 -0700 Subject: [PATCH 4/4] refactor(tools): read snapshot record lists inside their consuming helpers Sonar S5864 flagged the two helpers whose object-annotated parameters were iterated directly. Pass the snapshot and resolve the record list inside each helper, which is exactly how the pre-split code read them. Checker output, the 40-test suite, and the 73-scenario differential harness all remain byte-identical. Co-Authored-By: Claude Fable 5 --- tools/dsl_language_evaluation/_analysis.py | 8 ++++---- tools/dsl_language_evaluation/_measures.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/dsl_language_evaluation/_analysis.py b/tools/dsl_language_evaluation/_analysis.py index 93cba6db8..a661c2c32 100644 --- a/tools/dsl_language_evaluation/_analysis.py +++ b/tools/dsl_language_evaluation/_analysis.py @@ -370,8 +370,8 @@ def _all_gating_pass(gating_strata: list[dict[str, object]], gating_dimensions: ) -def _any_record_matches(records: object, field: str, value: str) -> bool: - return any(isinstance(item, Mapping) and item.get(field) == value for item in records) +def _any_record_matches(snapshot: Mapping[str, object], field_name: str, field: str, value: str) -> bool: + return any(isinstance(item, Mapping) and item.get(field) == value for item in snapshot.get(field_name, [])) def _gating_dimension_lists( @@ -393,8 +393,8 @@ def _gating_qualification( any_fail = any( _dimension_outcome(result, "fail") for dimension_results in gating_dimensions for result in dimension_results ) - unresolved = _any_record_matches(snapshot.get("disagreements", []), "status", "unresolved") - invalidating_deviation = _any_record_matches(snapshot.get("deviations", []), "severity", "invalidating") + unresolved = _any_record_matches(snapshot, "disagreements", "status", "unresolved") + invalidating_deviation = _any_record_matches(snapshot, "deviations", "severity", "invalidating") execution_complete = snapshot.get("execution_status") == "complete" qualifies_demonstrated = execution_complete and all_pass and not unresolved and not invalidating_deviation qualifies_refuted = execution_complete and any_fail diff --git a/tools/dsl_language_evaluation/_measures.py b/tools/dsl_language_evaluation/_measures.py index aa4a4034d..3c6931bd8 100644 --- a/tools/dsl_language_evaluation/_measures.py +++ b/tools/dsl_language_evaluation/_measures.py @@ -132,10 +132,10 @@ def _check_observed_keys( raise ValueError(f"observations without protocol-declared opportunities: {extras[:5]}") -def _subjects_by_id(subjects: object) -> dict[object, Mapping[str, object]]: +def _subjects_by_id(snapshot: Mapping[str, object]) -> dict[object, Mapping[str, object]]: return { subject.get("subject_id"): subject - for subject in subjects + for subject in snapshot.get("subjects", []) if isinstance(subject, Mapping) and isinstance(subject.get("subject_id"), str) } @@ -208,7 +208,7 @@ def _measure_opportunities( attempts = snapshot.get("attempts", []) observations = snapshot.get("observations", []) withdrawals = snapshot.get("withdrawals", []) - subjects_by_id = _subjects_by_id(snapshot.get("subjects", [])) + subjects_by_id = _subjects_by_id(snapshot) if not isinstance(attempts, list) or not isinstance(observations, list) or not isinstance(withdrawals, list): raise ValueError("snapshot execution records must be lists")