diff --git a/.github/scripts/harness_e2e_profiles.py b/.github/scripts/harness_e2e_profiles.py index 3d070dec5..704825596 100644 --- a/.github/scripts/harness_e2e_profiles.py +++ b/.github/scripts/harness_e2e_profiles.py @@ -124,6 +124,7 @@ def resolve_profile( requested: list[str], catalog_sha: str, expected_catalog_sha: str = "", + expected_profile_digest: str = "", ) -> dict[str, Any]: if profile not in VALID_PROFILES: raise ValueError(f"validation_profile must be one of {sorted(VALID_PROFILES)}") @@ -131,7 +132,15 @@ def resolve_profile( raise ValueError("code-defined scenarios must be non-empty and unique") if available != list(catalog.ids): raise ValueError("release catalog scenarios do not match harness-e2e list") - if expected_catalog_sha and expected_catalog_sha != catalog_sha: + if expected_profile_digest and expected_profile_digest != catalog.profile_digest: + raise ValueError( + "Harness E2E profile changed after preview; create a new preview" + ) + if ( + not expected_profile_digest + and expected_catalog_sha + and expected_catalog_sha != catalog_sha + ): raise ValueError( f"Harness E2E catalog moved from {expected_catalog_sha} to {catalog_sha}; create a new preview" ) @@ -168,6 +177,7 @@ def main() -> int: parser.add_argument("--scenarios-json", default="[]") parser.add_argument("--catalog-sha", required=True) parser.add_argument("--expected-catalog-sha", default="") + parser.add_argument("--expected-profile-digest", default="") parser.add_argument("--output", type=Path) args = parser.parse_args() try: @@ -178,6 +188,7 @@ def main() -> int: requested=parse_scenarios_json(args.scenarios_json, "requested scenarios"), catalog_sha=args.catalog_sha, expected_catalog_sha=args.expected_catalog_sha, + expected_profile_digest=args.expected_profile_digest, ) except (FileNotFoundError, ValueError) as error: raise SystemExit(f"invalid Harness E2E profile: {error}") from error diff --git a/.github/scripts/tests/test_harness_e2e_profiles.py b/.github/scripts/tests/test_harness_e2e_profiles.py index fde9773c1..e29a4049b 100644 --- a/.github/scripts/tests/test_harness_e2e_profiles.py +++ b/.github/scripts/tests/test_harness_e2e_profiles.py @@ -57,7 +57,7 @@ def test_custom_superset_is_promotable(): assert resolved["promotion_eligible"] is True -def test_rejects_unknown_scenario_and_stale_catalog(): +def test_rejects_unknown_scenario_and_changed_profile(): catalog = load_profile_catalog() with pytest.raises(ValueError, match="unknown Harness E2E scenarios"): resolve_profile( @@ -67,7 +67,7 @@ def test_rejects_unknown_scenario_and_stale_catalog(): requested=["missing"], catalog_sha="a" * 40, ) - with pytest.raises(ValueError, match="catalog moved"): + with pytest.raises(ValueError, match="profile changed"): resolve_profile( catalog, available=list(catalog.ids), @@ -75,9 +75,25 @@ def test_rejects_unknown_scenario_and_stale_catalog(): requested=[], catalog_sha="a" * 40, expected_catalog_sha="b" * 40, + expected_profile_digest="0" * 64, ) +def test_allows_release_bump_commits_when_profile_digest_is_unchanged(): + catalog = load_profile_catalog() + resolved = resolve_profile( + catalog, + available=list(catalog.ids), + profile="release", + requested=[], + catalog_sha="b" * 40, + expected_catalog_sha="a" * 40, + expected_profile_digest=catalog.profile_digest, + ) + assert resolved["catalog_sha"] == "b" * 40 + assert resolved["profile_digest"] == catalog.profile_digest + + def test_catalog_rejects_duplicate_scenarios(tmp_path: Path): path = tmp_path / "release-workers.yaml" path.write_text( diff --git a/.github/workflows/_harness-e2e.yml b/.github/workflows/_harness-e2e.yml index fbe784cdd..b3e3adb37 100644 --- a/.github/workflows/_harness-e2e.yml +++ b/.github/workflows/_harness-e2e.yml @@ -103,6 +103,11 @@ on: required: false type: string default: '' + profile_digest: + description: Canonical scenario profile digest used by the caller preview + required: false + type: string + default: '' coverage: description: Instrument the E2E stack and produce an LLVM coverage report required: false @@ -328,6 +333,7 @@ jobs: VALIDATION_PROFILE: ${{ inputs.validation_profile }} REQUESTED_SCENARIOS: ${{ inputs.scenarios_json }} EXPECTED_CATALOG_SHA: ${{ inputs.catalog_sha }} + EXPECTED_PROFILE_DIGEST: ${{ inputs.profile_digest }} # The instrumented binary would otherwise drop default.profraw in cwd. LLVM_PROFILE_FILE: ${{ inputs.coverage && format('{0}/target/coverage/build-profraw/%m_%p.profraw', github.workspace) || '' }} run: | @@ -338,7 +344,8 @@ jobs: --profile "$VALIDATION_PROFILE" \ --scenarios-json "$REQUESTED_SCENARIOS" \ --catalog-sha "${{ github.sha }}" \ - --expected-catalog-sha "$EXPECTED_CATALOG_SHA") + --expected-catalog-sha "$EXPECTED_CATALOG_SHA" \ + --expected-profile-digest "$EXPECTED_PROFILE_DIGEST") { echo "json=$(jq -c .scenarios <<<"$resolved")" echo "required=$(jq -c .required_scenarios <<<"$resolved")" diff --git a/.github/workflows/harness-e2e-deployed.yml b/.github/workflows/harness-e2e-deployed.yml index d44833527..49dbf9ce2 100644 --- a/.github/workflows/harness-e2e-deployed.yml +++ b/.github/workflows/harness-e2e-deployed.yml @@ -69,6 +69,10 @@ on: description: Release catalog SHA used by Release Control preview type: string default: '' + profile_digest: + description: Canonical scenario profile digest used by Release Control preview + type: string + default: '' permissions: actions: read @@ -97,6 +101,7 @@ jobs: validation_profile: ${{ inputs.validation_profile }} scenarios_json: ${{ inputs.scenarios_json }} catalog_sha: ${{ inputs.catalog_sha }} + profile_digest: ${{ inputs.profile_digest }} subjects: ${{ inputs.subjects || vars.HARNESS_E2E_SUBJECTS || '[{"id":"anthropic-sonnet","model":"claude-sonnet-4-6","provider":"anthropic"}]' }} judge_model: ${{ vars.HARNESS_E2E_JUDGE_MODEL || 'claude-sonnet-4-6' }} judge_provider: ${{ vars.HARNESS_E2E_JUDGE_PROVIDER || 'anthropic' }} diff --git a/harness/tests/e2e/run-deployed-ci.sh b/harness/tests/e2e/run-deployed-ci.sh index 44314e9db..3b2d81d37 100755 --- a/harness/tests/e2e/run-deployed-ci.sh +++ b/harness/tests/e2e/run-deployed-ci.sh @@ -311,11 +311,15 @@ printf 'workers: []\n' >"$project_dir/config.yaml" engine_pid=$! wait_for_engine -workers=("harness@$worker_tag" "database@$worker_tag" "fp@$worker_tag" "web@$worker_tag") +# The candidate channel belongs only to the workers pinned in stack_versions. +# Auxiliary E2E workers are not released as part of this operation and may not +# expose the candidate tag at all, so keep them on their stable channel. +support_worker_tag=latest +workers=("database@$support_worker_tag" "fp@$support_worker_tag" "web@$support_worker_tag") declare -A providers=() for provider in "$HARNESS_E2E_PROVIDER" "$HARNESS_E2E_JUDGE_PROVIDER"; do if [[ -z "${providers[$provider]:-}" ]]; then - workers+=("provider-$provider@$worker_tag") + workers+=("provider-$provider@$support_worker_tag") providers[$provider]=1 fi done @@ -337,14 +341,22 @@ add_with_retry() { return 1 } -log "Installing registry stack: ${workers[*]}" +log "Installing stable E2E support stack: ${workers[*]}" add_with_retry worker-add "${workers[@]}" +# Bring the exact dependencies up first and the released worker last. For a +# Harness candidate this prevents Harness from briefly booting against the +# stable dependency graph before its candidate dependencies replace it. while IFS=$'\t' read -r candidate_worker candidate_version; do log "Installing exact stack candidate: ${candidate_worker}@${candidate_version}" add_with_retry "candidate-${candidate_worker}" \ "${candidate_worker}@${candidate_version}" --force -done < <(jq -r 'to_entries | sort_by(.key)[] | [.key, .value] | @tsv' <<<"$stack_versions") +done < <(jq -r --arg release_worker "$HARNESS_E2E_RELEASE_WORKER" ' + to_entries + | sort_by([if .key == $release_worker then 1 else 0 end, .key])[] + | [.key, .value] + | @tsv +' <<<"$stack_versions") wait_for_functions \ harness::send harness::status worker::add database::query state::get \