Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion .github/scripts/harness_e2e_profiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,23 @@ def resolve_profile(
requested: list[str],
catalog_sha: str,
expected_catalog_sha: str = "",
expected_profile_digest: str = "",
) -> dict[str, Any]:
if profile not in VALID_PROFILES:
raise ValueError(f"validation_profile must be one of {sorted(VALID_PROFILES)}")
if len(available) != len(set(available)) or not available:
raise ValueError("code-defined scenarios must be non-empty and unique")
if available != list(catalog.ids):
raise ValueError("release catalog scenarios do not match harness-e2e list")
if expected_catalog_sha and expected_catalog_sha != catalog_sha:
if expected_profile_digest and expected_profile_digest != catalog.profile_digest:
raise ValueError(
"Harness E2E profile changed after preview; create a new preview"
)
if (
not expected_profile_digest
and expected_catalog_sha
and expected_catalog_sha != catalog_sha
):
raise ValueError(
f"Harness E2E catalog moved from {expected_catalog_sha} to {catalog_sha}; create a new preview"
)
Expand Down Expand Up @@ -168,6 +177,7 @@ def main() -> int:
parser.add_argument("--scenarios-json", default="[]")
parser.add_argument("--catalog-sha", required=True)
parser.add_argument("--expected-catalog-sha", default="")
parser.add_argument("--expected-profile-digest", default="")
parser.add_argument("--output", type=Path)
args = parser.parse_args()
try:
Expand All @@ -178,6 +188,7 @@ def main() -> int:
requested=parse_scenarios_json(args.scenarios_json, "requested scenarios"),
catalog_sha=args.catalog_sha,
expected_catalog_sha=args.expected_catalog_sha,
expected_profile_digest=args.expected_profile_digest,
)
except (FileNotFoundError, ValueError) as error:
raise SystemExit(f"invalid Harness E2E profile: {error}") from error
Expand Down
20 changes: 18 additions & 2 deletions .github/scripts/tests/test_harness_e2e_profiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_custom_superset_is_promotable():
assert resolved["promotion_eligible"] is True


def test_rejects_unknown_scenario_and_stale_catalog():
def test_rejects_unknown_scenario_and_changed_profile():
catalog = load_profile_catalog()
with pytest.raises(ValueError, match="unknown Harness E2E scenarios"):
resolve_profile(
Expand All @@ -67,17 +67,33 @@ def test_rejects_unknown_scenario_and_stale_catalog():
requested=["missing"],
catalog_sha="a" * 40,
)
with pytest.raises(ValueError, match="catalog moved"):
with pytest.raises(ValueError, match="profile changed"):
resolve_profile(
catalog,
available=list(catalog.ids),
profile="release",
requested=[],
catalog_sha="a" * 40,
expected_catalog_sha="b" * 40,
expected_profile_digest="0" * 64,
)


def test_allows_release_bump_commits_when_profile_digest_is_unchanged():
catalog = load_profile_catalog()
resolved = resolve_profile(
catalog,
available=list(catalog.ids),
profile="release",
requested=[],
catalog_sha="b" * 40,
expected_catalog_sha="a" * 40,
expected_profile_digest=catalog.profile_digest,
)
assert resolved["catalog_sha"] == "b" * 40
assert resolved["profile_digest"] == catalog.profile_digest


def test_catalog_rejects_duplicate_scenarios(tmp_path: Path):
path = tmp_path / "release-workers.yaml"
path.write_text(
Expand Down
9 changes: 8 additions & 1 deletion .github/workflows/_harness-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ on:
required: false
type: string
default: ''
profile_digest:
description: Canonical scenario profile digest used by the caller preview
required: false
type: string
default: ''
coverage:
description: Instrument the E2E stack and produce an LLVM coverage report
required: false
Expand Down Expand Up @@ -328,6 +333,7 @@ jobs:
VALIDATION_PROFILE: ${{ inputs.validation_profile }}
REQUESTED_SCENARIOS: ${{ inputs.scenarios_json }}
EXPECTED_CATALOG_SHA: ${{ inputs.catalog_sha }}
EXPECTED_PROFILE_DIGEST: ${{ inputs.profile_digest }}
# The instrumented binary would otherwise drop default.profraw in cwd.
LLVM_PROFILE_FILE: ${{ inputs.coverage && format('{0}/target/coverage/build-profraw/%m_%p.profraw', github.workspace) || '' }}
run: |
Expand All @@ -338,7 +344,8 @@ jobs:
--profile "$VALIDATION_PROFILE" \
--scenarios-json "$REQUESTED_SCENARIOS" \
--catalog-sha "${{ github.sha }}" \
--expected-catalog-sha "$EXPECTED_CATALOG_SHA")
--expected-catalog-sha "$EXPECTED_CATALOG_SHA" \
--expected-profile-digest "$EXPECTED_PROFILE_DIGEST")
{
echo "json=$(jq -c .scenarios <<<"$resolved")"
echo "required=$(jq -c .required_scenarios <<<"$resolved")"
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/harness-e2e-deployed.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ on:
description: Release catalog SHA used by Release Control preview
type: string
default: ''
profile_digest:
description: Canonical scenario profile digest used by Release Control preview
type: string
default: ''

permissions:
actions: read
Expand Down Expand Up @@ -97,6 +101,7 @@ jobs:
validation_profile: ${{ inputs.validation_profile }}
scenarios_json: ${{ inputs.scenarios_json }}
catalog_sha: ${{ inputs.catalog_sha }}
profile_digest: ${{ inputs.profile_digest }}
subjects: ${{ inputs.subjects || vars.HARNESS_E2E_SUBJECTS || '[{"id":"anthropic-sonnet","model":"claude-sonnet-4-6","provider":"anthropic"}]' }}
judge_model: ${{ vars.HARNESS_E2E_JUDGE_MODEL || 'claude-sonnet-4-6' }}
judge_provider: ${{ vars.HARNESS_E2E_JUDGE_PROVIDER || 'anthropic' }}
Expand Down
20 changes: 16 additions & 4 deletions harness/tests/e2e/run-deployed-ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -311,11 +311,15 @@ printf 'workers: []\n' >"$project_dir/config.yaml"
engine_pid=$!
wait_for_engine

workers=("harness@$worker_tag" "database@$worker_tag" "fp@$worker_tag" "web@$worker_tag")
# The candidate channel belongs only to the workers pinned in stack_versions.
# Auxiliary E2E workers are not released as part of this operation and may not
# expose the candidate tag at all, so keep them on their stable channel.
support_worker_tag=latest
workers=("database@$support_worker_tag" "fp@$support_worker_tag" "web@$support_worker_tag")
declare -A providers=()
for provider in "$HARNESS_E2E_PROVIDER" "$HARNESS_E2E_JUDGE_PROVIDER"; do
if [[ -z "${providers[$provider]:-}" ]]; then
workers+=("provider-$provider@$worker_tag")
workers+=("provider-$provider@$support_worker_tag")
providers[$provider]=1
fi
done
Expand All @@ -337,14 +341,22 @@ add_with_retry() {
return 1
}

log "Installing registry stack: ${workers[*]}"
log "Installing stable E2E support stack: ${workers[*]}"
add_with_retry worker-add "${workers[@]}"

# Bring the exact dependencies up first and the released worker last. For a
# Harness candidate this prevents Harness from briefly booting against the
# stable dependency graph before its candidate dependencies replace it.
while IFS=$'\t' read -r candidate_worker candidate_version; do
log "Installing exact stack candidate: ${candidate_worker}@${candidate_version}"
add_with_retry "candidate-${candidate_worker}" \
"${candidate_worker}@${candidate_version}" --force
done < <(jq -r 'to_entries | sort_by(.key)[] | [.key, .value] | @tsv' <<<"$stack_versions")
done < <(jq -r --arg release_worker "$HARNESS_E2E_RELEASE_WORKER" '
to_entries
| sort_by([if .key == $release_worker then 1 else 0 end, .key])[]
| [.key, .value]
| @tsv
' <<<"$stack_versions")

wait_for_functions \
harness::send harness::status worker::add database::query state::get \
Expand Down
Loading