diff --git a/.github/scripts/parse_release_tag.py b/.github/scripts/parse_release_tag.py index 68c8af9d0..cb3e73b18 100644 --- a/.github/scripts/parse_release_tag.py +++ b/.github/scripts/parse_release_tag.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Parse a release tag and emit setup outputs for release.yml. -Writes 12 keys to $GITHUB_OUTPUT: +Writes 13 keys to $GITHUB_OUTPUT: tag, worker, version, deploy, language, bin, manifest, - registry_tag, is_prerelease, dry_run, targets, experimental + registry_tag, is_prerelease, dry_run, targets, experimental, tag_sha """ from __future__ import annotations @@ -11,6 +11,7 @@ import os import pathlib import re +import subprocess import sys sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) @@ -19,7 +20,7 @@ TAG_RE = re.compile(r"^([a-z0-9][a-z0-9_-]*)/v(.+)$") DRY_RUN_RE = re.compile(r"-dry-run\.\d+$") -PRERELEASE_RE = re.compile(r"-[a-z]+\.\d+$") +STABLE_VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") def main(argv: list[str] | None = None) -> int: @@ -36,10 +37,10 @@ def main(argv: list[str] | None = None) -> int: if DRY_RUN_RE.search(version): dry_run, is_pre = "true", "true" - elif PRERELEASE_RE.search(version): - dry_run, is_pre = "false", "true" - else: + elif STABLE_VERSION_RE.fullmatch(version): dry_run, is_pre = "false", "false" + else: + dry_run, is_pre = "false", "true" worker_dir = pathlib.Path(worker) try: @@ -57,6 +58,9 @@ def main(argv: list[str] | None = None) -> int: annotation = _lib.read_tag_annotation(raw) registry_tag = annotation.get("registry-tag", "latest") or "latest" + tag_sha = subprocess.check_output( + ["git", "rev-list", "-n", "1", raw], text=True + ).strip() # Anything but a literal `true` is false: a lightweight tag, a missing # line, or a typo publishes as stable. Marking a worker experimental is @@ -87,6 +91,7 @@ def main(argv: list[str] | None = None) -> int: ("dry_run", dry_run), ("targets", targets), ("experimental", experimental), + ("tag_sha", tag_sha), ] gh_out = os.environ.get("GITHUB_OUTPUT") diff --git a/.github/scripts/registry_release.py b/.github/scripts/registry_release.py new file mode 100644 index 000000000..275c578d6 --- /dev/null +++ b/.github/scripts/registry_release.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Promote a staged worker release through the Registry HTTP API.""" + +from __future__ import annotations + +import argparse +import json +import os +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +class RegistryError(RuntimeError): + pass + + +def request_json( + method: str, + url: str, + payload: dict[str, Any], + *, + api_key: str | None = None, +) -> tuple[int, dict[str, Any]]: + body = json.dumps(payload).encode() + headers = {"Content-Type": "application/json"} + if api_key: + headers["X-API-Key"] = api_key + request = urllib.request.Request(url, data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return response.status, json.loads(response.read().decode()) + except urllib.error.HTTPError as error: + try: + response = json.loads(error.read().decode()) + except (json.JSONDecodeError, UnicodeDecodeError): + response = {"error": f"HTTP {error.code}"} + return error.code, response + + +def resolved_root_version(response: dict[str, Any]) -> str: + root = response.get("root") + version = root.get("version") if isinstance(root, dict) else None + if not isinstance(version, str) or not version: + raise RegistryError("Registry resolve response has no root.version") + return version + + +def resolve_version(api_url: str, worker: str, selector: str, *, allow_missing: bool = False) -> str | None: + status, response = request_json( + "POST", + f"{api_url.rstrip('/')}/resolve", + {"worker": worker, "version": selector}, + ) + if status == 200: + return resolved_root_version(response) + error = response.get("error") + code = error.get("code") if isinstance(error, dict) else None + if allow_missing and code in {"version_not_found", "worker_not_found"}: + return None + raise RegistryError(f"resolve {worker}@{selector} failed with HTTP {status}: {json.dumps(response)}") + + +def promotion_payload(version: str, current_latest: str | None) -> dict[str, str]: + payload = {"version": version, "expected_tag": "next"} + if current_latest is not None: + payload["expected_current_version"] = current_latest + return payload + + +def promote(api_url: str, api_key: str, worker: str, version: str) -> dict[str, Any]: + current_latest = resolve_version(api_url, worker, "latest", allow_missing=True) + current_next = resolve_version(api_url, worker, "next", allow_missing=True) + # A first promotion must still own `next`. Once Registry latest already + # points at the requested immutable version, allow an idempotent retry to + # repair GitHub/GHCR/Slack even if a newer candidate has moved `next`. + if current_latest != version and current_next != version: + raise RegistryError(f"next points to {current_next}, expected {version}") + encoded_worker = urllib.parse.quote(worker, safe="") + status, response = request_json( + "PUT", + f"{api_url.rstrip('/')}/w/{encoded_worker}/tags/latest", + promotion_payload(version, current_latest), + api_key=api_key, + ) + if status != 200: + raise RegistryError(f"promotion failed with HTTP {status}: {json.dumps(response)}") + + promoted = resolve_version(api_url, worker, "latest") + if promoted != version: + raise RegistryError(f"promotion verification resolved {promoted}, expected {version}") + + return { + "worker": worker, + "version": version, + "previous_latest": current_latest, + "next": current_next, + "latest": promoted, + "changed": bool(response.get("changed")), + "registry_response": response, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--api-url", default="https://api.workers.iii.dev") + parser.add_argument("--worker", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + api_key = os.environ.get("WORKERS_REGISTRY_API_KEY", "") + if not api_key: + raise SystemExit("WORKERS_REGISTRY_API_KEY is required") + try: + result = promote(args.api_url, api_key, args.worker, args.version) + except RegistryError as error: + raise SystemExit(str(error)) from error + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/release_candidate.py b/.github/scripts/release_candidate.py new file mode 100644 index 000000000..95c0fe143 --- /dev/null +++ b/.github/scripts/release_candidate.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Build and validate evidence for a staged worker release.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + + +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") + + +def parse_bool(value: str) -> bool: + normalized = value.strip().lower() + if normalized == "true": + return True + if normalized == "false": + return False + raise argparse.ArgumentTypeError("expected true or false") + + +def build_evidence(args: argparse.Namespace) -> dict: + results = { + "publish": args.publish_result, + "candidate_smoke": args.candidate_smoke_result, + "harness_quickstart": args.harness_quickstart_result, + "harness_e2e": args.harness_e2e_result, + } + candidate_ready = results["publish"] == "success" and results["candidate_smoke"] == "success" + if args.harness_gate_required: + candidate_ready = ( + candidate_ready + and results["harness_quickstart"] == "success" + and results["harness_e2e"] == "success" + ) + + return { + "schema_version": 1, + "repository": args.repository, + "release_run_id": args.release_run_id, + "run_attempt": args.run_attempt, + "tag_sha": args.tag_sha, + "release_tag": args.release_tag, + "worker": args.worker, + "version": args.version, + "deploy": args.deploy, + "registry_tag": args.registry_tag, + "harness_gate_required": args.harness_gate_required, + "promotable": args.promotable, + "candidate_ready": candidate_ready, + "results": results, + } + + +def validate_evidence(args: argparse.Namespace) -> dict: + evidence = json.loads(args.evidence.read_text()) + failures: list[str] = [] + + expected = { + "schema_version": 1, + "repository": args.repository, + "release_run_id": args.release_run_id, + "release_tag": f"{args.worker}/v{args.version}", + "worker": args.worker, + "version": args.version, + "registry_tag": "next", + "candidate_ready": True, + "promotable": True, + } + for key, value in expected.items(): + if evidence.get(key) != value: + failures.append(f"{key}: expected {value!r}, got {evidence.get(key)!r}") + + if not VERSION_RE.fullmatch(args.version): + failures.append("version must be stable semver MAJOR.MINOR.PATCH") + if not SHA_RE.fullmatch(str(evidence.get("tag_sha", ""))): + failures.append("tag_sha must be a full lowercase commit SHA") + if not isinstance(evidence.get("run_attempt"), int) or evidence["run_attempt"] < 1: + failures.append("run_attempt must be a positive integer") + + results = evidence.get("results") + if not isinstance(results, dict): + failures.append("results must be an object") + else: + if results.get("publish") != "success": + failures.append("publish gate did not succeed") + if results.get("candidate_smoke") != "success": + failures.append("candidate smoke gate did not succeed") + if evidence.get("harness_gate_required"): + if results.get("harness_quickstart") != "success": + failures.append("Harness quickstart gate did not succeed") + if results.get("harness_e2e") != "success": + failures.append("Harness E2E gate did not succeed") + + if failures: + raise SystemExit("invalid release candidate evidence:\n- " + "\n- ".join(failures)) + return evidence + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + build = subparsers.add_parser("build") + build.add_argument("--repository", required=True) + build.add_argument("--release-run-id", required=True) + build.add_argument("--run-attempt", type=int, required=True) + build.add_argument("--tag-sha", required=True) + build.add_argument("--release-tag", required=True) + build.add_argument("--worker", required=True) + build.add_argument("--version", required=True) + build.add_argument("--deploy", choices=("binary", "image", "bundle"), required=True) + build.add_argument("--registry-tag", required=True) + build.add_argument("--harness-gate-required", type=parse_bool, required=True) + build.add_argument("--promotable", type=parse_bool, required=True) + build.add_argument("--publish-result", required=True) + build.add_argument("--candidate-smoke-result", required=True) + build.add_argument("--harness-quickstart-result", required=True) + build.add_argument("--harness-e2e-result", required=True) + build.add_argument("--output", type=Path, required=True) + + validate = subparsers.add_parser("validate") + validate.add_argument("--evidence", type=Path, required=True) + validate.add_argument("--repository", required=True) + validate.add_argument("--release-run-id", required=True) + validate.add_argument("--worker", required=True) + validate.add_argument("--version", required=True) + validate.add_argument("--output", type=Path) + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.command == "build": + evidence = build_evidence(args) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n") + else: + evidence = validate_evidence(args) + rendered = json.dumps(evidence, sort_keys=True) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n") + print(rendered) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/tests/test_parse_release_tag.py b/.github/scripts/tests/test_parse_release_tag.py index d227e899a..18d9c7bb7 100644 --- a/.github/scripts/tests/test_parse_release_tag.py +++ b/.github/scripts/tests/test_parse_release_tag.py @@ -69,6 +69,7 @@ def test_stable_binary_tag(self, tmp_path): assert out["registry_tag"] == "latest" assert out["is_prerelease"] == "false" assert out["dry_run"] == "false" + assert len(out["tag_sha"]) == 40 def test_prerelease_sets_is_prerelease(self, tmp_path): repo = make_repo_with_tagged_worker(tmp_path, "smoke/v1.2.3-rc.1", "1.2.3-rc.1", @@ -82,6 +83,19 @@ def test_prerelease_sets_is_prerelease(self, tmp_path): assert out["dry_run"] == "false" assert out["registry_tag"] == "next" + def test_non_numeric_prerelease_suffix_is_not_promotable_stable(self, tmp_path): + repo = make_repo_with_tagged_worker( + tmp_path, + "smoke/v1.2.3-preview", + "1.2.3-preview", + registry_tag_line="registry-tag: next", + ) + out_path = tmp_path / "gh_output" + out_path.touch() + r = run_script(repo, "smoke/v1.2.3-preview", out_path) + assert r.returncode == 0 + assert parse_outputs(out_path)["is_prerelease"] == "true" + def test_dry_run_tag(self, tmp_path): repo = make_repo_with_tagged_worker(tmp_path, "smoke/v9.9.9-dry-run.1", "9.9.9-dry-run.1") out_path = tmp_path / "gh_output" diff --git a/.github/scripts/tests/test_registry_release.py b/.github/scripts/tests/test_registry_release.py new file mode 100644 index 000000000..4f339041f --- /dev/null +++ b/.github/scripts/tests/test_registry_release.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import pytest + +import registry_release +from registry_release import RegistryError, promotion_payload, resolved_root_version + + +def test_promotion_payload_uses_source_and_destination_preconditions(): + assert promotion_payload("1.2.3", "1.2.2") == { + "version": "1.2.3", + "expected_tag": "next", + "expected_current_version": "1.2.2", + } + + +def test_first_promotion_omits_missing_latest_precondition(): + assert promotion_payload("1.0.0", None) == { + "version": "1.0.0", + "expected_tag": "next", + } + + +def test_resolved_root_version_reads_resolver_contract(): + assert resolved_root_version({"root": {"name": "pdf", "version": "0.2.0"}}) == "0.2.0" + + +def test_resolved_root_version_rejects_malformed_response(): + with pytest.raises(RegistryError, match="root.version"): + resolved_root_version({"graph": []}) + + +def test_idempotent_retry_can_repair_metadata_after_next_moves(monkeypatch): + def resolve(_api_url, _worker, selector, *, allow_missing=False): + return "1.2.3" if selector == "latest" else "1.2.4" + + monkeypatch.setattr(registry_release, "resolve_version", resolve) + monkeypatch.setattr( + registry_release, + "request_json", + lambda *_args, **_kwargs: (200, {"changed": False}), + ) + result = registry_release.promote("https://registry.test", "key", "pdf", "1.2.3") + assert result["latest"] == "1.2.3" + assert result["next"] == "1.2.4" + assert result["changed"] is False + + +def test_stale_candidate_is_rejected_before_tag_update(monkeypatch): + def resolve(_api_url, _worker, selector, *, allow_missing=False): + assert allow_missing is True + return "1.2.2" if selector == "latest" else "1.2.4" + + monkeypatch.setattr(registry_release, "resolve_version", resolve) + called = False + + def request(*_args, **_kwargs): + nonlocal called + called = True + return 200, {} + + monkeypatch.setattr(registry_release, "request_json", request) + with pytest.raises(RegistryError, match="next points to 1.2.4"): + registry_release.promote("https://registry.test", "key", "pdf", "1.2.3") + assert called is False diff --git a/.github/scripts/tests/test_release_candidate.py b/.github/scripts/tests/test_release_candidate.py new file mode 100644 index 000000000..da874087a --- /dev/null +++ b/.github/scripts/tests/test_release_candidate.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pytest + +from release_candidate import build_evidence, validate_evidence + + +def build_args(**overrides): + values = { + "repository": "iii-hq/workers", + "release_run_id": "1234", + "run_attempt": 1, + "tag_sha": "a" * 40, + "release_tag": "harness/v1.2.3", + "worker": "harness", + "version": "1.2.3", + "deploy": "binary", + "registry_tag": "next", + "harness_gate_required": True, + "promotable": True, + "publish_result": "success", + "candidate_smoke_result": "success", + "harness_quickstart_result": "success", + "harness_e2e_result": "success", + } + values.update(overrides) + return argparse.Namespace(**values) + + +def validate_args(evidence: Path, **overrides): + values = { + "evidence": evidence, + "repository": "iii-hq/workers", + "release_run_id": "1234", + "worker": "harness", + "version": "1.2.3", + "output": None, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def write_evidence(tmp_path: Path, evidence: dict) -> Path: + path = tmp_path / "evidence.json" + path.write_text(json.dumps(evidence)) + return path + + +def test_build_marks_complete_harness_candidate_ready(): + evidence = build_evidence(build_args()) + assert evidence["candidate_ready"] is True + assert evidence["promotable"] is True + + +def test_build_requires_harness_e2e_when_applicable(): + evidence = build_evidence(build_args(harness_e2e_result="failure")) + assert evidence["candidate_ready"] is False + + +def test_non_harness_candidate_ignores_skipped_harness_gates(): + evidence = build_evidence( + build_args( + worker="pdf", + release_tag="pdf/v1.2.3", + harness_gate_required=False, + harness_quickstart_result="skipped", + harness_e2e_result="skipped", + ) + ) + assert evidence["candidate_ready"] is True + + +def test_validate_accepts_matching_promotable_evidence(tmp_path): + path = write_evidence(tmp_path, build_evidence(build_args())) + assert validate_evidence(validate_args(path))["worker"] == "harness" + + +@pytest.mark.parametrize( + ("override", "message"), + [ + ({"promotable": False}, "promotable"), + ({"candidate_ready": False}, "candidate_ready"), + ({"release_tag": "harness/v9.9.9"}, "release_tag"), + ({"registry_tag": "latest"}, "registry_tag"), + ], +) +def test_validate_rejects_mismatched_or_unready_evidence(tmp_path, override, message): + evidence = build_evidence(build_args()) + evidence.update(override) + path = write_evidence(tmp_path, evidence) + with pytest.raises(SystemExit, match=message): + validate_evidence(validate_args(path)) + + +def test_validate_rejects_prerelease_version(tmp_path): + evidence = build_evidence( + build_args(version="1.2.3-alpha.1", release_tag="harness/v1.2.3-alpha.1", promotable=False) + ) + path = write_evidence(tmp_path, evidence) + with pytest.raises(SystemExit, match="stable semver"): + validate_evidence(validate_args(path, version="1.2.3-alpha.1")) diff --git a/.github/workflows/_candidate-smoke.yml b/.github/workflows/_candidate-smoke.yml new file mode 100644 index 000000000..b41c33bdf --- /dev/null +++ b/.github/workflows/_candidate-smoke.yml @@ -0,0 +1,169 @@ +name: Worker candidate smoke + +on: + workflow_call: + inputs: + ref: + description: Git tag containing the worker release + required: true + type: string + worker: + description: Registry worker name + required: true + type: string + version: + description: Exact candidate version expected behind next + required: true + type: string + api_url: + description: Workers Registry base URL + required: false + type: string + default: https://api.workers.iii.dev + +permissions: + contents: read + +jobs: + smoke: + name: resolve, install, and boot ${{ inputs.worker }}@next + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + III_API_URL: ${{ inputs.api_url }} + SMOKE_DIR: ${{ github.workspace }}/target/release-candidate-smoke + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Install smoke dependencies + run: python3 -m pip install --quiet pyyaml + + - name: Install stable iii CLI + run: | + set -euo pipefail + curl -fsSL --retry 3 --retry-all-errors --retry-delay 5 \ + https://install.iii.dev/iii/main/install.sh -o /tmp/install-iii.sh + sh /tmp/install-iii.sh + { + echo "$HOME/.local/bin" + echo "$HOME/.iii/bin" + } >>"$GITHUB_PATH" + export PATH="$HOME/.local/bin:$HOME/.iii/bin:$PATH" + iii --version + + - name: Start empty engine + run: | + set -euo pipefail + mkdir -p "$SMOKE_DIR/project" + printf 'workers: []\n' >"$SMOKE_DIR/project/config.yaml" + cd "$SMOKE_DIR/project" + setsid iii -c config.yaml --no-update-check >"$SMOKE_DIR/engine.log" 2>&1 & + echo "$!" >"$SMOKE_DIR/engine.pid" + for _ in {1..120}; do + if ! kill -0 "$(cat "$SMOKE_DIR/engine.pid")" 2>/dev/null; then + echo "::error::iii engine exited before becoming ready" + cat "$SMOKE_DIR/engine.log" + exit 1 + fi + if iii trigger engine::workers::list --json '{}' >"$SMOKE_DIR/workers-baseline.json" 2>/dev/null; then + exit 0 + fi + sleep 1 + done + echo "::error::iii engine did not become ready" + cat "$SMOKE_DIR/engine.log" + exit 1 + + - name: Snapshot trigger baseline + run: | + set -euo pipefail + cd "$SMOKE_DIR/project" + iii trigger engine::triggers::list \ + --json '{"include_internal": false}' >"$SMOKE_DIR/trigger-types-baseline.json" + + - name: Install candidate through next + env: + WORKER: ${{ inputs.worker }} + run: | + set -euo pipefail + cd "$SMOKE_DIR/project" + timeout --signal=TERM --kill-after=15s 600 \ + iii worker add "${WORKER}@next" --force --reset-config \ + 2>&1 | tee "$SMOKE_DIR/worker-add.log" + + - name: Verify exact candidate lock + env: + WORKER: ${{ inputs.worker }} + VERSION: ${{ inputs.version }} + run: | + python3 .github/scripts/verify_registry_lock.py \ + --lock "$SMOKE_DIR/project/iii.lock" \ + --required "$WORKER" \ + --worker "$WORKER" \ + --version "$VERSION" \ + --output "$SMOKE_DIR/lock-verification.json" + + - name: Collect candidate interface + env: + WORKER: ${{ inputs.worker }} + run: | + python3 .github/scripts/collect_worker_interface.py \ + --worker "$WORKER" \ + --out "$SMOKE_DIR/worker-interface.json" \ + --wait-seconds 180 \ + --trigger-types-baseline "$SMOKE_DIR/trigger-types-baseline.json" \ + --workers-baseline "$SMOKE_DIR/workers-baseline.json" + python3 .github/scripts/collect_worker_interface.py \ + --assert-non-empty \ + --assert-typed-schemas \ + --assert-file "$SMOKE_DIR/worker-interface.json" + + - name: Write smoke context + if: always() + env: + WORKER: ${{ inputs.worker }} + VERSION: ${{ inputs.version }} + RELEASE_TAG: ${{ inputs.ref }} + run: | + mkdir -p "$SMOKE_DIR" + jq -n \ + --arg worker "$WORKER" \ + --arg version "$VERSION" \ + --arg release_tag "$RELEASE_TAG" \ + --arg registry_tag next \ + '{worker: $worker, version: $version, release_tag: $release_tag, registry_tag: $registry_tag}' \ + >"$SMOKE_DIR/release-context.json" + + - name: Show logs + if: failure() + run: | + for log in "$SMOKE_DIR"/*.log; do + [[ -f "$log" ]] || continue + echo "::group::$(basename "$log")" + tail -n 300 "$log" || true + echo "::endgroup::" + done + + - name: Upload smoke evidence + if: always() + uses: actions/upload-artifact@v6 + with: + name: release-candidate-smoke-${{ inputs.worker }}-${{ inputs.version }} + path: ${{ env.SMOKE_DIR }}/ + retention-days: 14 + if-no-files-found: error + + - name: Stop smoke stack + if: always() + run: | + set +e + if [[ -f "$SMOKE_DIR/project/config.yaml" ]]; then + cd "$SMOKE_DIR/project" + iii worker remove -y "${{ inputs.worker }}" >/dev/null 2>&1 || true + fi + if [[ -f "$SMOKE_DIR/engine.pid" ]]; then + kill -- "-$(cat "$SMOKE_DIR/engine.pid")" 2>/dev/null || \ + kill "$(cat "$SMOKE_DIR/engine.pid")" 2>/dev/null || true + fi diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index 6b78eafbc..ab3afa755 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -60,9 +60,9 @@ jobs: run: | OWNER="${GITHUB_REPOSITORY_OWNER,,}" BASE="ghcr.io/${OWNER}/${WORKER}" - echo "base=${BASE}" >> "$GITHUB_OUTPUT" - echo "image_tag=${BASE}:${VERSION}" >> "$GITHUB_OUTPUT" { + echo "base=${BASE}" + echo "image_tag=${BASE}:${VERSION}" echo "tags<> "$GITHUB_OUTPUT" - echo "version=$new_ver" >> "$GITHUB_OUTPUT" - echo "tag=${WORKER}/v${new_ver}" >> "$GITHUB_OUTPUT" + { + echo "current=$current" + echo "version=$new_ver" + echo "tag=${WORKER}/v${new_ver}" + } >> "$GITHUB_OUTPUT" echo "::notice::${WORKER}: ${current} -> ${new_ver}" - name: Validate manifest update @@ -218,7 +212,7 @@ jobs: - name: Create and push annotated tag env: TAG: ${{ steps.versions.outputs.tag }} - REGISTRY_TAG: ${{ inputs.tag }} + REGISTRY_TAG: next WORKER: ${{ inputs.worker }} NEW_VERSION: ${{ steps.versions.outputs.version }} EXPERIMENTAL: ${{ inputs.experimental }} diff --git a/.github/workflows/harness-e2e-deployed.yml b/.github/workflows/harness-e2e-deployed.yml index d7af02589..556e127b9 100644 --- a/.github/workflows/harness-e2e-deployed.yml +++ b/.github/workflows/harness-e2e-deployed.yml @@ -3,8 +3,14 @@ name: Harness E2E deployed on: workflow_dispatch: inputs: - channel: - description: Registry channel containing the published artifacts + cli_channel: + description: iii installer channel + required: true + type: choice + options: [latest, next] + default: latest + registry_tag: + description: Registry tag for the baseline Harness stack required: true type: choice options: [latest, next] @@ -51,7 +57,7 @@ permissions: contents: read concurrency: - group: harness-e2e-deployed-${{ inputs.release_worker }}-${{ inputs.release_version }}-${{ inputs.channel }} + group: harness-e2e-deployed-${{ inputs.release_worker }}-${{ inputs.release_version }}-${{ inputs.cli_channel }}-${{ inputs.registry_tag }} cancel-in-progress: false jobs: @@ -62,7 +68,8 @@ jobs: runs: ${{ inputs.runs }} max_parallel: 2 benchmark_lane: deployed - registry_tag: ${{ inputs.channel }} + cli_channel: ${{ inputs.cli_channel }} + registry_tag: ${{ inputs.registry_tag }} release_tag: ${{ inputs.release_tag }} release_worker: ${{ inputs.release_worker }} release_version: ${{ inputs.release_version }} diff --git a/.github/workflows/harness-quickstart.yml b/.github/workflows/harness-quickstart.yml index 0d60c138d..24bccd974 100644 --- a/.github/workflows/harness-quickstart.yml +++ b/.github/workflows/harness-quickstart.yml @@ -3,10 +3,45 @@ name: Harness quickstart on: schedule: - cron: "17 5 * * *" + workflow_call: + inputs: + cli_channel: + description: iii installer channel + type: string + default: latest + registry_tag: + description: Registry tag for the baseline Harness stack + type: string + default: latest + release_worker: + description: Worker whose exact candidate version must be installed + type: string + default: '' + release_version: + description: Exact candidate version + type: string + default: '' + release_tag: + description: Git tag associated with the candidate + type: string + default: '' + release_run_id: + description: Publishing Release workflow run id + type: string + default: '' + cascade_e2e: + description: Dispatch deployed E2E after quickstart + type: boolean + default: false workflow_dispatch: inputs: - channel: - description: Installer channel to validate + cli_channel: + description: iii installer channel to validate + type: choice + options: [latest, next] + default: latest + registry_tag: + description: Registry worker tag to validate type: choice options: [latest, next] default: latest @@ -35,20 +70,20 @@ permissions: contents: read concurrency: - group: harness-quickstart-${{ inputs.channel || github.event_name }}-${{ inputs.release_version || 'rolling' }} + group: harness-quickstart-${{ inputs.cli_channel || github.event_name }}-${{ inputs.registry_tag || github.event_name }}-${{ inputs.release_version || 'rolling' }} cancel-in-progress: ${{ inputs.release_version == '' }} jobs: validate: - name: harness quickstart / published registry (${{ matrix.channel }}) + name: harness quickstart / iii ${{ matrix.lane.cli }} / workers ${{ matrix.lane.registry }} strategy: fail-fast: false matrix: - channel: >- + lane: >- ${{ github.event_name == 'schedule' - && fromJSON('["latest","next"]') - || fromJSON(format('["{0}"]', inputs.channel)) + && fromJSON('[{"cli":"latest","registry":"latest"},{"cli":"next","registry":"next"}]') + || fromJSON(format('[{{"cli":"{0}","registry":"{1}"}}]', inputs.cli_channel, inputs.registry_tag)) }} runs-on: ubuntu-latest timeout-minutes: 20 @@ -60,7 +95,10 @@ jobs: env: HARNESS_QUICKSTART_ARTIFACTS_DIR: ${{ github.workspace }}/target/harness-quickstart HARNESS_QUICKSTART_TRACE: "1" - III_CHANNEL: ${{ matrix.channel }} + III_CLI_CHANNEL: ${{ matrix.lane.cli }} + III_WORKER_TAG: ${{ matrix.lane.registry }} + HARNESS_QUICKSTART_RELEASE_WORKER: ${{ inputs.release_worker }} + HARNESS_QUICKSTART_RELEASE_VERSION: ${{ inputs.release_version }} run: harness/tests/quickstart/run-ci.sh - name: Verify the released artifact was resolved @@ -70,7 +108,8 @@ jobs: RELEASE_VERSION: ${{ inputs.release_version }} RELEASE_TAG: ${{ inputs.release_tag }} RELEASE_RUN_ID: ${{ inputs.release_run_id }} - CHANNEL: ${{ matrix.channel }} + CLI_CHANNEL: ${{ matrix.lane.cli }} + REGISTRY_TAG: ${{ matrix.lane.registry }} run: | set -euo pipefail python3 -m pip install --quiet pyyaml @@ -87,8 +126,9 @@ jobs: --arg version "$RELEASE_VERSION" \ --arg tag "$RELEASE_TAG" \ --arg run_id "$RELEASE_RUN_ID" \ - --arg channel "$CHANNEL" \ - '{worker: $worker, version: $version, tag: $tag, release_run_id: $run_id, channel: $channel}' \ + --arg cli_channel "$CLI_CHANNEL" \ + --arg registry_tag "$REGISTRY_TAG" \ + '{worker: $worker, version: $version, tag: $tag, release_run_id: $run_id, cli_channel: $cli_channel, registry_tag: $registry_tag}' \ >target/harness-quickstart/release-context.json - name: Show quickstart logs @@ -110,9 +150,9 @@ jobs: echo if [[ -f target/harness-quickstart/result.json ]]; then jq -r ' - "| Status | Channel | CLI | Duration |", - "| --- | --- | --- | ---: |", - "| \(.status) | \(.channel) | \(.cli_version) | \(.elapsed_ms) ms |", + "| Status | CLI channel | Worker tag | CLI | Duration |", + "| --- | --- | --- | --- | ---: |", + "| \(.status) | \(.cli_channel) | \(.worker_tag) | \(.cli_version) | \(.elapsed_ms) ms |", (if .failure_reason != "" then "", "**Failure:** \(.failure_reason)" else empty end) ' target/harness-quickstart/result.json else @@ -124,7 +164,7 @@ jobs: if: always() uses: actions/upload-artifact@v6 with: - name: harness-quickstart-${{ matrix.channel }} + name: harness-quickstart-${{ matrix.lane.cli }}-${{ matrix.lane.registry }} path: target/harness-quickstart/ retention-days: 7 if-no-files-found: ignore @@ -141,7 +181,8 @@ jobs: - name: Dispatch deployed E2E CI env: GH_TOKEN: ${{ github.token }} - CHANNEL: ${{ inputs.channel }} + CLI_CHANNEL: ${{ inputs.cli_channel }} + REGISTRY_TAG: ${{ inputs.registry_tag }} WORKER: ${{ inputs.release_worker }} VERSION: ${{ inputs.release_version }} RELEASE_TAG: ${{ inputs.release_tag }} @@ -157,7 +198,8 @@ jobs: gh workflow run harness-e2e-deployed.yml \ --repo "$GITHUB_REPOSITORY" \ --ref "$WORKFLOW_REF" \ - --field "channel=$CHANNEL" \ + --field "cli_channel=$CLI_CHANNEL" \ + --field "registry_tag=$REGISTRY_TAG" \ --field "release_worker=$WORKER" \ --field "release_version=$VERSION" \ --field "release_tag=$RELEASE_TAG" \ diff --git a/.github/workflows/promote-worker.yml b/.github/workflows/promote-worker.yml new file mode 100644 index 000000000..035c108c3 --- /dev/null +++ b/.github/workflows/promote-worker.yml @@ -0,0 +1,257 @@ +name: Promote Worker + +on: + workflow_dispatch: + inputs: + worker: + description: Worker candidate to promote + required: true + type: string + version: + description: Stable semver candidate version + required: true + type: string + release_run_id: + description: Release workflow run containing the candidate evidence + required: true + type: string + +permissions: + actions: read + contents: write + packages: write + +concurrency: + group: promote-worker-${{ inputs.worker }} + cancel-in-progress: false + +jobs: + promote: + name: Promote ${{ inputs.worker }} v${{ inputs.version }} + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + WORKER: ${{ inputs.worker }} + VERSION: ${{ inputs.version }} + RELEASE_RUN_ID: ${{ inputs.release_run_id }} + API_URL: https://api.workers.iii.dev + steps: + - uses: actions/checkout@v5 + with: + ref: main + fetch-depth: 0 + fetch-tags: true + + - name: Validate dispatch context + run: | + set -euo pipefail + [[ "$GITHUB_REF_NAME" == main ]] || { + echo "::error::Promotions must be dispatched from main" + exit 2 + } + [[ "$WORKER" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || { + echo "::error::worker must be a Registry slug" + exit 2 + } + [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "::error::version must be stable semver MAJOR.MINOR.PATCH" + exit 2 + } + [[ "$RELEASE_RUN_ID" =~ ^[0-9]+$ ]] || { + echo "::error::release_run_id must be numeric" + exit 2 + } + + - name: Validate Release workflow run + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RELEASE_RUN_ID" >release-run.json + run_name=$(jq -r .name release-run.json) + [[ "$run_name" == Release ]] || { + echo "::error::run $RELEASE_RUN_ID belongs to '$run_name', expected 'Release'" + exit 1 + } + event=$(jq -r .event release-run.json) + [[ "$event" == push || "$event" == workflow_dispatch ]] || { + echo "::error::Release run event '$event' is not promotable" + exit 1 + } + + - name: Download candidate evidence + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + artifact="release-candidate-${WORKER}-${VERSION}" + mkdir -p evidence + gh run download "$RELEASE_RUN_ID" \ + --repo "$GITHUB_REPOSITORY" \ + --name "$artifact" \ + --dir evidence + test -s evidence/release-candidate.json + + - name: Validate candidate identity and gates + id: candidate + run: | + set -euo pipefail + python3 .github/scripts/release_candidate.py validate \ + --evidence evidence/release-candidate.json \ + --repository "$GITHUB_REPOSITORY" \ + --release-run-id "$RELEASE_RUN_ID" \ + --worker "$WORKER" \ + --version "$VERSION" \ + --output validated-candidate.json + + tag="${WORKER}/v${VERSION}" + git fetch origin "+refs/tags/${tag}:refs/tags/${tag}" + tag_sha=$(git rev-list -n 1 "$tag") + evidence_sha=$(jq -r .tag_sha validated-candidate.json) + [[ "$tag_sha" == "$evidence_sha" ]] || { + echo "::error::Git tag resolves to $tag_sha, evidence records $evidence_sha" + exit 1 + } + run_attempt=$(jq -r .run_attempt release-run.json) + evidence_attempt=$(jq -r .run_attempt validated-candidate.json) + [[ "$run_attempt" == "$evidence_attempt" ]] || { + echo "::error::Release run attempt is $run_attempt, evidence records $evidence_attempt" + exit 1 + } + + deploy=$(jq -r .deploy validated-candidate.json) + echo "tag=$tag" >>"$GITHUB_OUTPUT" + echo "deploy=$deploy" >>"$GITHUB_OUTPUT" + + - name: Validate GitHub prerelease + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.candidate.outputs.tag }} + run: | + set -euo pipefail + gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \ + --json tagName,isDraft,isPrerelease,url >github-release-before.json + jq -e --arg tag "$TAG" '.tagName == $tag and .isDraft == false' \ + github-release-before.json >/dev/null + + - name: Promote Registry release tag + env: + WORKERS_REGISTRY_API_KEY: ${{ secrets.WORKERS_REGISTRY_API_KEY }} + run: | + python3 .github/scripts/registry_release.py \ + --api-url "$API_URL" \ + --worker "$WORKER" \ + --version "$VERSION" \ + --output registry-promotion.json + + - name: Set up Docker Buildx + if: steps.candidate.outputs.deploy == 'image' + uses: docker/setup-buildx-action@v4 + + - name: Log in to ghcr.io + if: steps.candidate.outputs.deploy == 'image' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Promote image latest alias + if: steps.candidate.outputs.deploy == 'image' + run: | + set -euo pipefail + owner=${GITHUB_REPOSITORY_OWNER,,} + version_ref="ghcr.io/${owner}/${WORKER}:${VERSION}" + latest_ref="ghcr.io/${owner}/${WORKER}:latest" + docker buildx imagetools create --tag "$latest_ref" "$version_ref" + docker buildx imagetools inspect "$version_ref" --raw >version-manifest.json + docker buildx imagetools inspect "$latest_ref" --raw >latest-manifest.json + cmp version-manifest.json latest-manifest.json + + - name: Finalize GitHub Release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.candidate.outputs.tag }} + run: | + set -euo pipefail + gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" \ + --prerelease=false --latest=false + gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \ + --json tagName,isDraft,isPrerelease,url >github-release-after.json + jq -e --arg tag "$TAG" \ + '.tagName == $tag and .isDraft == false and .isPrerelease == false' \ + github-release-after.json >/dev/null + + - name: Announce promoted release + id: slack + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + CHANNEL: worker-releases + TAG: ${{ steps.candidate.outputs.tag }} + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + release_url=$(jq -r .url github-release-after.json) + text="๐Ÿš€ ${WORKER} v${VERSION} promoted to @latest โ€” <${release_url}|GitHub Release>" + payload=$(jq -n --arg channel "$CHANNEL" --arg text "$text" \ + '{channel: $channel, text: $text}') + resp=$(curl -sf -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$payload") + jq -e .ok <<<"$resp" >/dev/null || { + echo "::error::Slack API error: $(jq -r .error <<<"$resp")" + exit 1 + } + echo "ts=$(jq -r .ts <<<"$resp")" >>"$GITHUB_OUTPUT" + + notes=$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json body --jq .body 2>/dev/null || true) + if [[ -n "$notes" ]]; then + notes=$(printf '%s' "$notes" | head -c 2900) + thread_payload=$(jq -n \ + --arg channel "$CHANNEL" \ + --arg text "$notes" \ + --arg ts "$(jq -r .ts <<<"$resp")" \ + '{channel: $channel, text: $text, thread_ts: $ts}') + thread_resp=$(curl -sf -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$thread_payload") + jq -e .ok <<<"$thread_resp" >/dev/null || { + echo "::error::Slack thread API error: $(jq -r .error <<<"$thread_resp")" + exit 1 + } + fi + + - name: Write promotion summary + if: always() + run: | + { + echo "### Worker promotion" + echo + echo "- Candidate: \`${WORKER}@${VERSION}\`" + echo "- Release run: [${RELEASE_RUN_ID}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${RELEASE_RUN_ID})" + if [[ -f registry-promotion.json ]]; then + previous=$(jq -r '.previous_latest // "none"' registry-promotion.json) + changed=$(jq -r .changed registry-promotion.json) + echo "- Previous latest: \`${previous}\`" + echo "- Registry changed: \`${changed}\`" + fi + if [[ -f github-release-after.json ]]; then + echo "- GitHub Release: $(jq -r .url github-release-after.json)" + fi + } >>"$GITHUB_STEP_SUMMARY" + + - name: Upload promotion evidence + if: always() + uses: actions/upload-artifact@v6 + with: + name: promotion-${{ inputs.worker }}-${{ inputs.version }} + path: | + validated-candidate.json + release-run.json + github-release-before.json + github-release-after.json + registry-promotion.json + retention-days: 30 + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 335ef25d4..82f674d21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,6 +64,7 @@ on: type: string permissions: + actions: read contents: write packages: write @@ -91,9 +92,12 @@ jobs: dry_run: ${{ steps.meta.outputs.dry_run }} targets: ${{ steps.meta.outputs.targets }} experimental: ${{ steps.meta.outputs.experimental }} + tag_sha: ${{ steps.meta.outputs.tag_sha }} web_bundle: ${{ steps.web.outputs.needed }} interface_smoke: ${{ steps.smoke.outputs.interface_smoke }} harness_smoke: ${{ steps.harness_smoke.outputs.enabled }} + staged: ${{ steps.release_state.outputs.staged }} + promotable: ${{ steps.release_state.outputs.promotable }} steps: - uses: actions/checkout@v5 with: @@ -176,6 +180,27 @@ jobs: echo "enabled=$enabled" >> "$GITHUB_OUTPUT" echo "::notice::Harness quickstart smoke target=$enabled worker=$WORKER" + - name: Classify staged release + id: release_state + env: + REGISTRY_TAG: ${{ steps.meta.outputs.registry_tag }} + INTERFACE_SMOKE: ${{ steps.smoke.outputs.interface_smoke }} + IS_PRERELEASE: ${{ steps.meta.outputs.is_prerelease }} + DRY_RUN: ${{ steps.meta.outputs.dry_run }} + run: | + set -euo pipefail + staged=false + promotable=false + if [[ "$REGISTRY_TAG" == next && "$INTERFACE_SMOKE" == true && "$DRY_RUN" != true ]]; then + staged=true + if [[ "$IS_PRERELEASE" != true ]]; then + promotable=true + fi + fi + echo "staged=$staged" >>"$GITHUB_OUTPUT" + echo "promotable=$promotable" >>"$GITHUB_OUTPUT" + echo "::notice::staged=$staged promotable=$promotable" + # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # Create the GitHub Release shell (skipped on dry runs). # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -195,7 +220,8 @@ jobs: tag_name: ${{ needs.setup.outputs.tag }} name: ${{ needs.setup.outputs.worker }} ${{ needs.setup.outputs.version }} draft: false - prerelease: ${{ needs.setup.outputs.is_prerelease == 'true' }} + prerelease: ${{ needs.setup.outputs.is_prerelease == 'true' || needs.setup.outputs.staged == 'true' }} + make_latest: false generate_release_notes: true # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -211,7 +237,7 @@ jobs: bin_name: ${{ needs.setup.outputs.bin }} manifest_path: ${{ needs.setup.outputs.worker }}/${{ needs.setup.outputs.manifest }} tag_name: ${{ needs.setup.outputs.tag }} - is_prerelease: ${{ needs.setup.outputs.is_prerelease == 'true' }} + is_prerelease: ${{ needs.setup.outputs.is_prerelease == 'true' || needs.setup.outputs.staged == 'true' }} skip_create_release: true dry_run: ${{ needs.setup.outputs.dry_run == 'true' }} targets: ${{ needs.setup.outputs.targets }} @@ -249,7 +275,7 @@ jobs: version: ${{ needs.setup.outputs.version }} language: ${{ needs.setup.outputs.language }} tag_name: ${{ needs.setup.outputs.tag }} - is_prerelease: ${{ needs.setup.outputs.is_prerelease == 'true' }} + is_prerelease: ${{ needs.setup.outputs.is_prerelease == 'true' || needs.setup.outputs.staged == 'true' }} dry_run: ${{ needs.setup.outputs.dry_run == 'true' }} secrets: inherit @@ -273,48 +299,114 @@ jobs: experimental: ${{ needs.setup.outputs.experimental }} secrets: inherit - # Dispatch the independent published-artifact smoke CI immediately after - # the registry deployment of Harness or one of its mandatory dependencies. - # The smoke CI dispatches the deployed E2E CI after it passes. The release - # itself intentionally does not wait for either child workflow. - dispatch-smoke: - name: Dispatch Harness smoke CI + candidate-smoke: + name: Validate published candidate needs: [setup, publish] - if: >- - ${{ - !failure() && - !cancelled() && - needs.setup.outputs.harness_smoke == 'true' && - needs.setup.outputs.dry_run != 'true' && - needs.publish.result == 'success' && - (needs.setup.outputs.registry_tag == 'latest' || needs.setup.outputs.registry_tag == 'next') - }} + if: ${{ !failure() && !cancelled() && needs.setup.outputs.staged == 'true' && needs.publish.result == 'success' }} + uses: ./.github/workflows/_candidate-smoke.yml + with: + ref: ${{ inputs.tag || github.ref }} + worker: ${{ needs.setup.outputs.worker }} + version: ${{ needs.setup.outputs.version }} + + harness-quickstart: + name: Validate candidate in Harness quickstart + needs: [setup, candidate-smoke] + if: ${{ !failure() && !cancelled() && needs.setup.outputs.staged == 'true' && needs.setup.outputs.harness_smoke == 'true' }} + uses: ./.github/workflows/harness-quickstart.yml + with: + cli_channel: latest + registry_tag: latest + release_worker: ${{ needs.setup.outputs.worker }} + release_version: ${{ needs.setup.outputs.version }} + release_tag: ${{ needs.setup.outputs.tag }} + release_run_id: ${{ github.run_id }} + cascade_e2e: false + + harness-e2e: + name: Validate candidate in deployed Harness E2E + needs: [setup, harness-quickstart] + if: ${{ !failure() && !cancelled() && needs.setup.outputs.staged == 'true' && needs.setup.outputs.harness_smoke == 'true' }} + uses: ./.github/workflows/_harness-e2e.yml + with: + stack_mode: registry + runs: 1 + max_parallel: 2 + benchmark_lane: deployed + cli_channel: latest + registry_tag: latest + release_tag: ${{ needs.setup.outputs.tag }} + release_worker: ${{ needs.setup.outputs.worker }} + release_version: ${{ needs.setup.outputs.version }} + release_url: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ needs.setup.outputs.tag }} + release_run_id: ${{ github.run_id }} + subjects: ${{ vars.HARNESS_E2E_SUBJECTS || '[{"id":"anthropic-sonnet","model":"claude-sonnet-4-6","provider":"anthropic"}]' }} + judge_model: ${{ vars.HARNESS_E2E_JUDGE_MODEL || 'claude-sonnet-4-6' }} + judge_provider: ${{ vars.HARNESS_E2E_JUDGE_PROVIDER || 'anthropic' }} + secrets: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + openai_api_key: ${{ secrets.OPENAI_API_KEY }} + zai_api_key: ${{ secrets.ZAI_API_KEY }} + + candidate-ready: + name: Record candidate evidence + needs: [setup, publish, candidate-smoke, harness-quickstart, harness-e2e] + if: ${{ always() && needs.setup.result == 'success' && needs.setup.outputs.staged == 'true' }} + runs-on: ubuntu-latest permissions: - actions: write contents: read - runs-on: ubuntu-latest steps: - - name: Dispatch published-artifact smoke CI + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.tag || github.ref }} + + - name: Build candidate evidence env: - GH_TOKEN: ${{ github.token }} - CHANNEL: ${{ needs.setup.outputs.registry_tag }} + REPOSITORY: ${{ github.repository }} + RELEASE_RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + TAG_SHA: ${{ needs.setup.outputs.tag_sha }} + RELEASE_TAG: ${{ needs.setup.outputs.tag }} WORKER: ${{ needs.setup.outputs.worker }} VERSION: ${{ needs.setup.outputs.version }} - RELEASE_TAG: ${{ needs.setup.outputs.tag }} - RELEASE_RUN_ID: ${{ github.run_id }} - WORKFLOW_REF: ${{ github.ref_name }} + DEPLOY: ${{ needs.setup.outputs.deploy }} + REGISTRY_TAG: ${{ needs.setup.outputs.registry_tag }} + HARNESS_GATE_REQUIRED: ${{ needs.setup.outputs.harness_smoke }} + PROMOTABLE: ${{ needs.setup.outputs.promotable }} + PUBLISH_RESULT: ${{ needs.publish.result }} + CANDIDATE_SMOKE_RESULT: ${{ needs.candidate-smoke.result }} + HARNESS_QUICKSTART_RESULT: ${{ needs.harness-quickstart.result }} + HARNESS_E2E_RESULT: ${{ needs.harness-e2e.result }} run: | - set -euo pipefail - gh workflow run harness-quickstart.yml \ - --repo "$GITHUB_REPOSITORY" \ - --ref "$WORKFLOW_REF" \ - --field "channel=$CHANNEL" \ - --field "release_worker=$WORKER" \ - --field "release_version=$VERSION" \ - --field "release_tag=$RELEASE_TAG" \ - --field "release_run_id=$RELEASE_RUN_ID" \ - --field cascade_e2e=true - echo "Dispatched Harness smoke CI for $WORKER v$VERSION ($CHANNEL)." + python3 .github/scripts/release_candidate.py build \ + --repository "$REPOSITORY" \ + --release-run-id "$RELEASE_RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" \ + --tag-sha "$TAG_SHA" \ + --release-tag "$RELEASE_TAG" \ + --worker "$WORKER" \ + --version "$VERSION" \ + --deploy "$DEPLOY" \ + --registry-tag "$REGISTRY_TAG" \ + --harness-gate-required "$HARNESS_GATE_REQUIRED" \ + --promotable "$PROMOTABLE" \ + --publish-result "$PUBLISH_RESULT" \ + --candidate-smoke-result "$CANDIDATE_SMOKE_RESULT" \ + --harness-quickstart-result "$HARNESS_QUICKSTART_RESULT" \ + --harness-e2e-result "$HARNESS_E2E_RESULT" \ + --output release-candidate.json + cat release-candidate.json + + - name: Upload candidate evidence + uses: actions/upload-artifact@v6 + with: + name: release-candidate-${{ needs.setup.outputs.worker }}-${{ needs.setup.outputs.version }} + path: release-candidate.json + retention-days: 30 + if-no-files-found: error + + - name: Require all candidate gates + run: jq -e '.candidate_ready == true' release-candidate.json >/dev/null # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ # Announce the release in Slack (#worker-releases). Terminal job: @@ -323,8 +415,23 @@ jobs: # โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ announce: name: Announce on Slack - needs: [setup, binary-build, container-build, bundle-build, publish] - if: ${{ !failure() && !cancelled() && needs.setup.outputs.dry_run != 'true' }} + needs: [setup, binary-build, container-build, bundle-build, publish, candidate-ready] + if: >- + ${{ + always() && + needs.setup.result == 'success' && + needs.setup.outputs.dry_run != 'true' && + ( + (needs.setup.outputs.staged == 'true' && needs.candidate-ready.result == 'success') || + ( + needs.setup.outputs.staged != 'true' && + needs.binary-build.result != 'failure' && + needs.container-build.result != 'failure' && + needs.bundle-build.result != 'failure' && + (needs.setup.outputs.interface_smoke == 'false' || needs.publish.result == 'success') + ) + ) + }} runs-on: ubuntu-latest steps: - name: Post to Slack @@ -335,12 +442,21 @@ jobs: WORKER: ${{ needs.setup.outputs.worker }} VERSION: ${{ needs.setup.outputs.version }} IS_PRERELEASE: ${{ needs.setup.outputs.is_prerelease }} + STAGED: ${{ needs.setup.outputs.staged }} + PROMOTABLE: ${{ needs.setup.outputs.promotable }} RELEASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ needs.setup.outputs.tag }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -euo pipefail - suffix="" - if [[ "$IS_PRERELEASE" == "true" ]]; then suffix=" (pre-release)"; fi - text="๐Ÿš€ ${WORKER} v${VERSION}${suffix} released โ€” <${RELEASE_URL}|GitHub Release>" + if [[ "$STAGED" == true && "$PROMOTABLE" == true ]]; then + text="๐Ÿงช ${WORKER} v${VERSION} candidate passed on @next โ€” <${RUN_URL}|promote manually> ยท <${RELEASE_URL}|GitHub prerelease>" + elif [[ "$STAGED" == true ]]; then + text="๐Ÿงช ${WORKER} v${VERSION} prerelease passed on @next โ€” <${RELEASE_URL}|GitHub prerelease>" + else + suffix="" + if [[ "$IS_PRERELEASE" == "true" ]]; then suffix=" (pre-release)"; fi + text="๐Ÿš€ ${WORKER} v${VERSION}${suffix} released โ€” <${RELEASE_URL}|GitHub Release>" + fi payload=$(jq -n --arg channel "$CHANNEL" --arg text "$text" '{channel: $channel, text: $text}') resp=$(curl -sf -X POST https://slack.com/api/chat.postMessage \ -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \ diff --git a/docs/sops/release.md b/docs/sops/release.md index a58fcea31..29c44f6a1 100644 --- a/docs/sops/release.md +++ b/docs/sops/release.md @@ -8,6 +8,8 @@ [`.github/workflows/_container.yml`](../../.github/workflows/_container.yml), [`.github/workflows/_bundle.yml`](../../.github/workflows/_bundle.yml), [`.github/workflows/_publish-registry.yml`](../../.github/workflows/_publish-registry.yml), +[`.github/workflows/_candidate-smoke.yml`](../../.github/workflows/_candidate-smoke.yml), +[`.github/workflows/promote-worker.yml`](../../.github/workflows/promote-worker.yml), [`.github/scripts/parse_release_tag.py`](../../.github/scripts/parse_release_tag.py). On conflict, the workflow wins โ€” update this doc. @@ -20,7 +22,7 @@ verifying registry publish. One-time wiring is in [`new-worker.md`](new-worker.m - **Access:** GitHub Actions on this repo; org secrets configured (do not paste values): - `III_CI_APP_ID` / `III_CI_APP_PRIVATE_KEY` โ€” bot commit + tag push in Create Tag - - `WORKERS_REGISTRY_API_KEY` โ€” `POST /publish` and `POST /w//skills` + - `WORKERS_REGISTRY_API_KEY` โ€” publish, skills, and Registry tag promotion - **Worker wired:** `create-tag.yml` options + `release.yml` tag pattern (see [`new-worker.md`](new-worker.md) ยง6). - **Local green:** lint + tests for the worker; Rust binary: `--manifest` JSON valid. @@ -35,7 +37,6 @@ Actions โ†’ **Create Tag**: |---|---| | Worker | Folder name (must be in workflow options) | | Bump | `patch` / `minor` / `major` | -| Registry tag | `latest` or `next` โ€” channel for `iii worker add` resolution | | Experimental | Checkbox. Marks the worker experimental in the registry โ€” see [Experimental releases](#experimental-releases) | The workflow: @@ -43,7 +44,7 @@ The workflow: 1. Bumps version in the worker manifest (`Cargo.toml`, `package.json`, โ€ฆ). 2. Commits `chore(): bump to vX.Y.Z` to `main`. 3. Creates and pushes an **annotated** tag `/vX.Y.Z` with - `registry-tag: ` and `experimental: ` in the tag + `registry-tag: next` and `experimental: ` in the tag message. ### 2. Release pipeline @@ -53,24 +54,33 @@ Tag push triggers **Release** (`release.yml`): ```mermaid flowchart LR createTag[Create Tag] -->|"tag worker/vX.Y.Z"| setupJob[setup] - setupJob --> ghRelease[create GitHub Release] + setupJob --> ghRelease[create public GitHub prerelease] ghRelease --> buildBinary["binary: _rust-binary.yml"] ghRelease --> buildImage["image: _container.yml"] ghRelease --> buildBundle["bundle: _bundle.yml"] buildBinary --> publishJob[_publish-registry.yml] buildImage --> publishJob buildBundle --> publishJob - publishJob --> postPublish["POST /publish + skills"] + publishJob --> candidateSmoke[resolve / install / boot next] + candidateSmoke --> harnessGate{Harness or dependency?} + harnessGate -->|yes| quickstart[Harness quickstart] + quickstart --> e2e[Harness deployed E2E] + harnessGate -->|no| evidence[candidate evidence] + e2e --> evidence + evidence --> promotion[manual Promote Worker] + promotion --> latest[Registry latest + GitHub Release] ``` | Stage | Job | Output | |---|---|---| | setup | Parse tag + `iii.worker.yaml`; detect web bundle / smoke opt-out | worker, version, deploy, targets, โ€ฆ | -| create-release | GitHub Release shell | Release page for the tag | +| create-release | Public GitHub prerelease, never repository-global Latest | Release page and downloadable assets | | binary-build | `_rust-binary.yml` | Per-target `.tar.gz` / `.zip` + `.sha256` on the Release | | container-build | `_container.yml` | Multi-arch image at `ghcr.io//` | | bundle-build | `_bundle.yml` | `.tar.gz` + `.sha256` on the Release | | publish | `_publish-registry.yml` | Registry manifest + optional skills upload | +| candidate-smoke | Resolve `next`, install it, boot it, and verify the exact lock/interface | Published-artifact evidence | +| candidate-ready | Fold required gate results into one immutable artifact | `release-candidate--` | `deploy` from `iii.worker.yaml` selects exactly one build job. @@ -92,34 +102,62 @@ flowchart LR Workers with `interface_smoke: false` skip the entire publish job. -### 4. Registry tag semantics +### 4. Candidate gates + +Every Registry-published release resolves and installs `worker@next`, checks the +expected version in `iii.lock`, and verifies the registered interface. Harness +and its mandatory dependencies additionally run the published quickstart and +deployed E2E in the same Release run. Those gates use the stable CLI and stable +baseline stack, then replace the released worker with `worker@`. + +`interface_smoke: false` workers remain GitHub-only releases and do not enter +the staged Registry flow. + +### 5. Promote to latest + +After `candidate-ready` passes, run Actions โ†’ **Promote Worker** from `main` and +enter the worker, version, and Release run id. The workflow: + +1. Downloads and validates the candidate evidence and Git tag commit. +2. Confirms `next` still points to the exact candidate. +3. Moves the Registry `latest` tag with source and destination preconditions. +4. For image workers, moves `ghcr.io//:latest` to the immutable + version digest. +5. Converts the existing GitHub prerelease to a normal release without changing + the repository-global GitHub Latest release. +6. Posts the final Slack announcement. + +Registry, GitHub Release, and GHCR promotion operations are idempotent. If one +of those later steps fails, rerun the same promotion to repair the remaining +state. A rerun after Slack already accepted the root message can repeat the +announcement, so inspect `#worker-releases` before retrying a Slack-only +failure. + +### 6. Registry tag semantics | Channel | Typical use | |---|---| -| `latest` | Default; what most `iii worker add` installs resolve | -| `next` | Pre-release / risky channel; safer for first publish | +| `latest` | Last manually promoted stable worker version | +| `next` | Current candidate created by the release pipeline | -The channel is stored in the **annotated tag message** (`registry-tag:`). -`release.yml` refetches the annotated tag for this reason. Lightweight tags -lose the channel and default to `latest`. +After promotion, `next` and `latest` may point to the same immutable version. +The next release moves only `next`. The annotated Git tag records the initial +Registry tag; the regular Create Tag workflow always writes `next`. ## Experimental releases Tick **Experimental** on Create Tag to mark the worker unstable in the -registry. It is a badge and nothing else โ€” the version publishes to the -channel you picked, installs normally, and resolves normally. Nobody has to -opt in to see it, and nothing has to be promoted later. +registry. It is a badge and nothing else โ€” the version publishes to `next`, +installs normally, and resolves normally. Promotion does not clear the badge. It travels the same way the channel does: `experimental: true` in the annotated tag message, read by `parse_release_tag.py`, forwarded through `release.yml` to the publish payload. Anything but the literal `true` โ€” a missing line, a lightweight tag, a typo โ€” publishes as stable. -**Leave it unticked once the worker settles.** The registry treats a release -without the flag as the promotion signal and drops the badge, so there is no -separate "make it stable" step. Re-releasing while still experimental keeps -the original mark, so the badge records when the worker first shipped -experimental. +**Leave it unticked once the worker settles.** Publishing a later release +without the flag clears the badge. Registry tag promotion and experimental +maturity are independent states. For what the registry does with the flag, see [`EXPERIMENTAL_WORKERS.md`](https://github.com/iii-hq/registry/blob/main/docs/EXPERIMENTAL_WORKERS.md) @@ -132,6 +170,8 @@ in the registry repo. Actions โ†’ **Release** โ†’ `workflow_dispatch` โ†’ enter the existing tag (e.g. `session-manager/v0.1.0`). No new tag or version bump needed. Concurrency group `release-${{ github.ref }}` serializes per tag. +Duplicate `POST /publish` responses are accepted only when the exact version +already exists and the requested Registry tag still points to it. ### Prerelease @@ -142,7 +182,8 @@ Create Tag cannot produce prerelease suffixes. Push a manual **annotated** tag: ``` With tag message including `registry-tag: next`. Marks the GitHub Release as -prerelease; still builds and publishes (unless `interface_smoke: false`). +prerelease; still builds, publishes, and runs candidate gates, but cannot be +promoted by **Promote Worker** until a stable `MAJOR.MINOR.PATCH` is released. ### Alpha release from a feature branch @@ -217,6 +258,8 @@ refuses existing tags. | artifact resolution 404 | Build job didn't upload for that target | Check GitHub Release assets for the tag | | publish HTTP non-200 | Registry rejection or bad payload | Response body printed in job log; verify `WORKERS_REGISTRY_API_KEY` | | publish skipped entirely | `interface_smoke: false` | Expected for stdio/discovery-only workers | +| promotion evidence rejected | Wrong run/worker/version, failed gate, or prerelease semver | Use the candidate's successful Release run and exact stable version | +| promotion returns `409` | `next` advanced or `latest` changed concurrently | Do not promote the stale candidate; inspect the current Registry tags | On failure, publish dumps `iii-engine.log` and `worker-.log` (last 200 lines). @@ -225,8 +268,8 @@ On failure, publish dumps `iii-engine.log` and `worker-.log` (last 200 l There is **no unpublish**. Recovery: 1. Fix the issue on `main`. -2. Cut a new patch via Create Tag (registry `latest` moves forward). -3. Use `registry-tag: next` when uncertain before promoting to `latest`. +2. Cut a new patch via Create Tag; it publishes to `next`. +3. Validate and manually promote the replacement. GitHub Release assets for the bad version remain (immutable history). @@ -248,9 +291,9 @@ Confirm: ## Announce & organize -Slack announcement is automatic: the terminal `announce` job in -`release.yml` posts `๐Ÿš€ vX.Y.Z` to `#worker-releases` for every -successful non-dry-run release. `SLACK_BOT_TOKEN` is org-level (the same +Slack announcement is automatic: a successful candidate posts `๐Ÿงช` with its +`next` status, while **Promote Worker** posts the final `๐Ÿš€ ... promoted to +@latest` message. `SLACK_BOT_TOKEN` is org-level (the same bot as the iii engine release pipeline); the bot must be invited to `#worker-releases`. The GitHub release-notes body is posted as a thread reply under the announcement. Ticket association rides on PR titles โ€” diff --git a/harness/tests/e2e/run-deployed-ci.sh b/harness/tests/e2e/run-deployed-ci.sh index 94e5ac1ec..daeeea73d 100755 --- a/harness/tests/e2e/run-deployed-ci.sh +++ b/harness/tests/e2e/run-deployed-ci.sh @@ -17,19 +17,29 @@ repo_root=$(cd -- "$harness_root/.." && pwd) artifact_dir=${HARNESS_E2E_ARTIFACTS_DIR:-"$repo_root/target/harness-e2e"} e2e_bin=${HARNESS_E2E_BIN:-"$harness_root/target/release/harness-e2e"} install_url=${III_INSTALL_URL:-https://install.iii.dev/iii/main/install.sh} -channel=${III_CHANNEL:-latest} +cli_channel=${III_CLI_CHANNEL:-latest} +worker_tag=${III_WORKER_TAG:-latest} runs=${HARNESS_E2E_RUNS:-1} engine_port=49134 wait_seconds=180 add_timeout_seconds=600 -case "$channel" in +if [[ -n "${III_CHANNEL:-}" ]]; then + echo "III_CHANNEL was split into III_CLI_CHANNEL and III_WORKER_TAG" >&2 + exit 2 +fi + +case "$cli_channel" in latest | next) ;; *) - echo "III_CHANNEL must be latest or next" >&2 + echo "III_CLI_CHANNEL must be latest or next" >&2 exit 2 ;; esac +[[ "$worker_tag" =~ ^[A-Za-z0-9._-]+$ ]] || { + echo "III_WORKER_TAG must be a valid Registry tag" >&2 + exit 2 +} [[ -x "$e2e_bin" ]] || { echo "Harness E2E binary is not executable: $e2e_bin" >&2 exit 2 @@ -49,7 +59,8 @@ mkdir -p "$project_dir" "$e2e_home" export HOME="$e2e_home" export XDG_CONFIG_HOME="$e2e_home/.config" export PATH="$e2e_home/.local/bin:$e2e_home/.iii/bin:$PATH" -export III_CHANNEL="$channel" +export III_CLI_CHANNEL="$cli_channel" +export III_WORKER_TAG="$worker_tag" iii_bin="" engine_pid="" @@ -76,7 +87,8 @@ write_deployment_result() { --arg reason "$failure_reason" \ --arg phase "$failure_phase" \ --arg cli_version "$cli_version" \ - --arg channel "$channel" \ + --arg cli_channel "$cli_channel" \ + --arg worker_tag "$worker_tag" \ --arg release_worker "$HARNESS_E2E_RELEASE_WORKER" \ --arg release_version "$HARNESS_E2E_RELEASE_VERSION" \ --arg actual_release_version "$actual_release_version" \ @@ -89,7 +101,8 @@ write_deployment_result() { failure_reason: $reason, failure_phase: $phase, cli_version: $cli_version, - channel: $channel, + cli_channel: $cli_channel, + worker_tag: $worker_tag, release_worker: $release_worker, release_version: $release_version, actual_release_version: $actual_release_version, @@ -237,10 +250,10 @@ wait_for_model() { die "model $provider/$model did not resolve within ${wait_seconds}s" } -log "Installing iii from $channel" +log "Installing iii from $cli_channel" curl -fsSL --retry 3 --retry-all-errors --retry-delay 5 \ "$install_url" -o "$run_root/install.sh" -if [[ "$channel" == next ]]; then +if [[ "$cli_channel" == next ]]; then sh "$run_root/install.sh" --next 2>&1 | tee "$log_dir/install.log" else sh "$run_root/install.sh" 2>&1 | tee "$log_dir/install.log" @@ -255,11 +268,11 @@ printf 'workers: []\n' >"$project_dir/config.yaml" engine_pid=$! wait_for_engine -workers=(harness database) +workers=("harness@$worker_tag" "database@$worker_tag") declare -A providers=() for provider in "$HARNESS_E2E_PROVIDER" "$HARNESS_E2E_JUDGE_PROVIDER"; do if [[ -z "${providers[$provider]:-}" ]]; then - workers+=("provider-$provider") + workers+=("provider-$provider@$worker_tag") providers[$provider]=1 fi done @@ -268,6 +281,12 @@ log "Installing registry stack: ${workers[*]}" (cd "$project_dir" && timeout --signal=TERM --kill-after=15s "$add_timeout_seconds" \ "$iii_bin" worker add "${workers[@]}") 2>&1 | tee "$log_dir/worker-add.log" +log "Installing exact release candidate: ${HARNESS_E2E_RELEASE_WORKER}@${HARNESS_E2E_RELEASE_VERSION}" +(cd "$project_dir" && timeout --signal=TERM --kill-after=15s "$add_timeout_seconds" \ + "$iii_bin" worker add \ + "${HARNESS_E2E_RELEASE_WORKER}@${HARNESS_E2E_RELEASE_VERSION}" --force) \ + 2>&1 | tee "$log_dir/candidate-override.log" + wait_for_functions \ harness::send harness::status worker::add database::query state::get \ queue::define session::messages context::assemble router::models::get \ @@ -289,7 +308,7 @@ verify_args=( --output "$stack_dir/lock-verification.json" ) for worker in "${workers[@]}"; do - verify_args+=(--required "$worker") + verify_args+=(--required "${worker%@*}") done verification=$(python3 "$repo_root/.github/scripts/verify_registry_lock.py" "${verify_args[@]}") actual_release_version=$(jq -r '.actual_version' <<<"$verification") diff --git a/harness/tests/quickstart/README.md b/harness/tests/quickstart/README.md index c4571ebe5..aea07ea35 100644 --- a/harness/tests/quickstart/README.md +++ b/harness/tests/quickstart/README.md @@ -24,8 +24,11 @@ make -C harness quickstart-validate ``` The machine needs `curl` and `jq`. The default engine and Console ports (`49134` -and `3113`) must be available. The default installer channel is `latest`; set -`III_CHANNEL=next` to validate `next`. +and `3113`) must be available. The CLI installer and Registry worker selectors +are independent: `III_CLI_CHANNEL` chooses `latest` or `next` for `iii`, while +`III_WORKER_TAG` chooses the Registry tag used by `harness` and `console`. +The old combined `III_CHANNEL` variable is rejected to prevent a silent test +against the wrong side of the split. Set `HARNESS_QUICKSTART_TRACE=1` to print only the important external commands (`iii worker add`, `iii trigger`, installer, and engine) and save the list as `commands.log`. Polling attempts, assignments, cleanup, and other shell internals @@ -33,9 +36,10 @@ are omitted. The nightly/manual CI workflow preserves `result.json`, the generated project files, Console responses, raw logs, and the command trace. Release-triggered -runs also verify the exact released worker version in `iii.lock` before -dispatching the deployed Harness E2E workflow. +runs replace the released worker with its exact candidate version and verify it +in `iii.lock`. The Release workflow calls quickstart and deployed E2E +synchronously; manual quickstart runs may still request the E2E cascade. -The nightly schedule runs both `latest` and `next` as independent matrix jobs. -Manual runs select one of those channels through the workflow input. Behavioral -quality remains covered by the Harness E2E workflows. +The nightly schedule runs paired `latest/latest` and `next/next` CLI/worker lanes. +Manual runs select both values explicitly. Behavioral quality remains covered +by the Harness E2E workflows. diff --git a/harness/tests/quickstart/run-ci.sh b/harness/tests/quickstart/run-ci.sh index 5affbbd42..339846606 100755 --- a/harness/tests/quickstart/run-ci.sh +++ b/harness/tests/quickstart/run-ci.sh @@ -9,12 +9,20 @@ script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) repo_root=$(cd -- "$script_dir/../../.." && pwd) artifact_dir=${HARNESS_QUICKSTART_ARTIFACTS_DIR:-"$repo_root/target/harness-quickstart"} install_url=${III_INSTALL_URL:-https://install.iii.dev/iii/main/install.sh} -channel=${III_CHANNEL:-latest} +cli_channel=${III_CLI_CHANNEL:-latest} +worker_tag=${III_WORKER_TAG:-latest} +release_worker=${HARNESS_QUICKSTART_RELEASE_WORKER:-} +release_version=${HARNESS_QUICKSTART_RELEASE_VERSION:-} engine_port=49134 wait_seconds=${HARNESS_QUICKSTART_WAIT_SECONDS:-180} add_timeout_seconds=${HARNESS_QUICKSTART_ADD_TIMEOUT_SECONDS:-600} trace_enabled=${HARNESS_QUICKSTART_TRACE:-0} +if [[ -n "${III_CHANNEL:-}" ]]; then + echo "III_CHANNEL was split into III_CLI_CHANNEL and III_WORKER_TAG" >&2 + exit 2 +fi + case "$trace_enabled" in 0 | 1) ;; *) @@ -23,14 +31,29 @@ case "$trace_enabled" in ;; esac -case "$channel" in +case "$cli_channel" in + latest | next) ;; + *) + echo "III_CLI_CHANNEL must be 'latest' or 'next' (got: $cli_channel)" >&2 + exit 2 + ;; +esac + +case "$worker_tag" in latest | next) ;; *) - echo "III_CHANNEL must be 'latest' or 'next' (got: $channel)" >&2 + echo "III_WORKER_TAG must be 'latest' or 'next' (got: $worker_tag)" >&2 exit 2 ;; esac +if [[ -n "$release_worker" || -n "$release_version" ]]; then + [[ -n "$release_worker" && -n "$release_version" ]] || { + echo "HARNESS_QUICKSTART_RELEASE_WORKER and HARNESS_QUICKSTART_RELEASE_VERSION must be set together" >&2 + exit 2 + } +fi + for command_name in curl jq; do command -v "$command_name" >/dev/null 2>&1 || { echo "$command_name is required" >&2 @@ -119,7 +142,8 @@ write_result() { --arg reason "$failure_reason" \ --arg cli_version "$cli_version" \ --arg install_url "$install_url" \ - --arg channel "$channel" \ + --arg cli_channel "$cli_channel" \ + --arg worker_tag "$worker_tag" \ --argjson elapsed_ms "$(((SECONDS - started_at_seconds) * 1000))" \ --argjson engine_port "$engine_port" \ '{ @@ -127,7 +151,8 @@ write_result() { failure_reason: $reason, cli_version: $cli_version, install_url: $install_url, - channel: $channel, + cli_channel: $cli_channel, + worker_tag: $worker_tag, elapsed_ms: $elapsed_ms, engine_port: $engine_port }' >"$artifact_dir/result.json" @@ -281,11 +306,11 @@ start_engine() { cd "$project_dir" -log "Step 1/6: Install iii from $install_url (channel=$channel)" +log "Step 1/7: Install iii from $install_url (channel=$cli_channel)" log_command "curl -fsSL $install_url -o install.sh" curl -fsSL --retry 3 --retry-connrefused --retry-delay 5 \ "$install_url" -o "$run_root/install.sh" -if [[ "$channel" == "next" ]]; then +if [[ "$cli_channel" == "next" ]]; then log_command "sh install.sh --next" sh "$run_root/install.sh" --next 2>&1 | tee "$log_dir/install.log" else @@ -299,19 +324,28 @@ cli_version=$("$iii_bin" --version 2>&1) printf '%s\n' "$cli_version" >"$artifact_dir/cli-version.txt" ok "installed $cli_version" -log "Step 2/6: Start an empty engine" +log "Step 2/7: Start an empty engine" printf 'workers: []\n' >config.yaml start_engine wait_for_engine -log "Step 3/6: Add harness and Console" -run_worker_add harness console 2>&1 | tee "$log_dir/worker-add.log" +log "Step 3/7: Add harness and Console from worker tag $worker_tag" +run_worker_add "harness@$worker_tag" "console@$worker_tag" 2>&1 | tee "$log_dir/worker-add.log" ok "iii worker add harness console exited successfully" -log "Step 4/6: Verify registered functions" +log "Step 4/7: Apply exact release candidate override" +if [[ -n "$release_worker" ]]; then + run_worker_add "${release_worker}@${release_version}" --force \ + 2>&1 | tee "$log_dir/candidate-override.log" + ok "installed exact candidate ${release_worker}@${release_version}" +else + ok "no release candidate override requested" +fi + +log "Step 5/7: Verify registered functions" wait_for_functions -log "Step 5/6: Verify the Console HTTP surface" +log "Step 6/7: Verify the Console HTTP surface" log_command "iii trigger console::status --port $engine_port --json '{}'" console_status=$("$iii_bin" trigger console::status --port "$engine_port" \ --json '{}' 2>"$log_dir/console-status.log") @@ -323,7 +357,7 @@ curl -fsS --retry 10 --retry-all-errors --retry-delay 1 \ "http://127.0.0.1:$console_port/" -o "$artifact_dir/console.html" ok "Console answered on port $console_port" -log "Step 6/6: Verify generated project files" +log "Step 7/7: Verify generated project files" for output in config.yaml iii.lock; do [[ -s "$output" ]] || die "worker add did not write $output" grep -Eiq 'harness' "$output" || die "$output does not contain harness" @@ -331,4 +365,4 @@ for output in config.yaml iii.lock; do done ok "config.yaml and iii.lock reference harness + console" -log "ALL QUICKSTART ASSERTIONS PASSED ($cli_version, channel=$channel)" +log "ALL QUICKSTART ASSERTIONS PASSED ($cli_version, cli=$cli_channel, workers=$worker_tag)"