diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index d36cae9..57a901b 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -138,6 +138,8 @@ jobs: .release_version == $version and .slo.campaign_count == 4 and .slo.infrastructure_errors == 0 and + (.canaries | length == 3) and + .incident.completed == true and .operator.approval_mode == "solo-maintainer" and .operator.confirmed == true ' "${source_base}.json" >/dev/null diff --git a/docs/production-release-process.md b/docs/production-release-process.md index 557d222..fbf8082 100644 --- a/docs/production-release-process.md +++ b/docs/production-release-process.md @@ -85,10 +85,10 @@ unchanged, leaves the drill alias on the known-good digest, and uploads artifact as the rollback drill note. Download four chronological scheduled campaign artifacts, one expanded Falco -artifact, the attested candidate evidence, the rollback drill note, and the +artifact, the attested candidate evidence, all three RC canary manifests, the +rollback drill note, the fail-closed promotion incident note, and the operator's written promotion confirmation into one private working directory. -Create an -input manifest with paths relative to that directory: +Create an input manifest with paths relative to that directory: ```json { @@ -111,11 +111,33 @@ input manifest with paths relative to that directory: "evidence": "candidate/release-candidate-evidence.json", "evidence_sha256": "" }, + "canaries": [ + { + "milestone": "manual", + "evidence": "canary-manual/release-candidate-canary.json", + "evidence_sha256": "" + }, + { + "milestone": "t-plus-24h", + "evidence": "canary-t-plus-24h/release-candidate-canary.json", + "evidence_sha256": "" + }, + { + "milestone": "t-plus-72h", + "evidence": "canary-t-plus-72h/release-candidate-canary.json", + "evidence_sha256": "" + } + ], "rollback": { "completed": true, "evidence": "rollback/evidence.md", "evidence_sha256": "" }, + "incident": { + "completed": true, + "evidence": "incident/evidence.md", + "evidence_sha256": "" + }, "operator": { "login": "ErenAri", "approval_mode": "solo-maintainer", @@ -126,7 +148,12 @@ input manifest with paths relative to that directory: } ``` -The real manifest contains exactly four campaign entries. Run: +The real manifest contains exactly four campaign entries and exactly three +canary entries in `manual`, `t-plus-24h`, and `t-plus-72h` order. The two timed +canaries must come from `schedule` events, and every canary must bind the same +RC tag, commit, and image digest as the attested candidate. The incident note +must contain `[bpfcompat-promotion-incident:v1]` and describe the deliberately +rejected promotion run. Run: ```bash scripts/production-readiness-report.sh \ @@ -137,6 +164,8 @@ The command writes both Markdown and JSON. Add the reviewed outputs to `docs/releases/bpfcompat-0.4.0-readiness.{md,json}` in the final release pull request. The stable release workflow validates, checksums, attests, and publishes those files; a stable tag fails if they are absent or not `ready`. +The online candidate check requires an authenticated GitHub CLI version that +provides `gh attestation`; an older CLI fails closed. Do not use `BPFCOMPAT_SKIP_READINESS_ATTESTATION=1` outside the regression test; production evidence must verify the candidate attestation online. diff --git a/docs/production-slo-runbook.md b/docs/production-slo-runbook.md index e90b4ff..06edd40 100644 --- a/docs/production-slo-runbook.md +++ b/docs/production-slo-runbook.md @@ -54,8 +54,9 @@ boundary behavior change requires a new release candidate and restarts the window; documentation-only changes do not. Run the evidence aggregator after downloading the four campaign artifacts, -the expanded Falco report, candidate evidence, rollback note, and -solo-maintainer promotion confirmation: +the expanded Falco report, candidate evidence, manual/T+24h/T+72h canary +manifests, rollback and fail-closed incident notes, and solo-maintainer +promotion confirmation: ```bash scripts/production-readiness-report.sh \ diff --git a/docs/production-support-boundary.md b/docs/production-support-boundary.md index c4a081b..e3eb859 100644 --- a/docs/production-support-boundary.md +++ b/docs/production-support-boundary.md @@ -118,5 +118,6 @@ substitute for these technical controls. `scripts/production-readiness-report.sh` validates the campaign manifests, report hashes, Falco profile coverage, attested release-candidate evidence, -rollback evidence, and solo-operator promotion confirmation before producing -the graduation report. +manual/T+24h/T+72h canary evidence, rollback and fail-closed incident evidence, +and solo-operator promotion confirmation before producing the graduation +report. diff --git a/scripts/production-readiness-report.sh b/scripts/production-readiness-report.sh index 5c47933..c5995bb 100755 --- a/scripts/production-readiness-report.sh +++ b/scripts/production-readiness-report.sh @@ -59,7 +59,10 @@ jq -e ' (.campaigns | type == "array" and length == 4) and (.falco | type == "object") and (.candidate | type == "object") and + (.canaries | type == "array" and length == 3) and + ([.canaries[].milestone] == ["manual", "t-plus-24h", "t-plus-72h"]) and (.rollback.completed == true) and + (.incident.completed == true) and (.operator.approval_mode == "solo-maintainer") and (.operator.confirmed == true) ' "$input" >/dev/null || fail "manifest schema or required gates are invalid" @@ -201,16 +204,93 @@ jq -e --arg release_version "$release_version" ' if [[ "${BPFCOMPAT_SKIP_READINESS_ATTESTATION:-0}" != "1" ]]; then command -v gh >/dev/null || fail "gh is required to verify candidate evidence" + gh attestation --help >/dev/null 2>&1 || + fail "installed GitHub CLI lacks attestation support" gh attestation verify "$candidate" \ --repo Kernel-Guard/bpfcompat \ --signer-workflow Kernel-Guard/bpfcompat/.github/workflows/release-artifacts.yml \ >/dev/null || fail "release-candidate evidence attestation verification failed" fi +: >"$tmp/canary_rows" +: >"$tmp/canaries.ndjson" +: >"$tmp/canary_run_ids" +candidate_tag="$(jq -r '.tag' "$candidate")" +candidate_commit="$(jq -r '.commit_sha' "$candidate")" +candidate_image="$(jq -r '.image' "$candidate")" +candidate_digest="${candidate_image##*@}" + +while IFS= read -r canary; do + milestone="$(jq -r '.milestone' <<<"$canary")" + evidence_rel="$(jq -r '.evidence' <<<"$canary")" + canary_evidence="$(evidence_path "$evidence_rel")" + canary_sha="$(jq -r '.evidence_sha256' <<<"$canary")" + verify_hash "$canary_evidence" "$canary_sha" + + expected_event="schedule" + if [[ "$milestone" == "manual" ]]; then + expected_event="workflow_dispatch" + fi + jq -e \ + --arg milestone "$milestone" \ + --arg event "$expected_event" \ + --arg version "$candidate_tag" \ + --arg commit "$candidate_commit" \ + --arg digest "$candidate_digest" \ + ' + .schema_version == "v0.1" and + .marker == "[bpfcompat-rc-canary:v1]" and + .repository == "Kernel-Guard/bpfcompat" and + .milestone == $milestone and + .event == $event and + (.workflow_run_id | type == "number" and . > 0) and + (.external_consumer_run_id | type == "number" and . > 0) and + (.completed_at | fromdateiso8601 > 0) and + .version == $version and + .commit == $commit and + .image_digest == $digest and + .checks.clean_install == "pass" and + .checks.source_build == "pass" and + .checks.published_action == "pass" and + .checks.container == "pass" and + .checks.external_consumers == "pass" and + (.components | type == "array" and length > 0) and + ([.components[] | + (.sha256 | test("^[0-9a-f]{64}$")) and + (.path | type == "string" and length > 0) + ] | all) + ' "$canary_evidence" >/dev/null || + fail "RC canary ${milestone} evidence is invalid or does not match the candidate" + + canary_run_id="$(jq '.workflow_run_id' "$canary_evidence")" + canary_completed_at="$(jq -r '.completed_at' "$canary_evidence")" + printf '%s\n' "$canary_run_id" >>"$tmp/canary_run_ids" + printf "| %s | \`%s\` | %s |\n" \ + "$milestone" "$canary_run_id" "$canary_completed_at" >>"$tmp/canary_rows" + jq -n \ + --arg milestone "$milestone" \ + --argjson workflow_run_id "$canary_run_id" \ + --arg completed_at "$canary_completed_at" \ + --arg evidence_sha256 "$canary_sha" \ + '{ + milestone: $milestone, + workflow_run_id: $workflow_run_id, + completed_at: $completed_at, + evidence_sha256: $evidence_sha256 + }' >>"$tmp/canaries.ndjson" +done < <(jq -c '.canaries[]' "$input") + +[[ "$(sort -u "$tmp/canary_run_ids" | wc -l)" -eq 3 ]] || + fail "RC canary workflow run IDs must be unique" + rollback="$(evidence_path "$(jq -r '.rollback.evidence' "$input")")" verify_hash "$rollback" "$(jq -r '.rollback.evidence_sha256' "$input")" grep -Fq '[bpfcompat-rollback-drill:v1]' "$rollback" || fail "rollback evidence is missing its completion marker" +incident="$(evidence_path "$(jq -r '.incident.evidence' "$input")")" +verify_hash "$incident" "$(jq -r '.incident.evidence_sha256' "$input")" +grep -Fq '[bpfcompat-promotion-incident:v1]' "$incident" || + fail "incident evidence is missing its fail-closed completion marker" operator_evidence="$(evidence_path "$(jq -r '.operator.evidence' "$input")")" verify_hash "$operator_evidence" "$(jq -r '.operator.evidence_sha256' "$input")" operator="$(jq -r '.operator.login' "$input")" @@ -242,6 +322,7 @@ mkdir -p "$(dirname "$output_json")" echo "- Target executions: ${total_targets}" echo "- Infrastructure errors: 0" echo "- Target duration p95: ${p95_ms} ms" + echo "- Release-candidate canary observations: 3" echo "- Release operator: \`${operator}\`" echo "- Approval mode: \`${approval_mode}\` (no independent human approval)" echo @@ -251,26 +332,33 @@ mkdir -p "$(dirname "$output_json")" echo "|---:|---|---|---|---:|" cat "$tmp/campaign_rows" echo + echo "## Release-Candidate Canaries" + echo + echo "| Milestone | Workflow run | Completed (UTC) |" + echo "|---|---|---|" + cat "$tmp/canary_rows" + echo echo "## Required External Evidence" echo echo "- Falco expanded vendor-kernel matrix: PASS" echo "- Attested release candidate: PASS" - echo "- Rollback and incident exercise: PASS" + echo "- RC manual, T+24h, and T+72h canaries: PASS" + echo "- Rollback drill: PASS" + echo "- Fail-closed promotion incident: PASS" echo "- Deliberate solo-maintainer promotion confirmation: PASS" echo echo "Runtime loading, agent, API, registry, SaaS, Firecracker, and virtme-ng are excluded." } >"$output" campaigns_json="$(jq -s '.' "$tmp/campaigns.ndjson")" +canaries_json="$(jq -s '.' "$tmp/canaries.ndjson")" falco_run_id="$(jq '.falco.workflow_run_id' "$input")" falco_commit="$(jq -r '.falco.commit_sha' "$input")" falco_started_at="$(jq -r '.falco.started_at' "$input")" falco_sha="$(jq -r '.falco.report_sha256' "$input")" -candidate_tag="$(jq -r '.tag' "$candidate")" -candidate_commit="$(jq -r '.commit_sha' "$candidate")" -candidate_image="$(jq -r '.image' "$candidate")" candidate_sha="$(jq -r '.candidate.evidence_sha256' "$input")" rollback_sha="$(jq -r '.rollback.evidence_sha256' "$input")" +incident_sha="$(jq -r '.incident.evidence_sha256' "$input")" operator_sha="$(jq -r '.operator.evidence_sha256' "$input")" jq -n \ @@ -280,6 +368,7 @@ jq -n \ --arg supported_boundary "CLI + GitHub Action + disposable QEMU/KVM validation" \ --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --argjson campaigns "$campaigns_json" \ + --argjson canaries "$canaries_json" \ --argjson total_targets "$total_targets" \ --argjson p95_ms "$p95_ms" \ --argjson falco_run_id "$falco_run_id" \ @@ -291,6 +380,7 @@ jq -n \ --arg candidate_image "$candidate_image" \ --arg candidate_sha "$candidate_sha" \ --arg rollback_sha "$rollback_sha" \ + --arg incident_sha "$incident_sha" \ --arg operator "$operator" \ --arg approval_mode "$approval_mode" \ --arg operator_sha "$operator_sha" \ @@ -327,10 +417,15 @@ jq -n \ image: $candidate_image, evidence_sha256: $candidate_sha }, + canaries: $canaries, rollback: { completed: true, evidence_sha256: $rollback_sha }, + incident: { + completed: true, + evidence_sha256: $incident_sha + }, operator: { login: $operator, approval_mode: $approval_mode, diff --git a/scripts/production-readiness-report_test.sh b/scripts/production-readiness-report_test.sh index 85ae5fb..ed2a590 100755 --- a/scripts/production-readiness-report_test.sh +++ b/scripts/production-readiness-report_test.sh @@ -58,7 +58,12 @@ for index in 0 1 2 3; do '. + [{metadata: $metadata, report: $report}]' <<<"$campaigns")" done -mkdir -p "$tmp/falco" "$tmp/candidate" "$tmp/rollback" "$tmp/operator" +mkdir -p \ + "$tmp/falco" \ + "$tmp/candidate" \ + "$tmp/rollback" \ + "$tmp/incident" \ + "$tmp/operator" falco_report="$tmp/falco/modern-bpf-compat.json" make_report "$falco_report" \ ubuntu-22.04-5.15 \ @@ -79,13 +84,76 @@ jq -n --arg image "ghcr.io/kernel-guard/bpfcompat@sha256:$(printf 'b%.0s' {1..64 negative_report_sha256: ("f" * 64) }' >"$candidate" printf '%s\n' '[bpfcompat-rollback-drill:v1] completed' >"$tmp/rollback/evidence.md" +printf '%s\n' '[bpfcompat-promotion-incident:v1] invalid promotion failed closed' >"$tmp/incident/evidence.md" printf '%s\n' '[bpfcompat-solo-promotion:v1] ErenAri confirmed exact evidence' >"$tmp/operator/evidence.md" +canaries='[]' +for index in 0 1 2; do + case "$index" in + 0) + milestone="manual" + event="workflow_dispatch" + ;; + 1) + milestone="t-plus-24h" + event="schedule" + ;; + 2) + milestone="t-plus-72h" + event="schedule" + ;; + esac + dir="$tmp/canary-${index}" + mkdir -p "$dir" + jq -n \ + --arg milestone "$milestone" \ + --arg event "$event" \ + --argjson workflow_run_id "$((3000 + index))" \ + --argjson external_consumer_run_id "$((4000 + index))" \ + --arg completed_at "$(date -u -d "2026-07-30 15:05:34 UTC +$((index * (index + 1) / 2)) days" +%Y-%m-%dT%H:%M:%SZ)" \ + --arg image_digest "sha256:$(printf 'b%.0s' {1..64})" \ + '{ + schema_version: "v0.1", + marker: "[bpfcompat-rc-canary:v1]", + milestone: $milestone, + event: $event, + repository: "Kernel-Guard/bpfcompat", + workflow_run_id: $workflow_run_id, + external_consumer_run_id: $external_consumer_run_id, + completed_at: $completed_at, + version: "v0.4.0-rc.1", + commit: ("c" * 40), + image_digest: $image_digest, + checks: { + clean_install: "pass", + source_build: "pass", + published_action: "pass", + container: "pass", + external_consumers: "pass" + }, + components: [{ + sha256: ("a" * 64), + path: "evidence/component.json" + }] + }' >"$dir/release-candidate-canary.json" + canaries="$(jq \ + --arg milestone "$milestone" \ + --arg evidence "canary-${index}/release-candidate-canary.json" \ + --arg evidence_sha256 "$(sha256sum "$dir/release-candidate-canary.json" | awk '{print $1}')" \ + '. + [{ + milestone: $milestone, + evidence: $evidence, + evidence_sha256: $evidence_sha256 + }]' <<<"$canaries")" +done + jq -n \ --argjson campaigns "$campaigns" \ + --argjson canaries "$canaries" \ --arg falco_sha "$(sha256sum "$falco_report" | awk '{print $1}')" \ --arg candidate_sha "$(sha256sum "$candidate" | awk '{print $1}')" \ --arg rollback_sha "$(sha256sum "$tmp/rollback/evidence.md" | awk '{print $1}')" \ + --arg incident_sha "$(sha256sum "$tmp/incident/evidence.md" | awk '{print $1}')" \ --arg operator_sha "$(sha256sum "$tmp/operator/evidence.md" | awk '{print $1}')" \ '{ schema_version: "v0.1", @@ -102,11 +170,17 @@ jq -n \ evidence: "candidate/release-candidate-evidence.json", evidence_sha256: $candidate_sha }, + canaries: $canaries, rollback: { completed: true, evidence: "rollback/evidence.md", evidence_sha256: $rollback_sha }, + incident: { + completed: true, + evidence: "incident/evidence.md", + evidence_sha256: $incident_sha + }, operator: { login: "ErenAri", approval_mode: "solo-maintainer", @@ -121,11 +195,15 @@ BPFCOMPAT_SKIP_READINESS_ATTESTATION=1 \ grep -Fq 'Gate status: **ready**' "$tmp/readiness.md" grep -Fq 'Scheduled campaigns: 4' "$tmp/readiness.md" grep -Fq 'Falco expanded vendor-kernel matrix: PASS' "$tmp/readiness.md" +grep -Fq 'RC manual, T+24h, and T+72h canaries: PASS' "$tmp/readiness.md" +grep -Fq 'Fail-closed promotion incident: PASS' "$tmp/readiness.md" jq -e ' .gate_status == "ready" and .release_version == "0.4.0" and .slo.campaign_count == 4 and .slo.infrastructure_errors == 0 and + (.canaries | length == 3) and + .incident.completed == true and .operator.login == "ErenAri" and .operator.approval_mode == "solo-maintainer" and .operator.confirmed == true @@ -138,6 +216,41 @@ if BPFCOMPAT_SKIP_READINESS_ATTESTATION=1 \ exit 1 fi +jq 'del(.canaries[2])' "$tmp/input.json" >"$tmp/bad-missing-canary.json" +if BPFCOMPAT_SKIP_READINESS_ATTESTATION=1 \ + "$script" "$tmp/bad-missing-canary.json" "$tmp/bad.md" >/dev/null 2>&1; then + echo "[production-readiness-test] missing T+72h canary was accepted" >&2 + exit 1 +fi + +cp "$tmp/canary-1/release-candidate-canary.json" "$tmp/canary-1-valid.json" +jq '.event = "workflow_dispatch"' \ + "$tmp/canary-1/release-candidate-canary.json" >"$tmp/bad-canary.json" +mv "$tmp/bad-canary.json" "$tmp/canary-1/release-candidate-canary.json" +bad_canary_sha="$(sha256sum "$tmp/canary-1/release-candidate-canary.json" | awk '{print $1}')" +jq --arg sha "$bad_canary_sha" \ + '.canaries[1].evidence_sha256 = $sha' \ + "$tmp/input.json" >"$tmp/bad-canary-event.json" +if BPFCOMPAT_SKIP_READINESS_ATTESTATION=1 \ + "$script" "$tmp/bad-canary-event.json" "$tmp/bad.md" >/dev/null 2>&1; then + echo "[production-readiness-test] non-scheduled T+24h canary was accepted" >&2 + exit 1 +fi +mv "$tmp/canary-1-valid.json" "$tmp/canary-1/release-candidate-canary.json" + +cp "$tmp/incident/evidence.md" "$tmp/incident-valid.md" +printf '%s\n' 'incident marker missing' >"$tmp/incident/evidence.md" +bad_incident_sha="$(sha256sum "$tmp/incident/evidence.md" | awk '{print $1}')" +jq --arg sha "$bad_incident_sha" \ + '.incident.evidence_sha256 = $sha' \ + "$tmp/input.json" >"$tmp/bad-incident.json" +if BPFCOMPAT_SKIP_READINESS_ATTESTATION=1 \ + "$script" "$tmp/bad-incident.json" "$tmp/bad.md" >/dev/null 2>&1; then + echo "[production-readiness-test] unmarked incident evidence was accepted" >&2 + exit 1 +fi +mv "$tmp/incident-valid.md" "$tmp/incident/evidence.md" + jq '.targets[0].infra_error = "fixture failure" | .targets[0].status = "infra_error"' \ "$tmp/campaign-3/functional-execve-latest-kernel.json" >"$tmp/bad-report.json" mv "$tmp/bad-report.json" "$tmp/campaign-3/functional-execve-latest-kernel.json"