release: bind canary and incident evidence

.github/workflows/release-artifacts.yml

@@ -138,6 +138,8 @@ jobs:
             .release_version == $version and
             .slo.campaign_count == 4 and
             .slo.infrastructure_errors == 0 and
+            (.canaries | length == 3) and
+            .incident.completed == true and
             .operator.approval_mode == "solo-maintainer" and
             .operator.confirmed == true
           ' "${source_base}.json" >/dev/null

docs/production-release-process.md

@@ -85,10 +85,10 @@ unchanged, leaves the drill alias on the known-good digest, and uploads
 artifact as the rollback drill note.
 
 Download four chronological scheduled campaign artifacts, one expanded Falco
-artifact, the attested candidate evidence, the rollback drill note, and the
+artifact, the attested candidate evidence, all three RC canary manifests, the
+rollback drill note, the fail-closed promotion incident note, and the
 operator's written promotion confirmation into one private working directory.
-Create an
-input manifest with paths relative to that directory:
+Create an input manifest with paths relative to that directory:
 
 ```json
 {
@@ -111,11 +111,33 @@ input manifest with paths relative to that directory:
     "evidence": "candidate/release-candidate-evidence.json",
     "evidence_sha256": "<sha256>"
   },
+  "canaries": [
+    {
+      "milestone": "manual",
+      "evidence": "canary-manual/release-candidate-canary.json",
+      "evidence_sha256": "<sha256>"
+    },
+    {
+      "milestone": "t-plus-24h",
+      "evidence": "canary-t-plus-24h/release-candidate-canary.json",
+      "evidence_sha256": "<sha256>"
+    },
+    {
+      "milestone": "t-plus-72h",
+      "evidence": "canary-t-plus-72h/release-candidate-canary.json",
+      "evidence_sha256": "<sha256>"
+    }
+  ],
   "rollback": {
     "completed": true,
     "evidence": "rollback/evidence.md",
     "evidence_sha256": "<sha256>"
   },
+  "incident": {
+    "completed": true,
+    "evidence": "incident/evidence.md",
+    "evidence_sha256": "<sha256>"
+  },
   "operator": {
     "login": "ErenAri",
     "approval_mode": "solo-maintainer",
@@ -126,7 +148,12 @@ input manifest with paths relative to that directory:
 }
 ```
 
-The real manifest contains exactly four campaign entries. Run:
+The real manifest contains exactly four campaign entries and exactly three
+canary entries in `manual`, `t-plus-24h`, and `t-plus-72h` order. The two timed
+canaries must come from `schedule` events, and every canary must bind the same
+RC tag, commit, and image digest as the attested candidate. The incident note
+must contain `[bpfcompat-promotion-incident:v1]` and describe the deliberately
+rejected promotion run. Run:
 
 ```bash
 scripts/production-readiness-report.sh \
@@ -137,6 +164,8 @@ The command writes both Markdown and JSON. Add the reviewed outputs to
 `docs/releases/bpfcompat-0.4.0-readiness.{md,json}` in the final release pull
 request. The stable release workflow validates, checksums, attests, and
 publishes those files; a stable tag fails if they are absent or not `ready`.
+The online candidate check requires an authenticated GitHub CLI version that
+provides `gh attestation`; an older CLI fails closed.
 
 Do not use `BPFCOMPAT_SKIP_READINESS_ATTESTATION=1` outside the regression
 test; production evidence must verify the candidate attestation online.

docs/production-slo-runbook.md

@@ -54,8 +54,9 @@ boundary behavior change requires a new release candidate and restarts the
 window; documentation-only changes do not.
 
 Run the evidence aggregator after downloading the four campaign artifacts,
-the expanded Falco report, candidate evidence, rollback note, and
-solo-maintainer promotion confirmation:
+the expanded Falco report, candidate evidence, manual/T+24h/T+72h canary
+manifests, rollback and fail-closed incident notes, and solo-maintainer
+promotion confirmation:
 
 ```bash
 scripts/production-readiness-report.sh \

docs/production-support-boundary.md

@@ -118,5 +118,6 @@ substitute for these technical controls.
 
 `scripts/production-readiness-report.sh` validates the campaign manifests,
 report hashes, Falco profile coverage, attested release-candidate evidence,
-rollback evidence, and solo-operator promotion confirmation before producing
-the graduation report.
+manual/T+24h/T+72h canary evidence, rollback and fail-closed incident evidence,
+and solo-operator promotion confirmation before producing the graduation
+report.

scripts/production-readiness-report.sh

@@ -59,7 +59,10 @@ jq -e '
   (.campaigns | type == "array" and length == 4) and
   (.falco | type == "object") and
   (.candidate | type == "object") and
+  (.canaries | type == "array" and length == 3) and
+  ([.canaries[].milestone] == ["manual", "t-plus-24h", "t-plus-72h"]) and
   (.rollback.completed == true) and
+  (.incident.completed == true) and
   (.operator.approval_mode == "solo-maintainer") and
   (.operator.confirmed == true)
 ' "$input" >/dev/null || fail "manifest schema or required gates are invalid"
@@ -201,16 +204,93 @@ jq -e --arg release_version "$release_version" '
 
 if [[ "${BPFCOMPAT_SKIP_READINESS_ATTESTATION:-0}" != "1" ]]; then
   command -v gh >/dev/null || fail "gh is required to verify candidate evidence"
+  gh attestation --help >/dev/null 2>&1 ||
+    fail "installed GitHub CLI lacks attestation support"
   gh attestation verify "$candidate" \
     --repo Kernel-Guard/bpfcompat \
     --signer-workflow Kernel-Guard/bpfcompat/.github/workflows/release-artifacts.yml \
     >/dev/null || fail "release-candidate evidence attestation verification failed"
 fi
 
+: >"$tmp/canary_rows"
+: >"$tmp/canaries.ndjson"
+: >"$tmp/canary_run_ids"
+candidate_tag="$(jq -r '.tag' "$candidate")"
+candidate_commit="$(jq -r '.commit_sha' "$candidate")"
+candidate_image="$(jq -r '.image' "$candidate")"
+candidate_digest="${candidate_image##*@}"
+
+while IFS= read -r canary; do
+  milestone="$(jq -r '.milestone' <<<"$canary")"
+  evidence_rel="$(jq -r '.evidence' <<<"$canary")"
+  canary_evidence="$(evidence_path "$evidence_rel")"
+  canary_sha="$(jq -r '.evidence_sha256' <<<"$canary")"
+  verify_hash "$canary_evidence" "$canary_sha"
+
+  expected_event="schedule"
+  if [[ "$milestone" == "manual" ]]; then
+    expected_event="workflow_dispatch"
+  fi
+  jq -e \
+    --arg milestone "$milestone" \
+    --arg event "$expected_event" \
+    --arg version "$candidate_tag" \
+    --arg commit "$candidate_commit" \
+    --arg digest "$candidate_digest" \
+    '
+      .schema_version == "v0.1" and
+      .marker == "[bpfcompat-rc-canary:v1]" and
+      .repository == "Kernel-Guard/bpfcompat" and
+      .milestone == $milestone and
+      .event == $event and
+      (.workflow_run_id | type == "number" and . > 0) and
+      (.external_consumer_run_id | type == "number" and . > 0) and
+      (.completed_at | fromdateiso8601 > 0) and
+      .version == $version and
+      .commit == $commit and
+      .image_digest == $digest and
+      .checks.clean_install == "pass" and
+      .checks.source_build == "pass" and
+      .checks.published_action == "pass" and
+      .checks.container == "pass" and
+      .checks.external_consumers == "pass" and
+      (.components | type == "array" and length > 0) and
+      ([.components[] |
+        (.sha256 | test("^[0-9a-f]{64}$")) and
+        (.path | type == "string" and length > 0)
+      ] | all)
+    ' "$canary_evidence" >/dev/null ||
+    fail "RC canary ${milestone} evidence is invalid or does not match the candidate"
+
+  canary_run_id="$(jq '.workflow_run_id' "$canary_evidence")"
+  canary_completed_at="$(jq -r '.completed_at' "$canary_evidence")"
+  printf '%s\n' "$canary_run_id" >>"$tmp/canary_run_ids"
+  printf "| %s | \`%s\` | %s |\n" \
+    "$milestone" "$canary_run_id" "$canary_completed_at" >>"$tmp/canary_rows"
+  jq -n \
+    --arg milestone "$milestone" \
+    --argjson workflow_run_id "$canary_run_id" \
+    --arg completed_at "$canary_completed_at" \
+    --arg evidence_sha256 "$canary_sha" \
+    '{
+      milestone: $milestone,
+      workflow_run_id: $workflow_run_id,
+      completed_at: $completed_at,
+      evidence_sha256: $evidence_sha256
+    }' >>"$tmp/canaries.ndjson"
+done < <(jq -c '.canaries[]' "$input")
+
+[[ "$(sort -u "$tmp/canary_run_ids" | wc -l)" -eq 3 ]] ||
+  fail "RC canary workflow run IDs must be unique"
+
 rollback="$(evidence_path "$(jq -r '.rollback.evidence' "$input")")"
 verify_hash "$rollback" "$(jq -r '.rollback.evidence_sha256' "$input")"
 grep -Fq '[bpfcompat-rollback-drill:v1]' "$rollback" ||
   fail "rollback evidence is missing its completion marker"
+incident="$(evidence_path "$(jq -r '.incident.evidence' "$input")")"
+verify_hash "$incident" "$(jq -r '.incident.evidence_sha256' "$input")"
+grep -Fq '[bpfcompat-promotion-incident:v1]' "$incident" ||
+  fail "incident evidence is missing its fail-closed completion marker"
 operator_evidence="$(evidence_path "$(jq -r '.operator.evidence' "$input")")"
 verify_hash "$operator_evidence" "$(jq -r '.operator.evidence_sha256' "$input")"
 operator="$(jq -r '.operator.login' "$input")"
@@ -242,6 +322,7 @@ mkdir -p "$(dirname "$output_json")"
   echo "- Target executions: ${total_targets}"
   echo "- Infrastructure errors: 0"
   echo "- Target duration p95: ${p95_ms} ms"
+  echo "- Release-candidate canary observations: 3"
   echo "- Release operator: \`${operator}\`"
   echo "- Approval mode: \`${approval_mode}\` (no independent human approval)"
   echo
@@ -251,26 +332,33 @@ mkdir -p "$(dirname "$output_json")"
   echo "|---:|---|---|---|---:|"
   cat "$tmp/campaign_rows"
   echo
+  echo "## Release-Candidate Canaries"
+  echo
+  echo "| Milestone | Workflow run | Completed (UTC) |"
+  echo "|---|---|---|"
+  cat "$tmp/canary_rows"
+  echo
   echo "## Required External Evidence"
   echo
   echo "- Falco expanded vendor-kernel matrix: PASS"
   echo "- Attested release candidate: PASS"
-  echo "- Rollback and incident exercise: PASS"
+  echo "- RC manual, T+24h, and T+72h canaries: PASS"
+  echo "- Rollback drill: PASS"
+  echo "- Fail-closed promotion incident: PASS"
   echo "- Deliberate solo-maintainer promotion confirmation: PASS"
   echo
   echo "Runtime loading, agent, API, registry, SaaS, Firecracker, and virtme-ng are excluded."
 } >"$output"
 
 campaigns_json="$(jq -s '.' "$tmp/campaigns.ndjson")"
+canaries_json="$(jq -s '.' "$tmp/canaries.ndjson")"
 falco_run_id="$(jq '.falco.workflow_run_id' "$input")"
 falco_commit="$(jq -r '.falco.commit_sha' "$input")"
 falco_started_at="$(jq -r '.falco.started_at' "$input")"
 falco_sha="$(jq -r '.falco.report_sha256' "$input")"
-candidate_tag="$(jq -r '.tag' "$candidate")"
-candidate_commit="$(jq -r '.commit_sha' "$candidate")"
-candidate_image="$(jq -r '.image' "$candidate")"
 candidate_sha="$(jq -r '.candidate.evidence_sha256' "$input")"
 rollback_sha="$(jq -r '.rollback.evidence_sha256' "$input")"
+incident_sha="$(jq -r '.incident.evidence_sha256' "$input")"
 operator_sha="$(jq -r '.operator.evidence_sha256' "$input")"
 
 jq -n \
@@ -280,6 +368,7 @@ jq -n \
   --arg supported_boundary "CLI + GitHub Action + disposable QEMU/KVM validation" \
   --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
   --argjson campaigns "$campaigns_json" \
+  --argjson canaries "$canaries_json" \
   --argjson total_targets "$total_targets" \
   --argjson p95_ms "$p95_ms" \
   --argjson falco_run_id "$falco_run_id" \
@@ -291,6 +380,7 @@ jq -n \
   --arg candidate_image "$candidate_image" \
   --arg candidate_sha "$candidate_sha" \
   --arg rollback_sha "$rollback_sha" \
+  --arg incident_sha "$incident_sha" \
   --arg operator "$operator" \
   --arg approval_mode "$approval_mode" \
   --arg operator_sha "$operator_sha" \
@@ -327,10 +417,15 @@ jq -n \
       image: $candidate_image,
       evidence_sha256: $candidate_sha
     },
+    canaries: $canaries,
     rollback: {
       completed: true,
       evidence_sha256: $rollback_sha
     },
+    incident: {
+      completed: true,
+      evidence_sha256: $incident_sha
+    },
     operator: {
       login: $operator,
       approval_mode: $approval_mode,