feat: add verified compatibility preview

.github/workflows/external-consumer-canary.yml

@@ -131,9 +131,14 @@ jobs:
           if-no-files-found: warn
 
   falco:
-    name: Falco (scap-open modern_bpf loader)
+    name: Falco verified compatibility preview
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+      artifact-metadata: write
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
@@ -165,12 +170,18 @@ jobs:
             -DCREATE_TEST_TARGETS=OFF -DBUILD_BPF=OFF -DBUILD_DRIVER=OFF ..
           make scap-open -j"$(nproc)"
           file libscap/examples/01-open/scap-open
+          cd ../..
+          install -D -m 0755 \
+            libs/build/libscap/examples/01-open/scap-open \
+            reports/falco-conformance/subjects/scap-open
       - name: Validate scap-open across kernels (its real loader path)
+        id: validate
+        continue-on-error: true
         uses: ./
         with:
           command: $BPFCOMPAT_BIN --modern_bpf --num_events 10
           command-binary: libs/build/libscap/examples/01-open/scap-open
-          matrix: matrices/consumer-external.yaml
+          matrix: conformance/falco-modern-bpf-v0.1/matrix.yaml
           out: reports/falco.json
           markdown: reports/falco.md
           timeout: 20m
@@ -182,13 +193,89 @@ jobs:
           actual="$(jq -r '.command.binary.sha256 // empty' reports/falco.json)"
           test -n "$actual"
           test "$actual" = "$expected"
+          test "$(sha256sum reports/falco-conformance/subjects/scap-open | awk '{print $1}')" = "$actual"
+      - name: Evaluate the Falco conformance profile
+        id: evaluate
+        env:
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set +e
+          ./bin/bpfcompat conformance evaluate \
+            --profile conformance/falco-modern-bpf-v0.1/profile.yaml \
+            --report reports/falco.json \
+            --out-dir reports/falco-conformance \
+            --verifier-id https://github.com/Kernel-Guard/bpfcompat/tree/main/conformance/falco-modern-bpf-v0.1 \
+            --report-url "$RUN_URL"
+          rc=$?
+          set -e
+
+          test -s reports/falco-conformance/decision.json
+          test -s reports/falco-conformance/test-result.intoto.json
+          status="$(jq -er '.status' reports/falco-conformance/decision.json)"
+          case "$status:$rc" in
+            conformant:0|nonconformant:2|inconclusive:1) ;;
+            *) echo "evaluator status/exit mismatch: $status/$rc" >&2; exit 1 ;;
+          esac
+
+          statement_subject="$(jq -er 'select((.subject | length) == 1 and .subject[0].name == "scap-open") | .subject[0].digest.sha256' reports/falco-conformance/test-result.intoto.json)"
+          report_subject="$(jq -er '.subject.digest.sha256' reports/falco-conformance/decision.json)"
+          binary_subject="$(sha256sum reports/falco-conformance/subjects/scap-open | awk '{print $1}')"
+          test "$statement_subject" = "$report_subject"
+          test "$statement_subject" = "$binary_subject"
+
+          jq -e '.predicate' reports/falco-conformance/test-result.intoto.json \
+            > reports/falco-conformance/test-result.predicate.json
+          if [[ "$status" == "conformant" ]]; then
+            test -s reports/falco-conformance/verification-result.intoto.json
+            jq -e '.predicate' reports/falco-conformance/verification-result.intoto.json \
+              > reports/falco-conformance/verification-result.predicate.json
+          else
+            test ! -e reports/falco-conformance/verification-result.intoto.json
+          fi
+          echo "status=$status" >> "$GITHUB_OUTPUT"
+      - name: Attest the in-toto Test Result
+        id: test_attestation
+        uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
+        with:
+          subject-path: reports/falco-conformance/subjects/scap-open
+          predicate-type: https://in-toto.io/attestation/test-result/v0.1
+          predicate-path: reports/falco-conformance/test-result.predicate.json
+      - name: Attest the passing verification result
+        id: verification_attestation
+        if: steps.evaluate.outputs.status == 'conformant'
+        uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
+        with:
+          subject-path: reports/falco-conformance/subjects/scap-open
+          predicate-type: https://in-toto.io/attestation/svr/v0.2
+          predicate-path: reports/falco-conformance/verification-result.predicate.json
+      - name: Preserve signed attestation bundles
+        if: steps.test_attestation.outcome == 'success'
+        env:
+          TEST_BUNDLE: ${{ steps.test_attestation.outputs.bundle-path }}
+          VERIFICATION_BUNDLE: ${{ steps.verification_attestation.outputs.bundle-path }}
+        run: |
+          install -m 0644 "$TEST_BUNDLE" \
+            reports/falco-conformance/test-result.sigstore.json
+          if [[ -n "$VERIFICATION_BUNDLE" ]]; then
+            install -m 0644 "$VERIFICATION_BUNDLE" \
+              reports/falco-conformance/verification-result.sigstore.json
+          fi
       - name: Upload report
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
           name: canary-falco-${{ github.run_id }}
           path: reports/
           if-no-files-found: warn
+      - name: Enforce the conformance decision
+        if: always()
+        env:
+          STATUS: ${{ steps.evaluate.outputs.status }}
+        run: |
+          test "$STATUS" = "conformant" || {
+            echo "Falco conformance decision: ${STATUS:-evaluation unavailable}" >&2
+            exit 1
+          }
 
   report:
     name: File or close the canary tracking issue

README.md

@@ -711,6 +711,7 @@ Reference matrices (real, reproducible artifacts):
 
 - [`docs/integrations/`](docs/integrations/) — copy-paste templates: Go loader, Rust/Aya, OCI gadgets, and an in-process pre-load gate
 - [`docs/case-study-falco-modern-bpf.md`](docs/case-study-falco-modern-bpf.md) — Falco `modern_bpf` across 5 kernels
+- [`docs/verified-compatibility-preview.md`](docs/verified-compatibility-preview.md) — versioned Falco conformance profile, decisions, and signed evidence
 - [`docs/case-study-enterprise-kernels.md`](docs/case-study-enterprise-kernels.md) — RHEL/Oracle/Amazon/SUSE backported tier
 - [`docs/case-study-inspektor-gadget.md`](docs/case-study-inspektor-gadget.md) — published gadgets from OCI, zero config
 - [`docs/evidence-rhcos.md`](docs/evidence-rhcos.md) — RHEL CoreOS / OpenShift 4.14·4.16·4.18 × 6 artifacts (x86_64) + a real aarch64 boot

cmd/bpfcompat/conformance.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/kernel-guard/bpfcompat/internal/conformance"
+	"github.com/kernel-guard/bpfcompat/internal/runner"
+)
+
+func runConformance(args []string) int {
+	if len(args) == 0 {
+		printConformanceUsage()
+		return runner.ExitToolError
+	}
+	switch args[0] {
+	case "evaluate":
+		return runConformanceEvaluate(args[1:])
+	case "-h", "--help", "help":
+		printConformanceUsage()
+		return runner.ExitSuccess
+	default:
+		fmt.Fprintf(os.Stderr, "unknown conformance subcommand: %s\n\n", args[0])
+		printConformanceUsage()
+		return runner.ExitToolError
+	}
+}
+
+func runConformanceEvaluate(args []string) int {
+	fs := flag.NewFlagSet("conformance evaluate", flag.ContinueOnError)
+	fs.SetOutput(os.Stderr)
+	profilePath := fs.String("profile", "", "Path to a versioned conformance profile YAML")
+	reportPath := fs.String("report", "", "Path to the bpfcompat JSON report to evaluate")
+	outDir := fs.String("out-dir", "", "Directory for decision and in-toto statement outputs")
+	verifierID := fs.String("verifier-id", "", "Absolute HTTPS URI identifying the verifier")
+	reportURL := fs.String("report-url", "", "Optional absolute HTTPS URL for the originating test run")
+	fs.Usage = func() {
+		fmt.Fprintf(fs.Output(), "Usage:\n  bpfcompat conformance evaluate --profile <file> --report <file> --out-dir <dir> --verifier-id <uri> [--report-url <url>]\n\n")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return runner.ExitSuccess
+		}
+		return runner.ExitToolError
+	}
+	if fs.NArg() != 0 {
+		fmt.Fprintf(os.Stderr, "unexpected positional arguments: %v\n", fs.Args())
+		return runner.ExitToolError
+	}
+	for name, value := range map[string]string{
+		"--profile":     *profilePath,
+		"--report":      *reportPath,
+		"--out-dir":     *outDir,
+		"--verifier-id": *verifierID,
+	} {
+		if strings.TrimSpace(value) == "" {
+			fmt.Fprintf(os.Stderr, "%s is required\n", name)
+			return runner.ExitToolError
+		}
+	}
+
+	profile, err := conformance.LoadProfile(*profilePath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "load conformance profile: %v\n", err)
+		return runner.ExitToolError
+	}
+	report, err := conformance.LoadReport(*reportPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "load conformance report: %v\n", err)
+		return runner.ExitToolError
+	}
+	evaluation, err := conformance.Evaluate(profile, report, conformance.EvaluateOptions{
+		VerifierID: strings.TrimSpace(*verifierID),
+		ReportURL:  strings.TrimSpace(*reportURL),
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "evaluate conformance report: %v\n", err)
+		return runner.ExitToolError
+	}
+	paths, err := conformance.WriteEvaluation(*outDir, evaluation)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "write conformance evidence: %v\n", err)
+		return runner.ExitToolError
+	}
+
+	fmt.Printf("Conformance: %s\n", evaluation.Decision.Status)
+	fmt.Printf("Profile: %s sha256:%s\n", evaluation.Decision.Profile.ID, evaluation.Decision.Profile.SHA256)
+	fmt.Printf("Subject: %s sha256:%s\n", evaluation.Decision.Subject.Name, evaluation.Decision.Subject.Digest["sha256"])
+	fmt.Printf("Decision: %s\n", paths.Decision)
+	fmt.Printf("Test Result: %s\n", paths.TestResult)
+	if paths.VerificationResult != "" {
+		fmt.Printf("Verification Result: %s\n", paths.VerificationResult)
+	} else {
+		fmt.Println("Verification Result: not issued")
+	}
+
+	switch evaluation.Decision.Status {
+	case conformance.StatusConformant:
+		return runner.ExitSuccess
+	case conformance.StatusNonconformant:
+		return runner.ExitCompatibilityFailure
+	default:
+		return runner.ExitToolError
+	}
+}
+
+func printConformanceUsage() {
+	fmt.Println("Usage:")
+	fmt.Println("  bpfcompat conformance evaluate --profile <file> --report <file> --out-dir <dir> --verifier-id <uri> [--report-url <url>]")
+}

cmd/bpfcompat/main.go

@@ -1,3 +1,3 @@
 package main
 
 import (
@@ -61,6 +61,8 @@
 		return runKernelFreshness(args[1:])
 	case "kernel-sweep":
 		return runKernelSweep(args[1:])
+	case "conformance":
+		return runConformance(args[1:])
 	case "version", "--version":
 		return runVersion(args[1:])
 	case "env":
@@ -1456,6 +1458,7 @@
 	fmt.Println("  bpfcompat report-summary --report <file>")
 	fmt.Println("  bpfcompat kernel-freshness [--baselines <file>] [--fail-on-stale] [--update-from-report <file>]")
 	fmt.Println("  bpfcompat kernel-sweep --profile <id> [--count N] [--series <prefix>]")
+	fmt.Println("  bpfcompat conformance evaluate --profile <file> --report <file> --out-dir <dir> --verifier-id <uri>")
 	fmt.Println("  bpfcompat version [--json]")
 	fmt.Println("  bpfcompat env [--markdown|--json]")
 }

cmd/bpfcompat/main_test.go

@@ -109,3 +109,15 @@ func TestRunTestCommandDispatch(t *testing.T) {
 		t.Fatalf("run test-command -h = %d, want 0", got)
 	}
 }
+
+func TestRunConformanceDispatchAndValidation(t *testing.T) {
+	if got := run([]string{"conformance", "evaluate", "-h"}); got != runner.ExitSuccess {
+		t.Fatalf("run conformance evaluate -h = %d, want 0", got)
+	}
+	if got := run([]string{"conformance", "evaluate"}); got != runner.ExitToolError {
+		t.Fatalf("run conformance evaluate = %d, want tool error", got)
+	}
+	if got := run([]string{"conformance", "unknown"}); got != runner.ExitToolError {
+		t.Fatalf("run conformance unknown = %d, want tool error", got)
+	}
+}

conformance/README.md

@@ -0,0 +1,25 @@
+# bpfcompat Verified Compatibility Preview
+
+This directory contains versioned, machine-evaluable compatibility profiles.
+It is a preview conformance program, not an independent certification program.
+
+A conformance decision is scoped to:
+
+- one immutable subject digest;
+- one versioned profile and matrix;
+- one bpfcompat report and its exact test invocation;
+- the required kernels actually exercised by that report; and
+- the time and verifier identity recorded in the decision.
+
+The evaluator has three outcomes:
+
+- `conformant`: every required assertion passed;
+- `nonconformant`: the tested product behavior failed at least one required
+  assertion; and
+- `inconclusive`: the test contract, evidence, or infrastructure was incomplete.
+
+Only `conformant` decisions produce an in-toto Simple Verification Result.
+Every valid evaluation produces a decision and an in-toto Test Result, including
+failed and inconclusive evaluations.
+
+The first profile is [Falco modern_bpf v0.1](falco-modern-bpf-v0.1/spec.md).

conformance/falco-modern-bpf-v0.1/matrix.yaml

@@ -0,0 +1,16 @@
+# Canonical required matrix for the Falco modern_bpf compatibility preview.
+# This matches the upstream falcosecurity/libs proof-of-concept matrix. A
+# credential covers only these required profiles, even if a report contains
+# additional informational targets.
+name: falco-modern-bpf-v0.1
+profiles:
+  - id: ubuntu-22.04-5.15
+    required: true
+  - id: debian-12-6.1
+    required: true
+  - id: ubuntu-24.04-6.8
+    required: true
+  - id: almalinux-8-4.18
+    required: true
+  - id: almalinux-9-5.14
+    required: true