From 23723cd2c8dd3db392f2af85763502d0bef45c89 Mon Sep 17 00:00:00 2001 From: ErenAri Date: Sun, 9 Aug 2026 17:17:44 +0300 Subject: [PATCH] feat: add verified compatibility preview --- .../workflows/external-consumer-canary.yml | 91 ++- README.md | 1 + cmd/bpfcompat/conformance.go | 114 ++++ cmd/bpfcompat/main.go | 3 + cmd/bpfcompat/main_test.go | 12 + conformance/README.md | 25 + conformance/falco-modern-bpf-v0.1/matrix.yaml | 16 + .../falco-modern-bpf-v0.1/profile.yaml | 83 +++ conformance/falco-modern-bpf-v0.1/spec.md | 107 ++++ docs/case-study-falco-modern-bpf.md | 18 + docs/verified-compatibility-preview.md | 120 ++++ internal/conformance/attestation.go | 65 +++ internal/conformance/evaluate.go | 551 ++++++++++++++++++ internal/conformance/evaluate_test.go | 318 ++++++++++ internal/conformance/json.go | 83 +++ internal/conformance/profile.go | 310 ++++++++++ internal/conformance/profile_test.go | 89 +++ .../testdata/falco-pass-report.json | 87 +++ internal/conformance/types.go | 178 ++++++ internal/conformance/write.go | 67 +++ 20 files changed, 2336 insertions(+), 2 deletions(-) create mode 100644 cmd/bpfcompat/conformance.go create mode 100644 conformance/README.md create mode 100644 conformance/falco-modern-bpf-v0.1/matrix.yaml create mode 100644 conformance/falco-modern-bpf-v0.1/profile.yaml create mode 100644 conformance/falco-modern-bpf-v0.1/spec.md create mode 100644 docs/verified-compatibility-preview.md create mode 100644 internal/conformance/attestation.go create mode 100644 internal/conformance/evaluate.go create mode 100644 internal/conformance/evaluate_test.go create mode 100644 internal/conformance/json.go create mode 100644 internal/conformance/profile.go create mode 100644 internal/conformance/profile_test.go create mode 100644 internal/conformance/testdata/falco-pass-report.json create mode 100644 internal/conformance/types.go create mode 100644 internal/conformance/write.go diff --git a/.github/workflows/external-consumer-canary.yml b/.github/workflows/external-consumer-canary.yml index 9bcd013..843dfba 100644 --- a/.github/workflows/external-consumer-canary.yml +++ b/.github/workflows/external-consumer-canary.yml @@ -131,9 +131,14 @@ jobs: if-no-files-found: warn falco: - name: Falco (scap-open modern_bpf loader) + name: Falco verified compatibility preview runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + contents: read + id-token: write + attestations: write + artifact-metadata: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -165,12 +170,18 @@ jobs: -DCREATE_TEST_TARGETS=OFF -DBUILD_BPF=OFF -DBUILD_DRIVER=OFF .. make scap-open -j"$(nproc)" file libscap/examples/01-open/scap-open + cd ../.. + install -D -m 0755 \ + libs/build/libscap/examples/01-open/scap-open \ + reports/falco-conformance/subjects/scap-open - name: Validate scap-open across kernels (its real loader path) + id: validate + continue-on-error: true uses: ./ with: command: $BPFCOMPAT_BIN --modern_bpf --num_events 10 command-binary: libs/build/libscap/examples/01-open/scap-open - matrix: matrices/consumer-external.yaml + matrix: conformance/falco-modern-bpf-v0.1/matrix.yaml out: reports/falco.json markdown: reports/falco.md timeout: 20m @@ -182,6 +193,73 @@ jobs: actual="$(jq -r '.command.binary.sha256 // empty' reports/falco.json)" test -n "$actual" test "$actual" = "$expected" + test "$(sha256sum reports/falco-conformance/subjects/scap-open | awk '{print $1}')" = "$actual" + - name: Evaluate the Falco conformance profile + id: evaluate + env: + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set +e + ./bin/bpfcompat conformance evaluate \ + --profile conformance/falco-modern-bpf-v0.1/profile.yaml \ + --report reports/falco.json \ + --out-dir reports/falco-conformance \ + --verifier-id https://github.com/Kernel-Guard/bpfcompat/tree/main/conformance/falco-modern-bpf-v0.1 \ + --report-url "$RUN_URL" + rc=$? + set -e + + test -s reports/falco-conformance/decision.json + test -s reports/falco-conformance/test-result.intoto.json + status="$(jq -er '.status' reports/falco-conformance/decision.json)" + case "$status:$rc" in + conformant:0|nonconformant:2|inconclusive:1) ;; + *) echo "evaluator status/exit mismatch: $status/$rc" >&2; exit 1 ;; + esac + + statement_subject="$(jq -er 'select((.subject | length) == 1 and .subject[0].name == "scap-open") | .subject[0].digest.sha256' reports/falco-conformance/test-result.intoto.json)" + report_subject="$(jq -er '.subject.digest.sha256' reports/falco-conformance/decision.json)" + binary_subject="$(sha256sum reports/falco-conformance/subjects/scap-open | awk '{print $1}')" + test "$statement_subject" = "$report_subject" + test "$statement_subject" = "$binary_subject" + + jq -e '.predicate' reports/falco-conformance/test-result.intoto.json \ + > reports/falco-conformance/test-result.predicate.json + if [[ "$status" == "conformant" ]]; then + test -s reports/falco-conformance/verification-result.intoto.json + jq -e '.predicate' reports/falco-conformance/verification-result.intoto.json \ + > reports/falco-conformance/verification-result.predicate.json + else + test ! -e reports/falco-conformance/verification-result.intoto.json + fi + echo "status=$status" >> "$GITHUB_OUTPUT" + - name: Attest the in-toto Test Result + id: test_attestation + uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1 + with: + subject-path: reports/falco-conformance/subjects/scap-open + predicate-type: https://in-toto.io/attestation/test-result/v0.1 + predicate-path: reports/falco-conformance/test-result.predicate.json + - name: Attest the passing verification result + id: verification_attestation + if: steps.evaluate.outputs.status == 'conformant' + uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1 + with: + subject-path: reports/falco-conformance/subjects/scap-open + predicate-type: https://in-toto.io/attestation/svr/v0.2 + predicate-path: reports/falco-conformance/verification-result.predicate.json + - name: Preserve signed attestation bundles + if: steps.test_attestation.outcome == 'success' + env: + TEST_BUNDLE: ${{ steps.test_attestation.outputs.bundle-path }} + VERIFICATION_BUNDLE: ${{ steps.verification_attestation.outputs.bundle-path }} + run: | + install -m 0644 "$TEST_BUNDLE" \ + reports/falco-conformance/test-result.sigstore.json + if [[ -n "$VERIFICATION_BUNDLE" ]]; then + install -m 0644 "$VERIFICATION_BUNDLE" \ + reports/falco-conformance/verification-result.sigstore.json + fi - name: Upload report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -189,6 +267,15 @@ jobs: name: canary-falco-${{ github.run_id }} path: reports/ if-no-files-found: warn + - name: Enforce the conformance decision + if: always() + env: + STATUS: ${{ steps.evaluate.outputs.status }} + run: | + test "$STATUS" = "conformant" || { + echo "Falco conformance decision: ${STATUS:-evaluation unavailable}" >&2 + exit 1 + } report: name: File or close the canary tracking issue diff --git a/README.md b/README.md index 3a72273..c7c088d 100644 --- a/README.md +++ b/README.md @@ -711,6 +711,7 @@ Reference matrices (real, reproducible artifacts): - [`docs/integrations/`](docs/integrations/) — copy-paste templates: Go loader, Rust/Aya, OCI gadgets, and an in-process pre-load gate - [`docs/case-study-falco-modern-bpf.md`](docs/case-study-falco-modern-bpf.md) — Falco `modern_bpf` across 5 kernels +- [`docs/verified-compatibility-preview.md`](docs/verified-compatibility-preview.md) — versioned Falco conformance profile, decisions, and signed evidence - [`docs/case-study-enterprise-kernels.md`](docs/case-study-enterprise-kernels.md) — RHEL/Oracle/Amazon/SUSE backported tier - [`docs/case-study-inspektor-gadget.md`](docs/case-study-inspektor-gadget.md) — published gadgets from OCI, zero config - [`docs/evidence-rhcos.md`](docs/evidence-rhcos.md) — RHEL CoreOS / OpenShift 4.14·4.16·4.18 × 6 artifacts (x86_64) + a real aarch64 boot diff --git a/cmd/bpfcompat/conformance.go b/cmd/bpfcompat/conformance.go new file mode 100644 index 0000000..485d387 --- /dev/null +++ b/cmd/bpfcompat/conformance.go @@ -0,0 +1,114 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/kernel-guard/bpfcompat/internal/conformance" + "github.com/kernel-guard/bpfcompat/internal/runner" +) + +func runConformance(args []string) int { + if len(args) == 0 { + printConformanceUsage() + return runner.ExitToolError + } + switch args[0] { + case "evaluate": + return runConformanceEvaluate(args[1:]) + case "-h", "--help", "help": + printConformanceUsage() + return runner.ExitSuccess + default: + fmt.Fprintf(os.Stderr, "unknown conformance subcommand: %s\n\n", args[0]) + printConformanceUsage() + return runner.ExitToolError + } +} + +func runConformanceEvaluate(args []string) int { + fs := flag.NewFlagSet("conformance evaluate", flag.ContinueOnError) + fs.SetOutput(os.Stderr) + profilePath := fs.String("profile", "", "Path to a versioned conformance profile YAML") + reportPath := fs.String("report", "", "Path to the bpfcompat JSON report to evaluate") + outDir := fs.String("out-dir", "", "Directory for decision and in-toto statement outputs") + verifierID := fs.String("verifier-id", "", "Absolute HTTPS URI identifying the verifier") + reportURL := fs.String("report-url", "", "Optional absolute HTTPS URL for the originating test run") + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage:\n bpfcompat conformance evaluate --profile --report --out-dir --verifier-id [--report-url ]\n\n") + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return runner.ExitSuccess + } + return runner.ExitToolError + } + if fs.NArg() != 0 { + fmt.Fprintf(os.Stderr, "unexpected positional arguments: %v\n", fs.Args()) + return runner.ExitToolError + } + for name, value := range map[string]string{ + "--profile": *profilePath, + "--report": *reportPath, + "--out-dir": *outDir, + "--verifier-id": *verifierID, + } { + if strings.TrimSpace(value) == "" { + fmt.Fprintf(os.Stderr, "%s is required\n", name) + return runner.ExitToolError + } + } + + profile, err := conformance.LoadProfile(*profilePath) + if err != nil { + fmt.Fprintf(os.Stderr, "load conformance profile: %v\n", err) + return runner.ExitToolError + } + report, err := conformance.LoadReport(*reportPath) + if err != nil { + fmt.Fprintf(os.Stderr, "load conformance report: %v\n", err) + return runner.ExitToolError + } + evaluation, err := conformance.Evaluate(profile, report, conformance.EvaluateOptions{ + VerifierID: strings.TrimSpace(*verifierID), + ReportURL: strings.TrimSpace(*reportURL), + }) + if err != nil { + fmt.Fprintf(os.Stderr, "evaluate conformance report: %v\n", err) + return runner.ExitToolError + } + paths, err := conformance.WriteEvaluation(*outDir, evaluation) + if err != nil { + fmt.Fprintf(os.Stderr, "write conformance evidence: %v\n", err) + return runner.ExitToolError + } + + fmt.Printf("Conformance: %s\n", evaluation.Decision.Status) + fmt.Printf("Profile: %s sha256:%s\n", evaluation.Decision.Profile.ID, evaluation.Decision.Profile.SHA256) + fmt.Printf("Subject: %s sha256:%s\n", evaluation.Decision.Subject.Name, evaluation.Decision.Subject.Digest["sha256"]) + fmt.Printf("Decision: %s\n", paths.Decision) + fmt.Printf("Test Result: %s\n", paths.TestResult) + if paths.VerificationResult != "" { + fmt.Printf("Verification Result: %s\n", paths.VerificationResult) + } else { + fmt.Println("Verification Result: not issued") + } + + switch evaluation.Decision.Status { + case conformance.StatusConformant: + return runner.ExitSuccess + case conformance.StatusNonconformant: + return runner.ExitCompatibilityFailure + default: + return runner.ExitToolError + } +} + +func printConformanceUsage() { + fmt.Println("Usage:") + fmt.Println(" bpfcompat conformance evaluate --profile --report --out-dir --verifier-id [--report-url ]") +} diff --git a/cmd/bpfcompat/main.go b/cmd/bpfcompat/main.go index 7884c65..76d6e2a 100644 --- a/cmd/bpfcompat/main.go +++ b/cmd/bpfcompat/main.go @@ -61,6 +61,8 @@ func run(args []string) int { return runKernelFreshness(args[1:]) case "kernel-sweep": return runKernelSweep(args[1:]) + case "conformance": + return runConformance(args[1:]) case "version", "--version": return runVersion(args[1:]) case "env": @@ -1456,6 +1458,7 @@ func printRootUsage() { fmt.Println(" bpfcompat report-summary --report ") fmt.Println(" bpfcompat kernel-freshness [--baselines ] [--fail-on-stale] [--update-from-report ]") fmt.Println(" bpfcompat kernel-sweep --profile [--count N] [--series ]") + fmt.Println(" bpfcompat conformance evaluate --profile --report --out-dir --verifier-id ") fmt.Println(" bpfcompat version [--json]") fmt.Println(" bpfcompat env [--markdown|--json]") } diff --git a/cmd/bpfcompat/main_test.go b/cmd/bpfcompat/main_test.go index 53f26ec..36041f8 100644 --- a/cmd/bpfcompat/main_test.go +++ b/cmd/bpfcompat/main_test.go @@ -109,3 +109,15 @@ func TestRunTestCommandDispatch(t *testing.T) { t.Fatalf("run test-command -h = %d, want 0", got) } } + +func TestRunConformanceDispatchAndValidation(t *testing.T) { + if got := run([]string{"conformance", "evaluate", "-h"}); got != runner.ExitSuccess { + t.Fatalf("run conformance evaluate -h = %d, want 0", got) + } + if got := run([]string{"conformance", "evaluate"}); got != runner.ExitToolError { + t.Fatalf("run conformance evaluate = %d, want tool error", got) + } + if got := run([]string{"conformance", "unknown"}); got != runner.ExitToolError { + t.Fatalf("run conformance unknown = %d, want tool error", got) + } +} diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..eac2cee --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,25 @@ +# bpfcompat Verified Compatibility Preview + +This directory contains versioned, machine-evaluable compatibility profiles. +It is a preview conformance program, not an independent certification program. + +A conformance decision is scoped to: + +- one immutable subject digest; +- one versioned profile and matrix; +- one bpfcompat report and its exact test invocation; +- the required kernels actually exercised by that report; and +- the time and verifier identity recorded in the decision. + +The evaluator has three outcomes: + +- `conformant`: every required assertion passed; +- `nonconformant`: the tested product behavior failed at least one required + assertion; and +- `inconclusive`: the test contract, evidence, or infrastructure was incomplete. + +Only `conformant` decisions produce an in-toto Simple Verification Result. +Every valid evaluation produces a decision and an in-toto Test Result, including +failed and inconclusive evaluations. + +The first profile is [Falco modern_bpf v0.1](falco-modern-bpf-v0.1/spec.md). diff --git a/conformance/falco-modern-bpf-v0.1/matrix.yaml b/conformance/falco-modern-bpf-v0.1/matrix.yaml new file mode 100644 index 0000000..70f0069 --- /dev/null +++ b/conformance/falco-modern-bpf-v0.1/matrix.yaml @@ -0,0 +1,16 @@ +# Canonical required matrix for the Falco modern_bpf compatibility preview. +# This matches the upstream falcosecurity/libs proof-of-concept matrix. A +# credential covers only these required profiles, even if a report contains +# additional informational targets. +name: falco-modern-bpf-v0.1 +profiles: + - id: ubuntu-22.04-5.15 + required: true + - id: debian-12-6.1 + required: true + - id: ubuntu-24.04-6.8 + required: true + - id: almalinux-8-4.18 + required: true + - id: almalinux-9-5.14 + required: true diff --git a/conformance/falco-modern-bpf-v0.1/profile.yaml b/conformance/falco-modern-bpf-v0.1/profile.yaml new file mode 100644 index 0000000..4b648d0 --- /dev/null +++ b/conformance/falco-modern-bpf-v0.1/profile.yaml @@ -0,0 +1,83 @@ +schema_version: bpfcompat_conformance_profile.v0.1 +id: falco-modern-bpf-v0.1 +title: Falco modern_bpf loader compatibility +program_status: preview +profile_uri: https://github.com/Kernel-Guard/bpfcompat/blob/main/conformance/falco-modern-bpf-v0.1/profile.yaml +claim: >- + The exact scap-open loader binary identified by SHA-256 completed Falco's + modern_bpf initialization and bounded event capture contract on every + required kernel in the Falco modern_bpf v0.1 matrix. +limitations: + - This does not test the complete Falco daemon or rules engine. + - This does not certify kernels or configurations outside the recorded matrix. + - This preview is not affiliated with, sponsored by, or endorsed by Falco or CNCF. +report_schema_versions: + - v0.1 +subject: + kind: command_loader + binary_basename: scap-open + command: $BPFCOMPAT_BIN --modern_bpf --num_events 10 + expected_exit_code: 0 +matrix: + path: matrix.yaml + uri: https://github.com/Kernel-Guard/bpfcompat/blob/main/conformance/falco-modern-bpf-v0.1/matrix.yaml +environments: + - profile_id: ubuntu-22.04-5.15 + distro: ubuntu + version: "22.04" + kernel_family: "5.15" + arch: x86_64 + - profile_id: debian-12-6.1 + distro: debian + version: "12" + kernel_family: "6.1" + arch: x86_64 + - profile_id: ubuntu-24.04-6.8 + distro: ubuntu + version: "24.04" + kernel_family: "6.8" + arch: x86_64 + - profile_id: almalinux-8-4.18 + distro: almalinux + version: "8" + kernel_family: "4.18" + arch: x86_64 + - profile_id: almalinux-9-5.14 + distro: almalinux + version: "9" + kernel_family: "5.14" + arch: x86_64 +validity: + snapshot_days: 90 + continuous_max_age_days: 14 +assertions: + - id: FALCO-REPORT-001 + rule: report.schema + description: The report uses a schema allowed by this profile. + - id: FALCO-FRESHNESS-001 + rule: report.freshness + description: The underlying compatibility run is inside the profile's snapshot validity window. + - id: FALCO-SUBJECT-001 + rule: subject.loader_identity + description: The report identifies the exact scap-open loader bytes by SHA-256. + - id: FALCO-COMMAND-001 + rule: command.contract + description: The recorded command and expected exit code match the profile contract. + - id: FALCO-MATRIX-001 + rule: matrix.required_profiles + description: Every canonical profile appears exactly once and is marked required. + - id: FALCO-ENV-001 + rule: environment.actual_identity + description: Every required target records the requested environment and actual host kernel. + - id: FALCO-IMAGE-001 + rule: environment.image_digest + description: Every required target records the SHA-256 of the booted base image. + - id: FALCO-EVENT-001 + rule: behavior.command_pass + description: scap-open loaded, attached, and captured the bounded event count on every required target. +properties: + - BPFCOMPAT_FALCO_MODERN_BPF_V0_1 + - BPFCOMPAT_REAL_LOADER + - BPFCOMPAT_EVENT_CAPTURE + - BPFCOMPAT_ENTERPRISE_BACKPORT_KERNELS + - BPFCOMPAT_ARCH_X86_64 diff --git a/conformance/falco-modern-bpf-v0.1/spec.md b/conformance/falco-modern-bpf-v0.1/spec.md new file mode 100644 index 0000000..de61313 --- /dev/null +++ b/conformance/falco-modern-bpf-v0.1/spec.md @@ -0,0 +1,107 @@ +# Falco modern_bpf compatibility profile v0.1 + +Status: **Verified Compatibility Preview** + +This profile evaluates the kernel-compatibility behavior of the exact +`scap-open` binary built from `falcosecurity/libs`. It uses Falco's real +userspace loader path with the modern eBPF probe skeleton embedded. + +It is an independent compatibility test of publicly available software. It is +not affiliated with, sponsored by, or endorsed by the Falco project or CNCF. + +## Normative claim + +A `conformant` decision means: + +> The exact `scap-open` loader binary identified by SHA-256 completed Falco's +> `modern_bpf` initialization and bounded event-capture contract on every +> required kernel in `matrix.yaml`. + +The tested command MUST be: + +```bash +$BPFCOMPAT_BIN --modern_bpf --num_events 10 +``` + +and the expected exit code MUST be zero. `scap-open` exits successfully only +after `scap_open()` has exercised libpman's load path, attached the selected +programs, and captured the requested bounded event count. + +## Subject + +The subject is the loader binary, not the Falco organization, repository name, +branch, mutable tag, or every product built from the repository. The subject +MUST be named `scap-open` and MUST have a valid SHA-256 in the report's +`command.binary` identity. + +Source repository and commit metadata MAY accompany a submission, but source +metadata alone does not replace the binary digest. A future build-provenance +profile can establish the source-to-binary relationship separately. + +## Required matrix + +Every profile in `matrix.yaml` is mandatory. The report MAY contain additional +informational targets, but they do not expand the credential's scope. +The expected distro, release, kernel family, and architecture for every matrix +ID are separately declared in `profile.yaml`; matching a profile ID string alone +is not sufficient evidence. + +Each required target MUST: + +- appear exactly once and remain marked `required`; +- record requested distro, version, kernel family, and architecture; +- record the actual host kernel release and architecture; +- record the SHA-256 of the base image bytes used to boot the guest; and +- contain a passing required `command` functional result. + +The report MUST use a schema allowed by `profile.yaml` and MUST be evaluated +before its 90-day snapshot window expires. + +## Outcomes + +- `conformant`: all assertions pass. +- `nonconformant`: the prescribed loader command executes and fails on at least + one required target. +- `inconclusive`: the report is stale, incomplete, uses the wrong contract, + omits a required target or identity, or contains an infrastructure error. + +A known product failure is sufficient to establish `nonconformant` even when a +different target is inconclusive. Infrastructure failure by itself MUST NOT be +reported as Falco incompatibility. + +## Evidence and attestations + +Every evaluation writes: + +1. `decision.json`, binding the subject, report, profile, matrix, assertion + results, evaluated targets, verifier, creation time, and validity window; +2. `test-result.intoto.json`, using the + [in-toto Test Result v0.1](https://github.com/in-toto/attestation/blob/main/spec/predicates/test-result.md) + predicate. + +Only a conformant evaluation additionally writes +`verification-result.intoto.json`, using the +[in-toto Simple Verification Result v0.2](https://github.com/in-toto/attestation/blob/main/spec/predicates/svr.md) +predicate. Signing and transparency logging happen after evaluation so the +unsigned statement remains independently inspectable. + +## Validity + +A snapshot decision expires 90 days after the underlying compatibility run. +The report MUST still be within that window when evaluated. A registry MAY show +`continuous` only while the latest successful scheduled run is no more than 14 +days old. + +Any change to the subject digest, required matrix, command contract, profile, or +assertion semantics requires a new evaluation. A materially changed contract +requires a new profile version. + +## Explicit exclusions + +This profile does not establish: + +- complete Falco daemon startup or configuration correctness; +- Falco rules-engine or plugin behavior; +- correctness of every event field; +- support for kernels or configurations outside the recorded matrix; or +- endorsement by Falco, CNCF, or the eBPF Foundation. diff --git a/docs/case-study-falco-modern-bpf.md b/docs/case-study-falco-modern-bpf.md index 8d5c2de..e77bbb3 100644 --- a/docs/case-study-falco-modern-bpf.md +++ b/docs/case-study-falco-modern-bpf.md @@ -81,6 +81,24 @@ approach" These are public review comments quoted verbatim — technical preferences expressed during review, not an endorsement of the project. +## Verified Compatibility Preview + +The repository now uses that same real-loader path as the first pilot for a +versioned compatibility program. The normative contract is +[`falco-modern-bpf-v0.1`](../conformance/falco-modern-bpf-v0.1/spec.md), and the +weekly external-consumer canary evaluates its five required kernels. Every +valid evaluation emits an in-toto Test Result. Only a fully conformant run emits +and signs an in-toto Simple Verification Result for the exact `scap-open` +SHA-256. + +This is deliberately called a **Verified Compatibility Preview**, not a Falco +certification. Its subject is one exact loader binary and its scope is the +specified initialization and bounded event-capture contract. It does not cover +the full Falco daemon or rules engine, and it is not affiliated with or endorsed +by Falco or CNCF. See +[`verified-compatibility-preview.md`](verified-compatibility-preview.md) for the +evidence and verification workflow. + ## Reproduce it ```bash diff --git a/docs/verified-compatibility-preview.md b/docs/verified-compatibility-preview.md new file mode 100644 index 0000000..a83457f --- /dev/null +++ b/docs/verified-compatibility-preview.md @@ -0,0 +1,120 @@ +# Verified Compatibility Preview + +The preview turns a bpfcompat report into a narrow, reviewable compatibility +claim. It is the first step toward a certification-like program, but it is not +an independent certification or an endorsement by the project whose loader is +tested. + +The first profile is +[`falco-modern-bpf-v0.1`](../conformance/falco-modern-bpf-v0.1/spec.md). It tests +the exact `scap-open` bytes through Falco's real `modern_bpf` loader path on five +required x86-64 distribution kernels, including enterprise backport kernels. + +## Trust model + +The evidence separates four things that must not be conflated: + +1. The `scap-open` SHA-256 identifies the exact subject. +2. The bpfcompat report records the command contract, actual guest kernels, + boot-image SHA-256 values, and per-target results. +3. The versioned profile and matrix define what must pass. +4. A GitHub OIDC/Sigstore attestation identifies the repository workflow that + issued the result. + +Trusting an attestation is a policy decision. Consumers should verify its +subject digest, predicate type, signer repository, and workflow identity, then +enforce their own freshness requirement. The profile permits a 90-day snapshot; +the program may call the result continuous only while its latest successful +scheduled run is at most 14 days old. + +## Outcomes + +| Decision | Meaning | Signed Test Result | Signed verification result | +|---|---|---:|---:| +| `conformant` | Every required assertion passed | yes (`PASSED`) | yes | +| `nonconformant` | The prescribed loader command ran and failed on a required kernel | yes (`FAILED`) | no | +| `inconclusive` | Infrastructure, identity, freshness, or evidence was insufficient | yes (`WARNED`) | no | + +This distinction prevents a KVM, network, image, or runner outage from being +misrepresented as a Falco compatibility defect. If one target has a known +loader failure while another has an infrastructure error, the known product +failure is still `nonconformant`. + +## Evaluate a report locally + +Run the prescribed command mode against the canonical matrix: + +```bash +./bin/bpfcompat test \ + --command '$BPFCOMPAT_BIN --modern_bpf --num_events 10' \ + --command-binary ./scap-open \ + --matrix conformance/falco-modern-bpf-v0.1/matrix.yaml \ + --out reports/falco.json +``` + +Then evaluate the immutable report against the versioned policy: + +```bash +./bin/bpfcompat conformance evaluate \ + --profile conformance/falco-modern-bpf-v0.1/profile.yaml \ + --report reports/falco.json \ + --out-dir reports/falco-conformance \ + --verifier-id https://github.com/Kernel-Guard/bpfcompat/tree/main/conformance/falco-modern-bpf-v0.1 +``` + +Exit code `0` means conformant, `2` means nonconformant, and `1` means +inconclusive or a tool/input error. Inspect `decision.json` to distinguish an +inconclusive decision from a tool error. + +The evaluator writes: + +- `decision.json`, the detailed decision and assertion evidence; +- `test-result.intoto.json`, an unsigned in-toto Test Result v0.1 statement; +- `verification-result.intoto.json`, an unsigned in-toto Simple Verification + Result v0.2 statement, only for `conformant` decisions. + +The weekly [`external-consumer-canary`](../.github/workflows/external-consumer-canary.yml) +extracts the standard predicates, binds them to the same `scap-open` path, and +signs them with +[GitHub Artifact Attestations](https://github.com/actions/attest). It uploads the +subject binary, report, decision, unsigned statements, and signed Sigstore +bundles together for audit. + +## Verify a published result + +After downloading the `scap-open` subject from the same workflow run, verify +both the signer and predicate type: + +```bash +gh attestation verify ./scap-open \ + --repo Kernel-Guard/bpfcompat \ + --predicate-type https://in-toto.io/attestation/test-result/v0.1 + +gh attestation verify ./scap-open \ + --repo Kernel-Guard/bpfcompat \ + --predicate-type https://in-toto.io/attestation/svr/v0.2 +``` + +The second command is expected to succeed only for a conformant result. A +serious consumer should additionally pin the signer workflow and enforce the +maximum acceptable attestation age. Verification proves who issued the +statement and which bytes it covers; it does not broaden the profile's claim. + +## Promotion criteria + +The word `certified` remains out of scope until the program has all of the +following: + +- a documented appeals, revocation, and incident process; +- public verifier implementation and profile change control; +- multiple successful recurring cycles with measured infrastructure + reliability; +- independent governance or a clearly disclosed first-party self-verification + model; +- a public registry that derives current/expired/revoked state from signed + evidence; and +- at least one additional adopter profile proving the design is not + Falco-specific. + +Until then, the accurate claim is that bpfcompat issued a signed, scoped +compatibility verification for exact artifact bytes under a preview policy. diff --git a/internal/conformance/attestation.go b/internal/conformance/attestation.go new file mode 100644 index 0000000..9e53716 --- /dev/null +++ b/internal/conformance/attestation.go @@ -0,0 +1,65 @@ +package conformance + +func buildTestResultStatement(decision Decision) Statement { + result := "WARNED" + switch decision.Status { + case StatusConformant: + result = "PASSED" + case StatusNonconformant: + result = "FAILED" + } + + predicate := TestResultPredicate{ + Result: result, + Configuration: []ResourceDescriptor{ + resourceFromEvaluation(decision.Profile), + resourceFromEvaluation(decision.Matrix), + }, + URL: decision.Report.URL, + } + for _, assertion := range decision.Assertions { + switch assertion.Status { + case AssertionPass: + predicate.PassedTests = append(predicate.PassedTests, assertion.ID) + case AssertionFail: + predicate.FailedTests = append(predicate.FailedTests, assertion.ID) + case AssertionInconclusive: + predicate.WarnedTests = append(predicate.WarnedTests, assertion.ID) + } + } + return Statement{ + Type: StatementTypeV1, + Subject: []ResourceDescriptor{decision.Subject}, + PredicateType: TestResultPredicateTypeV01, + Predicate: predicate, + } +} + +func buildVerificationResultStatement(decision Decision) Statement { + return Statement{ + Type: StatementTypeV1, + Subject: []ResourceDescriptor{decision.Subject}, + PredicateType: SVRPredicateTypeV02, + Predicate: SVRPredicate{ + Verifier: SVRVerifier{ + ID: decision.Verifier.ID, + Policies: []ResourceDescriptor{ + resourceFromEvaluation(decision.Profile), + resourceFromEvaluation(decision.Matrix), + }, + }, + TimeCreated: decision.TimeCreated, + Properties: append([]string(nil), decision.Properties...), + }, + } +} + +func resourceFromEvaluation(resource EvaluatedResource) ResourceDescriptor { + return ResourceDescriptor{ + Name: resource.ID, + URI: resource.URI, + Digest: map[string]string{ + "sha256": resource.SHA256, + }, + } +} diff --git a/internal/conformance/evaluate.go b/internal/conformance/evaluate.go new file mode 100644 index 0000000..6855858 --- /dev/null +++ b/internal/conformance/evaluate.go @@ -0,0 +1,551 @@ +package conformance + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + "regexp" + "sort" + "strings" + "time" + + "github.com/kernel-guard/bpfcompat/pkg/schema" +) + +var sha256Pattern = regexp.MustCompile(`^[a-fA-F0-9]{64}$`) + +const maxConformanceReportBytes = 64 << 20 + +func LoadReport(path string) (ReportDocument, error) { + file, err := os.Open(path) + if err != nil { + return ReportDocument{}, fmt.Errorf("read conformance report: %w", err) + } + defer file.Close() + raw, err := io.ReadAll(io.LimitReader(file, maxConformanceReportBytes+1)) + if err != nil { + return ReportDocument{}, fmt.Errorf("read conformance report: %w", err) + } + if len(raw) > maxConformanceReportBytes { + return ReportDocument{}, fmt.Errorf("conformance report exceeds %d bytes", maxConformanceReportBytes) + } + if err := rejectDuplicateJSONKeys(raw); err != nil { + return ReportDocument{}, fmt.Errorf("parse conformance report JSON: %w", err) + } + var report schema.ReportV01 + if err := json.Unmarshal(raw, &report); err != nil { + return ReportDocument{}, fmt.Errorf("parse conformance report JSON: %w", err) + } + if strings.TrimSpace(report.Run.ID) == "" { + return ReportDocument{}, errors.New("conformance report run.id is required") + } + return ReportDocument{Report: report, Raw: raw, SHA256: digestBytes(raw)}, nil +} + +func Evaluate(profileDoc ProfileDocument, reportDoc ReportDocument, opts EvaluateOptions) (Evaluation, error) { + if err := validateVerifierURI(opts.VerifierID); err != nil { + return Evaluation{}, err + } + if opts.ReportURL != "" { + if err := validateAbsoluteHTTPSURI("report URL", opts.ReportURL); err != nil { + return Evaluation{}, err + } + } + if len(profileDoc.Raw) == 0 || profileDoc.SHA256 == "" || len(profileDoc.MatrixRaw) == 0 || profileDoc.MatrixSHA256 == "" { + return Evaluation{}, errors.New("loaded conformance profile document is incomplete") + } + if len(reportDoc.Raw) == 0 || reportDoc.SHA256 == "" { + return Evaluation{}, errors.New("loaded conformance report document is incomplete") + } + + now := time.Now().UTC() + if opts.Now != nil { + now = opts.Now().UTC() + } + ctx := newEvaluationContext(profileDoc, reportDoc, now) + results := make([]AssertionResult, 0, len(profileDoc.Profile.Assertions)) + for _, assertion := range profileDoc.Profile.Assertions { + result, err := ctx.evaluateAssertion(assertion) + if err != nil { + return Evaluation{}, err + } + results = append(results, result) + } + + status := decisionStatus(results) + subject, err := subjectDescriptor(reportDoc.Report, profileDoc.Profile.Subject) + if err != nil { + return Evaluation{}, err + } + validUntil := "" + if startedAt, err := time.Parse(time.RFC3339, reportDoc.Report.Run.StartedAt); err == nil { + validUntil = startedAt.UTC().AddDate(0, 0, profileDoc.Profile.Validity.SnapshotDays).Format(time.RFC3339) + } + + decision := Decision{ + SchemaVersion: DecisionSchemaVersion, + Status: status, + ProgramStatus: profileDoc.Profile.ProgramStatus, + Claim: strings.TrimSpace(profileDoc.Profile.Claim), + Limitations: append([]string(nil), profileDoc.Profile.Limitations...), + Subject: subject, + Profile: EvaluatedResource{ + ID: profileDoc.Profile.ID, + URI: profileDoc.Profile.ProfileURI, + SHA256: profileDoc.SHA256, + }, + Matrix: EvaluatedResource{ + ID: profileDoc.Matrix.Name, + URI: profileDoc.Profile.Matrix.URI, + SHA256: profileDoc.MatrixSHA256, + }, + Report: EvaluatedReport{ + SchemaVersion: reportDoc.Report.SchemaVersion, + RunID: reportDoc.Report.Run.ID, + StartedAt: reportDoc.Report.Run.StartedAt, + SHA256: reportDoc.SHA256, + URL: strings.TrimSpace(opts.ReportURL), + }, + Verifier: VerifierIdentity{ID: strings.TrimSpace(opts.VerifierID)}, + TimeCreated: now.Format(time.RFC3339), + ValidUntil: validUntil, + Assertions: results, + Targets: ctx.targetEvidence(), + } + if status == StatusConformant { + decision.Properties = append([]string(nil), profileDoc.Profile.Properties...) + } + + testResult := buildTestResultStatement(decision) + var verificationResult *Statement + if status == StatusConformant { + statement := buildVerificationResultStatement(decision) + verificationResult = &statement + } + return Evaluation{ + Decision: decision, + TestResult: testResult, + VerificationResult: verificationResult, + }, nil +} + +type evaluationContext struct { + profileDoc ProfileDocument + reportDoc ReportDocument + now time.Time + targets map[string][]*schema.Target +} + +func newEvaluationContext(profileDoc ProfileDocument, reportDoc ReportDocument, now time.Time) evaluationContext { + targets := make(map[string][]*schema.Target, len(reportDoc.Report.Targets)) + for i := range reportDoc.Report.Targets { + target := &reportDoc.Report.Targets[i] + id := strings.TrimSpace(target.ProfileID) + targets[id] = append(targets[id], target) + } + return evaluationContext{profileDoc: profileDoc, reportDoc: reportDoc, now: now, targets: targets} +} + +func (ctx evaluationContext) evaluateAssertion(assertion AssertionContract) (AssertionResult, error) { + var status, summary string + var evidence []string + switch assertion.Rule { + case RuleReportSchema: + status, summary, evidence = ctx.evaluateReportSchema() + case RuleReportFreshness: + status, summary, evidence = ctx.evaluateReportFreshness() + case RuleSubjectLoaderIdentity: + status, summary, evidence = ctx.evaluateSubjectIdentity() + case RuleCommandContract: + status, summary, evidence = ctx.evaluateCommandContract() + case RuleMatrixRequiredProfiles: + status, summary, evidence = ctx.evaluateMatrixCoverage() + case RuleEnvironmentIdentity: + status, summary, evidence = ctx.evaluateEnvironmentIdentity() + case RuleEnvironmentImageDigest: + status, summary, evidence = ctx.evaluateImageDigests() + case RuleBehaviorCommandPass: + status, summary, evidence = ctx.evaluateCommandBehavior() + default: + return AssertionResult{}, fmt.Errorf("unsupported conformance assertion rule %q", assertion.Rule) + } + return AssertionResult{ + ID: assertion.ID, + Rule: assertion.Rule, + Description: assertion.Description, + Status: status, + Summary: summary, + Evidence: evidence, + }, nil +} + +func (ctx evaluationContext) evaluateReportSchema() (string, string, []string) { + report := ctx.reportDoc.Report + if !containsString(ctx.profileDoc.Profile.ReportSchemaVersions, report.SchemaVersion) { + return AssertionInconclusive, "report schema is not allowed by the profile", []string{"schema_version=" + report.SchemaVersion} + } + if report.Summary.Status != "pass" && report.Summary.Status != "fail" { + return AssertionInconclusive, "report summary status is missing or invalid", []string{"summary.status=" + report.Summary.Status} + } + return AssertionPass, "report schema and summary status are valid", []string{ + "schema_version=" + report.SchemaVersion, + "summary.status=" + report.Summary.Status, + "report.sha256=" + ctx.reportDoc.SHA256, + } +} + +func (ctx evaluationContext) evaluateReportFreshness() (string, string, []string) { + startedAt, err := time.Parse(time.RFC3339, ctx.reportDoc.Report.Run.StartedAt) + if err != nil { + return AssertionInconclusive, "report run timestamp is missing or invalid", []string{"run.started_at=" + ctx.reportDoc.Report.Run.StartedAt} + } + startedAt = startedAt.UTC() + validUntil := startedAt.AddDate(0, 0, ctx.profileDoc.Profile.Validity.SnapshotDays) + evidence := []string{ + "run.started_at=" + startedAt.Format(time.RFC3339), + "valid_until=" + validUntil.Format(time.RFC3339), + "evaluated_at=" + ctx.now.Format(time.RFC3339), + } + if startedAt.After(ctx.now.Add(5 * time.Minute)) { + return AssertionInconclusive, "report run timestamp is in the future", evidence + } + if ctx.now.After(validUntil) { + return AssertionInconclusive, "report is outside the snapshot validity window", evidence + } + return AssertionPass, "report is inside the snapshot validity window", evidence +} + +func (ctx evaluationContext) evaluateSubjectIdentity() (string, string, []string) { + command := ctx.reportDoc.Report.Command + if command == nil || command.Binary == nil { + return AssertionInconclusive, "report does not identify a command loader binary", nil + } + binary := command.Binary + evidence := []string{ + "binary.basename=" + binary.BaseName, + "binary.sha256=" + strings.ToLower(binary.SHA256), + fmt.Sprintf("binary.size_bytes=%d", binary.SizeBytes), + } + if binary.BaseName != ctx.profileDoc.Profile.Subject.BinaryBaseName { + return AssertionInconclusive, "loader basename does not match the profile subject", evidence + } + if !isSHA256(binary.SHA256) || binary.SizeBytes <= 0 { + return AssertionInconclusive, "loader binary identity is incomplete", evidence + } + return AssertionPass, "exact loader binary identity is recorded", evidence +} + +func (ctx evaluationContext) evaluateCommandContract() (string, string, []string) { + command := ctx.reportDoc.Report.Command + if command == nil { + return AssertionInconclusive, "report has no command contract", nil + } + expectedDigest := commandInvocationDigest( + ctx.profileDoc.Profile.Subject.Command, + ctx.profileDoc.Profile.Subject.ExpectedExitCode, + ) + evidence := []string{ + "invocation.sha256=" + strings.ToLower(command.InvocationSHA256), + "expected_invocation.sha256=" + expectedDigest, + fmt.Sprintf("expected_exit_code=%d", command.ExpectedExitCode), + } + if !isSHA256(command.InvocationSHA256) || !strings.EqualFold(command.InvocationSHA256, expectedDigest) { + return AssertionInconclusive, "recorded invocation does not match the profile command", evidence + } + if command.ExpectedExitCode != ctx.profileDoc.Profile.Subject.ExpectedExitCode { + return AssertionInconclusive, "recorded expected exit code does not match the profile", evidence + } + return AssertionPass, "recorded invocation matches the profile command contract", evidence +} + +func (ctx evaluationContext) evaluateMatrixCoverage() (string, string, []string) { + reportProfiles := occurrences(ctx.reportDoc.Report.Matrix.Profiles) + var issues []string + evidence := make([]string, 0, len(ctx.profileDoc.Matrix.Profiles)) + for _, required := range ctx.profileDoc.Matrix.Profiles { + id := required.ID + evidence = append(evidence, "required_profile="+id) + if reportProfiles[id] != 1 { + issues = append(issues, fmt.Sprintf("matrix profile %s occurs %d times", id, reportProfiles[id])) + } + if len(ctx.targets[id]) != 1 { + issues = append(issues, fmt.Sprintf("target %s occurs %d times", id, len(ctx.targets[id]))) + continue + } + if !ctx.targets[id][0].Required { + issues = append(issues, "target "+id+" is not marked required") + } + } + if len(issues) > 0 { + sort.Strings(issues) + return AssertionInconclusive, "required matrix coverage is incomplete or ambiguous", append(evidence, issues...) + } + return AssertionPass, "every canonical matrix profile appears exactly once and is required", evidence +} + +func (ctx evaluationContext) evaluateEnvironmentIdentity() (string, string, []string) { + var issues []string + evidence := make([]string, 0, len(ctx.profileDoc.Profile.Environments)) + for _, expected := range ctx.profileDoc.Profile.Environments { + target, ok := ctx.singleTarget(expected.ProfileID) + if !ok { + issues = append(issues, expected.ProfileID+": target missing or duplicated") + continue + } + if target.Profile == nil || target.Host == nil { + issues = append(issues, expected.ProfileID+": requested or actual environment missing") + continue + } + requested := target.Profile + host := target.Host + if requested.Distro == "" || requested.Version == "" || requested.KernelFamily == "" || requested.Arch == "" || + host.Distro == "" || host.Version == "" || host.KernelFamily == "" || host.Kernel == "" || host.Arch == "" { + issues = append(issues, expected.ProfileID+": environment identity is incomplete") + continue + } + if requested.Distro != expected.Distro || requested.Version != expected.Version || + requested.KernelFamily != expected.KernelFamily || requested.Arch != expected.Arch { + issues = append(issues, expected.ProfileID+": requested environment does not match the profile contract") + continue + } + if host.Distro != expected.Distro || host.Version != expected.Version || + host.KernelFamily != expected.KernelFamily || host.Arch != expected.Arch { + issues = append(issues, expected.ProfileID+": actual environment does not match the profile contract") + continue + } + evidence = append(evidence, fmt.Sprintf("%s=%s/%s kernel=%s arch=%s", expected.ProfileID, host.Distro, host.Version, host.Kernel, host.Arch)) + } + if len(issues) > 0 { + sort.Strings(issues) + return AssertionInconclusive, "one or more target environment identities are incomplete", append(evidence, issues...) + } + return AssertionPass, "requested environments and actual host kernels are recorded", evidence +} + +func (ctx evaluationContext) evaluateImageDigests() (string, string, []string) { + var issues []string + evidence := make([]string, 0, len(ctx.profileDoc.Matrix.Profiles)) + for _, required := range ctx.profileDoc.Matrix.Profiles { + target, ok := ctx.singleTarget(required.ID) + if !ok { + issues = append(issues, required.ID+": target missing or duplicated") + continue + } + digest, err := imageDigestFromNotes(target.Notes) + if err != nil { + issues = append(issues, required.ID+": "+err.Error()) + continue + } + evidence = append(evidence, required.ID+".image.sha256="+digest) + } + if len(issues) > 0 { + sort.Strings(issues) + return AssertionInconclusive, "one or more booted image digests are unavailable", append(evidence, issues...) + } + return AssertionPass, "every required target records the booted image digest", evidence +} + +func (ctx evaluationContext) evaluateCommandBehavior() (string, string, []string) { + targetCount := len(ctx.profileDoc.Matrix.Profiles) + productFailures := make([]string, 0, targetCount) + inconclusive := make([]string, 0, targetCount) + evidence := make([]string, 0, targetCount) + for _, required := range ctx.profileDoc.Matrix.Profiles { + target, ok := ctx.singleTarget(required.ID) + if !ok { + inconclusive = append(inconclusive, required.ID+": target missing or duplicated") + continue + } + if target.Status == "infra_error" || strings.TrimSpace(target.InfraError) != "" { + inconclusive = append(inconclusive, required.ID+": infrastructure error") + continue + } + test, ok := commandFunctionalTest(target) + if !ok { + inconclusive = append(inconclusive, required.ID+": required command result missing or duplicated") + continue + } + if test.Command != ctx.profileDoc.Profile.Subject.Command || test.ExpectedExitCode != ctx.profileDoc.Profile.Subject.ExpectedExitCode { + inconclusive = append(inconclusive, required.ID+": per-target command contract differs") + continue + } + if target.Status == "pass" && target.Functional != nil && target.Functional.Status == "pass" && + test.Status == "pass" && test.ExitCode == test.ExpectedExitCode { + evidence = append(evidence, fmt.Sprintf("%s=pass exit=%d", required.ID, test.ExitCode)) + continue + } + if target.Status == "fail" && target.FailedStage == "command" && + target.ClassificationCode == "COMMAND_VALIDATION_FAILURE" && test.Status == "fail" { + productFailures = append(productFailures, fmt.Sprintf("%s=fail exit=%d expected=%d", required.ID, test.ExitCode, test.ExpectedExitCode)) + continue + } + inconclusive = append(inconclusive, required.ID+": command result is internally inconsistent") + } + if len(productFailures) > 0 { + sort.Strings(productFailures) + sort.Strings(inconclusive) + return AssertionFail, "the prescribed loader behavior failed on a required target", append(append(evidence, productFailures...), inconclusive...) + } + if len(inconclusive) > 0 { + sort.Strings(inconclusive) + return AssertionInconclusive, "the prescribed loader behavior was not completed on every required target", append(evidence, inconclusive...) + } + return AssertionPass, "the prescribed loader behavior passed on every required target", evidence +} + +func (ctx evaluationContext) singleTarget(id string) (*schema.Target, bool) { + targets := ctx.targets[id] + if len(targets) != 1 { + return nil, false + } + return targets[0], true +} + +func (ctx evaluationContext) targetEvidence() []TargetEvidence { + evidence := make([]TargetEvidence, 0, len(ctx.profileDoc.Matrix.Profiles)) + for _, required := range ctx.profileDoc.Matrix.Profiles { + target, ok := ctx.singleTarget(required.ID) + if !ok { + evidence = append(evidence, TargetEvidence{ProfileID: required.ID, Status: "missing", Required: true}) + continue + } + entry := TargetEvidence{ + ProfileID: target.ProfileID, + Status: target.Status, + Required: target.Required, + ClassificationCode: target.ClassificationCode, + } + if target.Host != nil { + entry.HostKernel = target.Host.Kernel + entry.Architecture = target.Host.Arch + } + if digest, err := imageDigestFromNotes(target.Notes); err == nil { + entry.ImageSHA256 = digest + } + evidence = append(evidence, entry) + } + return evidence +} + +func decisionStatus(results []AssertionResult) string { + status := StatusConformant + for _, result := range results { + if result.Status == AssertionFail { + return StatusNonconformant + } + if result.Status == AssertionInconclusive { + status = StatusInconclusive + } + } + return status +} + +func subjectDescriptor(report schema.ReportV01, contract SubjectContract) (ResourceDescriptor, error) { + if report.Command != nil && report.Command.Binary != nil && isSHA256(report.Command.Binary.SHA256) { + return ResourceDescriptor{ + Name: report.Command.Binary.BaseName, + Digest: map[string]string{"sha256": strings.ToLower(report.Command.Binary.SHA256)}, + }, nil + } + if isSHA256(report.Artifact.SHA256) { + name := report.Artifact.BaseName + if name == "" { + name = contract.BinaryBaseName + } + return ResourceDescriptor{ + Name: name, + Digest: map[string]string{"sha256": strings.ToLower(report.Artifact.SHA256)}, + }, nil + } + return ResourceDescriptor{}, errors.New("report has no immutable SHA-256 subject identity") +} + +func commandInvocationDigest(command string, expectedExit int) string { + invocation := fmt.Sprintf("%s\x00expected-exit=%d", strings.TrimSpace(command), expectedExit) + sum := sha256.Sum256([]byte(invocation)) + return hex.EncodeToString(sum[:]) +} + +func commandFunctionalTest(target *schema.Target) (*schema.FunctionalTest, bool) { + if target.Functional == nil { + return nil, false + } + var found []*schema.FunctionalTest + for i := range target.Functional.Tests { + test := &target.Functional.Tests[i] + if test.Name == "command" && test.Required { + found = append(found, test) + } + } + if len(found) != 1 { + return nil, false + } + return found[0], true +} + +func imageDigestFromNotes(notes []string) (string, error) { + const prefix = "base image sha256:" + values := make(map[string]struct{}) + for _, note := range notes { + trimmed := strings.TrimSpace(note) + if !strings.HasPrefix(strings.ToLower(trimmed), prefix) { + continue + } + value := strings.TrimSpace(trimmed[len(prefix):]) + if !isSHA256(value) { + return "", errors.New("base image SHA-256 note is malformed") + } + values[strings.ToLower(value)] = struct{}{} + } + if len(values) == 0 { + return "", errors.New("base image SHA-256 note is missing") + } + if len(values) != 1 { + return "", errors.New("base image SHA-256 notes disagree") + } + for value := range values { + return value, nil + } + panic("unreachable") +} + +func occurrences(values []string) map[string]int { + counts := make(map[string]int, len(values)) + for _, value := range values { + counts[strings.TrimSpace(value)]++ + } + return counts +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func isSHA256(value string) bool { + return sha256Pattern.MatchString(strings.TrimSpace(value)) +} + +func validateVerifierURI(raw string) error { + if strings.TrimSpace(raw) == "" { + return errors.New("verifier ID is required") + } + return validateAbsoluteHTTPSURI("verifier ID", raw) +} + +func validateAbsoluteHTTPSURI(field, raw string) error { + parsed, err := url.ParseRequestURI(strings.TrimSpace(raw)) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" { + return fmt.Errorf("%s must be an absolute https URI", field) + } + return nil +} diff --git a/internal/conformance/evaluate_test.go b/internal/conformance/evaluate_test.go new file mode 100644 index 0000000..4c1d0a4 --- /dev/null +++ b/internal/conformance/evaluate_test.go @@ -0,0 +1,318 @@ +package conformance + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kernel-guard/bpfcompat/pkg/schema" +) + +const testVerifierID = "https://bpfcompat.kernelguard.net/verifiers/program/v0.1" + +func TestEvaluateFalcoConformant(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + + evaluation, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatalf("evaluate conformant report: %v", err) + } + if evaluation.Decision.Status != StatusConformant { + t.Fatalf("decision status = %q, want %q: %+v", evaluation.Decision.Status, StatusConformant, evaluation.Decision.Assertions) + } + if evaluation.Decision.Subject.Name != "scap-open" || + evaluation.Decision.Subject.Digest["sha256"] != "1111111111111111111111111111111111111111111111111111111111111111" { + t.Fatalf("unexpected subject: %+v", evaluation.Decision.Subject) + } + if evaluation.Decision.ValidUntil != "2026-11-04T12:00:00Z" { + t.Fatalf("valid_until = %q", evaluation.Decision.ValidUntil) + } + if len(evaluation.Decision.Properties) != len(profile.Profile.Properties) { + t.Fatalf("properties = %v, want %v", evaluation.Decision.Properties, profile.Profile.Properties) + } + for _, assertion := range evaluation.Decision.Assertions { + if assertion.Status != AssertionPass { + t.Fatalf("assertion %s = %s, want pass", assertion.ID, assertion.Status) + } + } + if evaluation.VerificationResult == nil { + t.Fatal("conformant evaluation did not produce an SVR") + } + if evaluation.VerificationResult.PredicateType != SVRPredicateTypeV02 { + t.Fatalf("SVR predicate type = %q", evaluation.VerificationResult.PredicateType) + } + testResult, ok := evaluation.TestResult.Predicate.(TestResultPredicate) + if !ok { + t.Fatalf("test result predicate has type %T", evaluation.TestResult.Predicate) + } + if testResult.Result != "PASSED" || len(testResult.PassedTests) != len(profile.Profile.Assertions) { + t.Fatalf("unexpected test result: %+v", testResult) + } +} + +func TestEvaluateFalcoProductFailure(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + target := &report.Report.Targets[2] + target.Status = "fail" + target.FailedStage = "command" + target.ClassificationCode = "COMMAND_VALIDATION_FAILURE" + target.Functional.Status = "fail" + target.Functional.Tests[0].Status = "fail" + target.Functional.Tests[0].ExitCode = 2 + report.Report.Summary.Status = "fail" + report = reportDocumentFromStruct(t, report.Report) + + evaluation, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatalf("evaluate nonconformant report: %v", err) + } + if evaluation.Decision.Status != StatusNonconformant { + t.Fatalf("decision status = %q, want %q", evaluation.Decision.Status, StatusNonconformant) + } + if evaluation.VerificationResult != nil { + t.Fatal("nonconformant evaluation produced an SVR") + } + assertion := assertionForRule(t, evaluation.Decision, RuleBehaviorCommandPass) + if assertion.Status != AssertionFail { + t.Fatalf("behavior assertion = %q, want fail", assertion.Status) + } + testResult := evaluation.TestResult.Predicate.(TestResultPredicate) + if testResult.Result != "FAILED" || len(testResult.FailedTests) != 1 || testResult.FailedTests[0] != assertion.ID { + t.Fatalf("unexpected failed test result: %+v", testResult) + } +} + +func TestEvaluateFalcoInfrastructureFailureIsInconclusive(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + target := &report.Report.Targets[0] + target.Status = "infra_error" + target.FailedStage = "infra" + target.InfraError = "guest did not become reachable" + target.Host = nil + target.Functional = nil + report.Report.Summary.Status = "fail" + report = reportDocumentFromStruct(t, report.Report) + + evaluation, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatalf("evaluate inconclusive report: %v", err) + } + if evaluation.Decision.Status != StatusInconclusive { + t.Fatalf("decision status = %q, want %q", evaluation.Decision.Status, StatusInconclusive) + } + if evaluation.VerificationResult != nil { + t.Fatal("inconclusive evaluation produced an SVR") + } + assertion := assertionForRule(t, evaluation.Decision, RuleBehaviorCommandPass) + if assertion.Status != AssertionInconclusive { + t.Fatalf("behavior assertion = %q, want inconclusive", assertion.Status) + } + testResult := evaluation.TestResult.Predicate.(TestResultPredicate) + if testResult.Result != "WARNED" || len(testResult.WarnedTests) == 0 { + t.Fatalf("unexpected inconclusive test result: %+v", testResult) + } +} + +func TestEvaluateKnownProductFailureOutranksInfrastructureGap(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + failed := &report.Report.Targets[0] + failed.Status = "fail" + failed.FailedStage = "command" + failed.ClassificationCode = "COMMAND_VALIDATION_FAILURE" + failed.Functional.Status = "fail" + failed.Functional.Tests[0].Status = "fail" + failed.Functional.Tests[0].ExitCode = 7 + infrastructure := &report.Report.Targets[1] + infrastructure.Status = "infra_error" + infrastructure.FailedStage = "infra" + infrastructure.InfraError = "runner unavailable" + infrastructure.Functional = nil + report.Report.Summary.Status = "fail" + report = reportDocumentFromStruct(t, report.Report) + + evaluation, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatalf("evaluate mixed report: %v", err) + } + if evaluation.Decision.Status != StatusNonconformant { + t.Fatalf("decision status = %q, want %q", evaluation.Decision.Status, StatusNonconformant) + } +} + +func TestEvaluateWrongCommandAndExpiredReportAreInconclusive(t *testing.T) { + tests := []struct { + name string + mutate func(*ReportDocument) + now time.Time + rule string + }{ + { + name: "wrong command digest", + mutate: func(report *ReportDocument) { + report.Report.Command.InvocationSHA256 = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + now: time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC), + rule: RuleCommandContract, + }, + { + name: "expired report", + mutate: func(*ReportDocument) {}, + now: time.Date(2026, 11, 5, 12, 0, 1, 0, time.UTC), + rule: RuleReportFreshness, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + test.mutate(&report) + report = reportDocumentFromStruct(t, report.Report) + opts := testEvaluationOptions() + opts.Now = func() time.Time { return test.now } + evaluation, err := Evaluate(profile, report, opts) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if evaluation.Decision.Status != StatusInconclusive { + t.Fatalf("decision status = %q, want inconclusive", evaluation.Decision.Status) + } + if assertion := assertionForRule(t, evaluation.Decision, test.rule); assertion.Status != AssertionInconclusive { + t.Fatalf("assertion %s = %s, want inconclusive", assertion.ID, assertion.Status) + } + }) + } +} + +func TestEvaluateEnvironmentMustMatchProfileContract(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + report.Report.Targets[0].Profile.Distro = "debian" + report.Report.Targets[0].Host.Distro = "debian" + report = reportDocumentFromStruct(t, report.Report) + + evaluation, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatalf("evaluate mismatched environment: %v", err) + } + if evaluation.Decision.Status != StatusInconclusive { + t.Fatalf("decision status = %q, want inconclusive", evaluation.Decision.Status) + } + assertion := assertionForRule(t, evaluation.Decision, RuleEnvironmentIdentity) + if assertion.Status != AssertionInconclusive { + t.Fatalf("environment assertion = %q, want inconclusive", assertion.Status) + } +} + +func TestWriteEvaluationRemovesStaleSVR(t *testing.T) { + profile := loadFalcoProfile(t) + report := loadFalcoReport(t) + pass, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatal(err) + } + outDir := t.TempDir() + paths, err := WriteEvaluation(outDir, pass) + if err != nil { + t.Fatalf("write pass evaluation: %v", err) + } + if _, err := os.Stat(paths.VerificationResult); err != nil { + t.Fatalf("stat SVR: %v", err) + } + + report.Report.Targets[0].Status = "infra_error" + report.Report.Targets[0].InfraError = "no KVM" + report.Report.Targets[0].Functional = nil + report.Report.Summary.Status = "fail" + report = reportDocumentFromStruct(t, report.Report) + inconclusive, err := Evaluate(profile, report, testEvaluationOptions()) + if err != nil { + t.Fatal(err) + } + paths, err = WriteEvaluation(outDir, inconclusive) + if err != nil { + t.Fatalf("write inconclusive evaluation: %v", err) + } + if paths.VerificationResult != "" { + t.Fatalf("inconclusive paths contain SVR: %+v", paths) + } + if _, err := os.Stat(filepath.Join(outDir, VerificationResultFileName)); !os.IsNotExist(err) { + t.Fatalf("stale SVR still exists: %v", err) + } + for _, path := range []string{paths.Decision, paths.TestResult} { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + } +} + +func TestLoadReportRejectsDuplicateKeys(t *testing.T) { + path := filepath.Join(t.TempDir(), "report.json") + raw := []byte(`{"schema_version":"v0.1","schema_version":"v9","run":{"id":"run"}}`) + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadReport(path); err == nil || !strings.Contains(err.Error(), "duplicate object key") { + t.Fatalf("expected duplicate-key error, got %v", err) + } +} + +func loadFalcoProfile(t *testing.T) ProfileDocument { + t.Helper() + path := filepath.Join("..", "..", "conformance", "falco-modern-bpf-v0.1", "profile.yaml") + profile, err := LoadProfile(path) + if err != nil { + t.Fatalf("load Falco profile: %v", err) + } + return profile +} + +func loadFalcoReport(t *testing.T) ReportDocument { + t.Helper() + report, err := LoadReport(filepath.Join("testdata", "falco-pass-report.json")) + if err != nil { + t.Fatalf("load Falco report fixture: %v", err) + } + return report +} + +func reportDocumentFromStruct(t *testing.T, report schema.ReportV01) ReportDocument { + t.Helper() + raw, err := json.Marshal(report) + if err != nil { + t.Fatalf("marshal report fixture: %v", err) + } + return ReportDocument{Report: report, Raw: raw, SHA256: digestBytes(raw)} +} + +func testEvaluationOptions() EvaluateOptions { + return EvaluateOptions{ + VerifierID: testVerifierID, + ReportURL: "https://github.com/falcosecurity/libs/actions/runs/123456789", + Now: func() time.Time { + return time.Date(2026, 8, 7, 12, 0, 0, 0, time.UTC) + }, + } +} + +func assertionForRule(t *testing.T, decision Decision, rule string) AssertionResult { + t.Helper() + for _, assertion := range decision.Assertions { + if assertion.Rule == rule { + return assertion + } + } + t.Fatalf("decision has no assertion for rule %q", rule) + return AssertionResult{} +} diff --git a/internal/conformance/json.go b/internal/conformance/json.go new file mode 100644 index 0000000..7dd5fd2 --- /dev/null +++ b/internal/conformance/json.go @@ -0,0 +1,83 @@ +package conformance + +import ( + "bytes" + "encoding/json" + "fmt" + "io" +) + +// rejectDuplicateJSONKeys keeps security-sensitive evidence unambiguous across +// JSON implementations. encoding/json otherwise accepts duplicate object keys +// and retains the last value, while another verifier might retain the first. +func rejectDuplicateJSONKeys(raw []byte) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + if err := walkJSONValue(decoder, "$"); err != nil { + return err + } + if _, err := decoder.Token(); err != io.EOF { + if err == nil { + return fmt.Errorf("multiple top-level JSON values") + } + return err + } + return nil +} + +func walkJSONValue(decoder *json.Decoder, path string) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, ok := token.(json.Delim) + if !ok { + return nil + } + + switch delimiter { + case '{': + seen := make(map[string]struct{}) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + key, ok := keyToken.(string) + if !ok { + return fmt.Errorf("object key at %s is not a string", path) + } + if _, exists := seen[key]; exists { + return fmt.Errorf("duplicate object key %q at %s", key, path) + } + seen[key] = struct{}{} + if err := walkJSONValue(decoder, path+"."+key); err != nil { + return err + } + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim('}') { + return fmt.Errorf("object at %s is not closed", path) + } + case '[': + index := 0 + for decoder.More() { + if err := walkJSONValue(decoder, fmt.Sprintf("%s[%d]", path, index)); err != nil { + return err + } + index++ + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim(']') { + return fmt.Errorf("array at %s is not closed", path) + } + default: + return fmt.Errorf("unexpected JSON delimiter %q at %s", delimiter, path) + } + return nil +} diff --git a/internal/conformance/profile.go b/internal/conformance/profile.go new file mode 100644 index 0000000..0521dac --- /dev/null +++ b/internal/conformance/profile.go @@ -0,0 +1,310 @@ +package conformance + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/kernel-guard/bpfcompat/internal/matrix" + "gopkg.in/yaml.v3" +) + +const ( + RuleReportSchema = "report.schema" + RuleReportFreshness = "report.freshness" + RuleSubjectLoaderIdentity = "subject.loader_identity" + RuleCommandContract = "command.contract" + RuleMatrixRequiredProfiles = "matrix.required_profiles" + RuleEnvironmentIdentity = "environment.actual_identity" + RuleEnvironmentImageDigest = "environment.image_digest" + RuleBehaviorCommandPass = "behavior.command_pass" +) + +var ( + profileIDPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{0,127}$`) + assertionIDPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_-]{2,127}$`) + propertyPattern = regexp.MustCompile(`^[A-Z][A-Z0-9_]{2,127}$`) + knownRules = map[string]struct{}{ + RuleReportSchema: {}, + RuleReportFreshness: {}, + RuleSubjectLoaderIdentity: {}, + RuleCommandContract: {}, + RuleMatrixRequiredProfiles: {}, + RuleEnvironmentIdentity: {}, + RuleEnvironmentImageDigest: {}, + RuleBehaviorCommandPass: {}, + } +) + +type ProfileDocument struct { + Profile Profile + Raw []byte + SHA256 string + Path string + Matrix matrix.Matrix + MatrixRaw []byte + MatrixSHA256 string + MatrixPath string +} + +func LoadProfile(path string) (ProfileDocument, error) { + profilePath, err := filepath.Abs(path) + if err != nil { + return ProfileDocument{}, fmt.Errorf("resolve conformance profile path: %w", err) + } + raw, err := os.ReadFile(profilePath) + if err != nil { + return ProfileDocument{}, fmt.Errorf("read conformance profile: %w", err) + } + + var profile Profile + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + decoder.KnownFields(true) + if err := decoder.Decode(&profile); err != nil { + if !errors.Is(err, io.EOF) { + return ProfileDocument{}, fmt.Errorf("parse conformance profile YAML: %w", err) + } + } + var extraProfileDocument any + if err := decoder.Decode(&extraProfileDocument); !errors.Is(err, io.EOF) { + if err != nil { + return ProfileDocument{}, fmt.Errorf("parse conformance profile YAML: %w", err) + } + return ProfileDocument{}, errors.New("conformance profile must contain exactly one YAML document") + } + if err := validateProfile(profile); err != nil { + return ProfileDocument{}, err + } + + matrixPath, err := resolveProfileResource(profilePath, profile.Matrix.Path) + if err != nil { + return ProfileDocument{}, err + } + matrixRaw, err := os.ReadFile(matrixPath) + if err != nil { + return ProfileDocument{}, fmt.Errorf("read conformance matrix: %w", err) + } + if err := requireSingleYAMLDocument(matrixRaw, "conformance matrix"); err != nil { + return ProfileDocument{}, err + } + mx, err := matrix.LoadBytes(matrixRaw) + if err != nil { + return ProfileDocument{}, fmt.Errorf("load conformance matrix: %w", err) + } + for i := range mx.Profiles { + if !mx.Profiles[i].RequiredBool() { + return ProfileDocument{}, fmt.Errorf("conformance matrix profile %q must be required", mx.Profiles[i].ID) + } + } + if err := validateEnvironmentMatrix(profile.Environments, mx); err != nil { + return ProfileDocument{}, err + } + + return ProfileDocument{ + Profile: profile, + Raw: raw, + SHA256: digestBytes(raw), + Path: profilePath, + Matrix: mx, + MatrixRaw: matrixRaw, + MatrixSHA256: digestBytes(matrixRaw), + MatrixPath: matrixPath, + }, nil +} + +func requireSingleYAMLDocument(raw []byte, label string) error { + decoder := yaml.NewDecoder(bytes.NewReader(raw)) + var document any + if err := decoder.Decode(&document); err != nil { + return fmt.Errorf("parse %s YAML: %w", label, err) + } + if err := decoder.Decode(&document); !errors.Is(err, io.EOF) { + if err != nil { + return fmt.Errorf("parse %s YAML: %w", label, err) + } + return fmt.Errorf("%s must contain exactly one YAML document", label) + } + return nil +} + +func validateProfile(profile Profile) error { + if profile.SchemaVersion != ProfileSchemaVersion { + return fmt.Errorf("unsupported conformance profile schema_version %q", profile.SchemaVersion) + } + if !profileIDPattern.MatchString(profile.ID) { + return fmt.Errorf("profile id %q must match %s", profile.ID, profileIDPattern.String()) + } + if strings.TrimSpace(profile.Title) == "" { + return errors.New("profile title is required") + } + if profile.ProgramStatus != "preview" && profile.ProgramStatus != "active" { + return fmt.Errorf("profile program_status %q must be preview or active", profile.ProgramStatus) + } + if err := validateHTTPSURI("profile_uri", profile.ProfileURI); err != nil { + return err + } + if strings.TrimSpace(profile.Claim) == "" { + return errors.New("profile claim is required") + } + if len(profile.ReportSchemaVersions) == 0 { + return errors.New("profile report_schema_versions must not be empty") + } + if err := validateUniqueNonEmpty("report_schema_versions", profile.ReportSchemaVersions); err != nil { + return err + } + if profile.Subject.Kind != "command_loader" { + return fmt.Errorf("unsupported profile subject kind %q", profile.Subject.Kind) + } + if strings.TrimSpace(profile.Subject.BinaryBaseName) == "" { + return errors.New("profile subject.binary_basename is required") + } + if strings.TrimSpace(profile.Subject.Command) == "" { + return errors.New("profile subject.command is required") + } + if strings.TrimSpace(profile.Matrix.Path) == "" { + return errors.New("profile matrix.path is required") + } + if err := validateHTTPSURI("matrix.uri", profile.Matrix.URI); err != nil { + return err + } + if len(profile.Environments) == 0 { + return errors.New("profile environments must not be empty") + } + environmentIDs := make(map[string]struct{}, len(profile.Environments)) + for i, environment := range profile.Environments { + if !profileIDPattern.MatchString(environment.ProfileID) { + return fmt.Errorf("environments[%d].profile_id %q must match %s", i, environment.ProfileID, profileIDPattern.String()) + } + if _, exists := environmentIDs[environment.ProfileID]; exists { + return fmt.Errorf("duplicate environment profile_id %q", environment.ProfileID) + } + environmentIDs[environment.ProfileID] = struct{}{} + if strings.TrimSpace(environment.Distro) == "" || strings.TrimSpace(environment.Version) == "" || + strings.TrimSpace(environment.KernelFamily) == "" || strings.TrimSpace(environment.Arch) == "" { + return fmt.Errorf("environments[%d] must define distro, version, kernel_family, and arch", i) + } + } + if profile.Validity.SnapshotDays <= 0 { + return errors.New("profile validity.snapshot_days must be greater than zero") + } + if profile.Validity.ContinuousMaxAgeDays <= 0 { + return errors.New("profile validity.continuous_max_age_days must be greater than zero") + } + if profile.Validity.ContinuousMaxAgeDays > profile.Validity.SnapshotDays { + return errors.New("profile continuous_max_age_days must not exceed snapshot_days") + } + if len(profile.Assertions) == 0 { + return errors.New("profile assertions must not be empty") + } + assertionIDs := make(map[string]struct{}, len(profile.Assertions)) + rules := make(map[string]struct{}, len(profile.Assertions)) + for i := range profile.Assertions { + assertion := profile.Assertions[i] + if !assertionIDPattern.MatchString(assertion.ID) { + return fmt.Errorf("assertions[%d].id %q must match %s", i, assertion.ID, assertionIDPattern.String()) + } + if _, exists := assertionIDs[assertion.ID]; exists { + return fmt.Errorf("duplicate assertion id %q", assertion.ID) + } + assertionIDs[assertion.ID] = struct{}{} + if _, ok := knownRules[assertion.Rule]; !ok { + return fmt.Errorf("assertions[%d].rule %q is unsupported", i, assertion.Rule) + } + if _, exists := rules[assertion.Rule]; exists { + return fmt.Errorf("duplicate assertion rule %q", assertion.Rule) + } + rules[assertion.Rule] = struct{}{} + if strings.TrimSpace(assertion.Description) == "" { + return fmt.Errorf("assertions[%d].description is required", i) + } + } + for rule := range knownRules { + if _, ok := rules[rule]; !ok { + return fmt.Errorf("profile is missing required assertion rule %q", rule) + } + } + if len(profile.Properties) == 0 { + return errors.New("profile properties must not be empty") + } + if err := validateUniqueNonEmpty("properties", profile.Properties); err != nil { + return err + } + for i, property := range profile.Properties { + if !propertyPattern.MatchString(property) { + return fmt.Errorf("properties[%d] %q must match %s", i, property, propertyPattern.String()) + } + } + return nil +} + +func validateEnvironmentMatrix(environments []EnvironmentContract, mx matrix.Matrix) error { + matrixIDs := make(map[string]struct{}, len(mx.Profiles)) + for _, profile := range mx.Profiles { + matrixIDs[profile.ID] = struct{}{} + } + environmentIDs := make(map[string]struct{}, len(environments)) + for _, environment := range environments { + environmentIDs[environment.ProfileID] = struct{}{} + if _, ok := matrixIDs[environment.ProfileID]; !ok { + return fmt.Errorf("environment profile_id %q is not present in the conformance matrix", environment.ProfileID) + } + } + for _, profile := range mx.Profiles { + if _, ok := environmentIDs[profile.ID]; !ok { + return fmt.Errorf("conformance matrix profile %q has no expected environment", profile.ID) + } + } + return nil +} + +func resolveProfileResource(profilePath, resourcePath string) (string, error) { + if filepath.IsAbs(resourcePath) { + return "", errors.New("profile matrix.path must be relative to the profile") + } + baseDir := filepath.Dir(profilePath) + resolved := filepath.Clean(filepath.Join(baseDir, resourcePath)) + rel, err := filepath.Rel(baseDir, resolved) + if err != nil { + return "", fmt.Errorf("resolve profile matrix path: %w", err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", errors.New("profile matrix.path must stay inside the profile directory") + } + return resolved, nil +} + +func validateHTTPSURI(field, raw string) error { + parsed, err := url.ParseRequestURI(strings.TrimSpace(raw)) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" { + return fmt.Errorf("profile %s must be an absolute https URI", field) + } + return nil +} + +func validateUniqueNonEmpty(field string, values []string) error { + seen := make(map[string]struct{}, len(values)) + for i, value := range values { + value = strings.TrimSpace(value) + if value == "" { + return fmt.Errorf("%s[%d] must not be empty", field, i) + } + if _, exists := seen[value]; exists { + return fmt.Errorf("%s contains duplicate value %q", field, value) + } + seen[value] = struct{}{} + } + return nil +} + +func digestBytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/conformance/profile_test.go b/internal/conformance/profile_test.go new file mode 100644 index 0000000..bb6046b --- /dev/null +++ b/internal/conformance/profile_test.go @@ -0,0 +1,89 @@ +package conformance + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadFalcoProfile(t *testing.T) { + profile := loadFalcoProfile(t) + if profile.Profile.ID != "falco-modern-bpf-v0.1" { + t.Fatalf("profile id = %q", profile.Profile.ID) + } + if len(profile.Matrix.Profiles) != 5 { + t.Fatalf("matrix profiles = %d, want 5", len(profile.Matrix.Profiles)) + } + if !isSHA256(profile.SHA256) || !isSHA256(profile.MatrixSHA256) { + t.Fatalf("profile resources are not digest-bound: %+v", profile) + } +} + +func TestLoadProfileRejectsUnknownFieldsAndMatrixTraversal(t *testing.T) { + baseProfile, err := os.ReadFile(filepath.Join("..", "..", "conformance", "falco-modern-bpf-v0.1", "profile.yaml")) + if err != nil { + t.Fatal(err) + } + baseMatrix, err := os.ReadFile(filepath.Join("..", "..", "conformance", "falco-modern-bpf-v0.1", "matrix.yaml")) + if err != nil { + t.Fatal(err) + } + + t.Run("unknown field", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "matrix.yaml"), baseMatrix, 0o600); err != nil { + t.Fatal(err) + } + raw := append(append([]byte(nil), baseProfile...), []byte("unknown_field: true\n")...) + path := filepath.Join(dir, "profile.yaml") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfile(path); err == nil || !strings.Contains(err.Error(), "field unknown_field not found") { + t.Fatalf("expected strict YAML error, got %v", err) + } + }) + + t.Run("matrix traversal", func(t *testing.T) { + dir := t.TempDir() + raw := strings.Replace(string(baseProfile), "path: matrix.yaml", "path: ../matrix.yaml", 1) + path := filepath.Join(dir, "profile.yaml") + if err := os.WriteFile(path, []byte(raw), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfile(path); err == nil || !strings.Contains(err.Error(), "must stay inside") { + t.Fatalf("expected matrix traversal error, got %v", err) + } + }) + + t.Run("multiple profile documents", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "matrix.yaml"), baseMatrix, 0o600); err != nil { + t.Fatal(err) + } + raw := append(append([]byte(nil), baseProfile...), []byte("---\n{}\n")...) + path := filepath.Join(dir, "profile.yaml") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfile(path); err == nil || !strings.Contains(err.Error(), "exactly one YAML document") { + t.Fatalf("expected multiple-document error, got %v", err) + } + }) + + t.Run("multiple matrix documents", func(t *testing.T) { + dir := t.TempDir() + matrix := append(append([]byte(nil), baseMatrix...), []byte("---\n{}\n")...) + if err := os.WriteFile(filepath.Join(dir, "matrix.yaml"), matrix, 0o600); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "profile.yaml") + if err := os.WriteFile(path, baseProfile, 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfile(path); err == nil || !strings.Contains(err.Error(), "exactly one YAML document") { + t.Fatalf("expected multiple-document error, got %v", err) + } + }) +} diff --git a/internal/conformance/testdata/falco-pass-report.json b/internal/conformance/testdata/falco-pass-report.json new file mode 100644 index 0000000..714c601 --- /dev/null +++ b/internal/conformance/testdata/falco-pass-report.json @@ -0,0 +1,87 @@ +{ + "schema_version": "v0.1", + "run": { + "id": "20260806T120000Z-falco-preview", + "started_at": "2026-08-06T12:00:00Z" + }, + "artifact": { + "path": "command://scap-open", + "basename": "scap-open", + "sha256": "2222222222222222222222222222222222222222222222222222222222222222", + "size_bytes": 1048576 + }, + "command": { + "invocation_sha256": "bb266d2cad7ac34ea7324a97b77654eb159fee225d93d42165587c1aa750ab2b", + "expected_exit_code": 0, + "binary": { + "basename": "scap-open", + "sha256": "1111111111111111111111111111111111111111111111111111111111111111", + "size_bytes": 1048576 + } + }, + "matrix": { + "path": "test/bpfcompat/kernel-matrix.yaml", + "name": "falco-modern-bpf-poc", + "profiles": [ + "ubuntu-22.04-5.15", + "debian-12-6.1", + "ubuntu-24.04-6.8", + "almalinux-8-4.18", + "almalinux-9-5.14" + ] + }, + "targets": [ + { + "profile_id": "ubuntu-22.04-5.15", + "required": true, + "status": "pass", + "profile": {"distro": "ubuntu", "version": "22.04", "kernel_family": "5.15", "arch": "x86_64"}, + "host": {"distro": "ubuntu", "version": "22.04", "kernel_family": "5.15", "kernel": "5.15.0-201-generic", "arch": "x86_64"}, + "validation": {"load_status": "skipped"}, + "functional": {"status": "pass", "tests": [{"name": "command", "required": true, "status": "pass", "command": "$BPFCOMPAT_BIN --modern_bpf --num_events 10", "expected_exit_code": 0, "exit_code": 0}]}, + "notes": ["base image sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "command validation passed (exit code 0)"] + }, + { + "profile_id": "debian-12-6.1", + "required": true, + "status": "pass", + "profile": {"distro": "debian", "version": "12", "kernel_family": "6.1", "arch": "x86_64"}, + "host": {"distro": "debian", "version": "12", "kernel_family": "6.1", "kernel": "6.1.0-38-cloud-amd64", "arch": "x86_64"}, + "validation": {"load_status": "skipped"}, + "functional": {"status": "pass", "tests": [{"name": "command", "required": true, "status": "pass", "command": "$BPFCOMPAT_BIN --modern_bpf --num_events 10", "expected_exit_code": 0, "exit_code": 0}]}, + "notes": ["base image sha256: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "command validation passed (exit code 0)"] + }, + { + "profile_id": "ubuntu-24.04-6.8", + "required": true, + "status": "pass", + "profile": {"distro": "ubuntu", "version": "24.04", "kernel_family": "6.8", "arch": "x86_64"}, + "host": {"distro": "ubuntu", "version": "24.04", "kernel_family": "6.8", "kernel": "6.8.0-109-generic", "arch": "x86_64"}, + "validation": {"load_status": "skipped"}, + "functional": {"status": "pass", "tests": [{"name": "command", "required": true, "status": "pass", "command": "$BPFCOMPAT_BIN --modern_bpf --num_events 10", "expected_exit_code": 0, "exit_code": 0}]}, + "notes": ["base image sha256: cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "command validation passed (exit code 0)"] + }, + { + "profile_id": "almalinux-8-4.18", + "required": true, + "status": "pass", + "profile": {"distro": "almalinux", "version": "8", "kernel_family": "4.18", "arch": "x86_64"}, + "host": {"distro": "almalinux", "version": "8", "kernel_family": "4.18", "kernel": "4.18.0-553.75.1.el8_10.x86_64", "arch": "x86_64"}, + "validation": {"load_status": "skipped"}, + "functional": {"status": "pass", "tests": [{"name": "command", "required": true, "status": "pass", "command": "$BPFCOMPAT_BIN --modern_bpf --num_events 10", "expected_exit_code": 0, "exit_code": 0}]}, + "notes": ["base image sha256: dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "command validation passed (exit code 0)"] + }, + { + "profile_id": "almalinux-9-5.14", + "required": true, + "status": "pass", + "profile": {"distro": "almalinux", "version": "9", "kernel_family": "5.14", "arch": "x86_64"}, + "host": {"distro": "almalinux", "version": "9", "kernel_family": "5.14", "kernel": "5.14.0-570.26.1.el9_6.x86_64", "arch": "x86_64"}, + "validation": {"load_status": "skipped"}, + "functional": {"status": "pass", "tests": [{"name": "command", "required": true, "status": "pass", "command": "$BPFCOMPAT_BIN --modern_bpf --num_events 10", "expected_exit_code": 0, "exit_code": 0}]}, + "notes": ["base image sha256: eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", "command validation passed (exit code 0)"] + } + ], + "summary": {"status": "pass"}, + "paths": {"run_dir": ".bpfcompat/runs/20260806T120000Z-falco-preview", "json": "reports/falco.json"} +} diff --git a/internal/conformance/types.go b/internal/conformance/types.go new file mode 100644 index 0000000..a56a994 --- /dev/null +++ b/internal/conformance/types.go @@ -0,0 +1,178 @@ +package conformance + +import ( + "time" + + "github.com/kernel-guard/bpfcompat/pkg/schema" +) + +const ( + ProfileSchemaVersion = "bpfcompat_conformance_profile.v0.1" + DecisionSchemaVersion = "bpfcompat_conformance_decision.v0.1" + + StatusConformant = "conformant" + StatusNonconformant = "nonconformant" + StatusInconclusive = "inconclusive" + + AssertionPass = "pass" + AssertionFail = "fail" + AssertionInconclusive = "inconclusive" + + StatementTypeV1 = "https://in-toto.io/Statement/v1" + TestResultPredicateTypeV01 = "https://in-toto.io/attestation/test-result/v0.1" + SVRPredicateTypeV02 = "https://in-toto.io/attestation/svr/v0.2" +) + +type Profile struct { + SchemaVersion string `yaml:"schema_version"` + ID string `yaml:"id"` + Title string `yaml:"title"` + ProgramStatus string `yaml:"program_status"` + ProfileURI string `yaml:"profile_uri"` + Claim string `yaml:"claim"` + Limitations []string `yaml:"limitations,omitempty"` + ReportSchemaVersions []string `yaml:"report_schema_versions"` + Subject SubjectContract `yaml:"subject"` + Matrix MatrixContract `yaml:"matrix"` + Environments []EnvironmentContract `yaml:"environments"` + Validity ValidityContract `yaml:"validity"` + Assertions []AssertionContract `yaml:"assertions"` + Properties []string `yaml:"properties"` +} + +type SubjectContract struct { + Kind string `yaml:"kind"` + BinaryBaseName string `yaml:"binary_basename"` + Command string `yaml:"command"` + ExpectedExitCode int `yaml:"expected_exit_code"` +} + +type MatrixContract struct { + Path string `yaml:"path"` + URI string `yaml:"uri"` +} + +type EnvironmentContract struct { + ProfileID string `yaml:"profile_id"` + Distro string `yaml:"distro"` + Version string `yaml:"version"` + KernelFamily string `yaml:"kernel_family"` + Arch string `yaml:"arch"` +} + +type ValidityContract struct { + SnapshotDays int `yaml:"snapshot_days"` + ContinuousMaxAgeDays int `yaml:"continuous_max_age_days"` +} + +type AssertionContract struct { + ID string `yaml:"id"` + Rule string `yaml:"rule"` + Description string `yaml:"description"` +} + +type ResourceDescriptor struct { + Name string `json:"name,omitempty"` + URI string `json:"uri,omitempty"` + Digest map[string]string `json:"digest,omitempty"` +} + +type Decision struct { + SchemaVersion string `json:"schema_version"` + Status string `json:"status"` + ProgramStatus string `json:"program_status"` + Claim string `json:"claim"` + Limitations []string `json:"limitations,omitempty"` + Subject ResourceDescriptor `json:"subject"` + Profile EvaluatedResource `json:"profile"` + Matrix EvaluatedResource `json:"matrix"` + Report EvaluatedReport `json:"report"` + Verifier VerifierIdentity `json:"verifier"` + TimeCreated string `json:"time_created"` + ValidUntil string `json:"valid_until,omitempty"` + Properties []string `json:"properties,omitempty"` + Assertions []AssertionResult `json:"assertions"` + Targets []TargetEvidence `json:"targets,omitempty"` +} + +type EvaluatedResource struct { + ID string `json:"id"` + URI string `json:"uri,omitempty"` + SHA256 string `json:"sha256"` +} + +type EvaluatedReport struct { + SchemaVersion string `json:"schema_version"` + RunID string `json:"run_id"` + StartedAt string `json:"started_at"` + SHA256 string `json:"sha256"` + URL string `json:"url,omitempty"` +} + +type VerifierIdentity struct { + ID string `json:"id"` +} + +type AssertionResult struct { + ID string `json:"id"` + Rule string `json:"rule"` + Description string `json:"description"` + Status string `json:"status"` + Summary string `json:"summary"` + Evidence []string `json:"evidence,omitempty"` +} + +type TargetEvidence struct { + ProfileID string `json:"profile_id"` + Status string `json:"status"` + Required bool `json:"required"` + HostKernel string `json:"host_kernel,omitempty"` + Architecture string `json:"architecture,omitempty"` + ImageSHA256 string `json:"image_sha256,omitempty"` + ClassificationCode string `json:"classification_code,omitempty"` +} + +type Statement struct { + Type string `json:"_type"` + Subject []ResourceDescriptor `json:"subject"` + PredicateType string `json:"predicateType"` + Predicate any `json:"predicate"` +} + +type TestResultPredicate struct { + Result string `json:"result"` + Configuration []ResourceDescriptor `json:"configuration"` + URL string `json:"url,omitempty"` + PassedTests []string `json:"passedTests,omitempty"` + WarnedTests []string `json:"warnedTests,omitempty"` + FailedTests []string `json:"failedTests,omitempty"` +} + +type SVRPredicate struct { + Verifier SVRVerifier `json:"verifier"` + TimeCreated string `json:"timeCreated"` + Properties []string `json:"properties"` +} + +type SVRVerifier struct { + ID string `json:"id"` + Policies []ResourceDescriptor `json:"policies"` +} + +type Evaluation struct { + Decision Decision + TestResult Statement + VerificationResult *Statement +} + +type EvaluateOptions struct { + VerifierID string + ReportURL string + Now func() time.Time +} + +type ReportDocument struct { + Report schema.ReportV01 + Raw []byte + SHA256 string +} diff --git a/internal/conformance/write.go b/internal/conformance/write.go new file mode 100644 index 0000000..ede5b65 --- /dev/null +++ b/internal/conformance/write.go @@ -0,0 +1,67 @@ +package conformance + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +const ( + DecisionFileName = "decision.json" + TestResultFileName = "test-result.intoto.json" + VerificationResultFileName = "verification-result.intoto.json" +) + +type OutputPaths struct { + Decision string + TestResult string + VerificationResult string +} + +func WriteEvaluation(outDir string, evaluation Evaluation) (OutputPaths, error) { + absDir, err := filepath.Abs(outDir) + if err != nil { + return OutputPaths{}, fmt.Errorf("resolve conformance output directory: %w", err) + } + if err := os.MkdirAll(absDir, 0o755); err != nil { + return OutputPaths{}, fmt.Errorf("create conformance output directory: %w", err) + } + paths := OutputPaths{ + Decision: filepath.Join(absDir, DecisionFileName), + TestResult: filepath.Join(absDir, TestResultFileName), + } + if err := writeJSON(paths.Decision, evaluation.Decision); err != nil { + return OutputPaths{}, err + } + if err := writeJSON(paths.TestResult, evaluation.TestResult); err != nil { + return OutputPaths{}, err + } + + verificationPath := filepath.Join(absDir, VerificationResultFileName) + if evaluation.VerificationResult == nil { + // This path is owned by this evaluator. Removing it prevents a stale + // conformant result from surviving a failed or inconclusive re-evaluation. + if err := os.Remove(verificationPath); err != nil && !os.IsNotExist(err) { + return OutputPaths{}, fmt.Errorf("remove stale verification result: %w", err) + } + return paths, nil + } + if err := writeJSON(verificationPath, *evaluation.VerificationResult); err != nil { + return OutputPaths{}, err + } + paths.VerificationResult = verificationPath + return paths, nil +} + +func writeJSON(path string, value any) error { + raw, err := json.MarshalIndent(value, "", " ") + if err != nil { + return fmt.Errorf("marshal conformance JSON: %w", err) + } + raw = append(raw, '\n') + if err := os.WriteFile(path, raw, 0o600); err != nil { + return fmt.Errorf("write conformance JSON %s: %w", path, err) + } + return nil +}