diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0676a27..576114b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,11 @@ env: GOLANGCI_VERSION: "v2.12.2" GOVULNCHECK_VERSION: "v1.6.0" GORELEASER_VERSION: "v2.17.0" - HELM_VERSION: "v4.2.2" - HELM_LINUX_AMD64_SHA256: "9adafecab4d406853bba163a70e9f104f47dbbf65ce24b7653bae7e36150bcb6" - SYFT_VERSION: "v1.46.0" + HELM_VERSION: "v4.2.3" + HELM_LINUX_AMD64_SHA256: "e9b88b4ee95b18c706839c28d3a0220e5bc470e9cd9262410c90793c45ff8b7c" + PROMETHEUS_VERSION: "v3.13.1" + PROMETHEUS_LINUX_AMD64_SHA256: "962b812371aff838d152b6ff2d56fdb7a6396f5542f48ebf73421b9721f0d103" + SYFT_VERSION: "v1.49.0" jobs: build-test-lint: @@ -31,7 +33,7 @@ jobs: with: persist-credentials: false - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ env.GO_VERSION }} check-latest: false @@ -75,6 +77,9 @@ jobs: - name: Unit tests with race detector run: go test -race -count=1 -coverprofile=coverage.out ./... + - name: Operator safety policy tests + run: make test-scripts + - name: Multi-layer tenant-isolation suite run: make e2e-isolation @@ -84,6 +89,20 @@ jobs: - name: Binary integration smoke test run: go test -race -count=1 -tags=e2e ./tests/e2e + - name: Install pinned promtool + run: | + set -euo pipefail + version="${PROMETHEUS_VERSION#v}" + archive="$RUNNER_TEMP/prometheus.tar.gz" + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 --retry 3 --retry-all-errors \ + --output "$archive" "https://github.com/prometheus/prometheus/releases/download/${PROMETHEUS_VERSION}/prometheus-${version}.linux-amd64.tar.gz" + echo "${PROMETHEUS_LINUX_AMD64_SHA256} ${archive}" | sha256sum --check --status + tar -xzf "$archive" -C "$RUNNER_TEMP" + install -m 0755 "$RUNNER_TEMP/prometheus-${version}.linux-amd64/promtool" "$RUNNER_TEMP/promtool" + + - name: Portable Prometheus alert contract + run: make test-alert-rules PROMTOOL="$RUNNER_TEMP/promtool" + - name: Immutable OCI image contract run: make e2e-oci @@ -115,7 +134,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ env.GO_VERSION }} check-latest: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84dc383..e6d492d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,8 +15,8 @@ concurrency: env: GO_VERSION: "1.26.5" GORELEASER_VERSION: "v2.17.0" - SYFT_VERSION: "v1.46.0" - COSIGN_VERSION: "v3.0.6" + SYFT_VERSION: "v1.49.0" + COSIGN_VERSION: "v3.1.2" HUB_IMAGE: ghcr.io/ardurai/sith-hub jobs: @@ -58,7 +58,7 @@ jobs: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV" echo "RELEASE_KIND=${release_kind}" >> "$GITHUB_ENV" - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ env.GO_VERSION }} check-latest: false @@ -112,13 +112,13 @@ jobs: test -x "$IMAGE_CONTEXT/bin/linux/arm64/sith" - name: Set up QEMU for hub image platforms - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - name: Set up Buildx for hub image platforms - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - name: Authenticate to the GitHub Container Registry - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -143,7 +143,7 @@ jobs: - name: Publish immutable multi-platform hub image id: hub_image - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: ${{ runner.temp }}/sith-hub-image file: ${{ runner.temp }}/sith-hub-image/Containerfile @@ -184,7 +184,7 @@ jobs: - name: Attest hub image build provenance id: hub_provenance - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-name: ${{ env.HUB_IMAGE }} subject-digest: ${{ steps.hub_image.outputs.digest }} @@ -192,7 +192,7 @@ jobs: - name: Attest hub image SBOM id: hub_sbom - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-name: ${{ env.HUB_IMAGE }} subject-digest: ${{ steps.hub_image.outputs.digest }} @@ -210,34 +210,34 @@ jobs: - name: Generate SLSA build provenance id: provenance - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-checksums: dist/checksums.txt - name: Attest darwin amd64 SBOM id: sbom_darwin_amd64 - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/sith_${{ env.VERSION }}_darwin_amd64.tar.gz sbom-path: dist/sith_${{ env.VERSION }}_darwin_amd64.tar.gz.spdx.json - name: Attest darwin arm64 SBOM id: sbom_darwin_arm64 - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/sith_${{ env.VERSION }}_darwin_arm64.tar.gz sbom-path: dist/sith_${{ env.VERSION }}_darwin_arm64.tar.gz.spdx.json - name: Attest linux amd64 SBOM id: sbom_linux_amd64 - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/sith_${{ env.VERSION }}_linux_amd64.tar.gz sbom-path: dist/sith_${{ env.VERSION }}_linux_amd64.tar.gz.spdx.json - name: Attest linux arm64 SBOM id: sbom_linux_arm64 - uses: actions/attest@a1948c3f048ba23858d222213b7c278aabede763 # v4.1.1 + uses: actions/attest@f7c74d28b9d84cb8768d0b8ca14a4bac6ef463e6 # v4.2.0 with: subject-path: dist/sith_${{ env.VERSION }}_linux_arm64.tar.gz sbom-path: dist/sith_${{ env.VERSION }}_linux_arm64.tar.gz.spdx.json diff --git a/Makefile b/Makefile index 31b6457..53cf7a0 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,12 @@ CMD := ./cmd/sith BIN_DIR := bin GOLANGCI ?= golangci-lint GOVULNCHECK ?= govulncheck +PROMTOOL ?= promtool KIND ?= kind HELM ?= helm GORELEASER ?= goreleaser WAILS ?= wails -WAILS_VERSION ?= v2.12.0 +WAILS_VERSION ?= v2.13.0 CODESIGN ?= codesign PLISTBUDDY ?= /usr/libexec/PlistBuddy LIPO ?= lipo @@ -34,7 +35,7 @@ LDFLAGS := -s -w \ -X $(PKG)/internal/buildinfo.Commit=$(COMMIT) \ -X $(PKG)/internal/buildinfo.Date=$(DATE) -.PHONY: all build desktop-build test test-scripts perf e2e e2e-helm e2e-oci e2e-kind e2e-ocm e2e-postgres e2e-isolation lint vuln fmt fmt-check vet tidy clean run ci release-check help +.PHONY: all build desktop-build test test-scripts test-alert-rules perf e2e e2e-helm e2e-oci e2e-kind e2e-ocm e2e-postgres e2e-isolation lint vuln fmt fmt-check vet tidy clean run ci release-check help all: build @@ -43,8 +44,7 @@ build: ## Build the sith binary into bin/ go build -trimpath -ldflags '$(LDFLAGS)' -o $(BIN_DIR)/$(BINARY) $(CMD) desktop-build: ## Build the ad-hoc-signed macOS arm64 Sith.app development bundle - @command -v "$(WAILS)" >/dev/null || { echo "Wails $(WAILS_VERSION) is required" >&2; exit 1; } - @"$(WAILS)" version | grep -q '$(WAILS_VERSION)' || { echo "Wails $(WAILS_VERSION) is required" >&2; exit 1; } + @hack/verify-wails-version.sh "$(WAILS)" "$(WAILS_VERSION)" cd cmd/sith-desktop && "$(WAILS)" build -clean -m -nosyncgomod -s -trimpath -platform darwin/arm64 @set -euo pipefail; \ app='cmd/sith-desktop/build/bin/Sith.app'; \ @@ -59,11 +59,21 @@ test: ## Run unit tests with the race detector and report coverage go test -race -count=1 -coverprofile=coverage.out ./... test-scripts: ## Run focused safety tests for operator-facing shell harnesses + bash tests/scripts/wails_tooling_policy_test.sh + bash tests/scripts/release_tooling_policy_test.sh + bash tests/scripts/helm_tooling_policy_test.sh bash tests/scripts/m0_ocm_falsification_safety_test.sh bash tests/scripts/release_tag_identity_guide_test.sh bash tests/scripts/release_tag_policy_test.sh bash tests/scripts/release_pr_gate_policy_test.sh bash tests/scripts/release_hub_image_policy_test.sh + bash tests/scripts/prometheus_tooling_policy_test.sh + +test-alert-rules: ## Validate and unit-test the portable Prometheus alert contract + @promtool_path="$$(command -v "$(PROMTOOL)")" || { echo "promtool is required" >&2; exit 1; }; \ + case "$$promtool_path" in /*) ;; *) promtool_path="$$(pwd)/$$promtool_path" ;; esac; \ + cd monitoring && "$$promtool_path" check rules sith-hub.rules.yml && \ + "$$promtool_path" test rules sith-hub.rules.test.yml perf: ## Enforce the warm-cache TUI p95 latency budget without race overhead go test -count=1 -run '^TestWarmViewP95UnderOneHundredMilliseconds$$' ./internal/tui @@ -79,7 +89,7 @@ e2e-oci: ## Build and inspect the local immutable OCI image contract for linux/a e2e-kind: ## Exercise adapter and binary against two real kind clusters KIND_BIN="$(KIND)" KIND_NODE_IMAGE="$(KIND_NODE_IMAGE)" \ - go test -race -count=1 -timeout=15m -tags='e2e kind' -run '^Test(KindFleetFanout|KindOCIImageContract)$$' ./tests/e2e + go test -race -count=1 -timeout=15m -tags='e2e kind' -run '^Test(KindFleetFanout|KindOCIImageContract|KindArgoApplicationProjection)$$' ./tests/e2e e2e-ocm: ## Prove direct ClusterProxy transport in the pinned two-spoke M0 lab @set -euo pipefail; \ @@ -114,6 +124,9 @@ e2e-isolation: ## Run signed-identity, scoped-query, and real PostgreSQL isolati go test -run '^$$' -fuzz '^FuzzQueryScopedNeverLeaksForeignWorkspace$$' \ -fuzztime="$(ISOLATION_FUZZ_BUDGET)" -parallel="$(ISOLATION_FUZZ_WORKERS)" \ -timeout=2m ./internal/fleetcache + go test -run '^$$' -fuzz '^FuzzStoreForeignWorkspaceMutationCannotChangeEitherWorkspace$$' \ + -fuzztime="$(ISOLATION_FUZZ_BUDGET)" -parallel="$(ISOLATION_FUZZ_WORKERS)" \ + -timeout=2m ./internal/fleetcache lint: ## Run golangci-lint (v2) $(GOLANGCI) run ./... @@ -141,7 +154,7 @@ clean: ## Remove build and coverage artifacts run: build ## Build then run sith version $(BIN_DIR)/$(BINARY) version -ci: fmt-check vet lint vuln test test-scripts perf e2e build ## Run the full CI gate locally +ci: fmt-check vet lint vuln test test-scripts test-alert-rules perf e2e build ## Run the full CI gate locally release-check: ## Build, verify, and package the reproducible multi-platform release snapshot twice @command -v "$(GORELEASER)" >/dev/null || { echo "goreleaser is required" >&2; exit 1; } diff --git a/README.md b/README.md index 80d23b0..4e0fb9b 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,19 @@ brew tap ArdurAI/tap brew trust --formula ArdurAI/tap/sith brew install sith sith version +sith launch ``` Homebrew 6 requires explicit trust for third-party taps. The formula-scoped command keeps the trust boundary narrower than trusting every current and future formula in `ArdurAI/tap`. Older Homebrew versions that do not implement tap trust can omit that line. +`sith launch` is the one-command graphical entry point: it opens the native desktop on macOS and +the loopback-only browser UI on other supported platforms. Use `--mode desktop` or `--mode ui` to +override that selection. A demo machine without a default kubeconfig can import an existing bounded +directory for the session with `sith launch --kubeconfig-dir /path/to/kubeconfigs`; Sith neither +persists the selected path nor deploys anything to those clusters. + Release archives are also available for `darwin/amd64`, `darwin/arm64`, `linux/amd64`, and `linux/arm64`. Every archive has a checksum, an SPDX SBOM, a keyless Sigstore bundle, SLSA build provenance, and a platform-specific SBOM attestation. Verify those materials before installing; @@ -40,6 +47,8 @@ Sith requires a supported Go 1.26 toolchain. ```bash make build +./bin/sith launch # native desktop on macOS; loopback UI elsewhere +./bin/sith launch --mode ui --kubeconfig-dir /path/to/kubeconfigs ./bin/sith # interactive terminal: cache-first fleet view ./bin/sith tui # explicit equivalent ./bin/sith version @@ -50,6 +59,8 @@ make build ./bin/sith correlate 'deploy/payments status!=Healthy' ./bin/sith investigate # rank supported degraded signals across every context ./bin/sith investigate payments --context kind-dev --output json +./bin/sith audit verify ./sith-policy-audit.json # local integrity check; no hub or credentials +./bin/sith audit verify-pages ./audit-page-0001.json ./audit-page-0002.json # ordered snapshot ./bin/sith describe pod/api --context kind-dev -n apps ./bin/sith yaml secret/api-token --context kind-dev -n apps ./bin/sith logs api --context kind-dev -n apps --tail 100 -f @@ -93,9 +104,11 @@ handler includes a bounded per-process attempt limiter; a replicated hub must ad a shared limit at its ingress or gateway. Deployments must provide the HMAC pepper and Ed25519 private key through a secret manager, keep both out of logs and configuration repositories, and rotate them under an explicit operational procedure. These are E1 library and HTTP boundaries. -The P1 `sith hub` runtime now mounts only the session-authenticated fleet read/refresh surface -below; API-key, OIDC, and cloud-proof exchange handlers remain intentionally unmounted until their -ingress and operator lifecycle are composed. +The P1 `sith hub` runtime mounts the session-authenticated fleet read/refresh surface below. +API-key, raw OIDC-token, and cloud-proof exchange handlers remain intentionally unmounted until +their ingress and operator lifecycle are composed. The separately configured browser OIDC flow is +not a raw-token exchange: it completes authorization-code + PKCE server-side and only establishes +an HttpOnly browser session for the Hub console boundary. Pinned OIDC federation uses the same exchange model. Each endpoint is fixed to one requested workspace, and each provider configuration allowlists an exact HTTPS issuer, audience, token type, @@ -116,6 +129,65 @@ This follows [OpenID Connect Discovery 1.0](https://openid.net/specs/openid-conn [JWT BCP, RFC 8725](https://www.rfc-editor.org/rfc/rfc8725.html). Discovery and refresh add small outbound request and availability dependencies but create no cloud resources by themselves. +When all browser-OIDC deployment inputs are configured, the Hub additionally exposes a fixed +`GET /v1/workspaces/{workspace}/console/login` start route and the exact configured callback path. +It uses one issuer-pinned public client with Authorization Code + PKCE `S256`; the verifier, code, +upstream ID token, and minted Sith JWT remain server-side. A bounded, single-use, process-local +transaction binds the exact workspace, state, nonce, redirect URI, issuer, client ID, and verifier. +The callback consumes that transaction before redeeming the code, resolves the current membership +through forced RLS, and returns only a short-lived `__Host-sith-session` cookie (`Secure`, +`HttpOnly`, `Path=/`, no `Domain`, `SameSite=Lax`). Lax allows that new session to reach the safe +top-level console `GET` whose navigation began at the external IdP, while excluding unsafe +cross-site methods. Its short-lived `__Host-sith-oidc-tx` transaction cookie is also `SameSite=Lax` +so the top-level IdP callback can return; it contains an opaque random binding, not a credential. +Restart, expiry, replay, malformed/duplicate provider +JSON, failed PKCE, wrong state/nonce/issuer/audience, or an unavailable provider fails closed. No +login artifact, token, or proof is persisted or logged. On success, the callback redirects only to +the transaction-bound `GET /v1/workspaces/{workspace}/console` path; it accepts no return URL. + +That Hub-only console verifies the HttpOnly session server-side, resolves the workspace from signed +membership, and reads the existing persisted `hubfleet.Source` through the PEP. Its fixed +`GET /v1/workspaces/{workspace}/console/fleet` adapter requires a short-lived process-key-signed +CSRF token bound to the exact session, workspace, and expiry in a custom same-origin header. The +separate `GET /v1/workspaces/{workspace}/console/correlate` adapter accepts only one canonical exact +non-Secret resource identity and the fixed `health_not=Healthy` condition. It calls the existing +tenant-scoped `hubfleet.Correlator` once with a 257-row sentinel bound, then emits at most 256 +minimal health matches; an over-bound or malformed stored result fails closed instead of appearing +complete. The correlation proof has a distinct HMAC purpose and cannot be replaced by the fleet +proof. Its response omits raw observations, attributes, workspace fields, provenance, native IDs, +deep links, and source payloads. + +The separate `GET /v1/workspaces/{workspace}/console/inventory` adapter accepts one closed OCM +inventory kind (`Deployment`, `Pod`, or `Rollout`) plus optional exact namespace and name. It uses +the dedicated `fleet.inventory.search` PEP verb and its own non-interchangeable session/workspace +proof, performs one 257-row-sentinel persisted read, and emits at most 256 strictly decoded records. +The browser receives only resource identity, observation freshness, generation, and typed +availability/readiness counts. Pod image digests, raw observations, display hints, source payloads, +workspace fields, attributes, and provenance remain behind the evidence boundary; malformed or +over-bound stored results fail closed. + +The separate `GET /v1/workspaces/{workspace}/console/cves` adapter accepts one canonical uppercase +CVE identifier and reuses the existing tenant-scoped `hubfleet.CVESearcher` with its dedicated +`fleet.cve.identifier.search` PEP verb. Its fourth non-interchangeable session/workspace proof and +same-origin Fetch Metadata gate precede one 257-row-sentinel persisted read. The browser receives at +most 256 records containing only the cluster scope, immutable image digest, exact identifier, +canonical severity, observation time, and derived staleness. Raw observations, the complete +identifier list, display hints, source payloads, workspace fields, attributes, and provenance stay +behind the evidence boundary; mutable image references, selector mismatches, malformed or duplicate +stored facts, and over-bound results fail closed. + +The session JWT never enters HTML, JavaScript, a URL, browser storage, or a log. All console +responses are `no-store` and use a restrictive same-origin CSP; all data adapters reject +cross-site Fetch Metadata. The UI shows reachability, observation times, non-Healthy matches, +normalized inventory, and runtime CVE evidence beside stale/unreachable/truncated/unaccounted +coverage without claiming an empty or partial answer is healthy, complete, or CVE-free. +Correlation, inventory, and CVE lookup run only after explicit submit. The console performs no automatic +polling and cannot invoke collector refresh, a connector, local `exec`/edit/log/port-forward +operations, or any write. The bearer fleet API remains bearer-only, and no generic cookie +authentication middleware exists. Query identities are ordinary resource metadata, not secrets; +they may appear in access logs. Each submit adds one bounded Hub database read and no cloud, +scanner, or spoke egress. + Cloud-IAM identity starts from the same fail-closed exchange boundary. The foundation accepts only a verifier-normalized provider, explicit realm, immutable subject, audience, and bounded lifetime; it maps that identity through a current forced-RLS membership binding and consumes only an HMAC @@ -163,7 +235,16 @@ kubeconfig, endpoint, or token through the Sith collector contract. Only normali `health`, and bounded immutable-image `cve` facts are accepted, source-stamped, freshness-bounded, and stored behind forced RLS. Failed refreshes retain the last snapshot as explicitly stale evidence and record only a closed -failure category. The pinned direct OCM ClusterProxy adapter reads the exact rotating +failure category. Concurrent refresh requests are authorized independently and then coalesced only +within the same validated workspace. The shared refresh runs on a locally traced context detached +from individual callers but bounded by the hub lifecycle, so one canceled caller cannot cancel its +peers, shutdown cancels outstanding work, and no request credential, context value, or trace identity +crosses caller boundaries; completed and panicking refresh flights are removed. Within one refresh, +transport and validation use a bounded worker pool: `CollectorConfig` defaults to four +concurrent spokes and accepts only explicit limits from 1 through 64. Persistence and coverage +mutation remain serialized, so one store failure cancels the remaining workers before returning; +per-spoke deadlines and sorted stale/unreachable coverage remain unchanged. The pinned +direct OCM ClusterProxy adapter reads the exact rotating `sith-reader` managed-serviceaccount Secret for a registered spoke, opens a short-lived Konnectivity tunnel only to that spoke, and verifies both proxy mTLS and the spoke Kubernetes certificate; it never forwards a caller `Authorization` header, stores a credential, disables @@ -193,6 +274,70 @@ present and valid: `SITH_HUB_PROXY_CERT_FILE`, `SITH_HUB_PROXY_KEY_FILE`, and `SITH_HUB_KUBE_API_SERVER_NAME` for the direct ClusterProxy mTLS path. +Browser OIDC is disabled only when all four of these optional inputs are absent. Setting any one +requires all four before startup: `SITH_HUB_BROWSER_OIDC_ISSUER`, +`SITH_HUB_BROWSER_OIDC_CLIENT_ID`, `SITH_HUB_BROWSER_OIDC_REDIRECT_URI`, and +`SITH_HUB_SESSION_PRIVATE_KEY_FILE`. The client ID is the exact accepted ID-token audience; the +redirect URI must be the exact HTTPS callback registered with the issuer. The private PKCS#8 +Ed25519 key must be a read-only deployment mount and match the configured public key. This uses a +public OIDC client with PKCE, never a frontend client secret or refresh token. The pinned provider +must be reachable through the existing no-proxy, TLS 1.2+, public-network-only discovery/JWKS +transport; deployments must allow only that issuer's required endpoints. + +The existing TLS listener also serves two fixed, unauthenticated, body-free Kubernetes probe +routes. `GET /healthz` is dependency-free process liveness. `GET /readyz` performs one +application-pool PostgreSQL ping under a one-second server deadline; failure returns only an empty +`503`, never a database error or endpoint. Every other method, query, encoded variant, or path is +rejected. PostgreSQL deliberately affects readiness rather than liveness so an outage removes the +Pod from service without creating a dependency-driven restart storm. OCM/spoke reachability does +not participate because partial fleet coverage must remain visible rather than making the whole +Hub unready. + +Each valid completed `GET /readyz` database check also records one +`sith_hub_readiness_checks_total{outcome}` attempt and one +`sith_hub_readiness_check_duration_seconds{outcome}` observation in the existing isolated metrics +registry. `outcome` is only `ready` or `unavailable`; database errors, deadline expiry, caller +cancellation, and recovered checker panic collapse to `unavailable`. Rejected methods, queries, +paths, and encoded variants emit no observation. Invalid metric values are discarded, observer +panics cannot change probe behavior, and no request, endpoint, credential, tenant, spoke, error, or +panic detail becomes a label. These series are visible only through the opt-in loopback metrics +listener below and add no listener, Service, exporter, persistence, or remote telemetry path. + +The optional `SITH_HUB_METRICS_LISTEN_ADDR` is disabled unless it is exactly +`127.0.0.1:` or `[::1]:`. When configured, it exposes only a +separate plaintext `GET /metrics` listener backed by Sith's isolated, bounded-label registry. It +is not a tenant route, Service port, ingress, exporter, or telemetry backend, and cannot bind to a +hostname, wildcard, or cluster-routable address. A same-Pod collector may scrape it over +`localhost`; that preserves process-wide operational visibility without disclosing counters to a +workspace principal. This trade-off intentionally requires the operator to supply their own +collector and does not provide cross-Pod or remote scraping. + +Every Hub starts one restricted local child for the already-sanitized authentication-refusal +event. The request path writes only a two-byte closed record to a bounded Unix datagram socket; +when the socket is full or the child has died, the event is dropped immediately and increments the +unlabeled `sith_auth_refusal_delivery_drops_total` self-observation counter. Sith supplies the +child only that descriptor and stderr; it validates each fixed record and writes the single +structured warning there. It shares the container's mounts, UID, and network namespace, so this +is a bounded delivery and shutdown boundary rather than a filesystem or network sandbox. A blocked +stderr can therefore block only the child, which the Hub kills and reaps on shutdown. This adds no +listener, Service, exporter, queue, persistence, remote telemetry, request metadata, credential +data, or raw payload retention. The drop counter is scrapeable only when the same optional loopback +metrics endpoint above is enabled. + +The same already-sanitized bearer and browser-session boundaries increment exactly two +preinitialized `sith_auth_attempts_total{outcome="accepted|refused"}` series. `accepted` means the +local verifier succeeded and is emitted before workspace authorization; `refused` covers the +existing uniform authentication rejection paths. Every refusal also increments the legacy +unlabeled `sith_auth_refusals_total` counter exactly once. Runtime fanout is panic-isolated per +destination: metrics consume both outcomes, while the process audit child and structured-log +adapter remain refusal-only. No credential mode, failure reason, tenant, workspace, actor, +principal, token, IP, path, method, request, trace, or correlation value becomes a label or record. +These counters do not cover OIDC provider exchange/callback failures, authorization denials, +handler outcomes, or every future authentication mode. The portable package below consumes them +only for one aggregate refusal-only warning; they do not provide a generic refusal ratio, +brute-force detector, actor attribution, SLO, error budget, page, or complete security control, and +they add no new scrape or storage path. + Every referenced key, certificate, or CA file must be a read-only regular file from a deployment mount. The runtime obtains its Kubernetes identity only with in-cluster configuration; it has no kubeconfig fallback and uses that identity through the fixed `sith-reader` Secret reader. It serves @@ -200,10 +345,17 @@ only `POST /v1/workspaces/{workspace}/fleet:refresh`, `GET /v1/workspaces/{workspace}/fleet`, and `GET /v1/workspaces/{workspace}/fleet/images/{sha256:<64-lowercase-hex>}`, and `GET /v1/workspaces/{workspace}/fleet/images/{sha256:<64-lowercase-hex>}/cves`, and -`GET /v1/workspaces/{workspace}/fleet/cves/{CVE-YYYY-N...}`. Every route requires an -exact signed Sith session, derives the workspace scope from its signed memberships, carries that -scope through the PEP and RLS seams, accepts no query parameters, and returns only normalized -coverage/fleet data under `Cache-Control: no-store`. +`GET /v1/workspaces/{workspace}/fleet/cves/{CVE-YYYY-N...}`. The separate +`GET /v1/workspaces/{workspace}/audit/export` route returns the existing retained audit chain only +to a signed workspace admin. The distinct +`GET /v1/workspaces/{workspace}/audit/export/pages` route returns the first bounded page of a +larger immutable snapshot; a continuation accepts only the exact +`?cursor=` query. The complete export and fleet routes reject every query, and +the page route rejects every other key, duplicate, escaping, or alternate encoding. All bearer +routes require an exact signed Sith session, derive workspace scope from signed memberships, and +carry it through the PEP and RLS seams. Fleet routes return only normalized coverage/fleet data +under `Cache-Control: no-store`; both audit routes return versioned JSON attachments under the same +bearer-only, tenant-scoped cache boundary. After deriving the signed scope, the hub mints one opaque local trace ID for the governed request. It strips common caller-supplied trace and correlation carriers, never echoes or forwards them, @@ -213,11 +365,138 @@ actor, spoke, endpoint, resource, selector, argument digest, credential, raw err data in trace events. This is not a telemetry exporter: it adds no OpenTelemetry SDK, listener, network egress, queue, trace store, persistence, or action-intent protocol. +The PEP decision record itself is durable. Before an allowed governed operation can proceed, the +hub appends the privacy-minimized decision to a workspace-scoped PostgreSQL hash chain, then emits +the structured process log. The application role can insert and read immutable audit entries but +cannot update or delete them; forced RLS isolates both entries and their per-workspace chain heads. +Approval creation and consumption append distinct format-versioned lifecycle entries to that same +tenant chain in the exact transaction that mutates the single-use grant. An audit failure therefore +rolls back the approval mutation. Each lifecycle pair carries only a one-way, domain-separated +digest of the immutable grant binding—never raw targets, arguments, or justification content. +New grants use format 3: PostgreSQL mints one immutable absolute expiry exactly 10 minutes after +approval, checks `approved_at <= statement_timestamp() < expires_at` in the same conditional +consumption update, and binds both timestamps into the lifecycle evidence digest. An expired, +legacy, missing, foreign, mismatched, or replayed grant returns the same unavailable result; an +expired refusal retains the row, leaves `consumed_at` unset, and appends no success event. +Both audit routes use the dedicated `export-audit` action and `audit.export` PEP verb. Sith durably +appends an authorization decision before every read. The complete route verifies the head and all +retained history in one forced-RLS Repeatable Read snapshot and remains limited to 512 entries. For +larger chains, the page route fixes the first request's current head and returns at most 512 +consecutive entries. Every continuation is independently authenticated, authorized, and audited; +those later authorization rows remain in the live chain but cannot move the established snapshot +and appear in a future export. + +The continuation is a fixed-size, versioned, canonical base64url descriptor bound to a +domain-separated workspace digest, snapshot head sequence/hash, next sequence, and expected prior +hash. It is not a credential. The database rechecks the live and snapshot head anchors plus the +page boundary under forced RLS; a changed, foreign, skipped, or reordered continuation fails with +the same unavailable response. Each transaction commits before JSON serialization, so slow clients +cannot pin a database snapshot and late validation cannot produce a partial successful page. The +process admits at most four complete or paged exports concurrently. Each document contains only +the existing privacy-minimized policy and approval events plus their SHA-256 links. + +`sith audit verify ` provides the corresponding offline integrity check. It accepts +one non-symlink regular file of at most 1 MiB, rejects duplicate, unknown, case-mismatched, or +trailing JSON, and recomputes every versioned entry hash, sequence link, and declared head. The +command performs no hub request, database access, credential lookup, temporary-file creation, or +telemetry and emits only a bounded summary. A successful result means the document is internally +consistent; without an external anchor it does not prove origin or detect wholesale replacement by +a privileged store owner. Verification costs one bounded local read and at most 512 SHA-256 +computations, with no cloud, storage, egress, or recurring-service cost. +`sith audit verify-pages [page.json...]` performs the same strict bounded-file and +canonical-hash checks one page at a time, then requires one ordered same-workspace, +same-snapshot genesis-to-head sequence with no missing, replayed, or swapped page. Success is still +internal consistency, not external authenticity. The online page route adds one small audit row, +at most 512 entry reads plus fixed anchors, and one bounded response of egress per page. Offline +`verify-pages` instead performs one bounded local read and bounded local SHA-256 verification over +at most 512 entries per page, with no hub request or network egress. Both paths are linear in +retained entries and page count and add no object store, queue, worker, or recurring cloud resource. + +Responses use `X-Content-Type-Options: nosniff` and the fixed `sith-policy-audit.json` or +`sith-policy-audit-page.json` attachment filename. Authentication is bearer-only; neither route +uses browser cookies for authority, and the page route rejects every `Cookie` header. Filters, +raw-payload selection, asynchronous export, WORM retention, external anchoring, the future +intent-correlated Ardur decision ledger, and an E6-complete claim remain out of scope. +Either database or structured-process-log delivery failure blocks the operation, so production +database and logging availability and latency are part of the governed-read availability budget. +The optional loopback metrics surface exposes that boundary as +`sith_policy_audit_attempts_total{sink,outcome}` and +`sith_policy_audit_duration_seconds{sink,outcome}`. `sink` is only `durable` or `process`, and +`outcome` is only `success` or `error`; invalid observations are discarded, and no tenant, actor, +intent, trace, policy argument, or raw error becomes a label. A durable error intentionally has no +matching process attempt because the database append must succeed first. A process error therefore +appears after a durable success. Operators can build failure-rate and latency signals from +`rate(sith_policy_audit_attempts_total{outcome="error"}[5m])` and +`histogram_quantile(0.95, sum by (le, sink) +(rate(sith_policy_audit_duration_seconds_bucket[5m])))`. These fixed series add no listener, +exporter, persistence, or tenant-proportional cardinality beyond the existing opt-in metrics path. +Each authorized persisted fleet read also increments exactly one +`sith_federation_fleet_read_results_total{outcome}` series. `outcome` is the closed vocabulary +`complete`, `degraded`, `empty`, or `error`: internally inconsistent or incomplete coverage and a +result/coverage count mismatch are always `degraded`, and only a consistent zero-scope result is +`empty`. The observer runs after PEP +authorization, carries no workspace, spoke, resource, selector, principal, trace, age, or raw-error +label, and is panic-isolated from read behavior. This fixed four-series counter is an F10.1d +coverage-SLI substrate; it is not a read-freshness objective or error-budget policy. +The same authorized read also increments exactly one +`sith_federation_fleet_read_freshness_total{outcome}` series. The five closed outcomes are +`fresh`, `stale`, `unknown`, `empty`, and `error`. `fresh` requires a non-empty, internally +consistent, complete result where every returned cluster has a unique identity and non-zero +observation time. A structurally valid result with a proven stale retained scope is `stale`; unseen, +inconsistent, mismatched, or otherwise non-stale degraded coverage is `unknown`. Only a consistent zero-scope +result is `empty`, and a storage failure before a result exists is `error`. Coverage and freshness +are emitted as one validated pair, so an invalid dimension discards both observations rather than +fabricating a partial result. The fixed series carry no tenant-proportional labels and add no +listener, storage, exporter, background task, or network path. They are request-time SLI substrate: +a workspace with no reads emits no events, and this is not continuous snapshot-age monitoring, a +per-spoke series, an alert, SLO, or error budget. This follows Prometheus guidance to keep label +cardinality bounded and use counters for completed online-serving requests; see the +[instrumentation](https://prometheus.io/docs/practices/instrumentation/) and +[metric naming](https://prometheus.io/docs/practices/naming/) guidance. +The portable [hub alert rules](monitoring/sith-hub.rules.yml) turn the established policy-decision, +audit, auth-log delivery, aggregate snapshot failure, eligible fleet-read coverage, proven +fleet-read staleness, database-readiness, bounded authentication outcomes, and traffic-independent +build-info presence signals into nine bounded, fixture-tested alerts; the +[runbook](docs/runbooks/hub-alerts.md) documents installation and response. Load the rule file only +after arranging an operator-owned same-Pod scrape/forwarding path. Sith does not render a Service, +ServiceMonitor, PrometheusRule, Alertmanager receiver, exporter, or remote-write configuration. +These rules are an F10.4a/F10.4b/F10.4c/F10.4d/F10.4e/F10.4f/F10.4g baseline, not read-freshness, +dispatch-success, or PDP-latency SLOs or error budgets. Sustained `degraded|error` outcomes among +eligible `complete|degraded|error` fleet reads now produce one aggregate warning, but `complete` +remains a coverage-contract outcome rather +than a snapshot-age guarantee. Separately, sustained `stale` outcomes among at least twenty +freshness-eligible `fresh|stale` reads produce one aggregate warning; `unknown`, `error`, and `empty` +are excluded because none proves snapshot age. More than five percent `unavailable` among at least +twenty completed database-readiness checks over fifteen minutes produces one aggregate warning only +after a ten-minute hold; it is a control-plane dependency symptom, not a paging objective. Formal targets and budgets +require a separately reviewed F10.4 follow-up. If no `sith_build_info` sample reaches the evaluator +for ten minutes and the absence persists for five more, one aggregate warning reports loss of the +expected Hub telemetry path. Load the portable package only where that path is intentionally +installed. The warning cannot detect failure of its own evaluator, Alertmanager, or receiver, so an +external synthetic remains required for end-to-end metamonitoring. +More than five percent `error` among at least twenty eligible +`allow|deny|require-approval|error` decisions over fifteen minutes produces one aggregate warning +after a ten-minute hold. `deny` and `require-approval` remain valid decisions in its denominator, +not failures. This is a fail-closed PEP symptom, not an external Ardur PDP-latency SLI or SLO. +At least twenty aggregate `refused` authentication attempts with zero `accepted` attempts over +fifteen minutes, sustained for ten minutes, produce one aggregate warning. Any accepted attempt in +the same window suppresses it. At least one recent scraped sample from the preinitialized +accepted-outcome series must also have reached the rule evaluator during the last ten minutes; this +proves series visibility, not an accepted event. A missing or stale accepted series stays quiet +rather than turning partial telemetry into a refusal-only claim. This is not proof of brute force, +credential stuffing, account compromise, a specific actor, or a negotiated authentication SLO. +Chain verification detects retained-row edits, +deletion, reordering, broken links, and head mismatch. It does not make a WORM or non-repudiation +claim: detecting wholesale replacement by a privileged database owner requires a later externally +anchored checkpoint or immutable copy. + Before a signed scope exists, the authentication gate emits one local WARN record for every refusal with only the fixed `hub-auth` surface and `refused` outcome. It deliberately does not distinguish credential failure modes or carry a trace/correlation ID, token, header, path, client -address, workspace, principal, or verifier error. The record is a passive alerting signal, not an -audit record, rate limiter, telemetry export, or additional authentication decision. +address, workspace, principal, or verifier error. Because it precedes any signed scope or action +intent, this signal is outside the later E6 intent-correlated audit ledger contract. The record is a +passive alerting signal, not an audit record, rate limiter, telemetry export, or additional +authentication decision. The image route answers one exact, immutable runtime digest question across registered spokes. The direct reader accepts only canonical digests normalized from ordinary @@ -251,8 +530,26 @@ Run `sith hub migrate` as a short-lived deployment Job before starting `sith hub `SITH_HUB_MIGRATION_OWNER_DATABASE_URL` and `SITH_HUB_APPLICATION_DATABASE_ROLE`; mount the owner database URL from the deployment secret provider and set the application role explicitly. The command requires TLS for any non-local database target, applies the checksum-locked serializable -migration ledger, audits forced RLS, attempts to close its one owner connection, and exits. It never opens the -hub listener, creates a Kubernetes client, or starts collection. +migration ledger, creates the tenant-scoped policy-audit chain and exact single-use approval-grant +store, narrows the application role's audit and approval-table privileges, audits forced RLS plus +both immutable-entry contracts, attempts to close its one owner connection, and exits. Approval +rows contain only opaque identifiers, proposer/approver identity, the resolved proposal digest, +an evidence version, and lifecycle timestamps. The application role may insert them and update only +`consumed_at`; it cannot rewrite or delete the approved identity, digest, approval time, expiry, or +evidence version. The migration process never opens the hub listener, creates a Kubernetes client, +or starts collection. + +Migration 0011 preserves defaults for older format-1 audit writers, but older verifiers do not +understand format-2 approval lifecycle entries. During a rolling upgrade, run the migration, upgrade +all verifier-capable hub instances, and only then enable traffic that creates or consumes approvals. +Migration 0012 only extends the retained action constraint with the closed `export-audit` value; +deploy it before exposing the audit-export route so its authorizing decision can be appended. +Migration 0013 backfills legacy approval rows under one transactional access-exclusive owner lock, +immediately restores forced RLS, and marks those rows as legacy so they cannot be consumed by the +new evidence contract. It also enables audit format 3. Run the migration before deploying format-3 +writers and upgrade every verifier before enabling approval traffic; older writers fail closed +because the new immutable fields have no permissive defaults. Sith currently exposes no runtime +approval/dispatch path, so this ordering does not interrupt a supported write API. The normal hub process continues to use only `SITH_HUB_DATABASE_URL` for the non-owner application role. Do not reuse the migration-owner credential in the hub Deployment or place either database @@ -310,19 +607,168 @@ normalized in-memory records; partial results name stale/unreachable contexts. T persisted to disk, so raw workload specifications do not become a new plaintext credential-adjacent artifact. -`sith investigate` runs the deterministic R1-R6 catalog over the same cache: bad deploy, -OOMKilled, CrashLoopBackOff/repeated container failure, config drift, certificate expiry, and node -pressure. Every verdict includes its rule, exact cited signals, confidence state, missing lenses, -and an advisory command or PR change for the operator to inspect and run. The brain performs no -I/O and imports no connector planning, execution, intent, PEP, MCP, or local-operation path. +`sith investigate` runs the deterministic R1-R6 catalog plus adjacent R7 over the same cache: bad +deploy, OOMKilled, CrashLoopBackOff/repeated container failure, config drift, certificate expiry, +node pressure, and exact `ImagePullBackOff`/`ErrImagePull` detection. R7 proves only an image-pull +failure or backoff, not registry authentication, an invalid reference, reachability, rate limiting, +platform mismatch, or another underlying cause. Its sensitive-marked advisory is only a read-only +`kubectl describe pod` command; it retains no image reference, registry credential, Secret, Event +message, or raw payload. + +The brain package also exposes adjacent R8 to callers that already hold reviewed Argo CD graph +facts. R8 fails closed unless an attached, workspace-valid Application TIMELINE `FactChange` has +exact `argocd` source and provenance, protocol `1.0.0`, matching source/entity identity, a closed +payload with consistent failure kind and phase, an exact event time, and explicit caller-supplied +coverage. Only operation phase `Failed` or `Error` is accepted; missing or mismatched evidence, +`OutOfSync`, health, and successful or running operations do not prove failure. R8 discards the +revision and raw source payload, reports only that Argo recorded a failed operation, and offers the +sensitive read-only advisory +`kubectl --context {context} describe application.argoproj.io {name} -n {namespace}`. The +cache-backed `sith investigate` command does not fetch Argo Applications or infer Argo coverage, +so R8 appears there only after a future reader supplies the same validated graph evidence and +explicit TIMELINE coverage. + +Adjacent R9 accepts one already-authorized GitHub REST `Get a workflow run` response through a +bounded, API-versioned projector. It emits an unattached TIMELINE fact only when trusted caller +identity agrees with the response and GitHub reports exact status `completed` with conclusion +`failure`, `timed_out`, or `startup_failure`. Incomplete runs and completed non-failure conclusions +abstain; unknown states, identity mismatches, duplicate JSON members, malformed timestamps, and +ambiguous facts fail closed. The graph bridge admits only exact `github` source/provenance, +`workflow-runs/2026-03-10`, a closed payload, consistent run/attempt/native identity, matching event +time, and explicit caller-supplied TIMELINE coverage. R9 discards job, step, log, actor, branch, +commit, URL, and raw response data; it reports only that GitHub recorded a completed workflow-run +failure and leaves the cause unresolved. Its sensitive advisory tells a human to inspect the run's +failed jobs and logs before considering a rerun. It adds no GitHub client, token loading, fetch, +retention, repository-to-workload correlation, alert, typed intent, PEP handoff, mutation, or +execution. The cache-backed `sith investigate` path cannot produce R9 until a future reader supplies +validated workflow-run graph facts and explicit TIMELINE coverage. + +The brain's existing R3 CrashLoop rule can also consume the bounded Elasticsearch +`search/ecs-v1` facts produced from an already-fetched, complete Search API response. The graph +bridge requires exact Elasticsearch source/provenance, an attached Pod identity, a SHA-256 +native/resource identity that recomputes from the retained sanitized aggregate and Pod, and the +closed `logs.cause` values `panic`, `missing-config`, or +`dependency-failure`. It revalidates source bounds, discards count/container/window metadata, and +preserves only the Pod, cause, last classified event time, source, and stale flag. Fact presence +does not infer TELEMETRY coverage, and evidence for one Pod cannot strengthen another. Raw logs, +index/document IDs, query text, labels, URLs, credentials, and user data do not enter the brain or +CLI output. This path adds no Elasticsearch HTTP client, endpoint/index configuration, credential, +query execution, persistence, fleet correlation, typed intent, mutation, or execution; the +cache-backed `sith investigate` command still does not fetch Elasticsearch data. + +The E13 cost boundary likewise exposes a pure OpenCost `allocation/namespace-usd-v1` projector to +callers that already hold one authorized `/allocation` response for an explicit UTC window. It +accepts only the exact namespace aggregation contract with one set, disabled idle/sharing options, +and a trusted USD source assertion. Every valid Kubernetes namespace becomes exactly one +`FactCost` / `LensTelemetry` fact attached to its exact `{cluster, namespace}` identity, with +canonical five-decimal USD amounts and an observation time equal to the allocation-window end. The +projector validates the exact OpenCost component total with decimal arithmetic and discards provider +IDs, labels, annotations, workload identity, endpoints, and unknown fields. Any malformed, warned, +partial, duplicate-key, identity/window-mismatched, oversized, or invalid-total response is rejected +as a whole and emits zero facts; invalid rows are never filtered into a partial success. A successful +empty allocation map also emits zero facts, while missing OpenCost coverage is never estimated. This +package can preserve each successful projection in a per-scope snapshot, including an empty fact +set, and combine snapshots for one exact window into a deterministic workspace USD total. The +caller supplies the complete expected-scope set; output names every expected, reported, +successful-empty, and missing scope, and missing scopes never contribute synthetic zero cost. +Every fact is revalidated before all component and total amounts are summed with exact decimal +arithmetic. A rollup carries the source window end only when at least one scope reported and selects +no stale threshold. + +The rollup is an offline workspace computation core, not a live Hub feature. The OpenCost path adds +no OpenCost client, port-forward, service or ingress discovery, arbitrary endpoint, credentials, +Kubernetes Service-proxy RBAC, OCM transport, persistence, runtime wiring, per-team attribution, +UI/API, billing, optimization, mutation, currency conversion, freshness objective, or +GPU-utilization claim. The current CLI and Hub do not fetch, persist, roll up, or display these +facts yet. + +The bounded F13.3 GPU boundary separately exposes a pure DCGM +`prometheus/gpu-utilization-vector-v1` projector for callers that already hold one authorized +Prometheus instant-vector response and assert the exact query `DCGM_FI_DEV_GPU_UTIL`, with the API +series limit and per-query lookback override both disabled. This avoids a caller presenting an +API-truncated vector as complete while leaving the server's configured lookback/freshness policy +explicitly unresolved. It requires +the reviewed current dcgm-exporter whole-GPU identity labels, accepts paired `GPU_I_ID` and +`GPU_I_PROFILE` MIG evidence, and treats `namespace`/`pod`/`container` only as an all-or-nothing +`workload_best_effort` attribution. A fact always distinguishes its physical-GPU or MIG device +scope; exporter workload labels are never presented as precise per-workload accounting. Values +must be exact decimal percentages from 0 through 100 at the asserted query time. Unknown labels are +bounded but discarded, while raw GPU UUID, hostname, PCI bus, scrape target, job, instance, and +arbitrary pod-label data never enter the fact payload, display fields, or graph entity; selected +native identity is represented only by SHA-256. Duplicate projected identities, partial MIG or +workload labels, warnings or infos, non-vector or duplicate-key JSON, timestamp mismatch, +non-finite/out-of-range samples, and oversized input fail atomically. A successful empty vector +emits zero facts and does not by itself prove DCGM coverage. + +This DCGM core adds no Prometheus or DCGM client, discovery, endpoint or credential handling, new +RBAC, arbitrary PromQL, range aggregation, stale threshold, coverage rollup, persistence, cost or +idle-cost join, team mapping, UI/API, billing, optimization, rightsizing, mutation, or execution. +The current CLI and Hub do not fetch, persist, aggregate, or display these GPU utilization facts. + +Every verdict includes its rule, exact cited signals, confidence state, missing lenses, and an +advisory command or PR change for the operator to inspect and run. R1, R2, and R4 additionally name +an inert closed remediation verb plus the authoritative provenance a later reviewed renderer would +require; the candidate contains no target, arguments, approval, or execution capability. The brain +performs no I/O. It imports only the side-effect-free closed intent vocabulary—not connector +planning or execution, PEP, MCP, persistence, network, or local-operation paths. + +The contract-only GitOps provenance resolver is the next deliberately separate stage. It accepts +only a confirmed, entity-local R2/R4 `gitops.open-pr` candidate and exactly one immutable +`gitops-provenance/v1` bundle from the pinned canonical GitHub source contract. The bundle binds one +workspace and affected resource to a repository, non-symbolic base branch, exact base commit, +single update path, observed blob, exact desired content, evidence references, a maximum five-minute +validity interval, and the live planning handler's adapter version plus argument-schema digest. +Resolution reuses the GitHub handler's pure validation/canonicalization seam, rechecks the handler +contract, request cancellation, and bundle freshness after canonicalization, and returns only the +normalized repository target, exact canonical arguments and SHA-256 digest, copied evidence +references, or a closed abstention reason. It rejects missing, duplicate, stale, future, foreign, +unattached, drifted, handler-mutated, and handler-invalid inputs without I/O. + +Provenance readiness is not a PEP proposal, policy verdict, approval, credential, dispatch, Git +write, or execution capability. The resolver does not query GitHub and cannot independently prove +remote ref, commit, tree, or blob state; a future authorized read adapter must acquire those exact +facts and construct the bundle. That adapter's API calls, rate-limit/egress impact, credential +custody, and freshness policy are outside this offline slice. + +The approved next decomposition now defines the observed half independently as an immutable +`git-source-snapshot/v1`. It contains one workspace and affected resource, one pinned GitHub source +identity and repository, a configured non-symbolic base ref plus exact resolved commit, one safe +repository-relative path, and exact current file bytes plus the matching Git blob identity. The +constructor recomputes the blob object ID, copies and canonically orders attached subject/blob +evidence, and bounds content, evidence count, and validity. A pure trusted-time check classifies the +snapshot as fresh, future, or stale; it performs no I/O. + +The separately gated output half now exists as immutable `desired-change/v1`. A `DesiredChange` +defensively embeds one exact validated snapshot, one lowercase canonical `/` +identity, exact bounded non-NUL UTF-8 output bytes, and 2–32 unique stable evidence references that +must attach the snapshot's affected resource and observed blob. It preserves bytes without Unicode, +line-ending, whitespace, or YAML normalization, rejects exact no-op output, and canonically orders +copied evidence. + +There is deliberately no exported desired-change constructor. Until a concrete deterministic +transformer or declarative renderer is separately reviewed, request and runtime code cannot label +arbitrary bytes as trusted output. The snapshot still contains no desired bytes, and neither +contract contains PR metadata, handler binding, actor, intent, policy decision, approval, +credential, endpoint, persistence, dispatch, mutation, or execution state. Neither is wired into +the Brain, resolver, connector runtime, PEP, or Hub, so R2 and R4 remain advisory-only. These +offline contracts add no API request, egress, storage, cloud resource, telemetry cardinality, or +recurring cost; a future live adapter and transformer separately own credentials, rate limits, +freshness, format semantics, and evidence-to-output policy. Phase-L kubeconfig hydration supplies LIVE pod/workload/node evidence and discrete Kubernetes Events for TIMELINE when present. DESIRED and TELEMETRY remain unavailable unless a future connector supplies entity-attached facts. Consequently, an OOM or repeated failure is detected from LIVE evidence while its leak/spike/log-cause variant remains explicitly unconfirmed. A stale or missing required lens downgrades the dependent verdict rather than being treated as negative -evidence. Identical unhealthy image digests on two or more contexts produce a fleet-wide verdict -ahead of per-cluster findings. Advisory output never executes or dispatches anything. +evidence. For correlation-eligible canonical rules only, identical unhealthy image digests on two +or more contexts produce a fleet-wide verdict ahead of per-cluster findings. Adjacent R7, R8, and +R9 are never fleet-correlated; each remains entity-local. Advisory output never executes or +dispatches anything. + +Initial list-watch hydration is a complete, consistent snapshot rather than a bounded prefix: each +scope and kind uses 250-object pages under one request deadline, with hard limits of 10,000 objects +and 128 pages. Sith emits a watch error and does not open the stream when a continuation fails, +the collection changes resource version between pages, or either budget is exceeded. The TUI opens only when stdin and stdout are terminals; redirected bare invocations remain script-safe and print help. Tier-1 lenses are Pods, Deployments, Events, and Nodes. Use `:` for @@ -345,6 +791,10 @@ set of regular kubeconfig files for that UI session. It does not replace the sta supplied root must be an existing real directory; a file, symlink, or missing root fails before the local listener starts. Invalid, oversized, unreadable, or symlinked entries beneath a valid root are skipped with safe warnings that do not expose kubeconfig contents or an absolute local path. +Directory-imported configs must embed certificate-authority, client-certificate, client-key, and +token data rather than defer those reads to local file paths. Exec credential commands must be a +PATH-resolved program name, not a relative or absolute path. These constraints keep credential and +plugin resolution from reopening the directory replacement race after the bounded import completes. Each imported source is labeled by its relative filename; contexts with the same name remain isolated, and selecting a source in the context rail filters to its contexts. The import is limited to 128 traversed filesystem entries (including ignored symlinks and directories), 4 MiB per regular @@ -387,7 +837,7 @@ Run the full local quality gate with golangci-lint v2.12.2 and govulncheck v1.6. make ci ``` -Release changes additionally require GoReleaser v2.17.0 and Syft v1.46.0. This gate builds all +Release changes additionally require GoReleaser v2.17.0 and Syft v1.49.0. This gate builds all four archives twice and refuses the change if their SHA-256 digests differ: ```bash @@ -418,8 +868,11 @@ make e2e-kind The hub tenancy gate starts a temporary, digest-pinned PostgreSQL 18.4 container and proves the application role is a non-owner without `BYPASSRLS`, every current workspace table—including API -key verifier and OIDC binding rows—has forced RLS, direct unscoped reads are default-deny, foreign writes fail, and -transaction-local scope does not bleed through a reused pool connection. It also exercises real +key verifier, OIDC binding, and approval-grant rows—has forced RLS, direct unscoped reads are +default-deny, foreign writes fail, and transaction-local scope does not bleed through a reused pool +connection. It also proves same-workspace current-role checks, proposer/approver separation, +approve-then-swap refusal, immutable grant fields, replay refusal, and exactly one winner under a +concurrent approval-consumption race. It additionally exercises real API-key issuance, current-membership exchange, bounded rotation overlap, immediate revocation, OIDC binding resolution, and cross-workspace negative controls. It requires Docker and adds one official PostgreSQL image pull plus a short-lived local container: @@ -429,10 +882,11 @@ make e2e-postgres ``` The complete tenant-isolation suite combines the real PostgreSQL boundary with signed-token, -injected-header, scoped-query, and deterministic selector-fuzz invariants. It also removes and -weakens a live RLS policy and requires the suite to detect both mutations before restoring the -steady state. The native Go fuzzer runs exactly 50,000 generated selector mutations with four -workers; coverage no longer depends on CI runner speed, while a separate timeout catches hangs: +injected-header, scoped-query, and deterministic selector-fuzz invariants. It also removes, +weakens, and adds a second permissive live RLS policy and requires the suite to detect all three +mutations before restoring the steady state. The native Go fuzzer runs exactly 50,000 generated +selector mutations with four workers; coverage no longer depends on CI runner speed, while a +separate timeout catches hangs: ```bash make e2e-isolation diff --git a/charts/sith-hub/README.md b/charts/sith-hub/README.md index 6489e37..d4accb8 100644 --- a/charts/sith-hub/README.md +++ b/charts/sith-hub/README.md @@ -8,11 +8,17 @@ The chart creates no `Secret`, `data`, or `stringData` block. An E3-approved KMS | Value | Required Secret keys | Consumer | | --- | --- | --- | -| `runtime.existingSecret` | `database-url`, `session-public.pem`, `server-tls.crt`, `server-tls.key`, `proxy-ca.crt`, `proxy-tls.crt`, `proxy-tls.key` | long-running `sith hub` Deployment | +| `runtime.existingSecret` | `database-url`, `session-public.pem`, `server-tls.crt`, `server-tls.key`, `proxy-ca.crt`, `proxy-tls.crt`, `proxy-tls.key`; plus `session-private.pem` when browser OIDC is enabled | long-running `sith hub` Deployment | | `migration.existingSecret` | `owner-database-url` | short-lived `sith hub migrate` hook Job | `migration.applicationRole` is a non-secret PostgreSQL role name. The migration hook runs before install and upgrade, blocks the release if it fails, and receives no Kubernetes service-account token or runtime TLS material. The Deployment receives an in-cluster token only to read the fixed `sith-reader` managed-serviceaccount Secret; its ClusterRole permits exactly `get` on that one resource name and no list/watch or write verbs. +The same hook creates the forced-RLS approval-grant table used by F5.9a. The application role may +insert an exact digest-bound grant and atomically set only its `consumed_at` column; it cannot +rewrite or delete the workspace, intent, proposer, approver, or resolved digest. This adds one +small indexed row and one scoped transaction per approval, with no queue, polling loop, Secret, +Service, or cloud dependency. + The chart permits exactly two fixed profiles for both the hub and its migration hook: | Profile | Requests | Limits | Intended envelope | @@ -22,8 +28,52 @@ The chart permits exactly two fixed profiles for both the hub and its migration The heavy profile reserves five times the requested CPU and four times the requested memory, so it carries a correspondingly higher node-pool cost. These are fixed scheduling bounds, not measured capacity claims; no arbitrary resource override or third profile is accepted. Both profiles use the same immutable image requirement, existing Secret references, migration isolation, RBAC, probes, and pod/container hardening. They do not change replica count, database custody, or KMS materialization, so `heavy` does not claim unproven high availability. +Both profiles use HTTPS application probes on the existing `https` container port. `/healthz` +checks only process responsiveness. `/readyz` pings the existing least-privilege application +PostgreSQL pool under the Hub's one-second deadline. Probe responses are empty and reveal no +dependency or error details. A PostgreSQL outage therefore removes a Pod from Service endpoints +without making liveness fail and causing a restart storm. The probes add no plaintext listener, +Service port, credential, or OCM/spoke dependency check. + The chart pins workload hardening (UID/GID 65532, read-only root filesystem, RuntimeDefault seccomp, no privilege escalation, and all Linux capabilities dropped). It deliberately does not create a broad egress NetworkPolicy: the database and pinned OCM endpoints are deployment-specific, so operators must place the release in a namespace with an appropriate least-privilege egress policy. KMS provider resources, release-bound image publication, real install/upgrade proof, air-gap bundles, and addon packaging are later E9/E3 slices. +`runtime.browserOIDC` is disabled only when all three of `issuer`, `clientID`, and `redirectURI` are empty. Setting any one enables a fail-closed validation requiring all three. The configured client ID is the exact ID-token audience; the callback must be the same HTTPS URL registered at the issuer and supplied to the Hub. In that mode the Hub mounts `session-private.pem`, checks that it matches `session-public.pem`, completes authorization-code + PKCE (`S256`) server-side, and returns a short-lived `__Host-`, `Secure`, `HttpOnly`, `SameSite=Lax` session cookie. Lax permits the safe top-level console `GET` after the cross-site IdP callback but does not include the cookie on unsafe cross-site methods. The chart never renders any of this secret material. The Hub needs narrowly allowlisted egress only to the configured issuer's discovery, JWKS, and token endpoints; the operator's browser navigates to the issuer authorization endpoint. + +`runtime.metrics.listenAddress` is opt-in and must be exactly `127.0.0.1:` or +`[::1]:`, with a non-zero port. When set, the chart provides only the corresponding +container-port metadata and `SITH_HUB_METRICS_LISTEN_ADDR`; it creates no Service port, ingress, +scrape annotation, exporter, or automatic sidecar. The Hub serves only `GET /metrics` there from +its isolated low-cardinality registry. A same-Pod collector can reach it over `localhost` because +containers in a Pod share a network namespace; that is deliberately a local operational trade-off, +not a public or tenant-visible metrics endpoint. See the [Kubernetes Pod networking +documentation](https://kubernetes.io/docs/concepts/workloads/pods/#how-pods-manage-multiple-containers). +The registry includes fixed `sith_hub_readiness_checks_total{outcome}` and +`sith_hub_readiness_check_duration_seconds{outcome}` families for completed database-aware +readiness checks, where `outcome` is only `ready` or `unavailable`. These process-local series carry +no tenant, spoke, request, endpoint, credential, or raw-error label. +The portable repository rules include an install-neutral warning when more than five percent of at +least twenty aggregate checks are `unavailable` over fifteen minutes for ten minutes. The chart does +not install or configure the scraper, rule evaluator, Alertmanager, or receiver; see +[`docs/runbooks/hub-alerts.md`](../../docs/runbooks/hub-alerts.md). + +The Hub also starts a restricted same-container child for the one closed authentication-refusal +record. Sith supplies only an inherited bounded Unix datagram descriptor and stderr: no +configuration, environment, listener, socket pathname, network endpoint, or Secret value. The +child still shares the container's mounts, UID, and network namespace, so it is a bounded delivery +and shutdown boundary, not a filesystem or network sandbox. A full or dead local transport drops +the fixed record without delaying the governed response and increments the unlabeled +`sith_auth_refusal_delivery_drops_total` counter. Enabling `runtime.metrics` is the only way to +scrape that process-wide loss signal; the chart still renders no related Service port, ingress, +sidecar, queue, exporter, or remote telemetry path. + +The same optional endpoint exposes two fixed, preinitialized +`sith_auth_attempts_total{outcome="accepted|refused"}` series. A successful local verifier decision +counts as `accepted` before workspace authorization; every authentication rejection counts as +`refused` and also increments the legacy unlabeled `sith_auth_refusals_total` counter. The audit +child remains refusal-only. These counters contain no tenant, workspace, identity, credential, +request, network, error, trace, or authorization labels and do not add a Service, exporter, +persistence, remote write, alert, SLO, or cloud resource. + Validate supplied values before applying anything: ```bash diff --git a/charts/sith-hub/templates/deployment.yaml b/charts/sith-hub/templates/deployment.yaml index 924f64d..aebcd42 100644 --- a/charts/sith-hub/templates/deployment.yaml +++ b/charts/sith-hub/templates/deployment.yaml @@ -1,3 +1,19 @@ +{{- $browserOIDC := .Values.runtime.browserOIDC }} +{{- $browserEnabled := and $browserOIDC.issuer $browserOIDC.clientID $browserOIDC.redirectURI }} +{{- $metrics := default (dict "listenAddress" "") .Values.runtime.metrics }} +{{- $metricsListenAddress := default "" $metrics.listenAddress }} +{{- if and (or $browserOIDC.issuer $browserOIDC.clientID $browserOIDC.redirectURI) (not $browserEnabled) }} +{{- fail "runtime.browserOIDC.issuer, runtime.browserOIDC.clientID, and runtime.browserOIDC.redirectURI must be set together" }} +{{- end }} +{{- if $metricsListenAddress }} +{{- if not (regexMatch "^(127\\.0\\.0\\.1|\\[::1\\]):[1-9][0-9]{0,4}$" $metricsListenAddress) }} +{{- fail "runtime.metrics.listenAddress must use exact IPv4 or IPv6 loopback with a non-zero port" }} +{{- end }} +{{- $metricsPort := regexFind "[0-9]+$" $metricsListenAddress }} +{{- if gt (int $metricsPort) 65535 }} +{{- fail "runtime.metrics.listenAddress port must be at most 65535" }} +{{- end }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -34,9 +50,18 @@ spec: - name: https containerPort: 8443 protocol: TCP + {{- if $metricsListenAddress }} + - name: metrics + containerPort: {{ regexFind "[0-9]+$" $metricsListenAddress | int }} + protocol: TCP + {{- end }} env: - name: SITH_HUB_LISTEN_ADDR value: "0.0.0.0:8443" + {{- if $metricsListenAddress }} + - name: SITH_HUB_METRICS_LISTEN_ADDR + value: {{ $metricsListenAddress | quote }} + {{- end }} - name: SITH_HUB_DATABASE_URL valueFrom: secretKeyRef: @@ -50,6 +75,16 @@ spec: value: {{ required "runtime.sessionKeyID is required" .Values.runtime.sessionKeyID | quote }} - name: SITH_HUB_SESSION_PUBLIC_KEY_FILE value: /var/run/sith/runtime/session-public.pem + {{- if $browserEnabled }} + - name: SITH_HUB_BROWSER_OIDC_ISSUER + value: {{ $browserOIDC.issuer | quote }} + - name: SITH_HUB_BROWSER_OIDC_CLIENT_ID + value: {{ $browserOIDC.clientID | quote }} + - name: SITH_HUB_BROWSER_OIDC_REDIRECT_URI + value: {{ $browserOIDC.redirectURI | quote }} + - name: SITH_HUB_SESSION_PRIVATE_KEY_FILE + value: /var/run/sith/runtime/session-private.pem + {{- end }} - name: SITH_HUB_SERVER_TLS_CERT_FILE value: /var/run/sith/runtime/server-tls.crt - name: SITH_HUB_SERVER_TLS_KEY_FILE @@ -67,15 +102,25 @@ spec: - name: SITH_HUB_KUBE_API_SERVER_NAME value: {{ required "runtime.kubeAPIServerName is required" .Values.runtime.kubeAPIServerName | quote }} livenessProbe: - tcpSocket: + httpGet: + path: /healthz port: https + scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 readinessProbe: - tcpSocket: + httpGet: + path: /readyz port: https + scheme: HTTPS initialDelaySeconds: 2 periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 resources: {{- include "sith-hub.resources" . | nindent 12 }} securityContext: @@ -96,6 +141,10 @@ spec: items: - key: session-public.pem path: session-public.pem + {{- if $browserEnabled }} + - key: session-private.pem + path: session-private.pem + {{- end }} - key: server-tls.crt path: server-tls.crt - key: server-tls.key diff --git a/charts/sith-hub/values.schema.json b/charts/sith-hub/values.schema.json index 23111ef..df4cd56 100644 --- a/charts/sith-hub/values.schema.json +++ b/charts/sith-hub/values.schema.json @@ -23,12 +23,30 @@ "runtime": { "type": "object", "additionalProperties": false, - "required": ["existingSecret", "sessionIssuer", "sessionAudience", "sessionKeyID", "proxyAddress", "proxyServerName", "kubeAPIServerName"], + "required": ["existingSecret", "sessionIssuer", "sessionAudience", "sessionKeyID", "browserOIDC", "proxyAddress", "proxyServerName", "kubeAPIServerName"], "properties": { "existingSecret": {"type": "string", "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"}, "sessionIssuer": {"type": "string", "minLength": 1}, "sessionAudience": {"type": "string", "minLength": 1}, "sessionKeyID": {"type": "string", "minLength": 1}, + "browserOIDC": { + "type": "object", + "additionalProperties": false, + "required": ["issuer", "clientID", "redirectURI"], + "properties": { + "issuer": {"type": "string"}, + "clientID": {"type": "string"}, + "redirectURI": {"type": "string"} + } + }, + "metrics": { + "type": "object", + "additionalProperties": false, + "required": ["listenAddress"], + "properties": { + "listenAddress": {"type": "string", "pattern": "^(|127\\.0\\.0\\.1:[1-9][0-9]{0,4}|\\[::1\\]:[1-9][0-9]{0,4})$"} + } + }, "proxyAddress": {"type": "string", "minLength": 1}, "proxyServerName": {"type": "string", "minLength": 1}, "kubeAPIServerName": {"type": "string", "minLength": 1} diff --git a/charts/sith-hub/values.yaml b/charts/sith-hub/values.yaml index beedab8..70057d6 100644 --- a/charts/sith-hub/values.yaml +++ b/charts/sith-hub/values.yaml @@ -13,6 +13,16 @@ runtime: sessionIssuer: "" sessionAudience: "" sessionKeyID: "" + # browserOIDC is optional. When all values are set, the Hub enables a pinned public OIDC client + # and requires session-private.pem in runtime.existingSecret to issue the final HttpOnly cookie. + browserOIDC: + issuer: "" + clientID: "" + redirectURI: "" + # metrics is disabled unless listenAddress is the exact loopback address and non-zero port used + # by a same-Pod collector. The chart creates no Service, scrape annotation, or exporter. + metrics: + listenAddress: "" proxyAddress: "" proxyServerName: "" kubeAPIServerName: "" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 74deaf8..3746c9e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -270,6 +270,7 @@ erDiagram INTENT ||--o{ INTENT_TARGET : "fans out to" CLUSTER ||--o{ INTENT_TARGET : "is target of" INTENT ||--o{ AUDITLOG : "produces" + INTENT ||--o{ APPROVAL : "bound to exact digest" POLICY ||--o{ DECISION : "yields" INTENT ||--o{ DECISION : "adjudicated by" @@ -323,6 +324,17 @@ erDiagram json detail "what-happened" timestamp at } + APPROVAL { + id id PK + id intent_id + id workspace_id + string proposer + string approver + string resolved_digest "args + resolved target" + timestamp approved_at + timestamp expires_at "DB-derived approved_at + 10 minutes" + timestamp consumed_at "nullable, set once" + } MEMBERSHIP { id id PK id workspace_id FK @@ -341,9 +353,19 @@ erDiagram Design points: - **`Workspace` is the scoped tenancy object** — every query is scoped to a workspace, and a **database-level backstop** (RLS) enforces it independently of application code (see - [ADR-0003](adr/0003-tenancy-isolation.md)). + [ADR-0003](adr/0003-tenancy-isolation.md)). The catalog audit requires exactly one complete + `workspace_isolation` policy per table, so an additional permissive policy cannot silently + broaden access through PostgreSQL's OR-composition rule. - **`DECISION` (why-allowed, Ardur) and `AUDITLOG` (what-happened, Sith) are separate** and together form a complete agent-action ledger. +- **`APPROVAL` is mutable in exactly one direction** — its identity, resolved digest, + `approved_at`, and `expires_at` are immutable. PostgreSQL derives + `expires_at = approved_at + interval '10 minutes'`; callers cannot supply it, and a database + constraint plus column privileges reject later drift. One atomic update spends the row only when + `consumed_at IS NULL` and `approved_at <= statement_timestamp() < expires_at`. Missing, + mismatched, future, expired, or already-consumed grants affect no row and return the same stable + unavailable refusal. FORCE RLS isolates the row; the application role cannot delete it or update + any other column. - **`CLUSTER.last_seen` / `FLEET_FACT.observed_at`** are what abstention reads. ## 6. Enforcement pipeline (the PEP) diff --git a/docs/EPICS.md b/docs/EPICS.md index 0d9a1cf..3f7cbcd 100644 --- a/docs/EPICS.md +++ b/docs/EPICS.md @@ -1694,6 +1694,21 @@ sequenceDiagram - Approvals are per-action, single-use, and bound to the resolved-args hash. - Changing the args after approval invalidates it (approve-then-swap is blocked). +**Implementation status (F5.9a, 2026-07-18).** The durable server-side core persists a +same-workspace, distinct-approver grant bound to the existing immutable resolved proposal digest. +The non-owner application role can insert the forced-RLS row and atomically set only +`consumed_at`; it cannot rewrite or delete the intent, identities, or digest. Missing, foreign, +mismatched, and replayed grants share one fail-closed refusal, and a real PostgreSQL concurrency +test proves exactly one consumer wins. + +**Implementation status (F5.9b, 2026-07-21).** Every new grant has one immutable 10-minute +absolute lifetime minted from PostgreSQL statement time. Consumption checks the half-open +`approved_at <= consumed_at < expires_at` interval in the same conditional update that spends the +grant. Expiry is bound into versioned lifecycle evidence; legacy grants are retained but fail +closed, and format-1/2 audit records remain independently verifiable beside format 3. MCP +elicitation transport, Ardur PDP policy, multi-approver counting, credential minting, and dispatch +remain later slices; this status does not claim F5.9 complete. + **Key risk / guardrail.** Approve-then-swap (approve a benign action, then change args) is the classic agent bypass. Guardrail: the approval is bound to an arg-hash re-checked at dispatch, so a valid signature and a valid approval are both necessary but neither is sufficient if the args @@ -1852,6 +1867,20 @@ flowchart TD breach. Guardrail: query and export inherit E1's tenant scoping and DB RLS, so a record is only ever visible within its own workspace. +**Implemented F6.4 boundary (2026-07-18).** F6.4a and F6.4b expose and offline-verify one complete +privacy-minimized Sith policy/approval chain of at most 512 entries. F6.4c adds a distinct paged +export for larger retained chains without changing that complete-document route. The first page is +bound to the current head after its own admin-only `audit.export` decision is durably appended; +each later request is independently authenticated, authorized, and audited, while those later +audit rows remain outside the fixed snapshot. A versioned fixed-size canonical base64url +continuation carries a domain-separated workspace binding, snapshot head, next sequence, and prior +hash. It grants no authority and is revalidated against forced-RLS head and boundary rows in a +Repeatable Read transaction before at most 512 consecutive entries are returned. Every transaction +commits before encoding. `sith audit verify-pages` reads bounded stable files one at a time and +succeeds only for an ordered same-workspace, same-snapshot genesis-to-head sequence. This remains +an internal-continuity proof, not external authenticity, WORM retention, asynchronous export, the +Ardur decision ledger, the full action lifecycle, or E6 completion. + ### E6 exit criteria - Every phase of every action and read produces an audit entry; no unlogged action reaches a @@ -2107,6 +2136,16 @@ badges so coverage gaps are visible. 4. The view is read-only and holds no privileged path — it only shows what the API returns for the actor's workspace. +The first Hub rendering slice (#218) deliberately starts at the bounded `FleetResult` altitude: +cluster reachability, observation time, and honest coverage gaps. It uses its own cookie/session + +CSRF adapter over the existing PEP read and cannot mount local operations or trigger collection. +#220 adds the next F8.1 slice: an explicit exact-resource, fixed-not-Healthy query through the +existing tenant-scoped PEP correlator. The browser receives only cluster scope, resource identity, +normalized health, observation time, stale state, and coverage assessment; raw fact payloads and +provenance remain server-side. A 257-row sentinel makes over-bound results unavailable rather than +silently partial, and a separate session/workspace/purpose proof prevents fleet-proof reuse. +Inventory records and the service picker remain later read-only slices. + ```mermaid flowchart TD A["Operator opens fleet view"] --> B["Call read API (tenant-scoped, E2)"] @@ -2477,11 +2516,13 @@ and alerting · F10.5 crown-jewel hardening. ### F10.1 — Metrics **What it is.** Metrics about Sith's own health and behavior: control-plane liveness, federation -freshness, intent throughput, refusal and abstention rates, PDP latency. +freshness, intent throughput, bounded sanitized authentication-outcome counts, and future derived rates +where trustworthy denominators exist, abstention rates, and PDP latency. **How it works.** 1. The hub exposes metrics for scraping (control-plane health, DB, queue depths). -2. Federation metrics track per-spoke read freshness and dispatch success. +2. Federation metrics track bounded aggregate request-time freshness and, once write federation + exists, dispatch success without tenant-proportional labels. 3. Governance metrics track intents proposed/allowed/denied, abstention rate, and approval latency. 4. These describe Sith itself; Sith does not retain other systems' metric series. @@ -2489,7 +2530,7 @@ freshness, intent throughput, refusal and abstention rates, PDP latency. ```mermaid flowchart TD A["Sith hub"] --> B["Control-plane metrics: liveness, DB, queues"] - A --> C["Federation metrics: per-spoke freshness, dispatch success"] + A --> C["Federation metrics: aggregate read freshness, future dispatch success"] A --> D["Governance metrics: intents allowed/denied, abstention rate, approval latency"] B --> E["Exposed for scraping (about Sith itself)"] C --> E @@ -2505,6 +2546,56 @@ flowchart TD lake. Guardrail: metrics describe Sith's own behavior only; federated health reads stay a bounded cache (E2), not a series store. +**Implementation note (F10.1d).** Authorized persisted fleet reads emit one aggregate closed +coverage result: `complete`, `degraded`, `empty`, or `error`. Coverage inconsistency, result/count +mismatch, staleness, unreachability, truncation, and unaccounted scopes collapse to `degraded`; +only an internally consistent zero-scope result is `empty`. The fixed four-series counter has no tenant, spoke, +resource, selector, identity, trace, age, or raw-error label and uses only the existing opt-in +loopback scrape boundary. It is SLI substrate, not an SLO target, error budget, or alert. + +**Implementation note (F10.1f).** Each authorized persisted fleet read emits the existing coverage +outcome and one paired request-time freshness outcome: `fresh`, `stale`, `unknown`, `empty`, or +`error`. `fresh` requires a non-empty complete consistent result in which every returned cluster has +a unique identity and non-zero observation time. A structurally valid result with a proven stale +retained scope is `stale`; unobserved, inconsistent, mismatched, or otherwise non-stale degraded +coverage is `unknown`. +Only a consistent zero-scope result is `empty`, and a storage failure before a result exists is +`error`. The five preinitialized series carry no tenant, identity, trace, request, spoke, cluster, +resource, selector, credential, endpoint, age, or raw-error labels. The pair is validated before +either counter changes and observer panic remains isolated. This is request-time SLI substrate, not +continuous monitoring, a per-spoke series, alert, SLO, error budget, dispatch-success signal, or +PDP-latency signal. + +**Implementation note (F10.1e).** Each valid completed Hub database readiness check emits one +fixed `ready|unavailable` attempt and latency observation through the existing isolated loopback +registry. Dependency errors, deadline expiry, caller cancellation, and recovered checker panic +collapse to `unavailable`; invalid requests emit nothing and invalid observation values are +discarded. The two preinitialized outcome series carry no tenant, spoke, request, endpoint, +credential, error, or panic detail. Instrumentation is panic-isolated from the body-free probe and +adds no listener, Service, exporter, persistence, remote write, or cloud resource. + +**Implementation note (F10.1g).** Each refusal emitted by the existing sanitized bearer/session +middleware boundary increments one preinitialized, unlabeled `sith_auth_refusals_total` counter. +Runtime fanout independently reaches the existing process audit observer and the metric observer; +observer panics cannot suppress a later destination or alter the uniform HTTP 401 response. The +counter carries no reason, credential mode, tenant, workspace, actor, principal, token, IP, path, +request, trace, or correlation label. It does not count successful authentication, OIDC provider +exchange/callback failures, authorization denials, or every future authentication mode. The legacy +counter alone is not a denominator and remains compatible with existing scrapes. + +**Implementation note (F10.1h).** Each completed local bearer-token or browser-session verifier +decision increments one of exactly two preinitialized +`sith_auth_attempts_total{outcome="accepted|refused"}` series. `accepted` is emitted immediately +after verifier success and before workspace authorization; a later forbidden authorization is +therefore not misclassified as failed authentication. Every `refused` outcome also increments the +legacy unlabeled refusal counter exactly once. Metrics consume both outcomes, while the process +audit observer and structured-log adapter remain refusal-only and accepted observations cannot +write a datagram, log, or delivery-drop count. The outcome label is closed and carries no +credential mode, reason, tenant, workspace, actor, principal, token, IP, path, method, request, +trace, correlation, authorization, or handler-result dimension. The counters exclude provider +exchange/callback failures and define no ratio, alert, brute-force detector, SLO, error budget, +page, listener, exporter, persistence, remote write, or cloud resource. + ### F10.2 — Distributed tracing **What it is.** Traces that follow an intent's lifecycle across the PEP stages and the hub → spoke @@ -2601,6 +2692,65 @@ flowchart TD whole fleet. Guardrail: SLOs with error budgets and security alerting make degradation visible and actionable rather than silent. +**Implementation note (F10.4a).** The first portable rule contract covers only existing bounded +Hub symptoms: fail-closed policy-audit errors, lost authentication-refusal delivery records, and a +sustained aggregate snapshot failure ratio. It deliberately adds no remote scrape path or monitoring +CRD and does not claim the still-missing read-freshness, dispatch-success, or PDP-latency SLOs. + +**Implementation note (F10.4b).** A fourth portable warning consumes the bounded F10.1d outcome +counter and fires only when more than five percent of at least twenty aggregate eligible fleet reads +are `degraded` or `error` over fifteen minutes for ten minutes. Eligible outcomes are `complete`, +`degraded`, and `error`; legitimate `empty` reads are excluded from both numerator and denominator. +This is a user-visible coverage symptom, not a snapshot-age guarantee, SLO target, error budget, or +burn-rate page. + +**Implementation note (F10.4c).** A fifth portable warning consumes only the bounded F10.1e +`ready|unavailable` counter. It fires when more than five percent of at least twenty aggregate +database-readiness checks are `unavailable` over fifteen minutes and the condition persists for ten +minutes. The expression aggregates away all source labels, guards its denominator, and remains quiet +for missing, low-volume, all-ready, and transient data. It is a control-plane availability symptom, +not a read-freshness or paging SLO, and adds no scrape, rule-evaluation, notification, or cloud +infrastructure. + +**Implementation note (F10.4d).** A sixth portable warning uses the existing traffic-independent +`sith_build_info` gauge to detect when no expected Sith sample reaches the rule evaluator for +ten minutes and the absence persists for five more. The rule emits one fixed-label warning, depends +on no operator-specific target identity or Kubernetes metric, and is valid only where an operator +has intentionally installed the documented Hub scrape/forwarding path. It cannot detect failure of +its own evaluator, Alertmanager, or receiver; an external synthetic remains required for the full +notification path. + +**Implementation note (F10.4e).** A seventh portable warning consumes only the bounded F10.1f +`fresh|stale|unknown|empty|error` counter. It fires when more than five percent of at least twenty +aggregate freshness-eligible `fresh|stale` reads are proven `stale` over fifteen minutes and the +condition persists for ten minutes. `unknown`, `error`, and `empty` are excluded from both numerator +and denominator because none proves snapshot age. The expression aggregates away every source +label and adds no scrape, storage, remote-write, notification, or cloud infrastructure. This is a +request-time aggregate symptom, not continuous freshness monitoring, an SLO, an error budget, or a +page. + +**Implementation note (F10.4f).** An eighth portable warning consumes only the existing bounded +`sith_policy_decisions_total{verb,outcome}` counter. It fires when more than five percent of at least +twenty aggregate eligible `allow|deny|require-approval|error` decisions end in `error` over fifteen minutes +and the condition persists for ten minutes. `deny` and `require-approval` are valid decisions and +remain in the denominator, not the numerator. The expression aggregates away `verb` and every +source label and adds no runtime, scrape, storage, remote-write, notification, or cloud +infrastructure. This is a fail-closed PEP symptom, not external Ardur PDP latency, an SLO, an error +budget, a page, or a dispatch-success signal. + +**Implementation note (F10.4g).** A ninth portable warning consumes only the bounded +`sith_auth_attempts_total{outcome="accepted|refused"}` counter. It fires when at least twenty +aggregate attempts are `refused` and none are `accepted` over fifteen minutes, and the condition +persists for ten minutes. Any accepted attempt suppresses it. The freshness guard also requires at +least one recent scraped sample from the preinitialized accepted-outcome series during the most +recent ten minutes; that proves series visibility, not an accepted authentication event. Missing or +stale accepted-series data stays quiet rather than turning incomplete telemetry into a security +claim. +The expression aggregates away every source label and emits one fixed warning. It is refusal-only +traffic, not a generic refusal ratio, brute-force or credential-stuffing detector, actor +attribution, SLO, error budget, page, or complete authentication-monitoring claim, and it adds no +runtime, scrape, storage, remote-write, notification, or cloud infrastructure. + ### F10.5 — Crown-jewel hardening **What it is.** The hardening the hub demands as the highest-value target: signer-key protection, @@ -2638,6 +2788,14 @@ worst-case scenario. Guardrail: defense-in-depth — even full hub control canno spoke or bypass spoke-side re-validation, and the closed vocabulary plus per-spoke allowlists bound the damage (threat-model S1). +**Implementation note (F10.5a).** The Hub's existing TLS listener exposes fixed, body-free +`GET /healthz` and `GET /readyz` routes for Kubernetes. Liveness checks only process responsiveness; +readiness performs one least-privilege application-pool PostgreSQL ping under a one-second server +deadline. Database failure never enters the liveness decision, preventing dependency-driven restart +storms, and OCM/spoke reachability remains a fleet-coverage concern rather than a whole-Hub +readiness dependency. The contract adds no listener, Service port, credential, error detail, or +tenant-proportional signal. + ### E10 exit criteria - Sith exposes metrics, traces, and structured sanitized logs about its own behavior, correlated @@ -2898,18 +3056,21 @@ Guardrail: the taxonomy is closed; "re-skin a tool" is not an expressible connec ### F12.3 — Versioning and one-canonical-connector policy -**What it is.** A minor-additive protocol contract and a rule of one canonical connector per -tool — the Terraform discipline that prevents the Backstage redundancy/abandonment failure. +**What it is.** A structured, explicitly negotiated framework wire contract, a separate opaque +adapter/evidence contract version, and a rule of one canonical connector per tool — the Terraform +discipline that prevents the Backstage redundancy/abandonment failure. **How it works.** -1. Major protocol versions delineate compatibility; minor versions are strictly additive. -2. The registry admits **one** canonical connector per target tool, with declared ownership. -3. Breaking a connector's contract is a major-version, reviewed change — never a silent minor bump. +1. Endpoints advertise exact structured `{major, minor}` framework wire versions; the highest + exact common version is selected. No common major or no explicitly common minor fails closed. +2. Adapter/evidence contract versions remain opaque provenance and never drive wire compatibility. +3. The registry admits **one** canonical connector per target tool, with declared ownership. +4. Breaking the framework contract is a major-version, reviewed change — never a silent minor bump. ```mermaid flowchart TD A["Connector change"] --> B{"Breaking?"} - B -- "no" --> C["Minor: additive, compatible"] + B -- "no" --> C["Minor: additive and explicitly advertised"] B -- "yes" --> D["Major: reviewed compatibility break"] E["New connector for tool T"] --> F{"Canonical connector for T exists?"} F -- "yes" --> G["Improve the canonical one (no duplicate)"] @@ -2917,7 +3078,8 @@ flowchart TD ``` **Acceptance criteria.** -- Minor protocol changes are additive; breaks require a major version and review. +- Wire versions and opaque adapter versions are separate; only explicitly shared wire versions + negotiate, additive minors remain reviewed, and breaks require a major version and review. - The registry holds one canonical connector per tool with a named owner. **Key risk / guardrail.** Overlapping half-maintained connectors (the Backstage marketplace). @@ -2989,6 +3151,18 @@ flowchart TD **Key risk / guardrail.** Inventing costs for clusters that don't report them. Guardrail: no OpenCost → no cost fact; the gap is shown, never estimated silently. +**Current bounded slice (F13.1a, #282).** `internal/connector/opencost` accepts one +already-authorized OpenCost v1.120.2 `/allocation` response for an explicit UTC window, +`aggregate=namespace`, and one set covering the window. Idle, sharing, filtering, accumulation, +and aggregated metadata are disabled. Because the allocation response does not prove currency, +the slice requires a trusted USD assertion and rejects every other unit. It emits deterministic +namespace-attached TELEMETRY `cost` facts with exact five-decimal component validation and uses the +window end as source observation time. A successful empty allocation map returns zero facts; +malformed, +identity-mismatched, warned, partial, or synthetic/unscoped rows fail atomically. The slice performs +no network access and does not complete the live adapter, fleet/team rollup, freshness policy, +currency conversion, billing, optimization, or GPU-utilization work. + ### F13.2 — Hub fleet cost rollup (per-workspace / per-team) **What it is.** The capability none of the OSS tools ship free: aggregate per-cluster costs across @@ -3012,6 +3186,18 @@ flowchart TD **Key risk / guardrail.** A partial rollup read as complete. Guardrail: coverage is always shown. +**Current bounded slice (F13.2a, #284).** `internal/connector/opencost` preserves every successful +F13.1a projection in a per-scope snapshot, including a complete empty allocation set, and computes +one deterministic workspace USD total for an exact caller-bound window. The caller supplies the +unique expected cluster set; output separately names expected, reported, successful-empty, and +missing scopes, so missing OpenCost coverage never becomes zero cost or a complete rollup. Every +fact is revalidated against workspace, cluster, namespace, window, currency, lens, provenance, +canonical payload, and native identity before all monetary components and totals are summed with +exact decimal arithmetic. The rollup uses the window end as observation time only when at least one +scope reported. It adds no live transport, endpoint, credential, persistence, Hub/runtime wiring, +team/label attribution, UI, stale threshold, conversion, billing, optimization, GPU-efficiency +inference, or write path, and therefore does not complete F13.2. + ### F13.3 — GPU cost columns (DCGM) **What it is.** GPU cost/utilization columns in the fleet cost view where DCGM metrics exist — @@ -3035,6 +3221,21 @@ flowchart TD **Key risk / guardrail.** Over-claiming per-workload GPU precision. Guardrail: attribution is best-effort and labelled; physical-GPU-level data is not presented as per-pod truth. +**Current bounded slice (F13.3a, #286).** `internal/connector/dcgm` accepts one already-fetched +successful Prometheus instant vector for the exact caller-asserted expression +`DCGM_FI_DEV_GPU_UTIL`, with API series limiting and per-query lookback override disabled. It emits +deterministic TELEMETRY derived facts with explicit +`physical_gpu`, `mig_instance`, or `workload_best_effort` attribution. MIG ID/profile and +namespace/pod/container labels are each all-or-nothing; physical or MIG device scope remains +explicit when workload labels exist. Raw GPU UUID, hostname, PCI bus, scrape target, arbitrary pod +labels, endpoint data, and credentials are discarded; only a SHA-256 native identity survives. +Warnings, infos, ambiguous or duplicate identity, invalid percentages/timestamps, partial label +groups, malformed/duplicate JSON, and resource-bound violations fail atomically. A successful +empty vector returns zero facts and makes no coverage claim. The slice adds no client, network, +credentials, RBAC, arbitrary PromQL, range aggregation, stale policy, persistence, runtime wiring, +cost/idle-cost join, team mapping, UI/API, billing, optimization, mutation, or execution, and +therefore does not complete F13.3. + ### F13.4 — Freshness and non-goal guard **What it is.** The guard that keeps the overlay a *read* — freshness on every cost fact and a diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8ddee20..0e15aaf 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -27,6 +27,19 @@ executable evidence and dependency caveats are in [`experiments/M0-ocm-falsification.md`](experiments/M0-ocm-falsification.md). The bespoke transport/agent scope is deleted; the hub track proceeds to Phase 1. +> **Lifecycle-safe add-on convergence (2026-07-16).** #198 replaces the creation poll plus +> one-shot availability wait with one absolute deadline and current-object checks. Transient +> NotFound and delete/recreate transitions remain retryable, but the runner accepts availability +> only after the same current UID reports `Available=True` twice. Authorization, API, duplicate or +> malformed condition, and malformed identity failures remain terminal and do not print response +> bodies. + +> **Pinned Helm alignment (2026-07-16).** #197 aligns CI, the hub chart contract, and the M0 +> falsification runner on the official Helm `v4.2.3` patch release and its verified Linux amd64 +> archive checksum. Cross-file policy coverage rejects divergent pins, while both runtime gates +> reject prefix lookalikes and accept only the exact release or Helm's `+g` build +> metadata. + > **Phase-1 ClusterGateway authorization gate (2026-07-13).** M0 proves reverse-tunnel > connectivity and scoped-token RBAC; it does **not** authorize a Sith transport to use a > ClusterGateway proxy that forwards a hub caller's inbound `Authorization` header. The tracked @@ -129,6 +142,29 @@ always-green slices, each leaving the binary more useful than the last: | — | Local advisory Investigation Brain (R1–R6, reachable lenses) | #48 | | P | Packaging & supply chain (parallel; does not gate 1–6) | #27 | +> **Local fan-out hardening evidence (2026-07-16).** #181 contains client-go operations that +> outlive cancellation; #185 paginates Kubernetes resource lists within a deterministic +> fleet-wide materialization budget and reports incomplete scopes explicitly. #190 applies the +> same opaque-continuation discipline to generic server Tables, caps each response page at 4 MiB +> and each request at 16 MiB, rejects ignored limits and continuation cycles, and retains display +> fields only for selected facts. Unit adversarial coverage and a real second-page kind fixture +> prove that the presentation path remains bounded without dropping late-page server columns. +> #192 pages every list-watch bootstrap at 250 objects under one absolute request deadline and +> accepts a complete snapshot only within 10,000 objects and 128 pages per scope and kind. Empty +> or changed resource versions, ignored limits, continuation failures/cycles, cancellation, and +> budget exhaustion emit `WatchError` without opening a stream; a real late-page ConfigMap proves +> the watch starts from the completed consistent snapshot. +> #187 workspace-qualifies fleet-cache record identity, coverage, sync/pause/error state, and +> change notifications; missing or mixed-workspace replace/watch mutations fail closed, with +> race, fuzz, and real two-cluster kind coverage proving identical resource identities remain +> independent across workspaces. +> #196 anchors kubeconfig directory traversal and file reads to `os.Root`, rejects root or file +> identity replacement before parsing, refuses deferred local credential/plugin paths, and keeps +> all race diagnostics relative and content-free. +> #193 reads persisted cluster state and facts inside one workspace-scoped PostgreSQL +> `REPEATABLE READ, READ ONLY` transaction, so coverage and staleness always describe the same +> fact snapshot while transaction-local RLS remains the backstop. + **Exit criteria.** - First run to a populated cross-cluster answer in **< 10 minutes**, offline. - A correlation query returns a correct answer over **≥ 2 kubeconfig contexts**. @@ -163,6 +199,27 @@ kubeconfigs — the read source is abstracted so hub mode and local mode share o - A cross-cluster correlation query (e.g. "every cluster where deployment X is unhealthy"). - The **policy-hook seam** at the (future) intent boundary, returning "allow" for reads. +> **Hub refresh hardening evidence (2026-07-16).** #195 independently authorizes every caller, +> coalesces only concurrent refreshes for the same validated workspace, and runs shared work on a +> detached internal trace so leader/waiter cancellation and request context cannot cross caller +> boundaries. Completed, failed, and panicking flights are removed; different workspaces remain +> independent. #193 separately reads persisted coverage and facts from one repeatable-read +> workspace snapshot. #194 admits spoke transports through a validated 1-64 worker pool with a +> conservative default of four, serializes persistence and coverage mutation, and cancels all +> admitted peers before returning a parent-cancellation or store error. These boundaries are kept +> separate from refresh coordination so caller isolation, transport fan-out, and database snapshot +> consistency can each fail closed independently. + +> **P1 operator-console boundary.** #218 adds a separate Hub-only read console rather than mounting +> the capability-bearing local `sith ui`. Its HttpOnly session is verified server-side and bound to +> signed workspace membership; a short-lived session/workspace-bound CSRF token gates the one +> persisted-fleet read. The bearer API stays bearer-only. The view surfaces current, stale, +> unreachable, truncated, inconsistent, and unaccounted coverage without polling, collector +> refresh, connector access, local operations, or writes. #220 composes the existing PEP-governed +> `hubfleet.Correlator` into that boundary. One explicit exact-resource submit performs one bounded +> persisted health query and returns only a fail-closed minimal projection plus named coverage +> gaps; its purpose-specific proof cannot be replayed as the fleet-snapshot proof. + **Exit criteria.** - A single query returns a correct, tenant-scoped, cross-cluster answer over **≥ 2 spokes**. - Per-cluster **staleness is visible** in the result. @@ -258,6 +315,39 @@ transparent, abstaining** reasoner over E2's four-lens graph that *proposes, nev advisory in local mode, governed in the hub. One brain, two modes. The AI-SRE tools become *clients* of this governance, not competitors — their advice becomes a typed `plan` Sith gates. +The first adjacent rule, **R7**, adds exact `ImagePullBackOff` / `ErrImagePull` symptom detection +over the existing sanitized LIVE cache. It remains local, read-only, entity-scoped, and explicitly +uncertain about the underlying registry, reference, network, rate-limit, or platform cause. + +The second adjacent rule, **R8**, consumes only attached, workspace-valid Argo CD Application +TIMELINE facts emitted by the bounded `1.0.0` projector for operation phases `Failed` or `Error`, +with explicit caller-declared TIMELINE coverage; coverage is never inferred from fact presence. +It does not equate `OutOfSync` or degraded health with a failed operation, retain the source +revision or raw payload, diagnose the underlying cause, or correlate fleet-wide. R8 is available +to reviewed graph-fact callers; the cache-backed local CLI still has no Argo fetch path. + +The third adjacent rule, **R9**, consumes only an unattached, workspace-valid GitHub `WorkflowRun` +TIMELINE fact from the bounded `workflow-runs/2026-03-10` projector. The projector requires trusted +repository and run identity, exact status `completed`, and conclusion `failure`, `timed_out`, or +`startup_failure`; incomplete and completed non-failure runs abstain. The graph bridge requires +matching source/provenance, run/attempt/native identity, event time, a closed payload, and explicit +caller-declared TIMELINE coverage. R9 retains no job, step, log, actor, branch, commit, URL, or raw +response data, makes no root-cause claim, and remains source-native and entity-local. It offers only +sensitive human inspection guidance and adds no GitHub client, token, storage, alert, correlation, +typed intent, policy handoff, mutation, or execution. The cache-backed local CLI still has no +workflow-run fetch path. + +The existing **R3** rule now also accepts the reviewed Elasticsearch `search/ecs-v1` graph fact +through a narrow bridge. Only an attached Pod TELEMETRY `FactDerived` with exact Elasticsearch +source/provenance, matching scope and namespace, a revalidated SHA-256 native/resource identity +bound to the retained workspace, Pod, aggregate, and collection fields, and a closed `logs.cause` +payload can become an observation. The accepted values remain `panic`, +`missing-config`, and `dependency-failure`; count, container, event-window metadata, and the source +fact payload are discarded after validation. The last classified event time and stale flag are +preserved, while TELEMETRY coverage remains entirely caller-declared. Evidence attached to another +Pod cannot strengthen the CrashLoop verdict. This bridge adds no Elasticsearch client, endpoint, +index, query execution, credential, persistence, correlation, typed intent, mutation, or execution. + ## Integration waves (E12) — the connector coverage the brain needs Connectors ship in four waves (`docs/specs/E2-readfed-brain-integrations.md` §4), each scored by @@ -267,6 +357,73 @@ need:** - **W1 — daily core:** Kubernetes (the substrate) · GitHub · ArgoCD · Prometheus · Elasticsearch · AWS. With just this, R1/R2/R4/R5/R6 reach *confident* and R3 reaches *detect*. + Issue #206 establishes the first ArgoCD contract as a bounded, sanitized `Application`-to-graph + projector before any network adapter or out-of-process framework is generalized around it. + Issue #209 establishes the matching Prometheus `/api/v1/alerts` contract: already-fetched active + alerts become bounded TELEMETRY facts, annotations and unknown labels are discarded, and only one + unambiguous allowlisted Kubernetes identity can attach a fact to the graph. The endpoint remains + query-through; this slice adds no network client, series retention, credential loading, or writes. + Issue #212 establishes the GitHub merge-event contract: one already-fetched, API-versioned pull + request response becomes a bounded TIMELINE fact only when its merge evidence is internally + consistent. Caller-provided repository identity remains authoritative, sensitive response fields + are discarded, and the event stays unattached until an explicit repository-to-workload relation + exists. This slice adds no HTTP client, token loading, persistence, or GitHub write capability. + Issue #278 adds the equally bounded workflow-run failure contract: one already-fetched REST + response becomes an unattached TIMELINE fact only for an exact completed failure conclusion. + Unknown and ambiguous evidence fails closed, non-failure runs abstain, sensitive response fields + are discarded, and no fetch, token, persistence, correlation, alert, or write path is added. + Issue #214 establishes the Elasticsearch log-evidence contract: one already-fetched, complete + Search API response using the current ECS Kubernetes field profile becomes at most three bounded + TELEMETRY cause facts for R3. Cluster, namespace, and Pod identity must match the trusted caller. + A supplied container requires every hit to carry that exact container; an omitted container is a + deliberate Pod-wide query, accepts hits with any or no container field, and emits no container + identity. The trusted query window is the inclusive `[start, end]` interval, its duration cannot + exceed fifteen minutes, and its end cannot be more than five minutes ahead of collection time; + the duration cap is not a freshness claim. A future live reader must issue these same bounds. + Raw messages are classified in memory and discarded. + Missing cluster identity, partial or failed shards, `_source`, unknown fields, and ambiguous values + fail closed. This slice adds no HTTP client, index discovery, credentials, persistence, or writes. + Issue #280 connects those already-reviewed facts to R3 without widening the source contract: + exact `elasticsearch` / `search/ecs-v1` provenance and Pod identity are validated again, only the + closed cause classification enters the brain, cross-Pod evidence stays separate, and declared + TELEMETRY coverage and staleness remain authoritative. + Issue #216 establishes the AWS autoscaler-evidence contract: one already-fetched EKS + `DescribeNodegroup` response becomes a bounded LIVE inventory fact and a bounded LIVE + provider-health fact attached to an already-trusted Sith cluster. The response's partition, + account, region, cluster, nodegroup, and ARN must agree with the trusted request identity; + regions fail closed against the EKS endpoints currently documented for the commercial, + GovCloud, and China partitions, so an AWS region launch requires an intentional contract update; + scaling values, status, capacity type, and health issue codes fail closed against explicit + bounds and the AWS API's reviewed taxonomies. Facts retain only nodegroup name, capacity type, + min/desired/max counts, provider status, and sorted issue codes. IAM roles, account and region, + raw ARNs, Auto Scaling group names, resource IDs, health messages, tags, labels/taints, subnets, + launch templates, SSH/security-group data, and unknown fields are discarded. `ACTIVE` with no + EKS issues is provider-state evidence only; it does not prove Kubernetes nodes or workloads are + healthy. + + A future live AWS caller must enumerate only an explicit region allowlist, keep `ListClusters` + on its native-EKS default (connected external clusters are out of scope), exhaust opaque + `ListClusters` and `ListNodegroups` tokens with page and loop bounds, and use finite timeouts, + bounded concurrency, jittered retries, and API-quota-aware polling. It must use the standard AWS + SDK chain only for temporary, refreshable role, workload-identity, or federated credentials. + Long-lived static access keys are forbidden even when an environment variable or shared + credentials file exposes them; a live caller must fail closed when credentials cannot expire. + Least privilege is + `eks:ListClusters` on `Resource: "*"` (AWS exposes no resource type for that action), + `eks:DescribeCluster` plus `eks:ListNodegroups` on permitted cluster ARNs, and + `eks:DescribeNodegroup` on permitted nodegroup ARNs. CloudWatch, CloudTrail, Kubernetes token + minting, network access, credentials, persistence, and writes are not part of this projector. + The projector has no AWS-side effect or infrastructure cost; a future caller's polling cadence + still consumes EKS API quota and network traffic and must remain bounded. See the primary AWS + [ListClusters](https://docs.aws.amazon.com/eks/latest/APIReference/API_ListClusters.html), + [ListNodegroups](https://docs.aws.amazon.com/eks/latest/APIReference/API_ListNodegroups.html), + [DescribeNodegroup](https://docs.aws.amazon.com/eks/latest/APIReference/API_DescribeNodegroup.html), + [commercial and GovCloud endpoints](https://docs.aws.amazon.com/general/latest/gr/eks.html), + [China availability](https://docs.amazonaws.cn/en_us/aws/latest/userguide/eks.html), + [health issue](https://docs.aws.amazon.com/eks/latest/APIReference/API_Issue.html), + [service authorization](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonelastickubernetesservice.html), + and [credential provider](https://docs.aws.amazon.com/sdkref/latest/guide/standardized-credentials.html) + contracts. - **W2 — desired-state/diff:** Helm · Kustomize · kubectl-diff (readers, **not** action targets in v1). - **W3 — viz/tracing/clouds:** Grafana (deep-link only) · OTel (semconv key backbone) · OpenShift · Azure · GCP. diff --git a/docs/SITH-NOTION.md b/docs/SITH-NOTION.md index 75d07d4..89f0184 100644 --- a/docs/SITH-NOTION.md +++ b/docs/SITH-NOTION.md @@ -1756,6 +1756,15 @@ sequenceDiagram - Approvals are per-action, single-use, and bound to the resolved-args hash. - Changing the args after approval invalidates it (approve-then-swap is blocked). +**Implementation status (F5.9a/F5.9b, 2026-07-21).** The durable server-side core persists a +same-workspace, distinct-approver grant bound to the immutable resolved proposal digest and spends +it with one conditional PostgreSQL update. Every new grant has one immutable 10-minute absolute +lifetime minted from PostgreSQL statement time; consumption enforces the half-open +`approved_at <= consumed_at < expires_at` interval in that same update. Expiry is bound into +versioned lifecycle evidence, legacy grants are retained but fail closed, and historical audit +formats remain independently verifiable. MCP transport, Ardur PDP policy, multi-approver counting, +credential minting, and dispatch remain later slices; this does not claim F5.9 complete. + **Key risk / guardrail.** Approve-then-swap (approve a benign action, then change args) is the classic agent bypass. Guardrail: the approval is bound to an arg-hash re-checked at dispatch, so a valid signature and a valid approval are both necessary but neither is sufficient if the args @@ -2539,7 +2548,8 @@ and alerting · F10.5 crown-jewel hardening. ### F10.1 — Metrics **What it is.** Metrics about Sith's own health and behavior: control-plane liveness, federation -freshness, intent throughput, refusal and abstention rates, PDP latency. +freshness, intent throughput, bounded sanitized authentication-outcome counts, and future derived rates +where trustworthy denominators exist, abstention rates, and PDP latency. **How it works.** 1. The hub exposes metrics for scraping (control-plane health, DB, queue depths). @@ -2960,18 +2970,21 @@ Guardrail: the taxonomy is closed; "re-skin a tool" is not an expressible connec ### F12.3 — Versioning and one-canonical-connector policy -**What it is.** A minor-additive protocol contract and a rule of one canonical connector per -tool — the Terraform discipline that prevents the Backstage redundancy/abandonment failure. +**What it is.** A structured, explicitly negotiated framework wire contract, a separate opaque +adapter/evidence contract version, and a rule of one canonical connector per tool — the Terraform +discipline that prevents the Backstage redundancy/abandonment failure. **How it works.** -1. Major protocol versions delineate compatibility; minor versions are strictly additive. -2. The registry admits **one** canonical connector per target tool, with declared ownership. -3. Breaking a connector's contract is a major-version, reviewed change — never a silent minor bump. +1. Endpoints advertise exact structured `{major, minor}` framework wire versions; the highest + exact common version is selected. No common major or no explicitly common minor fails closed. +2. Adapter/evidence contract versions remain opaque provenance and never drive wire compatibility. +3. The registry admits **one** canonical connector per target tool, with declared ownership. +4. Breaking the framework contract is a major-version, reviewed change — never a silent minor bump. ```mermaid flowchart TD A["Connector change"] --> B{"Breaking?"} - B -- "no" --> C["Minor: additive, compatible"] + B -- "no" --> C["Minor: additive and explicitly advertised"] B -- "yes" --> D["Major: reviewed compatibility break"] E["New connector for tool T"] --> F{"Canonical connector for T exists?"} F -- "yes" --> G["Improve the canonical one (no duplicate)"] @@ -2979,7 +2992,8 @@ flowchart TD ``` **Acceptance criteria.** -- Minor protocol changes are additive; breaks require a major version and review. +- Wire versions and opaque adapter versions are separate; only explicitly shared wire versions + negotiate, additive minors remain reviewed, and breaks require a major version and review. - The registry holds one canonical connector per tool with a named owner. **Key risk / guardrail.** Overlapping half-maintained connectors (the Backstage marketplace). @@ -3051,6 +3065,18 @@ flowchart TD **Key risk / guardrail.** Inventing costs for clusters that don't report them. Guardrail: no OpenCost → no cost fact; the gap is shown, never estimated silently. +**Current bounded slice (F13.1a, #282).** `internal/connector/opencost` accepts one +already-authorized OpenCost v1.120.2 `/allocation` response for an explicit UTC window, +`aggregate=namespace`, and one set covering the window. Idle, sharing, filtering, accumulation, +and aggregated metadata are disabled. Because the allocation response does not prove currency, +the slice requires a trusted USD assertion and rejects every other unit. It emits deterministic +namespace-attached TELEMETRY `cost` facts with exact five-decimal component validation and uses the +window end as source observation time. A successful empty allocation map returns zero facts; +malformed, +identity-mismatched, warned, partial, or synthetic/unscoped rows fail atomically. The slice performs +no network access and does not complete the live adapter, fleet/team rollup, freshness policy, +currency conversion, billing, optimization, or GPU-utilization work. + ### F13.2 — Hub fleet cost rollup (per-workspace / per-team) **What it is.** The capability none of the OSS tools ship free: aggregate per-cluster costs across @@ -3074,6 +3100,18 @@ flowchart TD **Key risk / guardrail.** A partial rollup read as complete. Guardrail: coverage is always shown. +**Current bounded slice (F13.2a, #284).** `internal/connector/opencost` preserves every successful +F13.1a projection in a per-scope snapshot, including a complete empty allocation set, and computes +one deterministic workspace USD total for an exact caller-bound window. The caller supplies the +unique expected cluster set; output separately names expected, reported, successful-empty, and +missing scopes, so missing OpenCost coverage never becomes zero cost or a complete rollup. Every +fact is revalidated against workspace, cluster, namespace, window, currency, lens, provenance, +canonical payload, and native identity before all monetary components and totals are summed with +exact decimal arithmetic. The rollup uses the window end as observation time only when at least one +scope reported. It adds no live transport, endpoint, credential, persistence, Hub/runtime wiring, +team/label attribution, UI, stale threshold, conversion, billing, optimization, GPU-efficiency +inference, or write path, and therefore does not complete F13.2. + ### F13.3 — GPU cost columns (DCGM) **What it is.** GPU cost/utilization columns in the fleet cost view where DCGM metrics exist — @@ -3097,6 +3135,21 @@ flowchart TD **Key risk / guardrail.** Over-claiming per-workload GPU precision. Guardrail: attribution is best-effort and labelled; physical-GPU-level data is not presented as per-pod truth. +**Current bounded slice (F13.3a, #286).** `internal/connector/dcgm` accepts one already-fetched +successful Prometheus instant vector for the exact caller-asserted expression +`DCGM_FI_DEV_GPU_UTIL`, with API series limiting and per-query lookback override disabled. It emits +deterministic TELEMETRY derived facts with explicit +`physical_gpu`, `mig_instance`, or `workload_best_effort` attribution. MIG ID/profile and +namespace/pod/container labels are each all-or-nothing; physical or MIG device scope remains +explicit when workload labels exist. Raw GPU UUID, hostname, PCI bus, scrape target, arbitrary pod +labels, endpoint data, and credentials are discarded; only a SHA-256 native identity survives. +Warnings, infos, ambiguous or duplicate identity, invalid percentages/timestamps, partial label +groups, malformed/duplicate JSON, and resource-bound violations fail atomically. A successful +empty vector returns zero facts and makes no coverage claim. The slice adds no client, network, +credentials, RBAC, arbitrary PromQL, range aggregation, stale policy, persistence, runtime wiring, +cost/idle-cost join, team mapping, UI/API, billing, optimization, mutation, or execution, and +therefore does not complete F13.3. + ### F13.4 — Freshness and non-goal guard **What it is.** The guard that keeps the overlay a *read* — freshness on every cost fact and a diff --git a/docs/adr/0005-ai-mcp-ardur-pdp.md b/docs/adr/0005-ai-mcp-ardur-pdp.md index 8041c8c..bae7f82 100644 --- a/docs/adr/0005-ai-mcp-ardur-pdp.md +++ b/docs/adr/0005-ai-mcp-ardur-pdp.md @@ -13,9 +13,12 @@ governance, not the product.** Verified MCP facts (July 2026): - Tool annotations (`readOnlyHint`/`destructiveHint`/`idempotentHint`/`openWorldHint`) shipped in the **2025-03-26** spec and are **hints, not guarantees — enforce server-side**. -- **Elicitation** (a server requesting structured user input mid-flow via `elicitation/create` - + JSON schema) shipped in the **2025-06-18** spec — the native primitive for - human-in-the-loop approval. +- **Elicitation** (a server requesting structured user input mid-flow via `elicitation/create`) + shipped in the **2025-06-18** spec. The latest published + [**2025-11-25** contract](https://modelcontextprotocol.io/specification/2025-11-25/client/elicitation) + adds URL mode; + Sith uses form mode with a constrained schema for non-secret human approval. MCP does not define + the lifetime of the resulting server-side grant, so Sith enforces that boundary independently. Ardur (ArdurAI's runtime-governance runtime) is purpose-built to be a policy decision point, identity broker, and decision-ledger for agent actions. @@ -29,6 +32,9 @@ identity broker, and decision-ledger for agent actions. carry `destructiveHint: true` (+ correct `idempotentHint`), and require **Elicitation-based approval bound to a hash of the resolved args** (the agent cannot approve-then-swap). `intent.gitops-open-pr` ships first. +- The durable approval is single-use and valid for one immutable absolute 10-minute window. + PostgreSQL statement time mints and consumes it; the same atomic update requires + `approved_at <= consumed_at < expires_at`, and the lifecycle evidence digest binds the expiry. - **Annotations are hints ⇒ enforcement is server-side.** The MCP layer is a **thin adapter over the same PEP** the UI uses. There is no privileged agent path: an external agent (Claude Code, Codex, kagent) gets **exactly** the governance a human does. diff --git a/docs/adr/0008-deterministic-advisory-brain.md b/docs/adr/0008-deterministic-advisory-brain.md index 4259e22..2c0e7c2 100644 --- a/docs/adr/0008-deterministic-advisory-brain.md +++ b/docs/adr/0008-deterministic-advisory-brain.md @@ -24,9 +24,11 @@ bare workload name. 1. `internal/brain` consumes a closed normalized observation envelope: entity reference, evidence lens, key, value, source, time, and staleness. It performs no I/O. Kubernetes decoding stays in `fleetcache`; cache-to-rule projection stays in one adapter. -2. The catalog contains stable R1-R6 rules. Matching is exact and deterministic. Verdict ordering - is fleet-wide correlation, score, rule ID, then scope. R1 can compose as the cause of R3 when a - recent change and repeated failure are attached to the same entity. +2. The catalog contains stable canonical R1-R6 rules plus bounded adjacent R7 image-pull, R8 Argo + sync-operation failure, and R9 GitHub Actions workflow-run failure rules. Matching is exact and + deterministic. Verdict ordering is fleet-wide correlation, score, rule ID, then scope. R1 can + compose as the cause of R3 when a recent change and repeated failure are attached to the same + entity. 3. Required and strengthening lenses are checked per entity. Fleet-level connector availability cannot satisfy a workload's evidence gate. Missing or stale required evidence yields `unconfirmed`; missing variant evidence yields `detected` plus named missing lenses. @@ -39,6 +41,106 @@ bare workload name. rejects imports from connector, local-operation, and MCP packages; the brain has no plan, execute, intent, PEP, or dispatch seam. +### 2026-07-18 extension: adjacent R7 image-pull failure + +R7 is the first adjacent rule admitted by the existing observation schema. It matches only exact, +case-insensitive `ImagePullBackOff` or `ErrImagePull` values from the sanitized LIVE +`pod.reason` projection. The cited waiting reason proves an image-pull failure or backoff but does +not identify registry authentication, image-reference, reachability, rate-limit, platform, or any +other underlying cause. R7 therefore emits only a sensitive-marked, read-only +`kubectl describe pod` advisory and is explicitly excluded from fleet-wide image-digest +correlation. It adds no registry probe, credential access, Event-message retention, connector, +write path, storage, or network egress. + +This boundary follows Kubernetes' documented container-state contract: a waiting container is +still completing startup operations such as pulling its image, its `Reason` summarizes that state, +and `kubectl describe pod` is the documented inspection surface. See the upstream +[Pod lifecycle](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-states) +and [Pod debugging](https://kubernetes.io/docs/tasks/debug/debug-application/debug-pods/) guides. + +### 2026-07-18 extension: adjacent R8 Argo sync-operation failure + +R8 consumes the already-reviewed graph contract rather than raw Argo CRDs. `FromGraphFacts` +first validates the workspace-bounded graph and then admits only an attached TIMELINE +`FactChange` whose source kind and provenance adapter are `argocd`, protocol is `1.0.0`, and source +and entity identify the same Application. Its closed payload must pair `change_kind=sync-failed` +with operation phase `Failed` or `Error`, and its event time must equal the fact observation time. +Only canonical `change.kind=sync-failed` enters the evaluator; revision, repository, messages, +conditions, raw CRD data, and operation payload fields are discarded. + +Graph projection copies caller-declared lens coverage but never infers it from fact presence. +Missing, unavailable, stale, or observation-stale TIMELINE evidence therefore produces an +`unconfirmed` R8 verdict. `OutOfSync` is desired-state drift and is not a failed-operation signal. +R8 states only that Argo CD reported a failed operation and leaves rendering, validation, +authorization, hooks, networking, Kubernetes API, resource, and other causes unresolved. Its only +advisory is a sensitive-marked, read-only `kubectl describe application.argoproj.io` command. + +This boundary follows Argo CD's stable notification trigger, where failed synchronization is +defined by `app.status?.operationState.phase` being `Error` or `Failed`, and the stable catalog's +`on-sync-failed` operation-failure contract. See the upstream +[notification triggers](https://argo-cd.readthedocs.io/en/stable/operator-manual/notifications/triggers/) +and [notification catalog](https://argo-cd.readthedocs.io/en/stable/operator-manual/notifications/catalog/). + +### 2026-07-18 extension: adjacent R9 GitHub Actions workflow-run failure + +R9 consumes one already-authorized GitHub REST `Get a workflow run` response through a pure +projector. Trusted host, owner, repository, run ID, and collection time remain caller-supplied. The +response must agree with that identity, contain a positive workflow ID and attempt, use a recognized +status and conclusion, and provide a bounded event timestamp. Only exact status `completed` paired +with `failure`, `timed_out`, or `startup_failure` emits an unattached TIMELINE `FactChange`; +incomplete and completed non-failure runs abstain, while unknown or internally inconsistent input +fails closed. Unknown source fields are deliberately discarded, but duplicate JSON members are +rejected before decoding. + +`FromGraphFacts` admits that fact only when source kind and provenance adapter are `github`, protocol +is `workflow-runs/2026-03-10`, the resource is an unattached `WorkflowRun`, host/owner/run/attempt and +native identities agree, the closed payload carries an accepted failure conclusion, and its event +time equals the fact observation time. The evaluator sees only canonical +`change.kind=workflow-run-failed`; conclusion, workflow ID, job and step data, logs, actors, branches, +commits, URLs, unknown response fields, and raw JSON do not enter its observation or output. + +R9 states only that GitHub reported a completed failed run. It does not choose among code, +configuration, credential, permission, capacity, dependency, or other causes, and it does not infer +repository-to-workload or cross-host relationships. Its only advisory is sensitive human guidance +to inspect failed jobs and logs before considering a rerun. GitHub documents the REST response +contract in [Workflow runs](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2026-03-10) +and the closed conclusion vocabulary in +[Checks](https://docs.github.com/en/rest/guides/using-the-rest-api-to-interact-with-checks). + +### 2026-07-18 extension: Elasticsearch log-cause graph bridge for R3 + +R3 continues to use its existing generic `logs.cause` TELEMETRY strengthener. The extension is a +narrow graph admission path for the already-reviewed Elasticsearch projector, not a new rule. +`FromGraphFacts` first validates the workspace-bounded graph, then admits only an attached +TELEMETRY `FactDerived` whose resource is an Elasticsearch `LogSignal`, source kind and provenance +adapter are both `elasticsearch`, protocol is `search/ecs-v1`, source/scope/namespace agree, and +the entity carries exactly one Pod identity. The source hash uses only the retained workspace, +scope, namespace, Pod, optional container, cause, count, first/last event times, and collection +time, allowing the bridge to recompute both the full native ID and resource-name prefix. Extra +resource attributes, display fields, entity dimensions, cross-Pod retargeting, and malformed, +noncanonical, or mismatched SHA-256 identities fail closed. + +The exact-case payload is closed to `key`, `value`, `count`, `first_event_at`, `last_event_at`, and +optional `container`. The bridge revalidates the source projector's byte, count, event-window, +clock-skew, and text bounds, accepts only `logs.cause` with `panic`, `missing-config`, or +`dependency-failure`, then discards count, container, and the source payload. The resulting +observation retains only exact Pod identity, the closed cause, last classified event time, source, +and stale flag. Graph fact presence never creates TELEMETRY coverage; missing, unavailable, stale, +or observation-stale TELEMETRY evidence therefore cannot produce a fully confirmed R3 result. +Entity grouping uses exact scope, namespace, kind, and name, so evidence attached to one Pod cannot +strengthen another Pod or create a fleet-wide cause claim. + +Raw messages are classified and discarded by the source projector before this bridge runs. Index +and document IDs, query text, labels, URLs, credentials, user data, and container/count metadata do +not enter the observation, citation, replay, or renderer. The extension adds bounded in-memory JSON +validation only: no HTTP/TLS client, endpoint/index configuration, API key, mapping discovery, +query execution, pagination, retention, socket, filesystem, database, process, correlation, typed +intent, dispatch, mutation, or cloud resource. Elastic documents the underlying Search API and +selected-field behavior in [Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) +and [Retrieve selected fields](https://www.elastic.co/docs/reference/elasticsearch/rest-apis/retrieve-selected-fields), +and the normalized identity fields in [ECS orchestrator fields](https://www.elastic.co/docs/reference/ecs/ecs-orchestrator) +and [Filebeat Kubernetes fields](https://www.elastic.co/docs/reference/beats/filebeat/exported-fields-kubernetes-processor). + ## Consequences - Investigations are offline, reproducible, replayable, and inspectable. The same observation @@ -47,8 +149,12 @@ bare workload name. telemetry-dependent cause variant. This is intentionally less confident than an opaque guess. - Phase L does not auto-port-forward to Prometheus/Loki, retain telemetry series, read Git desired state, or infer absent evidence. Those connectors can later emit the same observation contract. +- The existing cache-backed CLI does not fetch Argo Applications, GitHub workflow runs, or + Elasticsearch logs. R8, R9, and Elasticsearch-strengthened R3 are available only to callers that + already possess validated graph facts and explicitly declare the relevant lens coverage. - There is no hosted or cloud cost. Runtime cost is one existing tier-1 hydration pass plus - in-memory rule evaluation over the returned records. No extra Kubernetes watch is introduced. + in-memory rule evaluation over the returned records or an existing bounded graph. No extra + Kubernetes watch, Argo/GitHub/Elasticsearch request, storage, or network egress is introduced. - Advisory strings are not shell execution. Operators remain responsible for reviewing and running a suggestion with their own kubeconfig identity. diff --git a/docs/adr/0009-release-supply-chain.md b/docs/adr/0009-release-supply-chain.md index e27f2e2..822c61b 100644 --- a/docs/adr/0009-release-supply-chain.md +++ b/docs/adr/0009-release-supply-chain.md @@ -26,8 +26,8 @@ long-lived cross-repository credential. 2. CI performs two complete snapshot builds and compares archive SHA-256 digests. It separately verifies checksum coverage, exact archive shape, native `sith version` metadata, and SPDX 2.3 documents. SBOM timestamps and transparency-log signatures are not called reproducible. -3. Syft v1.46.0 creates one SPDX SBOM per archive. The checksum manifest covers both archives and - SBOMs. Cosign v3.0.6 signs every archive, SBOM, and the checksum manifest with GitHub's short-lived +3. Syft v1.49.0 creates one SPDX SBOM per archive. The checksum manifest covers both archives and + SBOMs. Cosign v3.1.2 signs every archive, SBOM, and the checksum manifest with GitHub's short-lived OIDC identity and emits self-contained Sigstore bundles. 4. `actions/attest` v4 creates one SLSA provenance statement over the checksum manifest's subjects and one SPDX predicate binding for each archive/SBOM pair. Action dependencies are pinned to diff --git a/docs/adr/0011-opencost-namespace-cost-facts.md b/docs/adr/0011-opencost-namespace-cost-facts.md new file mode 100644 index 0000000..8b256c0 --- /dev/null +++ b/docs/adr/0011-opencost-namespace-cost-facts.md @@ -0,0 +1,86 @@ +# ADR 0011: Exact-decimal USD boundary for OpenCost namespace cost facts + +**Status:** Accepted +**Date:** 2026-07-18 +**Decision owners:** E13 / F13.1a ([#282](https://github.com/ArdurAI/sith/issues/282)) + +## Context + +E13 needs a cost fact that can later be rolled up across clusters without inventing coverage, +mixing currencies, or turning a read integration into a billing or optimization engine. OpenCost's +allocation API returns an array of allocation sets. A namespace aggregation identifies each row by +its map key, allocation name, and properties. In v1.120.2 the response rounds amount fields to five +fractional digits, returns invalid non-finite values as `null`, and computes `totalCost` from CPU, +GPU, RAM, persistent-volume, network, load-balancer, shared, external, and adjustment values. + +The allocation JSON does not carry a currency field. Collection time is also not the source +observation time: rereading an old window must not make historical cost appear fresh. Raw +allocation properties can contain provider IDs, labels, annotations, controllers, Pods, nodes, and +other metadata that this slice does not need. + +## Decision + +1. `internal/connector/opencost` is a pure projector over one already-authorized response. The + trusted caller records one explicit UTC window, a step equal to that window, exact namespace + aggregation, no filter or accumulation, and disabled idle, sharing, proportional-asset, and + aggregated-metadata options. The projector performs no I/O or mutation. +2. The first normalized protocol is `allocation/namespace-usd-v1`. The caller must assert that the + source values are USD; every other currency fails closed. Currency inference and conversion are + forbidden. +3. The response must be a code-200 success with exactly one allocation set and no non-empty message + or warning. A complete empty set produces zero facts. Every invalid row aborts the projection and + returns no prefix. +4. The map key, allocation name, and `properties.namespace` must be identical and pass Kubernetes' + DNS-label validation. `properties.cluster` must equal the trusted Sith scope. Synthetic idle, + unmounted, unallocated, or other unscoped names are rejected. +5. Amounts are exact JSON decimals with at most five fractional digits and a USD 1 trillion bound. + Base and total amounts are non-negative; adjustment fields may be negative. The projector + recomputes OpenCost's component total with rational arithmetic and accepts at most `0.00010` + difference, covering the upstream response-rounding envelope without binary floating-point + drift. +6. Each row emits one deterministic `FactCost` / `LensTelemetry` fact attached to the exact cluster + and namespace. `ObservedAt` is the allocation-window end. The allowlisted payload contains only + namespace, window, USD, canonical five-decimal components, adjustments, and total. +7. Provider IDs, labels, annotations, controller/Pod/node identity, endpoints, collectors, unknown + fields, and raw response metadata are discarded. Duplicate members, mixed-case aliases, + trailing data, invalid UTF-8, excessive size/count/depth, `null` or malformed amounts, window or + identity mismatch, and inconsistent totals fail closed. + +## Consequences + +- Namespace cost evidence is deterministic, private, bounded, and safe for later typed consumers. +- Historical responses retain their true window end, allowing a later F13.4 policy to derive + staleness without a projector-selected objective. +- The USD-only contract is deliberately narrower than OpenCost deployments with another configured + currency. Those clusters produce no fact until a source-bound currency contract is reviewed. +- GPU monetary cost is retained as a total component, but the fact makes no utilization, + efficiency, idle-cost precision, DCGM, or MIG attribution claim. +- This slice creates no infrastructure, network call, storage, egress, logging-volume, cloud API, + or recurring cost. A future transport must use least-privilege read-only OpenCost access and + preserve the exact query contract. +- F13.1 is not complete until a reviewed live read path surfaces unavailable OpenCost coverage. + F13.2 fleet/team rollup, F13.3 GPU columns, and F13.4 freshness/non-goal presentation remain open. + +## Alternatives considered + +- **Build the HTTP client and projector together:** rejected because endpoint discovery, + authorization, TLS, request budgets, and availability are separate operational contracts. +- **Use `float64`:** rejected because stable fact identity and total validation require + deterministic decimal behavior. +- **Infer USD or accept an unspecified currency:** rejected because future rollups must never mix + unproven units. +- **Use collection time as `ObservedAt`:** rejected because it would make a historical window look + fresh after rereading it. +- **Retain all OpenCost fields for future flexibility:** rejected because unneeded provider and + workload metadata widens privacy and correlation risk. +- **Project synthetic idle or unmounted rows:** rejected because F13.1a promises only facts attached + to real Kubernetes namespaces. + +## Primary references + +- [OpenCost allocation API](https://opencost.io/docs/integrations/api/) +- [OpenCost v1.120.2](https://github.com/opencost/opencost/releases/tag/v1.120.2) +- [OpenCost allocation response implementation](https://github.com/opencost/opencost/blob/v1.120.2/core/pkg/opencost/allocation_json.go) +- [OpenCost total-cost implementation](https://github.com/opencost/opencost/blob/v1.120.2/core/pkg/opencost/allocation.go) +- [OpenCost HTTP response envelope](https://github.com/opencost/opencost/blob/v1.120.2/core/pkg/protocol/http.go) +- [OpenCost API schema](https://github.com/opencost/opencost/blob/v1.120.2/docs/swagger.json) diff --git a/docs/adr/0012-opencost-coverage-aware-workspace-rollup.md b/docs/adr/0012-opencost-coverage-aware-workspace-rollup.md new file mode 100644 index 0000000..a21436d --- /dev/null +++ b/docs/adr/0012-opencost-coverage-aware-workspace-rollup.md @@ -0,0 +1,92 @@ +# ADR 0012: Coverage-aware workspace rollup for OpenCost cost facts + +**Status:** Accepted +**Date:** 2026-07-18 +**Decision owners:** E13 / F13.2a ([#284](https://github.com/ArdurAI/sith/issues/284)) + +## Context + +ADR 0011 and issue 282 define an exact-decimal USD projector for one already-authorized OpenCost +namespace-allocation response. A successful response may contain an empty allocation map and +correctly emit zero facts. A later fleet rollup cannot infer from zero facts whether the cluster +reported an empty result or never reported at all. Treating both cases as zero cost would violate +E13's central coverage guardrail. + +The live access path is also unresolved. OpenCost documents its API on port 9003 through an +operator-run Kubernetes port-forward and notes that deployments may expose a Service or Ingress. +Sith has local, Hub, and security-held OCM environments, but no accepted contract assigns OpenCost +endpoint discovery, authentication, TLS, or credential forwarding to one of them. Those choices +must not leak into the normalization or aggregation core. + +Per-team grouping is similarly premature: the normalized cost fact deliberately discards labels, +and Sith does not yet have a canonical team-attribution identity for a namespace. Guessing from a +workload or arbitrary label would create unstable cross-tenant accounting semantics. + +## Decision + +1. `ProjectNamespaceCostSnapshot` wraps a successful F13.1a projection in a value-only envelope + containing its exact workspace, cluster scope, UTC window, trusted USD unit, and facts. Presence + of the snapshot is the reporting signal; an empty fact slice is a successful empty report. +2. `RollupWorkspaceCosts` accepts one explicit expected-scope set plus at most one successful + snapshot per reporting scope. Expected scopes are bounded, unique, and caller-authoritative. +3. Every snapshot must match the requested workspace, exact UTC window, and USD unit. Every fact is + revalidated against the closed cost taxonomy, TELEMETRY lens, cluster/namespace entity, + OpenCost provenance and protocol, canonical payload bytes, native SHA-256 identity, and source + observation time. +4. Invalid, duplicate, foreign, stale-marked, oversized, or ambiguous input aborts the entire + operation and returns no partial rollup. Duplicate namespaces within one cluster are rejected. +5. All fifteen component, adjustment, and total values are parsed as exact rational decimals and + summed independently. Output uses canonical five-decimal strings; binary floating point is not + used. +6. Coverage separately names expected, reported, successful-empty, and missing scopes. A missing + scope contributes no fact and no synthetic zero. `complete` is true only when every expected + scope has a successful snapshot. +7. The rollup preserves the allocation-window end as `observed_at` when at least one scope + reported. With no report, `observed_at` is absent. No collection time or stale objective is + invented. +8. The computation is bounded to 256 scopes, 1,024 facts per scope, 4,096 facts total, 8 MiB of + normalized payload, and a 256 KiB result. Each of the fifteen cost fields is accumulated and + checked independently: one fact's absolute field value cannot exceed `1,000,000,000,000`, and + one rollup's absolute field total cannot exceed `4,096,000,000,000,000` (the per-fact limit + multiplied by the 4,096-fact limit). + +## Consequences + +- A workspace total can never silently present partial OpenCost coverage as complete. +- Successful empty reports remain distinguishable from unavailable OpenCost without retaining raw + responses or adding a sentinel fact. +- The result retains aggregate amounts, coverage metadata (expected, reported, successful-empty, + and missing categories plus `complete`), and optional `observed_at` only. Namespace names, + provider IDs, labels, annotations, workload identity, endpoints, credentials, and unknown source + fields do not survive. +- Historical evidence remains tied to its source window, allowing F13.4 to select a freshness + objective later without retroactively changing fact semantics. +- This is an offline workspace computation core. It does not provide the live F13.1 adapter, + persistence, Hub/runtime composition, an API or UI, team rollups, or F13.2 completion. +- Runtime expense is bounded local CPU and memory. The slice creates no cloud resource, network + call, storage, telemetry-volume, egress, or recurring-service cost. + +## Alternatives considered + +- **Treat zero facts as zero cost:** rejected because it conflates successful empty coverage with a + missing cluster. +- **Emit a synthetic zero-cost fact:** rejected because a sentinel would look like observed + namespace cost and contaminate the fact model. +- **Accept arbitrary OpenCost URLs and credentials in the core:** held because this requires an + explicit SSRF, redirect, TLS, endpoint-provenance, and credential-forwarding decision. +- **Use the Kubernetes Service proxy:** held because it adds `services/proxy` RBAC and does not + solve the security-held OCM transport. +- **Group by an arbitrary team label now:** rejected because no canonical, tenant-scoped team + identity exists and F13.1a intentionally discards labels. +- **Use `float64`:** rejected because deterministic fleet totals require exact decimal behavior. +- **Stamp collection time or choose a stale threshold:** rejected because rereading historical + evidence must not make it fresh, and the objective belongs to F13.4. + +## Primary references + +- [OpenCost allocation API](https://opencost.io/docs/integrations/api/) +- [OpenCost installation and access](https://opencost.io/docs/installation/install/) +- [OpenCost v1.120.2](https://github.com/opencost/opencost/releases/tag/v1.120.2) +- [ADR 0011](0011-opencost-namespace-cost-facts.md) +- [F13.1a issue 282](https://github.com/ArdurAI/sith/issues/282) +- [E13 transport escalation](https://github.com/ArdurAI/sith/issues/31#issuecomment-5013914477) diff --git a/docs/adr/0013-dcgm-gpu-utilization-facts.md b/docs/adr/0013-dcgm-gpu-utilization-facts.md new file mode 100644 index 0000000..a58d1f2 --- /dev/null +++ b/docs/adr/0013-dcgm-gpu-utilization-facts.md @@ -0,0 +1,91 @@ +# ADR 0013: Privacy-minimized DCGM GPU utilization facts + +- **Status:** Accepted +- **Date:** 2026-07-18 +- **Issue:** [#286](https://github.com/ArdurAI/sith/issues/286) + +## Context + +E13 requires GPU cost/utilization columns where DCGM evidence exists, while explicitly forbidding a +physical-GPU sample from being presented as exact per-workload accounting. Sith also has no accepted +runtime owner for Prometheus/DCGM discovery, endpoint selection, authorization, persistence, cost +joins, or UI composition. + +Prometheus documents `/api/v1/query` as its stable instant-query endpoint. A successful vector +contains a label set and one `[unix_timestamp, value]` sample per series, and the response can carry +warnings or info annotations alongside data. NVIDIA dcgm-exporter 4.6.0-4.8.3 was the current +release reviewed for this decision. Its default counters define `DCGM_FI_DEV_GPU_UTIL` as a percent +gauge with a product-dependent sample period. Its renderer emits whole-GPU labels and paired +`GPU_I_ID`/`GPU_I_PROFILE` labels for MIG; its Kubernetes transformation uses exact `pod`, +`namespace`, and `container` attributes. dcgm-exporter can use different collection paths for +exclusive, shared, and MIG workloads, so the presence of workload labels is evidence of exporter +attribution, not uniform accounting precision. + +## Decision + +1. Add a value-only `internal/connector/dcgm` projector for one already-authorized Prometheus + instant-vector response. The caller must assert the exact expression + `DCGM_FI_DEV_GPU_UTIL`, evaluation time, collection time, workspace, and source scope. The + Prometheus API series limit and per-query lookback override must both be disabled, preventing an + API-truncated vector from being asserted as complete without pretending to know server-level + scrape freshness. +2. Accept only successful warning-free and info-free vector responses. Bind every sample timestamp + to the asserted evaluation time and accept only canonical decimal utilization from 0 through + 100 percent. +3. Require current whole-GPU identity labels. Treat `GPU_I_ID` and `GPU_I_PROFILE` as a complete + pair and distinguish `physical_gpu` from `mig_instance` device scope. +4. Treat `namespace`, `pod`, and `container` as an all-or-nothing workload identity. When present, + emit `workload_best_effort` while retaining the underlying physical or MIG device scope. Never + claim exact per-workload accounting. +5. Retain only utilization, device scope, attribution, model, paired MIG metadata, and explicit + workload identity. Hash the selected native series identity with SHA-256. Do not retain raw GPU + UUID, hostname, PCI bus, scrape target, job, instance, arbitrary pod labels, endpoint data, or + credentials in payloads, display fields, or graph entities. +6. Validate unknown labels but discard them. If two native series differ only in discarded labels, + their selected identities collide deliberately and the whole response fails instead of silently + double-counting or collapsing evidence. +7. Bound response bytes, series, labels, label sizes, JSON depth, timestamps, encoded facts, and + identity text. Reject malformed or duplicate-key JSON, partial label groups, ambiguous identity, + non-finite/out-of-range values, and every late invalid series atomically. +8. Treat a successful empty vector as zero facts. It does not prove DCGM absence or complete + coverage; a future runtime must model coverage explicitly. + +## Consequences + +- Sith gains a deterministic, reviewable GPU-utilization normalization seam without acquiring + network, credential, Kubernetes RBAC, process, persistence, billing, optimization, or mutation + authority. +- Whole-GPU, MIG, and exporter-attributed workload observations remain distinguishable without + leaking raw hardware or scrape identity. +- Current CLI and Hub paths still cannot fetch, persist, aggregate, join with GPU cost, or display + these facts. F13.3 and E13 remain incomplete. +- The projector does not know the exporter scrape age hidden behind Prometheus lookback semantics. + Freshness/coverage and query-runtime design remain separate accepted-contract requirements. +- The projector cannot prove that a central Prometheus response contains only the caller-asserted + scope; a future runtime must bind the authorized query target and coverage to exactly one scope. +- Runtime cost is bounded local CPU and memory. A future live query path will consume Prometheus + compute and series cardinality and must be scoped, rate-limited, and observed independently. +- A future dcgm-exporter label-contract change requires a reviewed projector protocol revision; + permissive aliasing is deliberately avoided. + +## Alternatives considered + +- **Add a direct Prometheus or DCGM client now:** held because endpoint provenance, TLS, + authorization, retries, limits, and runtime ownership are unresolved. +- **Accept arbitrary PromQL:** rejected because transformed expressions can erase metric identity, + change units, aggregate scopes, and overstate provenance. +- **Enable and retain arbitrary pod labels:** rejected because this adds pod-read RBAC, + attacker-controlled metadata, inventory disclosure, and cardinality cost. +- **Present any pod-labelled series as exact per-pod utilization:** rejected because dcgm-exporter + exclusive, shared, and MIG paths do not share one precision contract. +- **Infer idle GPU cost in this slice:** held until utilization, OpenCost cost, time-window, + coverage, runtime composition, and presentation contracts are accepted together. + +## Primary sources + +- [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/) +- [NVIDIA dcgm-exporter 4.6.0-4.8.3 release](https://github.com/NVIDIA/dcgm-exporter/releases/tag/4.6.0-4.8.3) +- [NVIDIA default counters](https://github.com/NVIDIA/dcgm-exporter/blob/4.6.0-4.8.3/etc/default-counters.csv) +- [NVIDIA dcgm-exporter Kubernetes and MIG documentation](https://docs.nvidia.com/datacenter/dcgm/latest/gpu-telemetry/dcgm-exporter.html) +- [NVIDIA current renderer labels](https://github.com/NVIDIA/dcgm-exporter/blob/4.6.0-4.8.3/internal/pkg/rendermetrics/render_metrics.go) +- [NVIDIA current Kubernetes attribute names](https://github.com/NVIDIA/dcgm-exporter/blob/4.6.0-4.8.3/internal/pkg/transformation/const.go) diff --git a/docs/adr/0014-connector-wire-adapter-version-split.md b/docs/adr/0014-connector-wire-adapter-version-split.md new file mode 100644 index 0000000..156f246 --- /dev/null +++ b/docs/adr/0014-connector-wire-adapter-version-split.md @@ -0,0 +1,76 @@ +# ADR 0014: Split connector wire compatibility from adapter provenance + +- **Status:** Accepted +- **Date:** 2026-07-19 +- **Issue:** [#288](https://github.com/ArdurAI/sith/issues/288) +- **Parent:** [#30](https://github.com/ArdurAI/sith/issues/30) + +## Context + +E12 requires an out-of-process, typed, versioned connector framework. The existing internal +connector descriptor has one string field, `ProtocolV`, but landed code uses that field for two +different domains: + +- the original source-adapter spec describes one connector-wide semver where minor changes are + additive and major changes are reviewed; and +- adapters and projectors use opaque evidence or behavior identifiers such as `1.0.0`, + `alerts/v1`, `search/ecs-v1`, and `gitops-open-pr/2026-03-10`. + +Those opaque values are important provenance, but they cannot truthfully drive a future gRPC +compatibility handshake. Repurposing them would require an unrelated migration of existing facts, +fixtures, and persisted evidence before transport exists. + +Protocol Buffers allows additive fields as a wire-safe change, but wire safety is narrower than +application-semantic compatibility. It also forbids reusing field numbers and strongly discourages +field-type changes. Sith therefore needs an explicit compatibility policy before it commits a +protobuf service definition. + +## Decision + +1. Add a structured `WireVersion{Major, Minor}` framework domain. Major zero is invalid. +2. Connector descriptors advertise an explicit set of 1 through 32 supported wire versions. The + registry rejects malformed, duplicate, or oversized offers and stores a deterministic sorted + copy. +3. `NegotiateWireVersion` chooses the highest exact version advertised by both endpoints. +4. No common major returns a distinct major-mismatch error. A common major without an explicitly + shared minor returns a distinct unsupported-minor error. Same major alone never implies support. +5. Rename connector-descriptor `ProtocolV` to opaque `AdapterVersion`. This value continues to + identify the adapter's evidence and behavior contract and is not parsed as semver. +6. Preserve `fleet.Provenance.ProtocolV` and its serialized `protocol_version` field. Existing + evidence, persisted facts, and projector protocol identifiers do not migrate in this slice. +7. The initial framework offer is `{major: 1, minor: 0}`. +8. Keep protobuf, generated code, gRPC dependencies, subprocess launch, IPC authentication, + credentials, networking, persistence, and execution out of this slice. + +## Consequences + +- The framework can evolve transport independently from every adapter's native/evidence contract. +- Compatibility is deterministic and fail-closed; an opaque provenance value can no longer be + mistaken for a transport version. +- Version negotiation has a fixed 32-entry-per-endpoint allocation and comparison bound. +- A connector adding minor version 1 must explicitly keep minor version 0 in its offer if it still + supports it. This is more verbose than range inference but prevents accidental semantic claims. +- Registry descriptor JSON changes before any public out-of-process SDK exists. The package is + internal and current descriptor consumers are migrated in the same change. +- Existing fleet evidence JSON and database rows remain stable. +- This adds bounded in-memory comparison only. It adds no process, listener, permission, cloud + resource, telemetry cardinality, or recurring cost. +- Future protobuf work must preserve field numbers, reserve removed fields, avoid required fields, + and independently define authenticated local IPC, deadlines, health, and bounded restart policy. + +## Alternatives considered + +- **Repurpose `ProtocolV` as strict semver and migrate all evidence identifiers:** rejected as a + broad breaking migration before transport exists. +- **Use opaque adapter identifiers directly for the wire handshake:** rejected because major/minor + compatibility would be undefined and untestable. +- **Infer support for every lower minor in the same major:** rejected because protobuf wire safety + does not prove application-semantic support. +- **Delay the split until protobuf lands:** rejected because schema work without an accepted + version-domain boundary would bake the ambiguity into the wire contract. + +## Primary sources + +- [Protocol Buffers proto3 language and compatibility guide](https://protobuf.dev/programming-guides/proto3/) +- [Protocol Buffers best practices](https://protobuf.dev/best-practices/dos-donts/) +- [gRPC core concepts](https://grpc.io/docs/what-is-grpc/core-concepts/) diff --git a/docs/adr/README.md b/docs/adr/README.md index 5cd0dcf..dd24d99 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -20,6 +20,10 @@ decision rests on an external fact, that fact is web-verified and cited (see als | [0008](0008-deterministic-advisory-brain.md) | Deterministic local advisory brain and evidence contract | Accepted | | [0009](0009-release-supply-chain.md) | Reproducible and identity-bound release supply chain | Accepted | | [0010](0010-native-local-desktop-shell.md) | Native local desktop shell | Accepted | +| [0011](0011-opencost-namespace-cost-facts.md) | Exact-decimal USD boundary for OpenCost namespace cost facts | Accepted | +| [0012](0012-opencost-coverage-aware-workspace-rollup.md) | Coverage-aware workspace rollup for OpenCost cost facts | Accepted | +| [0013](0013-dcgm-gpu-utilization-facts.md) | Privacy-minimized DCGM GPU utilization facts | Accepted | +| [0014](0014-connector-wire-adapter-version-split.md) | Split connector wire compatibility from adapter provenance | Accepted | Planning ADRs remain **Proposed** until their implementation lane accepts or rejects them. Implementation-specific ADRs may be **Accepted** when the corresponding shipped slice provides diff --git a/docs/experiments/M0-ocm-falsification.md b/docs/experiments/M0-ocm-falsification.md index a3cf8ed..edb5d0a 100644 --- a/docs/experiments/M0-ocm-falsification.md +++ b/docs/experiments/M0-ocm-falsification.md @@ -122,13 +122,17 @@ kubeconfig: its only per-spoke credentials are the projected `sith-reader` token | clusteradm / OCM core | `v1.3.1-0-g90bdc31` / `1.3.1` | | cluster-proxy chart | `0.10.0`, SHA-256 `30128f5f…481c75d` | | managed-serviceaccount chart | `0.10.0`, SHA-256 `ddd8b7da…0b06dbf` | -| Helm | `v4.1.4` | +| Helm | `v4.2.3` | | Go fixture toolchain | `go1.26.5` | The addon releases and their publication dates are available from the upstream [`cluster-proxy` v0.10.0 release](https://github.com/open-cluster-management-io/cluster-proxy/releases/tag/v0.10.0) and [`managed-serviceaccount` v0.10.0 release](https://github.com/open-cluster-management-io/managed-serviceaccount/releases/tag/v0.10.0). +The Helm pin follows the official +[`v4.2.3` release](https://github.com/helm/helm/releases/tag/v4.2.3). The runner requires the exact +release version and accepts only Helm's `+g` build-metadata form; prefix lookalikes and +vendor suffixes fail closed. The OCM registration procedure follows the official [`clusteradm` registration guidance](https://open-cluster-management.io/docs/getting-started/installation/register-a-cluster/), including `--force-internal-endpoint-lookup` for local kind clusters. @@ -208,7 +212,12 @@ argv exposure imposed by `clusteradm` 1.3.1 and makes a captured registration to before the reach tests begin. It downloads both 0.10.0 charts into isolated scratch, verifies their exact archive hashes, -installs them, and waits for all four `ManagedClusterAddOn` conditions. +installs them, and waits for all four `ManagedClusterAddOn` conditions. Creation and availability +share one absolute five-minute deadline per add-on. The runner polls only the current object's +machine-formatted UID and `Available` condition, tolerates NotFound and delete/recreate transitions, +and requires two consecutive `Available=True` observations of the same UID. Authorization, API, +duplicate-condition, malformed-status, and malformed-identity failures stop the run without printing +the server response body. ### Scoped reach and negative controls diff --git a/docs/runbooks/hub-alerts.md b/docs/runbooks/hub-alerts.md new file mode 100644 index 0000000..468d8ea --- /dev/null +++ b/docs/runbooks/hub-alerts.md @@ -0,0 +1,250 @@ +# Sith hub alert runbook + +These alerts consume only Sith's fixed-cardinality, process-wide metrics. They are a portable +F10.4a/F10.4b/F10.4c/F10.4d/F10.4e/F10.4f/F10.4g baseline, not the complete F10.4 SLO and error-budget +contract. +The two fleet-read alerts cover sustained aggregate coverage degradation and proven stale reads; +neither defines a freshness objective or continuously monitors snapshot age without request traffic. +The rules do not cover governed dispatch success, PDP latency, or KMS/signing. The missing-telemetry +warning detects loss of expected Sith samples only while its rule evaluator remains healthy; it +cannot prove the evaluator, Alertmanager, receiver, or complete notification path is available. +The authentication warning reports only sustained refusal-only aggregate traffic. It cannot +attribute a caller or distinguish an attack from expired credentials, rollout drift, or another +authentication-path outage. + +The rules aggregate away every input-series label before producing an alert. Configure stable +Prometheus external labels if the notification must identify an environment. Do not add workspace, +spoke, actor, trace, intent, request, credential, endpoint, selector, or raw-error labels. + +## Installation and validation + +Sith does not install Prometheus, Alertmanager, a Prometheus Operator CRD, or a remotely reachable +metrics Service. First arrange an operator-owned same-Pod collector that scrapes the exact-loopback +endpoint described in the README and forwards the existing series to a rule evaluator. Then copy +`monitoring/sith-hub.rules.yml` into that evaluator's rule-file path and reload it using the +evaluator's documented mechanism. Loading this package declares that the environment expects a Sith +Hub telemetry path. Do not load it in an environment where no Hub is expected; otherwise the +missing-telemetry warning is correctly indistinguishable from a broken scrape/forwarding path. + +Validate the unchanged file before loading it: + +```bash +promtool check rules monitoring/sith-hub.rules.yml +cd monitoring && promtool test rules sith-hub.rules.test.yml +``` + +Notification routing and receivers remain operator-owned. Confirm the complete scrape → rule +evaluation → Alertmanager → receiver path with an external synthetic test; these white-box rules +cannot detect failure of their own monitoring path. + +## SithHubPolicyAuditFailure + +**Meaning.** A durable database append or structured process-log delivery failed. Sith deliberately +fails the governed read closed, so this is an immediate user-visible availability symptom. + +**Triage.** + +1. Check whether the `durable` or `process` error counter increased; do not attach raw errors or + tenant identifiers to the alert. +2. For `durable`, check PostgreSQL availability, connection saturation, forced-RLS migration state, + and storage health through the database operator's own observability surface. +3. For `process`, check whether the supervised local audit consumer is running and draining stderr; + verify its configured command and resource limits without printing credentials or request data. +4. Restore the failed sink, then perform one authorized read and confirm both sink success counters + increase in database-before-process order. + +Do not bypass either sink to recover availability. That would turn an observable outage into an +unlogged authorization path. + +## SithHubPolicyDecisionErrorRatioHigh + +**Meaning.** More than 5% of at least 20 aggregate eligible +`allow|deny|require-approval|error` policy decisions ended in `error` over 15 minutes, and the +condition persisted for 10 minutes. `error` means the policy hook failed, returned an invalid +decision, or mandatory policy-audit delivery failed. Every case blocks the governed operation. +`deny` and `require-approval` are valid decisions and remain in the denominator, not the numerator. +This is a PEP symptom, not an external Ardur PDP-latency signal or SLO. + +**Triage.** + +1. Compare aggregate `error` with `allow`, `deny`, and `require-approval`. Do not add verb, tenant, + workspace, actor, intent, trace, request, reason, credential, endpoint, selector, or raw-error + labels to the alert. +2. Check the policy hook's availability and decision-contract validation through operator-owned + logs and traces. Keep hook errors and decision payloads out of metric labels and annotations. +3. Check both mandatory audit sinks and PostgreSQL health. The existing critical + `SithHubPolicyAuditFailure` alert is the more immediate signal for any individual sink error. +4. Confirm valid decisions resume and the ratio resolves naturally. Do not bypass policy or audit, + reinterpret errors as denies, or weaken the fail-closed path to recover availability. + +## SithHubAuthRefusalDeliveryDrop + +**Meaning.** Authentication was still refused, but its bounded structured warning could not be +delivered to the supervised local consumer. Authorization remains fail-safe; security evidence is +degraded. + +**Triage.** + +1. Check the supervised consumer's process state, stderr drain, restart behavior, and resource + pressure. Do not log the rejected credential, header, path, client address, principal, or + verifier error. +2. Confirm the drop counter stops increasing after delivery recovers. +3. Review the operator-owned log pipeline for matching gaps. Treat the metric as a signal, not as a + reconstruction of the missing records. + +## SithHubAuthenticationRefusalOnly + +**Meaning.** At least 20 aggregate completed authentication attempts were `refused` and none were +`accepted` over 15 minutes, and that condition persisted for 10 minutes. Any accepted attempt in +the same window suppresses the warning. At least one recent scraped sample from the preinitialized +accepted-outcome series must have reached the evaluator during the last 10 minutes. That sample +proves series visibility, not an accepted authentication event. The rule stays quiet when that +series is absent or stale, because partial telemetry cannot prove refusal-only traffic. This is an +operational symptom, not proof of brute force, credential stuffing, account compromise, or a +specific actor. + +**Triage.** + +1. First confirm `sith_build_info` and both preinitialized `accepted|refused` outcome series are + current. Check the operator-owned scrape and forwarding path without widening the loopback + listener or exposing credentials. +2. Verify whether a recent credential rotation, session-key change, deployment rollout, verifier + configuration change, or clock problem could make every completed verifier decision fail. Use + sanitized operator-owned logs; do not add credential, reason, tenant, workspace, actor, + principal, IP, path, request, endpoint, or verifier-error data to the metric or alert. +3. Confirm known-valid bearer and browser-session authentication through an authorized smoke test. + Keep the later workspace-authorization result separate: a valid credential that receives a 403 + is still an `accepted` authentication attempt. +4. If repeated refusals remain suspicious, investigate through the approved security-log and + identity-provider surfaces. Do not infer an attack, block an identity, or weaken authentication + from this process-wide aggregate alone. + +## SithHubFederationSnapshotFailureRatioHigh + +**Meaning.** More than 5% of at least 20 aggregate snapshot attempts failed over 15 minutes, and +the condition persisted for 10 minutes. The alert is a warning because current metrics intentionally +do not identify a tenant or spoke, and the threshold is an operational baseline rather than a +formal read-freshness SLO. + +**Triage.** + +1. Break down `sith_federation_spoke_snapshot_attempts_total` by its closed `outcome` label to + distinguish transport, deadline, invalid-snapshot, store-error, and canceled failures. +2. Use the tenant-scoped fleet APIs and coverage assessment—not metric labels—to identify stale or + unreachable scopes. Keep authorization and RLS boundaries intact. +3. For transport/deadline failures, inspect OCM addon health, projected service-account rotation, + tunnel connectivity, and the hub's egress policy. For store errors, inspect PostgreSQL health. +4. Confirm the success counter resumes and the failure ratio falls naturally. Do not delete stale + evidence or force a refresh outside the governed read path. + +## SithHubFleetReadCoverageDegradationHigh + +**Meaning.** More than 5% of at least 20 aggregate eligible fleet reads were `degraded` or `error` +over 15 minutes, and the condition persisted for 10 minutes. Eligible outcomes are `complete`, +`degraded`, and `error`. `degraded` includes incomplete or internally inconsistent coverage; +`error` means the authorized persisted read failed. A `complete` outcome means the existing coverage +contract reported no gaps—it does not guarantee snapshot age. Legitimate internally consistent +`empty` reads are excluded from the ratio. + +**Triage.** + +1. Compare the aggregate `degraded` and `error` counters. Do not add tenant, workspace, spoke, + resource, principal, trace, endpoint, age, or raw-error labels to the metric or alert. +2. For `degraded`, use the tenant-scoped fleet API and its named stale, unreachable, truncated, + unaccounted, and inconsistent coverage gaps to identify affected scopes. Keep PEP and RLS + boundaries intact; the process-wide alert cannot safely identify them. +3. For `error`, inspect PostgreSQL availability, connection saturation, migrations, and the Hub + process through their operator-owned observability surfaces. Do not expose wrapped reader errors + as metric labels or alert annotations. +4. Confirm eligible reads resume with `complete` outcomes and the ratio falls naturally. Do not + suppress evidence by treating degraded reads as complete, excluding scopes, bypassing the PEP, + or weakening RLS. + +## SithHubFleetReadStalenessHigh + +**Meaning.** More than 5% of at least 20 aggregate freshness-eligible fleet reads were proven +`stale` over 15 minutes, and the condition persisted for 10 minutes. Eligible outcomes are only +`fresh` and `stale`: both require a structurally valid result with unique cluster identities and +non-zero observation times. `unknown`, `error`, and `empty` are excluded because they do not prove +snapshot age. This warning is request-time evidence, not continuous age monitoring or a formal SLO. + +**Triage.** + +1. Compare the aggregate `fresh` and `stale` counters. Do not add tenant, workspace, spoke, cluster, + resource, principal, trace, endpoint, age, or raw-error labels to the metric or alert. +2. Use the tenant-scoped fleet API and retained observation timestamps—not process-wide metric + labels—to identify the stale authorized scope. Keep PEP and RLS boundaries intact. +3. Inspect the relevant OCM addon health, projected service-account rotation, tunnel connectivity, + Hub egress policy, snapshot persistence, and PostgreSQL health through operator-owned surfaces. +4. Confirm new authorized reads return `fresh` and the ratio resolves naturally. Do not hide stale + evidence by rewriting timestamps, excluding registered clusters, treating unknown results as + fresh, bypassing the PEP, or weakening RLS. + +## SithHubDatabaseReadinessDegradationHigh + +**Meaning.** More than 5% of at least 20 aggregate completed database readiness checks were +`unavailable` over 15 minutes, and the condition persisted for 10 minutes. The alert is a warning: +it reports sustained failure of the Hub's application-pool PostgreSQL ping, not a formal availability +objective, fleet-freshness signal, or proof that the scrape and notification path is healthy. + +**Triage.** + +1. Compare the aggregate `ready` and `unavailable` counters. Do not add request, endpoint, tenant, + workspace, spoke, credential, error, panic, or database detail to the metric or alert. +2. Inspect PostgreSQL availability, connection saturation, forced-RLS migration state, storage, and + network policy through operator-owned surfaces. Keep credentials and query/error text out of logs. +3. Confirm Kubernetes is removing unready Pods from service and that healthy replicas retain enough + capacity. Do not weaken or bypass `/readyz` to recover traffic. +4. After the database path recovers, confirm `ready` checks resume and the ratio resolves naturally. + Separately test the external scrape-to-receiver path because this white-box rule cannot detect its + own absence. + +## SithHubTelemetryMissing + +**Meaning.** The rule evaluator received no `sith_build_info` sample for ten minutes and the absence +continued through the five-minute hold. Build info is set to one when the Hub constructs its +isolated registry and requires no requests, so this is a traffic-independent warning that the +expected Hub metrics, loopback scrape, or forwarding path is missing. + +**Triage.** + +1. Confirm the Hub process and its opt-in exact-loopback metrics listener are running with the + intended version and configuration. Do not widen the listener to a wildcard or cluster-routable + address. +2. Check the operator-owned same-Pod collector's loopback scrape, forwarding queue, authentication, + backpressure, and last successful sample timestamp. Keep credentials and response bodies out of + logs and alert labels. +3. Confirm the rule evaluator is accepting the forwarded `sith_build_info` series. Do not add a + fixed `job`, `instance`, Pod, workspace, or endpoint dependency to the portable rule. +4. Confirm the alert resolves when any current Sith build-info sample arrives. Separately exercise + an external synthetic from collection through Alertmanager to the receiver; this rule cannot fire + if its own evaluator is down. + +## Threshold and cost notes + +The nine expressions evaluate once per minute over existing series and produce at most nine alert +instances per rule evaluator. They create no recording series, listener, exporter, remote-write +path, or storage. The five ratio alerts use a 5% threshold and a 10-minute hold, with the minimum +volume defined independently by each expression. The snapshot ratio requires 20 aggregate attempts. The +fleet-read ratio requires 20 eligible `complete|degraded|error` reads and excludes `empty` from +numerator and denominator, so legitimate zero-scope traffic neither fires nor hides the alert. +The fleet-read staleness ratio separately requires 20 eligible `fresh|stale` reads. It excludes +`unknown`, `error`, and `empty` from both numerator and denominator, so unproven age cannot trigger +or dilute the warning. +The database-readiness ratio requires 20 aggregate `ready|unavailable` checks, uses the same 5% +threshold and 10-minute hold, and cannot divide by zero. +The policy-decision ratio requires 20 aggregate `allow|deny|require-approval|error` decisions and +counts only `error` as failure. It aggregates away the closed verb and every source label. +The refusal-only authentication warning requires at least 20 aggregate refusals, zero accepted +attempts over the same 15-minute window, at least one recent scraped sample from the preinitialized +accepted-outcome series during the most recent 10 minutes, and a 10-minute hold. That sample proves +series visibility, not an accepted event. It is deliberately not a refusal ratio: without an +environment-specific baseline, a generic percentage would create an arbitrary security threshold. +Missing or stale accepted-series data cannot satisfy the expression. +The missing-telemetry warning evaluates the existing traffic-independent `sith_build_info` gauge, +tolerates the most recent ten minutes of samples, and then requires five continuous minutes of +absence. Its installation precondition is the operator's explicit expectation signal; the rule +does not require or preserve source labels. +Change these thresholds only through a reviewed local override backed by observed traffic and an +explicit response owner. Full SLO thresholds and burn-rate alerts wait for negotiated objectives, +continuous freshness monitoring, and the missing dispatch and PDP production signals. diff --git a/docs/specs/E2-readfed-brain-integrations.md b/docs/specs/E2-readfed-brain-integrations.md index c2fdb39..289dbdb 100644 --- a/docs/specs/E2-readfed-brain-integrations.md +++ b/docs/specs/E2-readfed-brain-integrations.md @@ -403,6 +403,20 @@ hub). Weights are indicative, to be tuned against real incidents. advisory* accordingly. - **Coverage gate.** LIVE + logs(TELEMETRY) are the crux; DESIRED/TIMELINE disambiguate. +**Implemented Elasticsearch graph seam (#280).** The bounded `search/ecs-v1` projector described +in #214 can feed this existing signal without exposing raw logs to the brain. `FromGraphFacts` +accepts only an attached Pod TELEMETRY `FactDerived` whose source kind and provenance adapter are +both `elasticsearch`, protocol is exact, source/scope/namespace agree, and the SHA-256 +native/resource identity recomputes from the retained workspace, Pod, aggregate, and collection +fields. The payload contains only `key`, `value`, `count`, +`first_event_at`, `last_event_at`, and optional `container`. The key must be `logs.cause`; the value +must be `panic`, `missing-config`, or `dependency-failure`; source projector count, time-window, +clock-skew, and optional-container bounds are revalidated. Only the Pod identity, cause, last event +time, source, and stale flag survive as an observation. Count and container metadata are discarded. +Caller-declared coverage is copied exactly and never inferred from fact presence. Different Pods +remain different evaluator entities, so log evidence for one Pod cannot strengthen another Pod's +CrashLoop. This is an in-memory graph bridge, not an Elasticsearch reader or freshness claim. + #### R4 — Config drift (live diverged from desired) - **Symptom.** Live ≠ desired for a workload/Application (Argo `OutOfSync`, or `diff` verb shows a @@ -471,9 +485,53 @@ hub). Weights are indicative, to be tuned against real incidents. --- -**Adjacent rules (same pattern, add as coverage lands):** `ImagePullBackOff / registry-auth`, -`FailingDependency` (mesh/trace-driven), `Pipeline/SyncFailure` (Argo/CI), `HPA-thrash`, -`PVC-full / volume-bind`. The schema in §3.3 admits them without change. +**Future adjacent rules (same pattern, add as coverage lands):** `FailingDependency` +(mesh/trace-driven), `HPA-thrash`, and `PVC-full / volume-bind`. +The schema in §3.3 admits them without change. + +**Implemented adjacent rule R7 (2026-07-18):** R7 consumes only the existing sanitized LIVE +`pod.reason` projection and matches exact, case-insensitive `ImagePullBackOff` or `ErrImagePull`. +Those reasons establish the image-pull symptom, not its underlying cause; R7 does not claim +registry authentication, bad reference, reachability, rate-limit, platform, or another variant. +It cites the single Pod observation and emits only a sensitive-marked, read-only +`kubectl describe pod` advisory. It retains no image reference, registry credential, Secret, +Event message, or raw payload and performs no probe, connector call, write, governed-plan handoff, +or fleet-wide correlation. Stale or unavailable LIVE evidence produces an `unconfirmed` verdict +with the LIVE gap named. + +**Implemented adjacent rule R8 (2026-07-18):** R8 completes only the Argo half of the earlier +`Pipeline/SyncFailure` candidate. It consumes an attached, workspace-valid TIMELINE `FactChange` +from the bounded Argo CD `1.0.0` projector when `change_kind=sync-failed`, operation phase is +`Failed` or `Error`, source/provenance/entity identity agrees, and payload event time equals the +fact observation time. The bridge emits only canonical `change.kind=sync-failed`; it discards the +revision and all raw payload fields and copies explicit caller coverage without inferring it. +`OutOfSync`, degraded health, successful/running operations, arbitrary strings, malformed or +ambiguous facts, and unrelated connector changes do not prove R8. The hypothesis states only that +Argo reported a failed operation, remains uncertain among rendering, validation, authorization, +hook, network, Kubernetes API, resource, and other causes, and offers only a sensitive-marked, +read-only `kubectl describe application.argoproj.io` advisory. R8 performs no Argo fetch, client +call, storage, alert, SLO, typed intent, PEP handoff, dispatch, mutation, execution, or fleet +correlation. The cache-backed local CLI cannot produce R8 until a future reader supplies validated +Argo graph facts and explicit TIMELINE coverage. + +**Implemented adjacent rule R9 (2026-07-18):** R9 completes the GitHub Actions half of the earlier +`Pipeline/SyncFailure` candidate. A bounded `workflow-runs/2026-03-10` projector consumes one +already-authorized GitHub REST `Get a workflow run` response and emits one unattached TIMELINE fact +only when trusted host/owner/repository/run identity agrees with the response, IDs and event time are +valid, status is exact `completed`, and conclusion is `failure`, `timed_out`, or `startup_failure`. +Incomplete and completed non-failure runs abstain; unknown states, duplicate JSON members, +identity/time inconsistencies, malformed input, and ambiguous graph facts fail closed. The bridge +requires exact `github` source/provenance, protocol, WorkflowRun identity, a closed payload, +consistent run/attempt/native identity, matching event time, and explicit caller-declared TIMELINE +coverage. It emits only canonical `change.kind=workflow-run-failed`, so workflow ID, conclusion, +jobs, steps, logs, actors, branches, commits, URLs, unknown source fields, and raw response data are +discarded before evaluation and output. R9 states only that GitHub reported a completed failed run, +does not diagnose code/configuration/credential/permission/capacity/dependency or another cause, +and offers only sensitive human guidance to inspect failed jobs and logs before considering a +rerun. It performs no client call, token loading, fetch, retention, alerting, SLO evaluation, +repository-to-workload or fleet correlation, typed intent, PEP handoff, dispatch, mutation, or +execution. The cache-backed local CLI cannot produce R9 until a future reader supplies validated +workflow-run graph facts and explicit TIMELINE coverage. ### 3.5 Rule composition and arbitration @@ -508,6 +566,107 @@ rendering* and the *gating* differ. This is the "one engine, two modes" thesis a the local wedge ships "k9s for your whole fleet **that also tells you why payments is down**," and the same brain, in the hub, proposes a governed remediation. +#### 3.6.1 Staged governed-plan boundary + +F14.6 does not translate rendered local advisory strings into writes. Its first contract is an +inert, rule-owned `RemediationCandidate` containing only one reviewed closed `intent.Verb` and an +ordered set of closed provenance requirements. It carries no resolved target, handler arguments, +actor, workspace, credential, signature, dispatch state, or execution capability. Candidate +presence is not evidence readiness or authorization; the existing advisory remains the verdict's +operator-facing output. + +The initial reviewed mappings are deliberately narrow: + +- R1 may name `argocd.rollback` only with requirements for an authoritative Argo Application target + and exact revision. +- R2 and R4 may name `gitops.open-pr` only with requirements for repository, base ref, expected base + commit, file path, observed blob identity, and exact bounded desired content. +- R3, R5, R6, R7, R8, and R9 remain advisory-only. + +A later reviewed provenance adapter may materialize a PEP proposal only after every requirement is +satisfied from authoritative evidence and every handler-owned argument validates. Until then no PEP +import, proposal, approval, persistence, network, dispatch, mutation, or execution belongs in the +Brain. This preserves the same deterministic rules across local and hub modes without allowing +human prose to become an implicit action contract. + +The first post-Brain resolver contract is GitOps-only and remains pre-PEP. For a confirmed, +entity-local R2 or R4 candidate, it requires exactly one immutable `gitops-provenance/v1` bundle +owned by the pinned GitHub source adapter contract. That bundle binds one workspace and cited +resource to one repository, a non-symbolic configured base branch, exact base commit, one update +path, observed blob identity, exact desired content, bounded PR metadata, immutable evidence +references, and a validity interval of at most five minutes. It also pins the exact +`gitops.open-pr` handler adapter version and raw argument-schema digest. + +Exact desired content is the source adapter's validated UTF-8 byte sequence. Neither the bundle nor +resolver performs Unicode, line-ending, whitespace, or YAML normalization; the handler embeds those +bytes as one JSON string in its canonical argument document. The returned `ArgumentsDigest` is +SHA-256 over that complete canonical JSON byte sequence, so it binds repository preconditions, PR +metadata, path, blob, and content together. There is intentionally no second standalone content +digest in this contract. + +The bundle carries `ObservedAt` and `ValidUntil`, normalized to UTC at construction. It requires +`ObservedAt < ValidUntil` and permits an interval of at most five minutes, inclusive. Resolution +uses an injected trusted server clock normalized to UTC: `now < ObservedAt` is future provenance, +`now >= ValidUntil` is stale, and only `ObservedAt <= now < ValidUntil` is fresh. This contract has +no implicit clock-skew allowance; a future adapter needing one must make it an explicit reviewed +source policy rather than silently widening the resolver window. + +The pure resolver fails closed on zero or multiple bundles, stale or future observations, +cross-workspace or unattached resources, noncanonical candidates or descriptors, handler/schema +drift, unsafe handler arguments, target mismatch, or any canonical output that changes the source's +repository, base, commit, path, blob, content, or PR metadata. GitHub argument semantics stay owned +by the planning handler: the resolver calls its I/O-free canonicalization seam and checks the +contract, request cancellation, and source validity window again immediately before returning. A +ready result contains only the normalized repository target, canonical arguments, their SHA-256 +digest, and evidence references. It contains no actor, role, intent ID, credential, signature, +policy decision, approval, persistence, dispatch, mutation, or execution state. + +This contract does not fetch GitHub state. A later authorized canonical read adapter must use exact +reference, commit, and tree/blob observations to construct the bundle; the offline resolver treats +that immutable adapter output as the source claim and never substitutes caller data. The later Hub +composition must separately derive workspace, actor, role, and intent ID from authenticated server +state, invoke the planner, and call the PEP. Therefore this stage is provenance-complete only; +F14.6 and the local-versus-hub exit criterion remain open. + +The owner-approved follow-on decomposition starts with the observed side only: +`git-source-snapshot/v1`. A `GitSourceSnapshot` binds one validated workspace and affected resource +to exactly one `github-git-source-snapshot/2026-03-10` source identity, one repository, one configured +non-symbolic base ref, its exact resolved commit object ID, one repository-relative path, the exact +current UTF-8 bytes, and their exact blob object ID. Construction recomputes the Git blob identity +over `blob \0` and rejects a mismatched claim. Forty-hex SHA-1 identifiers match +GitHub's current Git database API; 64-hex SHA-256 identifiers are accepted under Git's hash-transition +format. SHA-1 here is an interoperability identity, not a security digest. + +Snapshot input is bounded to 64 KiB of non-NUL UTF-8 content, a five-minute validity interval, and +2–32 unique stable evidence references. The canonical evidence set must attach both the affected +resource and the exact repository blob. Mutable resource references and evidence slices are copied, +evidence is deterministically ordered, timestamps are normalized to UTC, and all validated snapshot +fields remain private. Its only state-dependent operation is a pure trusted-time classification: +`now < ObservedAt` is future, `now >= ValidUntil` is stale, and the half-open interval between them is +fresh. A zero clock or internally invalid snapshot fails closed. + +`GitSourceSnapshot` deliberately has no desired bytes, PR title/body, commit message, handler +contract, actor, role, intent ID, policy or approval decision, credential, endpoint, signature, +persistence, dispatch, mutation, or execution state. + +The separately reviewed output contract is `desired-change/v1`. One opaque `DesiredChange` binds a +defensive copy of one valid snapshot to one lowercase canonical `/` identity, +exact proposed UTF-8 bytes, and 2–32 unique stable evidence references. Construction preserves the +output byte sequence exactly, applies the same 64 KiB and non-NUL bounds as the snapshot, rejects an +exact no-op against current content, canonically sorts copied evidence, and requires both the +affected resource and exact observed blob to remain attached. The embedded snapshot preserves the +repository, base ref, commit, path, blob, current bytes, evidence, and validity window as one exact +later composition precondition. + +Desired-change construction is package-private. No concrete R2 memory-limit transformer, R4 +live-to-Git reconciler, YAML/Helm/Kustomize renderer, or file mapper is approved in this contract, +so callers cannot relabel supplied replacement bytes as trusted output. The change exposes only its +closed contract version and has no PR metadata, handler binding, actor, role, intent ID, policy or +approval decision, credential, endpoint, signature, persistence, dispatch, mutation, or execution +state. Neither half is wired into the existing resolver, Brain, connector runtime, PEP, or Hub. +R2 and R4 therefore remain operator-facing advisory rules; this split adds no production read or +write path. + ### 3.7 Where the Brain lives (open decision) Two placements, both viable: @@ -548,10 +707,10 @@ the *reasoning*. Decision deferred to the owner (see §7). | Connector | Kind | Verbs | Lenses | Tier | Mode | Notes | |---|---|---|---|---|---|---| | **Kubernetes** (core) | RA + TA | di, rd, qy, df, (pl/ex/vf via actions) | LIVE, TIMELINE (Events/RS history), DESIRED (last-applied), GRAPH | **T0** | both | The substrate — this *is* F2.1/F11.1. Feeds every rule. | -| **GitHub** | RA + TA | di, rd, qy, df, **pl/ex** (`gitops.open-pr`), vf | DESIRED (manifests), TIMELINE (commits/PR/deploys) | **T1** | read=local, write=hub | Read (desired/timeline) local via user token; `gitops.open-pr` is the first governed write (P2). | +| **GitHub** | RA + TA | di, rd, qy, df, **pl/ex** (`gitops.open-pr`), vf | DESIRED (manifests), TIMELINE (commits/PR/deploys) | **T1** | read=local, write=hub | Bounded pure projectors now normalize caller-fetched merged-PR and completed workflow-run failure evidence; the HTTP/token reader remains future. `gitops.open-pr` is the first governed write (P2). | | **ArgoCD** | RA + BR + TA | di, rd, qy, **df**, pl/ex (`argocd.sync`,`argocd.rollback`), vf | DESIRED, LIVE, TIMELINE (sync history), drift | **T1** | read=local, sync=hub | Richest single connector — 3 lenses + the exemplar of the `diff` verb. Central to R1, R4. Application CRDs read via kubeconfig. | | **Prometheus** | RA + query-through | di, **qy**, rd (alerts) | TELEMETRY | **T1** | local-if-reachable / hub | Query-through, not retained. Central to R2, R5, R6 and R1 validation. | -| **Elasticsearch** | RA + query-through | di, **qy** | TELEMETRY (logs) | **T2** | hub (local if creds) | Log search for R3. Auth/index-mapping variance → T2. | +| **Elasticsearch** | RA + query-through | di, **qy** | TELEMETRY (logs) | **T2** | hub (local if creds) | Bounded sanitized `search/ecs-v1` cause facts now bridge into R3; HTTP/auth/index/query execution remains future. Auth/index-mapping variance → T2. | | **AWS** | RA (enum/cred) | di, rd, (qy CloudWatch later) | LIVE (nodes/infra), TIMELINE (CloudTrail later) | **T2** | cluster-enum local; deep facts hub | Enumeration + short-lived token minting (no long-lived keys). Feeds R6 (nodegroup/autoscaler). | #### Wave 2 — the desired-state / diff pipeline @@ -597,9 +756,10 @@ lenses are covered by an installed connector. Otherwise it **abstains** (§2.6, | R6 node pressure | K8s (Node) | — | K8s Events + cloud autoscaler | **Prometheus** (quantify) | **Kubernetes** (detect) **+ Prometheus** (quantify) + cloud (autoscaler) | **Reading the map:** with only the **Wave-1 core (Kubernetes + Argo + GitHub + Prometheus + a log -store)**, all six rules reach at least a *detect* verdict, and R1/R2/R4/R5/R6 reach *confident*. R3's -confidence needs a log connector. This is why W1 is the daily core — it is precisely the coverage the -six rules need. +store)**, all six rules reach at least a *detect* verdict, and R1/R2/R3/R4/R5 reach *confident*. +R6 reaches detection plus Prometheus-backed quantification; it requires cloud-autoscaler facts for +a *confident* verdict. This is why W1 is the daily core — it provides the common coverage while +keeping cloud-specific R6 confidence conditional on the cloud adapter. ### 4.4 Scope-discipline call-outs (anti-drift, from SCOPE §10) @@ -673,8 +833,9 @@ the plan-renderer forks. - [ ] Every connector in §4.2 declares its **kind** (RA/BR/TA), its **verb subset**, and the **lenses** it feeds; the framework rejects an out-of-taxonomy connector (E12/F12.2). -- [ ] With **only the Wave-1 core** installed, R1/R2/R4/R5/R6 reach a *confident* verdict and R3 - reaches at least *detect* — matching the coverage map (§4.3). +- [ ] With **only the Wave-1 core** installed, R1/R2/R3/R4/R5 reach a *confident* verdict and R6 + reaches detection plus Prometheus-backed quantification; cloud-autoscaler facts are required + for R6 confidence — matching the coverage map (§4.3). - [ ] **Scope discipline holds:** Fluentd/FluentBit expose LIVE health only (no log ingestion through them); Grafana is deep-link only; Helm/Kustomize expose no action verbs in v1. - [ ] Each connector's **mode** (local / hub / both) is honored — a local-only run uses no @@ -711,9 +872,9 @@ the plan-renderer forks. "what changed recently" useful without drifting toward a store. Recommendation: a small fixed window (e.g. last N events / last 24–72h), tuned against real incidents; explicitly *not* a series. 4. **Rule weights and thresholds.** The §3.4 weights are indicative. A versioned, synthetic replay - harness in `internal/brain/testdata/replays` now guards the six canonical rules, coverage - abstention, cause chaining, and safe fleet correlation. Future tuning should add sanitized incident - shapes to that corpus and preserve its exact expected verdict contract. + harness in `internal/brain/testdata/replays` now guards the six canonical rules, adjacent R7-R9, + coverage abstention, cause chaining, and safe fleet correlation. Future tuning should add + sanitized incident shapes to that corpus and preserve its exact expected verdict contract. 5. **Cross-cluster remediation surface for node/cluster-level causes (R6).** Node/nodegroup/autoscaler actions are outside the v1 closed vocabulary — kept advisory now. Whether/when to add a cloud-scoped typed verb is a later E4 decision. diff --git a/docs/specs/F2.1-source-adapter-contract.md b/docs/specs/F2.1-source-adapter-contract.md index f29fbad..04e51af 100644 --- a/docs/specs/F2.1-source-adapter-contract.md +++ b/docs/specs/F2.1-source-adapter-contract.md @@ -154,7 +154,7 @@ type Evidence struct { // of re-skinning it (E12/F12.2), and lets audit trace any fact to its origin. type Provenance struct { Adapter string `json:"adapter"` // Connector.Kind() - ProtocolV string `json:"protocol_version"` // connector protocol version that produced it + ProtocolV string `json:"protocol_version"` // opaque adapter/evidence contract; not E12 wire compatibility NativeID string `json:"native_id,omitempty"` // the source's own id (UID, series, object id) DeepLink string `json:"deep_link,omitempty"` // URL into the tool's own UI (brokered read-through) Collector string `json:"collector,omitempty"` // read-session / worker id, for debugging @@ -313,12 +313,18 @@ type Connector interface { // Descriptor pins a connector into E12's closed taxonomy and versioning discipline. type Descriptor struct { - Kind string `json:"kind"` - ConnKind ConnectorKind `json:"connector_kind"` // read-adapter | brokered-read-through | typed-action (F12.2) - ProtocolV string `json:"protocol_version"` // semver; minor is additive, major is a reviewed break (F12.3) - Owner string `json:"owner"` // named owner — one canonical connector per tool - Capabilities []Capability `json:"capabilities"` - Verbs []string `json:"verbs,omitempty"` // closed-vocab verbs this adapter hosts (typed-action only) + Kind string `json:"kind"` // canonical target-tool identity; one connector per tool + ConnKind ConnectorKind `json:"connector_kind"` // read-adapter | brokered-read-through | typed-action (F12.2) + WireVersions []WireVersion `json:"wire_versions"` // exact framework versions explicitly supported + AdapterVersion string `json:"adapter_version"` // opaque evidence/behavior contract; not transport semver + Owner string `json:"owner"` // named owner — one canonical connector per tool + Capabilities []Capability `json:"capabilities"` + Verbs []string `json:"verbs,omitempty"` // closed-vocab verbs this adapter hosts (typed-action only) +} + +type WireVersion struct { + Major uint32 `json:"major"` // must be greater than zero + Minor uint32 `json:"minor"` } // ConnectorKind is E12/F12.2's closed taxonomy — nothing outside these three exists. @@ -331,6 +337,16 @@ const ( ) ``` +`Kind` is the canonical target-tool identity used by registry uniqueness; a second connector for +the same tool is rejected even if it declares a different connector taxonomy. `WireVersions` and +`AdapterVersion` are separate domains. Every wire version must have `Major > 0`. The framework +negotiates the highest exact `{major, minor}` value advertised by both endpoints. Each offer is +bounded to 32 versions. It rejects malformed, duplicate, or oversized offers, no common major, and +a common major without an explicitly shared minor. Protobuf wire-safe +additions do not by themselves prove application-semantic compatibility, so Sith never infers +support for an unadvertised minor. Existing fleet evidence keeps its serialized +`protocol_version` field as opaque adapter/evidence provenance. + ### 3.2 The read verbs — `Reader` (discover · read · query) Every source implements at least `Reader`. These are the day-0 verbs, and the only verbs the @@ -549,13 +565,15 @@ type Registry struct { /* unexported: map[string]entry + mutex */ } func NewRegistry() *Registry // Register builds the connector, then FAIL-SAFE checks: -// 1. Descriptor.Kind is non-empty and not already registered (one canonical per tool). +// 1. Descriptor.Kind is the non-empty canonical target-tool identity and is not already registered. // 2. Descriptor.ConnKind is one of the three (F12.2) — else refused. -// 3. Every declared Capability maps to an interface the concrete type satisfies +// 3. WireVersions has 1..32 entries, every Major is > 0, values are unique and canonicalized; +// AdapterVersion is non-empty. +// 4. Every declared Capability maps to an interface the concrete type satisfies // (e.g. CapExecute => value asserts to Executor). A declared-but-unimplemented // capability is a registration error. A capability implemented but not declared // is ignored (declaration is the source of truth; you opt in explicitly). -// 4. typed-action connectors declare their closed-vocab Verbs; read/brokered do not. +// 5. typed-action connectors declare their closed-vocab Verbs; read/brokered do not. // Any failure returns an error and registers nothing. func (r *Registry) Register(f Factory) error @@ -607,11 +625,12 @@ connector executions). ```go Descriptor{ - Kind: "local-kubeconfig", - ConnKind: KindReadAdapter, - ProtocolV: "1.0.0", - Owner: "sith-core", - Capabilities: []Capability{CapDiscover, CapRead, CapQuery}, + Kind: "local-kubeconfig", + ConnKind: KindReadAdapter, + WireVersions: []WireVersion{{Major: 1, Minor: 0}}, + AdapterVersion: "1.0.0", + Owner: "sith-core", + Capabilities: []Capability{CapDiscover, CapRead, CapQuery}, // no Verbs — read adapter } ``` @@ -801,7 +820,7 @@ Consequences: --- -## 9. Open decisions (recorded, not blocking) +## 9. Decisions and open questions (recorded, not blocking) 1. **`Observed` typing — `json.RawMessage` vs. a typed union.** Chosen: `json.RawMessage`, decoded by `FactKind`, to keep `fleet` dependency-free and additive. Revisit if a typed `oneof`/generics @@ -814,9 +833,11 @@ Consequences: 3. **Reachability probe cost.** `/version` vs. `SelfSubjectAccessReview` vs. lazy (probe on first real read). Leaning: cheap `/version` at `Discover`, lazy upgrade on `Read`/`Query`. *Tunable; not contract-affecting.* -4. **`ProtocolV` granularity.** One protocol version for the whole contract vs. per-verb. Chosen: - one per connector (F12.3 semver), simplest that satisfies "minor is additive". *Revisit at E12 - when gRPC wire format lands.* +4. **Framework wire vs. adapter provenance (resolved for E12 in #288).** Chosen: an explicit list + of structured framework `{major, minor}` versions plus a separate opaque adapter contract + version. Negotiate only exact values both endpoints advertise; reject major mismatch and + same-major/no-common-minor cases. The legacy fleet-evidence `protocol_version` field remains + opaque provenance and is never used for transport compatibility. 5. **Where `Workspace` is stamped in local mode.** A single implicit local workspace constant vs. threading E1's type down. Leaning: a `fleet.LocalWorkspace` constant so the type is present from day 0 and the hub swap is mechanical. *Confirm against E1's final Workspace type.* diff --git a/go.mod b/go.mod index 4bb888f..c14a243 100644 --- a/go.mod +++ b/go.mod @@ -7,21 +7,24 @@ toolchain go1.26.5 require ( charm.land/bubbletea/v2 v2.0.8 github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/google/jsonschema-go v0.4.3 github.com/jackc/pgx/v5 v5.10.0 github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 - github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/client_golang v1.24.0 github.com/spf13/cobra v1.10.2 - github.com/wailsapp/wails/v2 v2.12.0 + github.com/wailsapp/wails/v2 v2.13.0 github.com/zalando/go-keyring v0.2.8 go.yaml.in/yaml/v3 v3.0.4 + golang.org/x/sys v0.47.0 golang.org/x/term v0.45.0 - google.golang.org/grpc v1.79.3 + google.golang.org/grpc v1.82.1 k8s.io/api v0.36.2 k8s.io/apimachinery v0.36.2 k8s.io/client-go v0.36.2 k8s.io/streaming v0.36.2 - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 sigs.k8s.io/yaml v1.6.0 ) @@ -49,7 +52,6 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/jsonschema-go v0.4.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -78,8 +80,8 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/samber/lo v1.49.1 // indirect github.com/segmentio/asm v1.1.3 // indirect @@ -93,15 +95,14 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/crypto v0.51.0 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/oauth2 v0.35.0 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -109,7 +110,6 @@ require ( k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect - sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) diff --git a/go.sum b/go.sum index cf62e0f..3e64870 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -149,14 +149,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= -github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= +github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -195,8 +195,8 @@ github.com/wailsapp/go-webview2 v1.0.22 h1:YT61F5lj+GGaat5OB96Aa3b4QA+mybD0Ggq6N github.com/wailsapp/go-webview2 v1.0.22/go.mod h1:qJmWAmAmaniuKGZPWwne+uor3AHMB5PFhqiK0Bbj8kc= github.com/wailsapp/mimetype v1.4.1 h1:pQN9ycO7uo4vsUUuPeHEYoUkLVkaRntMnHJxVwYhwHs= github.com/wailsapp/mimetype v1.4.1/go.mod h1:9aV5k31bBOv5z6u+QP8TltzvNGJPmNJD4XlAL3U+j3o= -github.com/wailsapp/wails/v2 v2.12.0 h1:BHO/kLNWFHYjCzucxbzAYZWUjub1Tvb4cSguQozHn5c= -github.com/wailsapp/wails/v2 v2.12.0/go.mod h1:mo1bzK1DEJrobt7YrBjgxvb5Sihb1mhAY09hppbibQg= +github.com/wailsapp/wails/v2 v2.13.0 h1:S7OgXWpj72V91unF8iDWJKbcS9ZpwCT3R0QVru4v2Mg= +github.com/wailsapp/wails/v2 v2.13.0/go.mod h1:nVr/wSIEZ7xxKPkzK65mjpKpaOPQI2k4pvLwGR/i4kc= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= @@ -207,31 +207,31 @@ github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPc github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= -go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -246,19 +246,19 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -285,8 +285,8 @@ k8s.io/streaming v0.36.2 h1:NSKthPPg9UFSKsRauVJUVGH2Dvn8fhKmY4qrMkw/p98= k8s.io/streaming v0.36.2/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0 h1:/YpDJ4vReG7ZmzSpBGxduXgywWkJU9zHubgJG03MT+Y= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.36.0/go.mod h1:tJo1aepTXyR+8Xs3sUsGBDk4Ub2AM5dPAPKJx0mpm5c= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/hack/experiments/m0-ocm-falsification.sh b/hack/experiments/m0-ocm-falsification.sh index 10ce9dc..666fd9e 100755 --- a/hack/experiments/m0-ocm-falsification.sh +++ b/hack/experiments/m0-ocm-falsification.sh @@ -11,7 +11,7 @@ readonly CLUSTER_PROXY_SHA256="30128f5f211d3c3d6ab1040929e0d1ca7565869935aa60130 readonly MANAGED_SERVICEACCOUNT_VERSION="0.10.0" readonly MANAGED_SERVICEACCOUNT_SHA256="ddd8b7da55667b534102397abd2e61988f9ead8a0e97b942731f869ed0b06dbf" readonly GO_VERSION="go1.26.5" -readonly HELM_VERSION="v4.1.4" +readonly HELM_VERSION="v4.2.3" readonly FIREWALL_CHAIN="SITH_M0_HUB_DENY" readonly KIND_BIN="${KIND_BIN:-kind}" @@ -258,7 +258,7 @@ check_tools() { die "clusteradm ${CLUSTERADM_VERSION} is required" helm_version="$(${HELM_BIN} version --short 2>/dev/null)" - [[ "${helm_version}" == "${HELM_VERSION}"* ]] || + helm_version_is_pinned_release "${helm_version}" || die "Helm ${HELM_VERSION} is required; got: ${helm_version}" go_version="$(${GO_BIN} env GOVERSION)" @@ -269,6 +269,15 @@ check_tools() { "${DOCKER_BIN}" info >/dev/null 2>&1 || die "Docker engine is unavailable" } +helm_version_is_pinned_release() { + local actual=$1 + local pattern + + [[ "${HELM_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || return 1 + pattern="^${HELM_VERSION//./\\.}(\\+g[0-9a-f]{7,40})?$" + [[ "${actual}" =~ ${pattern} ]] +} + prepare_scratch() { prepare_default_scratch_parent enter_scratch_parent @@ -481,16 +490,60 @@ wait_for_addon_creation() { local cluster=$1 local addon=$2 local deadline=$((SECONDS + 300)) - - while ! "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" get \ - "managedclusteraddon/${addon}" >/dev/null 2>&1; do + local observation + local uid + local condition + local ready_uid="" + local remaining + + while (( SECONDS < deadline )); do + remaining=$((deadline - SECONDS)) + if ! observation="$("${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" get \ + "managedclusteraddon/${addon}" --ignore-not-found --request-timeout="${remaining}s" \ + -o=go-template='{{.metadata.uid}}{{"\t"}}{{range .status.conditions}}{{if eq .type "Available"}}{{.status}}{{"|"}}{{end}}{{end}}' \ + 2>/dev/null)"; then + if (( SECONDS >= deadline )); then + die "timed out waiting for ${cluster} managedclusteraddon/${addon} availability" + fi + die "cannot read current ${cluster} managedclusteraddon/${addon} state" + fi if (( SECONDS >= deadline )); then - die "timed out waiting for ${cluster} managedclusteraddon/${addon} creation" + die "timed out waiting for ${cluster} managedclusteraddon/${addon} availability" fi + + if [[ -z "${observation}" ]]; then + ready_uid="" + sleep 1 + continue + fi + if [[ "${observation}" != *$'\t'* ]]; then + die "${cluster} managedclusteraddon/${addon} returned malformed status" + fi + uid="${observation%%$'\t'*}" + condition="${observation#*$'\t'}" + if [[ ! "${uid}" =~ ^[A-Za-z0-9._:-]{1,128}$ ]]; then + die "${cluster} managedclusteraddon/${addon} returned malformed identity" + fi + + case "${condition}" in + 'True|') + if [[ "${ready_uid}" == "${uid}" ]]; then + log "${cluster} managedclusteraddon/${addon} is Available" + return 0 + fi + ready_uid="${uid}" + continue + ;; + '' | 'False|' | 'Unknown|') + ready_uid="" + ;; + *) + die "${cluster} managedclusteraddon/${addon} returned malformed Available status" + ;; + esac sleep 1 done - "${KUBECTL_BIN}" --context "${HUB_CONTEXT}" -n "${cluster}" wait \ - "managedclusteraddon/${addon}" --for=condition=Available --timeout=300s + die "timed out waiting for ${cluster} managedclusteraddon/${addon} availability" } install_addons() { diff --git a/hack/verify-release-hub-image.sh b/hack/verify-release-hub-image.sh index 595366f..250fae0 100755 --- a/hack/verify-release-hub-image.sh +++ b/hack/verify-release-hub-image.sh @@ -13,8 +13,10 @@ if [[ "$#" != 2 || "$1" != "--dist" ]]; then usage fi -readonly REPOSITORY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" -readonly DIST_DIRECTORY="$(cd "$2" && pwd -P)" +REPOSITORY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +readonly REPOSITORY_ROOT +DIST_DIRECTORY="$(cd "$2" && pwd -P)" +readonly DIST_DIRECTORY readonly DOCKER_BIN="${DOCKER_BIN:-docker}" command -v "$DOCKER_BIN" >/dev/null diff --git a/hack/verify-wails-version.sh b/hack/verify-wails-version.sh new file mode 100755 index 0000000..d8fcdf2 --- /dev/null +++ b/hack/verify-wails-version.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +if [[ "$#" -ne 2 ]]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi + +readonly WAILS_COMMAND="$1" +readonly EXPECTED_VERSION="$2" + +if [[ ! "${EXPECTED_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf 'invalid expected Wails version: %s\n' "${EXPECTED_VERSION}" >&2 + exit 2 +fi + +if ! wails_path="$(command -v "${WAILS_COMMAND}")"; then + printf 'Wails %s is required\n' "${EXPECTED_VERSION}" >&2 + exit 1 +fi +readonly wails_path + +if ! version_output="$("${wails_path}" version)"; then + printf 'failed to execute Wails version check\n' >&2 + exit 1 +fi +readonly version_output + +actual_version="${version_output%%$'\n'*}" +readonly actual_version +if [[ "${actual_version}" != "${EXPECTED_VERSION}" ]]; then + printf 'Wails %s is required; got: %s\n' "${EXPECTED_VERSION}" "${actual_version:-}" >&2 + exit 1 +fi diff --git a/internal/auditdelivery/process.go b/internal/auditdelivery/process.go new file mode 100644 index 0000000..a84f527 --- /dev/null +++ b/internal/auditdelivery/process.go @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package auditdelivery provides the Hub's bounded, process-supervised local delivery path for +// already-sanitized authentication-refusal events. +package auditdelivery + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + + "golang.org/x/sys/unix" + + "github.com/ArdurAI/sith/internal/hubserver" +) + +const ( + // ChildArgument starts the internal fixed-record audit sink. It is not a Cobra command and + // bypasses normal configuration loading so the child inherits no deployment secrets. + ChildArgument = "__sith-hub-auth-audit-sink" + + authRecordVersion byte = 1 + authRecordRefused byte = 1 +) + +var authRefusalLine = []byte(`{"level":"WARN","msg":"authentication refused","surface":"hub-auth","auth_outcome":"refused"}` + "\n") + +// DropObserver observes a failed local send without receiving event contents or request metadata. +type DropObserver interface { + ObserveAuthRefusalDeliveryDrop() +} + +// Config fixes the trusted executable, inherited stderr, and low-cardinality drop observer for +// one process-supervised sink. The production default is the current Sith executable with only +// ChildArgument; Arguments exists solely to support isolated process lifecycle tests. +type Config struct { + Executable string + Arguments []string + Stderr *os.File + Drops DropObserver +} + +// ProcessObserver delivers the one closed authentication-refusal event through a nonblocking +// Unix datagram socket. It owns its child process and must be closed by the Hub runtime. +type ProcessObserver struct { + mu sync.Mutex + parent *os.File + command *exec.Cmd + drops DropObserver + close sync.Once + closeErr error +} + +var _ hubserver.AuthObserver = (*ProcessObserver)(nil) + +// NewProcessObserver starts one restricted child with only an inherited Unix datagram descriptor +// and stderr. It never starts a goroutine, opens a listener, creates a socket pathname, or retains +// event data outside the kernel's bounded socket buffer. +func NewProcessObserver(config Config) (*ProcessObserver, error) { + executable, err := trustedExecutable(config.Executable) + if err != nil { + return nil, err + } + arguments := config.Arguments + if arguments == nil { + arguments = []string{ChildArgument} + } + if len(arguments) == 0 { + return nil, fmt.Errorf("construct process audit observer: child arguments are required") + } + stderr := config.Stderr + if stderr == nil { + stderr = os.Stderr + } + if stderr == nil { + return nil, fmt.Errorf("construct process audit observer: stderr is required") + } + + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + if err != nil { + return nil, fmt.Errorf("construct process audit observer: create datagram socket pair: %w", err) + } + unix.CloseOnExec(fds[0]) + unix.CloseOnExec(fds[1]) + parent := os.NewFile(uintptr(fds[0]), "sith-hub-auth-audit-parent") + child := os.NewFile(uintptr(fds[1]), "sith-hub-auth-audit-child") + if parent == nil || child == nil { + if parent != nil { + _ = parent.Close() + } else { + _ = unix.Close(fds[0]) + } + if child != nil { + _ = child.Close() + } else { + _ = unix.Close(fds[1]) + } + return nil, fmt.Errorf("construct process audit observer: own datagram descriptors") + } + + // #nosec G204 -- production resolves the current absolute Sith executable and supplies only the + // fixed internal child argument; injectable values exist solely for the isolated lifecycle test. + command := exec.Command(executable, arguments...) + command.Env = []string{} + command.Stderr = stderr + command.ExtraFiles = []*os.File{child} + if err := command.Start(); err != nil { + _ = parent.Close() + _ = child.Close() + return nil, fmt.Errorf("construct process audit observer: start child: %w", err) + } + if err := child.Close(); err != nil { + _ = parent.Close() + _ = command.Process.Kill() + _ = command.Wait() + return nil, fmt.Errorf("construct process audit observer: close parent child descriptor: %w", err) + } + return &ProcessObserver{parent: parent, command: command, drops: config.Drops}, nil +} + +func trustedExecutable(value string) (string, error) { + if value == "" { + var err error + value, err = os.Executable() + if err != nil { + return "", fmt.Errorf("construct process audit observer: resolve current executable: %w", err) + } + } + if !filepath.IsAbs(value) { + return "", fmt.Errorf("construct process audit observer: executable must be absolute") + } + info, err := os.Stat(value) + if err != nil || !info.Mode().IsRegular() { + return "", fmt.Errorf("construct process audit observer: executable is unavailable") + } + return value, nil +} + +// ObserveAuth sends only the fixed wire form of a valid refusal. A full, dead, or closed socket +// records one unlabeled drop and returns immediately; it can never delay authentication. +func (observer *ProcessObserver) ObserveAuth(event hubserver.AuthEvent) { + if observer == nil || event.Validate() != nil { + return + } + payload := encodeAuthRefusal(event) + if payload == nil { + return + } + observer.mu.Lock() + defer observer.mu.Unlock() + if observer.parent == nil || sendDatagram(observer.parent, payload) != nil { + if observer.drops != nil { + observer.drops.ObserveAuthRefusalDeliveryDrop() + } + } +} + +func encodeAuthRefusal(event hubserver.AuthEvent) []byte { + if event.Validate() != nil || event.Outcome != hubserver.AuthOutcomeRefused { + return nil + } + return []byte{authRecordVersion, authRecordRefused} +} + +func sendDatagram(file *os.File, payload []byte) error { + if file == nil || len(payload) != 2 { + return fmt.Errorf("send audit datagram: payload is invalid") + } + raw, err := file.SyscallConn() + if err != nil { + return fmt.Errorf("send audit datagram: access descriptor: %w", err) + } + if writeErr := raw.Write(func(fd uintptr) bool { + err = unix.Send(int(fd), payload, unix.MSG_DONTWAIT|unix.MSG_NOSIGNAL) + return true + }); writeErr != nil { + return fmt.Errorf("send audit datagram: access descriptor: %w", writeErr) + } + return err +} + +// Close closes the parent socket, terminates the child, and reaps it. No copied stdio pipes or +// delivery goroutines exist, so shutdown cannot wait on a blocked local stderr write. +func (observer *ProcessObserver) Close() error { + if observer == nil { + return nil + } + observer.close.Do(func() { + observer.mu.Lock() + parent, command := observer.parent, observer.command + observer.parent = nil + observer.mu.Unlock() + if parent != nil { + if err := parent.Close(); err != nil && observer.closeErr == nil { + observer.closeErr = fmt.Errorf("stop process audit observer: close parent datagram: %w", err) + } + } + if command == nil || command.Process == nil { + return + } + if err := command.Process.Kill(); err != nil && !errors.Is(err, os.ErrProcessDone) && observer.closeErr == nil { + observer.closeErr = fmt.Errorf("stop process audit observer: terminate child: %w", err) + } + if err := command.Wait(); err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) && observer.closeErr == nil { + observer.closeErr = fmt.Errorf("stop process audit observer: reap child: %w", err) + } + } + }) + return observer.closeErr +} + +// RunChild consumes and validates the fixed inherited datagrams, then writes a single structured +// local line. It intentionally has no configuration, environment, listener, persistence, or +// network path; a blocked stderr affects only this child process. +func RunChild(stderr *os.File) error { + if stderr == nil { + return fmt.Errorf("run process audit child: stderr is required") + } + socket := os.NewFile(uintptr(3), "sith-hub-auth-audit-child") + if socket == nil { + return fmt.Errorf("run process audit child: inherited descriptor is unavailable") + } + defer func() { _ = socket.Close() }() + return runChild(socket, stderr) +} + +func runChild(socket, stderr *os.File) error { + if socket == nil || stderr == nil { + return fmt.Errorf("run process audit child: socket and stderr are required") + } + fd := int(socket.Fd()) + buffer := make([]byte, 16) + for { + count, _, err := unix.Recvfrom(fd, buffer, 0) + if err != nil { + if errors.Is(err, unix.EINTR) { + continue + } + if errors.Is(err, unix.ECONNRESET) || errors.Is(err, unix.ENOTCONN) { + return nil + } + return fmt.Errorf("run process audit child: receive datagram: %w", err) + } + if count == 0 { + return nil + } + if !validAuthRefusalRecord(buffer[:count]) { + continue + } + if _, err := stderr.Write(authRefusalLine); err != nil { + return fmt.Errorf("run process audit child: write structured event: %w", err) + } + } +} + +func validAuthRefusalRecord(record []byte) bool { + return len(record) == 2 && record[0] == authRecordVersion && record[1] == authRecordRefused +} diff --git a/internal/auditdelivery/process_test.go b/internal/auditdelivery/process_test.go new file mode 100644 index 0000000..97f410d --- /dev/null +++ b/internal/auditdelivery/process_test.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auditdelivery + +import ( + "bufio" + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/ArdurAI/sith/internal/hubserver" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestProcessObserverDeliversOnlyFixedRecordAndReapsChild(t *testing.T) { + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer reader.Close() + defer writer.Close() + observer := newTestProcessObserver(t, writer, nil) + + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeAccepted}) + observer.ObserveAuth(hubserver.AuthEvent{Outcome: "token=secret"}) + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + line := make(chan string, 1) + go func() { + value, readErr := bufio.NewReader(reader).ReadString('\n') + if readErr != nil { + line <- "read error: " + readErr.Error() + return + } + line <- value + }() + select { + case value := <-line: + if value != string(authRefusalLine) || strings.Contains(value, "secret") { + t.Fatalf("child record = %q", value) + } + case <-time.After(time.Second): + t.Fatal("child did not deliver fixed authentication refusal record") + } + closeObserverWithin(t, observer) + if observer.command.ProcessState == nil { + t.Fatal("child process was not reaped") + } +} + +func TestProcessObserverNeverDeliversOrDropsAcceptedAuthentication(t *testing.T) { + var drops atomic.Uint64 + observer := &ProcessObserver{drops: dropObserverFunc(func() { drops.Add(1) })} + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeAccepted}) + if drops.Load() != 0 { + t.Fatalf("accepted authentication delivery drops = %d, want 0", drops.Load()) + } + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + if drops.Load() != 1 { + t.Fatalf("refused authentication delivery drops = %d, want 1", drops.Load()) + } +} + +func TestProcessObserverDropsFullDatagramBufferWithoutBlocking(t *testing.T) { + parent, child := socketPair(t) + defer child.Close() + var drops atomic.Uint64 + observer := &ProcessObserver{parent: parent, drops: dropObserverFunc(func() { drops.Add(1) })} + started := time.Now() + for range 50000 { + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("full datagram buffer delayed authentication observer for %s", elapsed) + } + if drops.Load() == 0 { + t.Fatal("full datagram buffer did not produce an observable drop") + } + closeObserverWithin(t, observer) +} + +func TestBlockedChildCannotDelayAuthenticationRefusalOrEscapeShutdown(t *testing.T) { + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer reader.Close() + defer writer.Close() + fillPipe(t, writer) + observer := newTestProcessObserver(t, writer, nil) + handler, err := hubserver.AuthenticateWithObserver(rejectingVerifier{}, observer, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("authentication refusal reached next handler") + })) + if err != nil { + closeObserverWithin(t, observer) + t.Fatal(err) + } + started := time.Now() + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/fleet?token=secret", nil)) + if elapsed := time.Since(started); elapsed > 250*time.Millisecond { + closeObserverWithin(t, observer) + t.Fatalf("blocked child delayed authentication refusal for %s", elapsed) + } + if response.Code != http.StatusUnauthorized || response.Body.String() != "{\"error\":\"unauthorized\"}\n" { + closeObserverWithin(t, observer) + t.Fatalf("authentication refusal = %d/%q", response.Code, response.Body.String()) + } + // The child receives the datagram, then blocks attempting its first stderr write because the + // inherited pipe is already full. Parent shutdown must still kill and reap it promptly. + time.Sleep(50 * time.Millisecond) + closeObserverWithin(t, observer) + if observer.command.ProcessState == nil { + t.Fatal("blocked child process was not reaped") + } +} + +func TestAuthRefusalRecordRejectsMalformedRecords(t *testing.T) { + for _, record := range [][]byte{ + nil, + {}, + []byte("token=secret"), + {authRecordVersion}, + {authRecordVersion, authRecordRefused, 0}, + {0, authRecordRefused}, + {authRecordVersion, 0}, + } { + if validAuthRefusalRecord(record) { + t.Fatalf("validAuthRefusalRecord(%q) = true", record) + } + } + if !validAuthRefusalRecord([]byte{authRecordVersion, authRecordRefused}) { + t.Fatal("validAuthRefusalRecord() rejected the fixed record") + } +} + +// TestProcessAuditChildHelper runs only in the restricted subprocess started by +// newTestProcessObserver. It uses the same inherited-FD entrypoint as the shipped Sith binary. +func TestProcessAuditChildHelper(t *testing.T) { + if !slicesContain(os.Args, "--audit-child") { + return + } + if err := RunChild(os.Stderr); err != nil { + os.Exit(2) + } + os.Exit(0) +} + +func newTestProcessObserver(t *testing.T, stderr *os.File, drops DropObserver) *ProcessObserver { + t.Helper() + observer, err := NewProcessObserver(Config{ + Executable: os.Args[0], + Arguments: []string{"-test.run=^TestProcessAuditChildHelper$", "--", "--audit-child"}, + Stderr: stderr, + Drops: drops, + }) + if err != nil { + t.Fatal(err) + } + return observer +} + +func socketPair(t *testing.T) (*os.File, *os.File) { + t.Helper() + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + if err != nil { + t.Fatal(err) + } + if err := unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_SNDBUF, 256); err != nil { + _ = unix.Close(fds[0]) + _ = unix.Close(fds[1]) + t.Fatal(err) + } + return os.NewFile(uintptr(fds[0]), "test-parent"), os.NewFile(uintptr(fds[1]), "test-child") +} + +func fillPipe(t *testing.T, writer *os.File) { + t.Helper() + raw, err := writer.SyscallConn() + if err != nil { + t.Fatal(err) + } + if err := raw.Control(func(fd uintptr) { + if setErr := unix.SetNonblock(int(fd), true); setErr != nil { + err = setErr + return + } + defer func() { + if setErr := unix.SetNonblock(int(fd), false); setErr != nil && err == nil { + err = setErr + } + }() + buffer := make([]byte, 4096) + for { + _, writeErr := unix.Write(int(fd), buffer) + if errors.Is(writeErr, unix.EAGAIN) || errors.Is(writeErr, unix.EWOULDBLOCK) { + return + } + if writeErr != nil { + err = writeErr + return + } + } + }); err != nil { + t.Fatal(err) + } + if err != nil { + t.Fatal(err) + } +} + +func closeObserverWithin(t *testing.T, observer *ProcessObserver) { + t.Helper() + done := make(chan error, 1) + go func() { done <- observer.Close() }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("process observer shutdown exceeded one second") + } +} + +type dropObserverFunc func() + +func (function dropObserverFunc) ObserveAuthRefusalDeliveryDrop() { function() } + +type rejectingVerifier struct{} + +func (rejectingVerifier) Verify(context.Context, string) (tenancy.Principal, error) { + return tenancy.Principal{}, errors.New("invalid token") +} + +func slicesContain(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/internal/auditrecord/export.go b/internal/auditrecord/export.go new file mode 100644 index 0000000..44d0422 --- /dev/null +++ b/internal/auditrecord/export.go @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package auditrecord defines the portable, privacy-minimized audit export contract shared by the +// durable store and authenticated HTTP boundary. +package auditrecord + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + // SchemaV1 identifies the first portable retained-chain document. + SchemaV1 = "sith.policy-audit-export/v1" + // HashAlgorithm identifies the digest used by the retained chain. + HashAlgorithm = "sha-256" + // MaxEntries bounds one synchronous online export before database or network work begins. + MaxEntries = 512 + // MaxDocumentBytes bounds one portable JSON document before offline parsing. + MaxDocumentBytes = 1 << 20 + + policyAuditHashDomain = "sith-policy-audit-chain/v1" + approvalAuditHashDomain = "sith-approval-audit-chain/v2" + approvalExpiryAuditHashDomain = "sith-approval-audit-chain/v3" +) + +// Export is one complete, verified workspace snapshot. It is constructed only after the backing +// repeatable-read transaction commits, so encoding it cannot pin a database transaction. +type Export struct { + Schema string `json:"schema"` + WorkspaceID string `json:"workspace_id"` + Chain Chain `json:"chain"` + Entries []Entry `json:"entries"` +} + +// Chain identifies the exact retained-history head covered by an export. +type Chain struct { + HashAlgorithm string `json:"hash_algorithm"` + HeadSequence int64 `json:"head_sequence"` + HeadHash string `json:"head_hash"` +} + +// Entry is the privacy-minimized, independently rehashable projection of one retained event. The +// workspace is carried once by Export and is nevertheless bound into each entry hash. +type Entry struct { + Sequence int64 `json:"sequence"` + FormatVersion int16 `json:"format_version"` + RecordedAt time.Time `json:"recorded_at"` + TraceID string `json:"trace_id"` + Actor string `json:"actor"` + Role string `json:"role"` + Action string `json:"action"` + Verb string `json:"verb"` + Verdict string `json:"verdict"` + ReasonCode string `json:"reason_code"` + EventKind string `json:"event_kind"` + // EvidenceDigest is opaque at this privacy-minimized boundary. Offline verification binds it + // into the versioned chain; the database writer proves its grant-field semantics before append. + EvidenceDigest string `json:"evidence_digest"` + PreviousHash string `json:"previous_hash"` + EntryHash string `json:"entry_hash"` +} + +// ValidateForWorkspace rechecks the portable disclosure boundary independently of the backing +// store. Cryptographic recomputation remains the exporter's responsibility; this check rejects a +// foreign, oversized, discontinuous, or unsupported document before HTTP serialization. +func (export Export) ValidateForWorkspace(workspaceID tenancy.WorkspaceID) error { + return export.validateForWorkspace(workspaceID, false) +} + +func (export Export) validateForWorkspace(workspaceID tenancy.WorkspaceID, verifyHashes bool) error { + if export.Schema != SchemaV1 || export.WorkspaceID != string(workspaceID) || + workspaceInvalid(workspaceID) || export.Chain.HashAlgorithm != HashAlgorithm || + len(export.Entries) == 0 || len(export.Entries) > MaxEntries || + export.Chain.HeadSequence != int64(len(export.Entries)) || !validHash(export.Chain.HeadHash) { + return fmt.Errorf("audit export envelope is invalid") + } + previous, err := validateEntrySequence( + workspaceID, export.Entries, 1, zeroHash(), verifyHashes, + ) + if err != nil { + return fmt.Errorf("audit export: %w", err) + } + if previous != export.Chain.HeadHash { + return fmt.Errorf("audit export head is invalid") + } + return nil +} + +// Verify recomputes every entry digest from the serialized document and checks the complete +// retained chain. It proves internal consistency only: without an external anchor, a privileged +// store owner could still replace an entire chain and head with a different self-consistent one. +func (export Export) Verify() error { + return export.VerifyForWorkspace(tenancy.WorkspaceID(export.WorkspaceID)) +} + +// VerifyForWorkspace binds the document to an expected tenant before recomputing its hashes. +func (export Export) VerifyForWorkspace(workspaceID tenancy.WorkspaceID) error { + return export.validateForWorkspace(workspaceID, true) +} + +func validateEntrySequence( + workspaceID tenancy.WorkspaceID, + entries []Entry, + startSequence int64, + previousHash string, + verifyHashes bool, +) (string, error) { + if workspaceInvalid(workspaceID) || len(entries) == 0 || startSequence <= 0 || !validHash(previousHash) { + return "", fmt.Errorf("audit entry sequence boundary is invalid") + } + previous := previousHash + for index, entry := range entries { + sequence := startSequence + int64(index) + if sequence <= 0 || entry.Sequence != sequence || entry.RecordedAt.IsZero() || + entry.RecordedAt.Location() != time.UTC || + !entry.RecordedAt.Equal(entry.RecordedAt.Truncate(time.Microsecond)) || + !validTraceID(entry.TraceID) || !safeText(entry.Actor, 256) || !validRole(entry.Role) || + !validVerdict(entry.Verdict) || !validReasonCode(entry.ReasonCode) || + entry.PreviousHash != previous || !validHash(entry.EntryHash) || !validEntryShape(entry) { + return "", fmt.Errorf("audit entry %d is invalid", sequence) + } + if verifyHashes { + recomputed, err := RecomputeEntryHash(workspaceID, entry) + if err != nil || recomputed != entry.EntryHash { + return "", fmt.Errorf("audit entry %d hash is invalid", sequence) + } + } + previous = entry.EntryHash + } + return previous, nil +} + +func zeroHash() string { return "sha256:" + strings.Repeat("0", 64) } + +// RecomputeEntryHash returns the versioned canonical SHA-256 digest for one portable entry. The +// database writer and offline verifier share this primitive so format framing cannot drift. +func RecomputeEntryHash(workspaceID tenancy.WorkspaceID, entry Entry) (string, error) { + if workspaceInvalid(workspaceID) || entry.Sequence <= 0 || entry.RecordedAt.IsZero() || + entry.RecordedAt.Location() != time.UTC || + !entry.RecordedAt.Equal(entry.RecordedAt.Truncate(time.Microsecond)) || + !validTraceID(entry.TraceID) || !safeText(entry.Actor, 256) || !validRole(entry.Role) || + !validVerdict(entry.Verdict) || !validReasonCode(entry.ReasonCode) || + !validHash(entry.PreviousHash) || !validEntryShape(entry) { + return "", fmt.Errorf("audit export entry hash input is invalid") + } + previousHash, err := hex.DecodeString(strings.TrimPrefix(entry.PreviousHash, "sha256:")) + if err != nil || len(previousHash) != sha256.Size { + return "", fmt.Errorf("audit export previous hash is invalid") + } + + domain := policyAuditHashDomain + switch entry.FormatVersion { + case 2: + domain = approvalAuditHashDomain + case 3: + domain = approvalExpiryAuditHashDomain + } + canonical := make([]byte, 0, 512) + canonical = appendCanonicalString(canonical, domain) + canonical = appendCanonicalString(canonical, strconv.FormatInt(int64(entry.FormatVersion), 10)) + canonical = appendCanonicalString(canonical, strconv.FormatInt(entry.Sequence, 10)) + canonical = append(canonical, previousHash...) + for _, value := range []string{ + entry.RecordedAt.UTC().Truncate(time.Microsecond).Format(time.RFC3339Nano), + entry.TraceID, string(workspaceID), entry.Actor, entry.Role, entry.Action, entry.Verb, + entry.Verdict, entry.ReasonCode, + } { + canonical = appendCanonicalString(canonical, value) + } + if entry.FormatVersion == 2 || entry.FormatVersion == 3 { + canonical = appendCanonicalString(canonical, entry.EventKind) + canonical = appendCanonicalString(canonical, entry.EvidenceDigest) + } + digest := sha256.Sum256(canonical) + return "sha256:" + hex.EncodeToString(digest[:]), nil +} + +func workspaceInvalid(workspaceID tenancy.WorkspaceID) bool { + return tenancy.ValidateWorkspaceID(workspaceID) != nil +} + +func validEntryShape(entry Entry) bool { + role := tenancy.Role(entry.Role) + action := tenancy.Action(entry.Action) + verdict := pep.Verdict(entry.Verdict) + switch entry.FormatVersion { + case 1: + if entry.EventKind != "policy-decision" || entry.EvidenceDigest != "" { + return false + } + verb := pep.Verb(entry.Verb) + if verb == "invalid" { + switch action { + case tenancy.ActionRead, tenancy.ActionExportAudit, tenancy.ActionProposeIntent: + default: + return false + } + return verdict == pep.VerdictDeny && entry.ReasonCode == "invalid-request" + } + switch action { + case tenancy.ActionRead: + if !verb.Valid() || verb == pep.VerbAuditExport { + return false + } + case tenancy.ActionExportAudit: + if verb != pep.VerbAuditExport { + return false + } + case tenancy.ActionProposeIntent: + if !intent.Verb(entry.Verb).Valid() { + return false + } + default: + return false + } + if !role.Allows(action) && (verdict != pep.VerdictDeny || (entry.ReasonCode != "role-denied" && entry.ReasonCode != "invalid-request")) { + return false + } + return true + case 2, 3: + if !validHash(entry.EvidenceDigest) || entry.Verb != "approval.grant" || verdict != pep.VerdictAllow || + entry.ReasonCode != entry.EventKind { + return false + } + return (entry.EventKind == "approval-created" && role == tenancy.RoleApprover && action == tenancy.ActionApproveIntent) || + (entry.EventKind == "approval-consumed" && role == tenancy.RoleOperator && action == tenancy.ActionProposeIntent) + default: + return false + } +} + +func validRole(value string) bool { return tenancy.Role(value).Valid() } + +func validVerdict(value string) bool { + switch pep.Verdict(value) { + case pep.VerdictAllow, pep.VerdictDeny, pep.VerdictRequireApproval: + return true + default: + return false + } +} + +func validTraceID(value string) bool { + if len(value) != 32 { + return false + } + return lowercaseHex(value) +} + +func validHash(value string) bool { + return len(value) == 71 && strings.HasPrefix(value, "sha256:") && lowercaseHex(value[7:]) +} + +func lowercaseHex(value string) bool { + for _, character := range value { + if (character < '0' || character > '9') && (character < 'a' || character > 'f') { + return false + } + } + return true +} + +func safeText(value string, maximum int) bool { + if value == "" || !utf8.ValidString(value) || strings.TrimSpace(value) != value || len(value) > maximum { + return false + } + for _, character := range value { + if unicode.IsControl(character) { + return false + } + } + return true +} + +func validReasonCode(value string) bool { + if !safeText(value, 64) { + return false + } + for _, character := range value { + if (character < 'a' || character > 'z') && (character < '0' || character > '9') && + character != '-' && character != '_' && character != '.' { + return false + } + } + return true +} + +func appendCanonicalString(target []byte, value string) []byte { + target = strconv.AppendInt(target, int64(len(value)), 10) + target = append(target, ':') + return append(target, value...) +} diff --git a/internal/auditrecord/export_test.go b/internal/auditrecord/export_test.go new file mode 100644 index 0000000..e8b0314 --- /dev/null +++ b/internal/auditrecord/export_test.go @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auditrecord + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/tenancy" +) + +const expiringApprovalEvidenceFixture = "sha256:25edbb61ecc55494ed155e14b30733b08ab090469a789b79eed3bf871ddbd1b4" + +func TestExportValidateForWorkspaceAcceptsClosedPortableChain(t *testing.T) { + t.Parallel() + + exported := validTestExport() + if err := exported.ValidateForWorkspace("workspace-a"); err != nil { + t.Fatalf("ValidateForWorkspace() error = %v", err) + } + + approval := validTestExport() + approval.Entries[0].FormatVersion = 2 + approval.Entries[0].Role = "approver" + approval.Entries[0].Action = "approve-intent" + approval.Entries[0].Verb = "approval.grant" + approval.Entries[0].ReasonCode = "approval-created" + approval.Entries[0].EventKind = "approval-created" + approval.Entries[0].EvidenceDigest = hash("b") + if err := approval.ValidateForWorkspace("workspace-a"); err != nil { + t.Fatalf("approval ValidateForWorkspace() error = %v", err) + } + approval.Entries[0].FormatVersion = 3 + approval.Entries[0].EvidenceDigest = expiringApprovalEvidenceFixture + if err := approval.ValidateForWorkspace("workspace-a"); err != nil { + t.Fatalf("expiring approval ValidateForWorkspace() error = %v", err) + } +} + +func TestExportValidateForWorkspaceRejectsForeignAndMalformedDocuments(t *testing.T) { + t.Parallel() + + tests := map[string]func(*Export){ + "foreign workspace": func(value *Export) { value.WorkspaceID = "workspace-b" }, + "unknown schema": func(value *Export) { value.Schema = "caller-schema" }, + "wrong head count": func(value *Export) { value.Chain.HeadSequence = 2 }, + "wrong head hash": func(value *Export) { value.Chain.HeadHash = hash("b") }, + "wrong sequence": func(value *Export) { value.Entries[0].Sequence = 2 }, + "broken link": func(value *Export) { value.Entries[0].PreviousHash = hash("b") }, + "sub-microsecond time": func(value *Export) { + value.Entries[0].RecordedAt = value.Entries[0].RecordedAt.Add(time.Nanosecond) + }, + "non-UTC time": func(value *Export) { + value.Entries[0].RecordedAt = value.Entries[0].RecordedAt.In(time.FixedZone("offset", -5*60*60)) + }, + "unsafe actor": func(value *Export) { value.Entries[0].Actor = "user:alice\ntoken=secret" }, + "invalid UTF-8 actor": func(value *Export) { + value.Entries[0].Actor = string([]byte{'u', 's', 'e', 'r', ':', 0xff}) + }, + "unknown role": func(value *Export) { value.Entries[0].Role = "owner" }, + "ordinary audit verb as read": func(value *Export) { + value.Entries[0].Action = "read" + value.Entries[0].Verb = "audit.export" + }, + "approval payload": func(value *Export) { + value.Entries[0].FormatVersion = 2 + value.Entries[0].EvidenceDigest = "token=secret" + }, + "unknown invalid-request action": func(value *Export) { + value.Entries[0].Action = "caller-action" + value.Entries[0].Verb = "invalid" + value.Entries[0].Verdict = "deny" + value.Entries[0].ReasonCode = "invalid-request" + }, + "oversized": func(value *Export) { + value.Entries = make([]Entry, MaxEntries+1) + value.Chain.HeadSequence = int64(len(value.Entries)) + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + value := validTestExport() + mutate(&value) + if err := value.ValidateForWorkspace(tenancy.WorkspaceID("workspace-a")); err == nil { + t.Fatalf("malformed export accepted: %#v", value) + } + }) + } +} + +func TestRecomputeEntryHashGoldenFormats(t *testing.T) { + t.Parallel() + + mixed := validMixedTestExport() + tests := []struct { + name string + entry Entry + want string + }{ + {name: "format 1 policy decision", entry: mixed.Entries[0], want: "sha256:67544ba8ac180f834bc221aa136c7d121c0e63228b02bc7c7dce2508de26c4ea"}, + {name: "format 2 approval lifecycle", entry: mixed.Entries[1], want: "sha256:dfbfb98dda5768b259314faa5ed57f40ae4575c466e899ad79c23cea06277ece"}, + {name: "format 3 expiring approval lifecycle", entry: mixed.Entries[2], want: "sha256:3cd45877bc466f0a61700a8f13b91e1f5b31c2b9ac382032be2410131d4da338"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := RecomputeEntryHash("workspace-a", test.entry) + if err != nil { + t.Fatalf("RecomputeEntryHash() error = %v", err) + } + if got != test.want { + t.Fatalf("RecomputeEntryHash() = %q, want %q", got, test.want) + } + }) + } +} + +func TestExportVerifyRecomputesEveryEntryAndHead(t *testing.T) { + t.Parallel() + + if err := validMixedTestExport().Verify(); err != nil { + t.Fatalf("Verify() error = %v", err) + } + + tests := map[string]func(*Export){ + "workspace": func(value *Export) { value.WorkspaceID = "workspace-b" }, + "recorded time": func(value *Export) { value.Entries[0].RecordedAt = value.Entries[0].RecordedAt.Add(time.Second) }, + "sub-microsecond time": func(value *Export) { + value.Entries[0].RecordedAt = value.Entries[0].RecordedAt.Add(time.Nanosecond) + }, + "non-UTC time": func(value *Export) { + value.Entries[0].RecordedAt = value.Entries[0].RecordedAt.In(time.FixedZone("offset", -5*60*60)) + }, + "trace identifier": func(value *Export) { value.Entries[0].TraceID = strings.Repeat("2", 32) }, + "actor": func(value *Export) { value.Entries[0].Actor = "user:mallory" }, + "role": func(value *Export) { value.Entries[0].Role = "operator" }, + "action": func(value *Export) { value.Entries[0].Action = "read" }, + "verb": func(value *Export) { value.Entries[0].Verb = "fleet.read" }, + "verdict": func(value *Export) { value.Entries[0].Verdict = "deny" }, + "reason": func(value *Export) { value.Entries[0].ReasonCode = "role-denied" }, + "event kind": func(value *Export) { value.Entries[1].EventKind = "approval-consumed" }, + "evidence": func(value *Export) { value.Entries[1].EvidenceDigest = hash("c") }, + "entry hash": func(value *Export) { value.Entries[1].EntryHash = hash("d"); value.Chain.HeadHash = hash("d") }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + value := validMixedTestExport() + mutate(&value) + if err := value.Verify(); err == nil { + t.Fatal("Verify() accepted a tampered export") + } + }) + } +} + +func FuzzExportVerifyBindsCanonicalFields(f *testing.F) { + f.Add(uint8(0), "seed") + f.Fuzz(func(t *testing.T, field uint8, seed string) { + value := validMixedTestExport() + value.Entries = append([]Entry(nil), value.Entries...) + digest := sha256.Sum256([]byte(seed)) + encoded := hex.EncodeToString(digest[:]) + switch field % 14 { + case 0: + value.Entries[0].Sequence++ + case 1: + entry := &value.Entries[0] + entry.FormatVersion = 2 + entry.Actor, entry.Role, entry.Action = "user:approver", "approver", "approve-intent" + entry.Verb, entry.Verdict, entry.ReasonCode = "approval.grant", "allow", "approval-created" + entry.EventKind, entry.EvidenceDigest = "approval-created", hash("e") + case 2: + value.Entries[0].RecordedAt = value.Entries[0].RecordedAt.Add(time.Microsecond) + case 3: + value.Entries[0].TraceID = encoded[:32] + case 4: + value.WorkspaceID = "workspace-" + encoded[:16] + case 5: + value.Entries[0].Actor = "user:" + encoded + case 6: + entry := &value.Entries[0] + entry.Role, entry.Action, entry.Verb = "operator", "propose-intent", "deployment.restart" + case 7: + entry := &value.Entries[0] + entry.Action, entry.Verb = "read", "fleet.read" + case 8: + value.Entries[0].Verdict, value.Entries[0].ReasonCode = "deny", "policy-deny" + case 9: + value.Entries[0].ReasonCode = "phase-1-read" + case 10: + value.Entries[0].PreviousHash = hash("f") + case 11: + value.Entries[1].EntryHash, value.Chain.HeadHash = hash("d"), hash("d") + case 12: + entry := &value.Entries[1] + entry.Actor, entry.Role, entry.Action = "user:operator", "operator", "propose-intent" + entry.ReasonCode, entry.EventKind = "approval-consumed", "approval-consumed" + case 13: + value.Entries[1].EvidenceDigest = "sha256:" + encoded + } + if err := value.Verify(); err == nil { + t.Fatal("Verify() accepted a canonical-field mutation without rehashing") + } + }) +} + +func validTestExport() Export { + exported := Export{ + Schema: SchemaV1, WorkspaceID: "workspace-a", + Chain: Chain{HashAlgorithm: HashAlgorithm, HeadSequence: 1}, + Entries: []Entry{{ + Sequence: 1, FormatVersion: 1, RecordedAt: time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC), + TraceID: strings.Repeat("1", 32), Actor: "user:alice", Role: "admin", Action: "export-audit", + Verb: "audit.export", Verdict: "allow", ReasonCode: "phase-1-audit-export", + EventKind: "policy-decision", PreviousHash: hash("0"), + }}, + } + entryHash, err := RecomputeEntryHash("workspace-a", exported.Entries[0]) + if err != nil { + panic(err) + } + exported.Entries[0].EntryHash = entryHash + exported.Chain.HeadHash = entryHash + return exported +} + +func validMixedTestExport() Export { + exported := validTestExport() + second := Entry{ + Sequence: 2, FormatVersion: 2, + RecordedAt: time.Date(2026, time.July, 18, 9, 31, 0, 123456000, time.UTC), + TraceID: strings.Repeat("2", 32), Actor: "user:bob", Role: "approver", Action: "approve-intent", + Verb: "approval.grant", Verdict: "allow", ReasonCode: "approval-created", + EventKind: "approval-created", EvidenceDigest: hash("b"), PreviousHash: exported.Entries[0].EntryHash, + } + secondHash, err := RecomputeEntryHash("workspace-a", second) + if err != nil { + panic(err) + } + second.EntryHash = secondHash + exported.Entries = append(exported.Entries, second) + third := Entry{ + Sequence: 3, FormatVersion: 3, + RecordedAt: time.Date(2026, time.July, 18, 9, 32, 0, 123456000, time.UTC), + TraceID: strings.Repeat("3", 32), Actor: "user:alice", Role: "operator", Action: "propose-intent", + Verb: "approval.grant", Verdict: "allow", ReasonCode: "approval-consumed", + EventKind: "approval-consumed", EvidenceDigest: expiringApprovalEvidenceFixture, PreviousHash: secondHash, + } + thirdHash, err := RecomputeEntryHash("workspace-a", third) + if err != nil { + panic(err) + } + third.EntryHash = thirdHash + exported.Entries = append(exported.Entries, third) + exported.Chain.HeadSequence = 3 + exported.Chain.HeadHash = thirdHash + return exported +} + +func hash(character string) string { return "sha256:" + strings.Repeat(character, 64) } diff --git a/internal/auditrecord/page.go b/internal/auditrecord/page.go new file mode 100644 index 0000000..32dca85 --- /dev/null +++ b/internal/auditrecord/page.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auditrecord + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + "math" + + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + // PageSchemaV1 identifies one bounded interval from an immutable retained-chain snapshot. + PageSchemaV1 = "sith.policy-audit-page/v1" + // PageCursorChars is the exact canonical base64url length of one continuation descriptor. + PageCursorChars = 151 + + pageCursorVersion byte = 1 + pageCursorBytes = 113 + pageCursorDomain = "sith-policy-audit-page-cursor/v1" +) + +// Page is one bounded, consecutive interval from a declared workspace chain snapshot. An +// intermediate page proves only its own internal links; PageSequenceVerifier establishes complete +// genesis-to-head continuity across an ordered set of pages. +type Page struct { + Schema string `json:"schema"` + WorkspaceID string `json:"workspace_id"` + Snapshot Chain `json:"snapshot"` + StartSequence int64 `json:"start_sequence"` + PreviousHash string `json:"previous_hash"` + Entries []Entry `json:"entries"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// PageRequest is a validated first-page or continuation request. Its fields remain private so +// storage code cannot accidentally trust caller-controlled cursor components without parsing and +// workspace binding first. +type PageRequest struct { + workspaceDigest [sha256.Size]byte + initial bool + headSequence int64 + headHash string + nextSequence int64 + previousHash string +} + +// FirstPage creates a request that will bind itself to the current workspace head in the backing +// repeatable-read transaction. +func FirstPage(workspaceID tenancy.WorkspaceID) (PageRequest, error) { + if workspaceInvalid(workspaceID) { + return PageRequest{}, fmt.Errorf("construct first audit page request: workspace is invalid") + } + return PageRequest{workspaceDigest: pageWorkspaceDigest(workspaceID), initial: true}, nil +} + +// ContinuePage strictly parses one canonical, fixed-size base64url continuation descriptor and +// binds it to the expected workspace. The descriptor is not a credential; the database must still +// validate its snapshot and boundary hashes after normal authentication and authorization. +func ContinuePage(workspaceID tenancy.WorkspaceID, encoded string) (PageRequest, error) { + if workspaceInvalid(workspaceID) || len(encoded) != PageCursorChars { + return PageRequest{}, fmt.Errorf("parse audit page cursor: cursor is invalid") + } + payload, err := base64.RawURLEncoding.Strict().DecodeString(encoded) + if err != nil || len(payload) != pageCursorBytes || + base64.RawURLEncoding.EncodeToString(payload) != encoded || payload[0] != pageCursorVersion { + return PageRequest{}, fmt.Errorf("parse audit page cursor: cursor is invalid") + } + wantWorkspace := pageWorkspaceDigest(workspaceID) + if !bytes.Equal(payload[1:33], wantWorkspace[:]) { + return PageRequest{}, fmt.Errorf("parse audit page cursor: cursor is invalid") + } + headUnsigned := binary.BigEndian.Uint64(payload[33:41]) + nextUnsigned := binary.BigEndian.Uint64(payload[73:81]) + if headUnsigned > math.MaxInt64 || nextUnsigned > math.MaxInt64 { + return PageRequest{}, fmt.Errorf("parse audit page cursor: cursor is invalid") + } + request := PageRequest{ + workspaceDigest: wantWorkspace, + headSequence: int64(headUnsigned), + headHash: "sha256:" + hex.EncodeToString(payload[41:73]), + nextSequence: int64(nextUnsigned), + previousHash: "sha256:" + hex.EncodeToString(payload[81:113]), + } + if err := request.ValidateForWorkspace(workspaceID); err != nil { + return PageRequest{}, fmt.Errorf("parse audit page cursor: cursor is invalid") + } + return request, nil +} + +// EncodePageCursor creates the only accepted continuation encoding. The next sequence must still +// belong to the declared snapshot; final pages carry no continuation. +func EncodePageCursor( + workspaceID tenancy.WorkspaceID, + headSequence int64, + headHash string, + nextSequence int64, + previousHash string, +) (string, error) { + if workspaceInvalid(workspaceID) || headSequence <= 1 || nextSequence <= 1 || + nextSequence > headSequence || !validHash(headHash) || !validHash(previousHash) { + return "", fmt.Errorf("encode audit page cursor: cursor boundary is invalid") + } + headBytes, err := decodeHash(headHash) + if err != nil { + return "", fmt.Errorf("encode audit page cursor: cursor boundary is invalid") + } + previousBytes, err := decodeHash(previousHash) + if err != nil { + return "", fmt.Errorf("encode audit page cursor: cursor boundary is invalid") + } + payload := make([]byte, pageCursorBytes) + payload[0] = pageCursorVersion + workspaceDigest := pageWorkspaceDigest(workspaceID) + copy(payload[1:33], workspaceDigest[:]) + binary.BigEndian.PutUint64(payload[33:41], uint64(headSequence)) + copy(payload[41:73], headBytes) + binary.BigEndian.PutUint64(payload[73:81], uint64(nextSequence)) + copy(payload[81:113], previousBytes) + encoded := base64.RawURLEncoding.EncodeToString(payload) + if len(encoded) != PageCursorChars { + return "", fmt.Errorf("encode audit page cursor: canonical encoding is invalid") + } + return encoded, nil +} + +// ValidateForWorkspace rechecks that a typed request is complete and belongs to one workspace. +func (request PageRequest) ValidateForWorkspace(workspaceID tenancy.WorkspaceID) error { + if workspaceInvalid(workspaceID) { + return fmt.Errorf("audit page request workspace is invalid") + } + wantWorkspace := pageWorkspaceDigest(workspaceID) + if !bytes.Equal(request.workspaceDigest[:], wantWorkspace[:]) { + return fmt.Errorf("audit page request workspace is invalid") + } + if request.initial { + if request.headSequence != 0 || request.headHash != "" || request.nextSequence != 0 || request.previousHash != "" { + return fmt.Errorf("first audit page request is invalid") + } + return nil + } + if request.headSequence <= 1 || request.nextSequence <= 1 || request.nextSequence > request.headSequence || + !validHash(request.headHash) || !validHash(request.previousHash) { + return fmt.Errorf("continuation audit page request is invalid") + } + return nil +} + +// Initial reports whether the backing store should capture a new snapshot head. +func (request PageRequest) Initial() bool { return request.initial } + +// HeadSequence returns the immutable snapshot head sequence for a continuation. +func (request PageRequest) HeadSequence() int64 { return request.headSequence } + +// HeadHash returns the immutable snapshot head hash for a continuation. +func (request PageRequest) HeadHash() string { return request.headHash } + +// NextSequence returns the first entry requested by a continuation. +func (request PageRequest) NextSequence() int64 { return request.nextSequence } + +// PreviousHash returns the expected hash immediately before NextSequence. +func (request PageRequest) PreviousHash() string { return request.previousHash } + +// ValidatePage binds a returned page to the exact parsed request. A first request must start at +// genesis; a continuation must preserve its declared snapshot, next sequence, and prior hash. +func (request PageRequest) ValidatePage(page Page) error { + workspaceID := tenancy.WorkspaceID(page.WorkspaceID) + if err := request.ValidateForWorkspace(workspaceID); err != nil { + return fmt.Errorf("audit page does not match request") + } + if err := page.ValidateForWorkspace(workspaceID); err != nil { + return fmt.Errorf("audit page does not match request") + } + if request.initial { + if page.StartSequence != 1 || page.PreviousHash != zeroHash() { + return fmt.Errorf("audit page does not match first request") + } + return nil + } + if page.Snapshot.HeadSequence != request.headSequence || page.Snapshot.HeadHash != request.headHash || + page.StartSequence != request.nextSequence || page.PreviousHash != request.previousHash { + return fmt.Errorf("audit page does not match continuation") + } + return nil +} + +// ValidateForWorkspace checks a page's closed schema, snapshot, bounds, links, and continuation +// without recomputing entry hashes. +func (page Page) ValidateForWorkspace(workspaceID tenancy.WorkspaceID) error { + return page.validateForWorkspace(workspaceID, false) +} + +// Verify checks one page's internal canonical hashes and links. Complete-chain proof requires an +// ordered PageSequenceVerifier reaching the declared snapshot head. +func (page Page) Verify() error { + return page.VerifyForWorkspace(tenancy.WorkspaceID(page.WorkspaceID)) +} + +// VerifyForWorkspace binds a page to an expected tenant before recomputing every contained hash. +func (page Page) VerifyForWorkspace(workspaceID tenancy.WorkspaceID) error { + return page.validateForWorkspace(workspaceID, true) +} + +func (page Page) validateForWorkspace(workspaceID tenancy.WorkspaceID, verifyHashes bool) error { + if page.Schema != PageSchemaV1 || page.WorkspaceID != string(workspaceID) || workspaceInvalid(workspaceID) || + page.Snapshot.HashAlgorithm != HashAlgorithm || page.Snapshot.HeadSequence <= 0 || + !validHash(page.Snapshot.HeadHash) || page.StartSequence <= 0 || !validHash(page.PreviousHash) || + len(page.Entries) == 0 || len(page.Entries) > MaxEntries { + return fmt.Errorf("audit page envelope is invalid") + } + endSequence := page.StartSequence + int64(len(page.Entries)) - 1 + if endSequence < page.StartSequence || endSequence > page.Snapshot.HeadSequence { + return fmt.Errorf("audit page range is invalid") + } + lastHash, err := validateEntrySequence( + workspaceID, page.Entries, page.StartSequence, page.PreviousHash, verifyHashes, + ) + if err != nil { + return fmt.Errorf("audit page: %w", err) + } + if endSequence == page.Snapshot.HeadSequence { + if page.NextCursor != "" || lastHash != page.Snapshot.HeadHash { + return fmt.Errorf("final audit page head is invalid") + } + return nil + } + if len(page.Entries) != MaxEntries || page.NextCursor == "" { + return fmt.Errorf("intermediate audit page boundary is invalid") + } + next, err := ContinuePage(workspaceID, page.NextCursor) + if err != nil || next.HeadSequence() != page.Snapshot.HeadSequence || + next.HeadHash() != page.Snapshot.HeadHash || next.NextSequence() != endSequence+1 || + next.PreviousHash() != lastHash { + return fmt.Errorf("intermediate audit page cursor is invalid") + } + return nil +} + +// PageSequenceResult is the bounded summary of one fully verified ordered page sequence. +type PageSequenceResult struct { + Schema string + WorkspaceID string + Pages int + Entries int64 + Chain Chain +} + +// PageSequenceVerifier incrementally verifies ordered page documents without retaining earlier +// page payloads in memory. +type PageSequenceVerifier struct { + initialized bool + complete bool + workspaceID tenancy.WorkspaceID + chain Chain + nextSequence int64 + previousHash string + pages int +} + +// Add verifies and consumes the next page in sequence. +func (verifier *PageSequenceVerifier) Add(page Page) error { + if verifier == nil || verifier.complete { + return fmt.Errorf("audit page sequence cannot accept another page") + } + workspaceID := tenancy.WorkspaceID(page.WorkspaceID) + if err := page.VerifyForWorkspace(workspaceID); err != nil { + return fmt.Errorf("verify audit page sequence: %w", err) + } + if !verifier.initialized { + if page.StartSequence != 1 || page.PreviousHash != zeroHash() { + return fmt.Errorf("verify audit page sequence: first page does not start at genesis") + } + verifier.initialized = true + verifier.workspaceID = workspaceID + verifier.chain = page.Snapshot + verifier.nextSequence = 1 + verifier.previousHash = zeroHash() + } else if workspaceID != verifier.workspaceID || page.Snapshot != verifier.chain { + return fmt.Errorf("verify audit page sequence: page snapshot is inconsistent") + } + if page.StartSequence != verifier.nextSequence || page.PreviousHash != verifier.previousHash { + return fmt.Errorf("verify audit page sequence: page order or continuity is invalid") + } + endSequence := page.StartSequence + int64(len(page.Entries)) - 1 + verifier.nextSequence = endSequence + 1 + verifier.previousHash = page.Entries[len(page.Entries)-1].EntryHash + verifier.pages++ + if page.NextCursor == "" { + verifier.complete = true + } + return nil +} + +// Finish succeeds only after the sequence reaches its declared snapshot head. +func (verifier *PageSequenceVerifier) Finish() (PageSequenceResult, error) { + if verifier == nil || !verifier.initialized || !verifier.complete || + verifier.nextSequence-1 != verifier.chain.HeadSequence || verifier.previousHash != verifier.chain.HeadHash { + return PageSequenceResult{}, fmt.Errorf("audit page sequence is incomplete") + } + return PageSequenceResult{ + Schema: PageSchemaV1, WorkspaceID: string(verifier.workspaceID), Pages: verifier.pages, + Entries: verifier.chain.HeadSequence, Chain: verifier.chain, + }, nil +} + +func pageWorkspaceDigest(workspaceID tenancy.WorkspaceID) [sha256.Size]byte { + canonical := make([]byte, 0, len(pageCursorDomain)+len(workspaceID)+16) + canonical = appendCanonicalString(canonical, pageCursorDomain) + canonical = appendCanonicalString(canonical, string(workspaceID)) + return sha256.Sum256(canonical) +} + +func decodeHash(value string) ([]byte, error) { + if !validHash(value) { + return nil, fmt.Errorf("hash is invalid") + } + decoded, err := hex.DecodeString(value[7:]) + if err != nil || len(decoded) != sha256.Size { + return nil, fmt.Errorf("hash is invalid") + } + return decoded, nil +} diff --git a/internal/auditrecord/page_test.go b/internal/auditrecord/page_test.go new file mode 100644 index 0000000..4f12800 --- /dev/null +++ b/internal/auditrecord/page_test.go @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auditrecord + +import ( + "encoding/base64" + "strings" + "testing" + "time" +) + +func TestPageCursorRoundTripIsCanonicalAndWorkspaceBound(t *testing.T) { + t.Parallel() + cursor, err := EncodePageCursor("workspace-a", 513, hash("f"), 513, hash("e")) + if err != nil { + t.Fatal(err) + } + if len(cursor) != PageCursorChars || strings.ContainsAny(cursor, "+/=") { + t.Fatalf("cursor = %q", cursor) + } + request, err := ContinuePage("workspace-a", cursor) + if err != nil { + t.Fatal(err) + } + if request.Initial() || request.HeadSequence() != 513 || request.HeadHash() != hash("f") || + request.NextSequence() != 513 || request.PreviousHash() != hash("e") { + t.Fatalf("request = %#v", request) + } + if _, err := ContinuePage("workspace-b", cursor); err == nil { + t.Fatal("foreign workspace accepted cursor") + } + first, err := FirstPage("workspace-a") + if err != nil || !first.Initial() || first.ValidateForWorkspace("workspace-a") != nil || + first.ValidateForWorkspace("workspace-b") == nil { + t.Fatalf("first page request/error = %#v/%v", first, err) + } +} + +func TestPageRequestBindsReturnedPage(t *testing.T) { + t.Parallel() + pages := validTestPages(t, MaxEntries+1) + first, err := FirstPage("workspace-a") + if err != nil || first.ValidatePage(pages[0]) != nil { + t.Fatalf("first request/page error = %v", err) + } + continuation, err := ContinuePage("workspace-a", pages[0].NextCursor) + if err != nil || continuation.ValidatePage(pages[1]) != nil { + t.Fatalf("continuation request/page error = %v", err) + } + if continuation.ValidatePage(pages[0]) == nil || first.ValidatePage(pages[1]) == nil { + t.Fatal("page request accepted the wrong interval") + } +} + +func TestPageCursorRejectsMalformedAndNonCanonicalEncodings(t *testing.T) { + t.Parallel() + cursor, err := EncodePageCursor("workspace-a", 513, hash("f"), 513, hash("e")) + if err != nil { + t.Fatal(err) + } + payload, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + t.Fatal(err) + } + tests := map[string]string{ + "short": cursor[:len(cursor)-1], + "padded": cursor + "=", + "invalid alphabet": strings.Repeat("+", PageCursorChars), + "unknown version": base64.RawURLEncoding.EncodeToString(append([]byte{2}, payload[1:]...)), + } + for name, encoded := range tests { + t.Run(name, func(t *testing.T) { + if _, err := ContinuePage("workspace-a", encoded); err == nil { + t.Fatal("invalid cursor was accepted") + } + }) + } + for _, test := range []struct { + head, next int64 + headHash, previousHash string + }{ + {head: 1, next: 1, headHash: hash("a"), previousHash: hash("0")}, + {head: 513, next: 514, headHash: hash("a"), previousHash: hash("b")}, + {head: 513, next: 513, headHash: "bad", previousHash: hash("b")}, + } { + if _, err := EncodePageCursor("workspace-a", test.head, test.headHash, test.next, test.previousHash); err == nil { + t.Fatalf("invalid boundary accepted: %#v", test) + } + } +} + +func TestPageVerifyAndSequenceVerifierAcceptCompleteSnapshot(t *testing.T) { + t.Parallel() + pages := validTestPages(t, MaxEntries+1) + if len(pages) != 2 || pages[0].NextCursor == "" || pages[1].NextCursor != "" { + t.Fatalf("pages = %#v", pages) + } + for index, page := range pages { + if err := page.Verify(); err != nil { + t.Fatalf("page %d Verify() error = %v", index, err) + } + } + var verifier PageSequenceVerifier + for _, page := range pages { + if err := verifier.Add(page); err != nil { + t.Fatal(err) + } + } + result, err := verifier.Finish() + if err != nil { + t.Fatal(err) + } + if result.Schema != PageSchemaV1 || result.WorkspaceID != "workspace-a" || result.Pages != 2 || + result.Entries != MaxEntries+1 || result.Chain != pages[0].Snapshot { + t.Fatalf("result = %#v", result) + } + if err := verifier.Add(pages[1]); err == nil { + t.Fatal("complete verifier accepted another page") + } +} + +func TestPageSequenceVerifierRejectsIncompleteReorderedAndForeignPages(t *testing.T) { + t.Parallel() + pages := validTestPages(t, MaxEntries+1) + var incomplete PageSequenceVerifier + if err := incomplete.Add(pages[0]); err != nil { + t.Fatal(err) + } + if _, err := incomplete.Finish(); err == nil { + t.Fatal("incomplete sequence verified") + } + + for name, candidate := range map[string][]Page{ + "starts after genesis": {pages[1]}, + "reordered": {pages[1], pages[0]}, + "replayed": {pages[0], pages[0]}, + "missing first link": func() []Page { + changed := pages[0] + changed.PreviousHash = hash("a") + return []Page{changed} + }(), + "foreign workspace": func() []Page { + changed := pages[0] + changed.WorkspaceID = "workspace-b" + return []Page{changed} + }(), + } { + t.Run(name, func(t *testing.T) { + var verifier PageSequenceVerifier + for _, page := range candidate { + if err := verifier.Add(page); err != nil { + return + } + } + if _, err := verifier.Finish(); err == nil { + t.Fatal("invalid sequence verified") + } + }) + } +} + +func TestPageVerifyRejectsTamperedRangeCursorAndHash(t *testing.T) { + t.Parallel() + base := validTestPages(t, MaxEntries+1)[0] + mutations := map[string]func(*Page){ + "workspace": func(page *Page) { page.WorkspaceID = "workspace-b" }, + "start": func(page *Page) { page.StartSequence = 2 }, + "previous": func(page *Page) { page.PreviousHash = hash("a") }, + "entry": func(page *Page) { page.Entries[0].Actor = "user:mallory" }, + "head": func(page *Page) { page.Snapshot.HeadHash = hash("b") }, + "missing cursor": func(page *Page) { page.NextCursor = "" }, + "cursor": func(page *Page) { page.NextCursor = strings.Repeat("A", PageCursorChars) }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + page := base + page.Entries = append([]Entry(nil), base.Entries...) + mutate(&page) + if err := page.Verify(); err == nil { + t.Fatal("tampered page verified") + } + }) + } +} + +func FuzzPageCursorMutationCannotBindOriginalPage(f *testing.F) { + f.Add(uint8(0), uint8(1)) + f.Add(uint8(41), uint8(0xff)) + f.Add(uint8(112), uint8(0x80)) + f.Fuzz(func(t *testing.T, offset, delta uint8) { + cursor, page := validContinuationPage(t) + payload, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + t.Fatal(err) + } + change := delta + if change == 0 { + change = 1 + } + payload[int(offset)%len(payload)] ^= change + request, err := ContinuePage("workspace-a", base64.RawURLEncoding.EncodeToString(payload)) + if err != nil { + return + } + if request.ValidatePage(page) == nil { + t.Fatal("mutated continuation bound the original page") + } + }) +} + +func validContinuationPage(t *testing.T) (string, Page) { + t.Helper() + previous := hash("e") + entry := Entry{ + Sequence: 513, FormatVersion: 1, + RecordedAt: time.Date(2026, time.July, 18, 10, 0, 0, 0, time.UTC), + TraceID: strings.Repeat("1", 32), Actor: "user:alice", Role: "admin", Action: "export-audit", + Verb: "audit.export", Verdict: "allow", ReasonCode: "phase-1-audit-export", + EventKind: "policy-decision", PreviousHash: previous, + } + head, err := RecomputeEntryHash("workspace-a", entry) + if err != nil { + t.Fatal(err) + } + entry.EntryHash = head + page := Page{ + Schema: PageSchemaV1, WorkspaceID: "workspace-a", + Snapshot: Chain{HashAlgorithm: HashAlgorithm, HeadSequence: entry.Sequence, HeadHash: head}, + StartSequence: entry.Sequence, PreviousHash: previous, Entries: []Entry{entry}, + } + cursor, err := EncodePageCursor("workspace-a", entry.Sequence, head, entry.Sequence, previous) + if err != nil { + t.Fatal(err) + } + return cursor, page +} + +func validTestPages(t *testing.T, count int) []Page { + t.Helper() + entries := make([]Entry, count) + previous := zeroHash() + for index := range entries { + entry := Entry{ + Sequence: int64(index + 1), FormatVersion: 1, + RecordedAt: time.Date(2026, time.July, 18, 10, 0, 0, index*1000, time.UTC).Truncate(time.Microsecond), + TraceID: strings.Repeat("1", 32), Actor: "user:alice", Role: "admin", Action: "export-audit", + Verb: "audit.export", Verdict: "allow", ReasonCode: "phase-1-audit-export", + EventKind: "policy-decision", PreviousHash: previous, + } + hashValue, err := RecomputeEntryHash("workspace-a", entry) + if err != nil { + t.Fatal(err) + } + entry.EntryHash = hashValue + entries[index] = entry + previous = hashValue + } + snapshot := Chain{HashAlgorithm: HashAlgorithm, HeadSequence: int64(count), HeadHash: previous} + pages := make([]Page, 0, (count+MaxEntries-1)/MaxEntries) + for start := 0; start < count; start += MaxEntries { + end := start + MaxEntries + if end > count { + end = count + } + page := Page{ + Schema: PageSchemaV1, WorkspaceID: "workspace-a", Snapshot: snapshot, + StartSequence: int64(start + 1), PreviousHash: entries[start].PreviousHash, + Entries: append([]Entry(nil), entries[start:end]...), + } + if end < count { + cursor, err := EncodePageCursor("workspace-a", int64(count), previous, int64(end+1), entries[end-1].EntryHash) + if err != nil { + t.Fatal(err) + } + page.NextCursor = cursor + } + pages = append(pages, page) + } + return pages +} diff --git a/internal/brain/boundary_test.go b/internal/brain/boundary_test.go index 219366d..8a0cd53 100644 --- a/internal/brain/boundary_test.go +++ b/internal/brain/boundary_test.go @@ -30,11 +30,49 @@ func TestBrainHasNoWritePathImports(t *testing.T) { if err != nil { t.Fatalf("unquote import: %v", err) } - for _, forbidden := range []string{"/connector", "/localops", "/mcpserver"} { + for _, forbidden := range []string{"/connector", "/hubdb", "/localops", "/mcpserver", "/pep"} { if strings.Contains(path, forbidden) { t.Fatalf("brain imports forbidden write-capable package %q", path) } } + if forbiddenBrainSideEffectImport(path) { + t.Fatalf("brain imports forbidden side-effect package %q", path) + } + } + } +} + +func TestForbiddenBrainSideEffectImportRejectsPackageTrees(t *testing.T) { + t.Parallel() + tests := []struct { + path string + forbidden bool + }{ + {path: "database/sql", forbidden: true}, + {path: "database/sql/driver", forbidden: true}, + {path: "net", forbidden: true}, + {path: "net/http", forbidden: true}, + {path: "net/rpc", forbidden: true}, + {path: "os", forbidden: true}, + {path: "os/exec", forbidden: true}, + {path: "os/user", forbidden: true}, + {path: "google.golang.org/grpc", forbidden: true}, + {path: "google.golang.org/grpc/credentials", forbidden: true}, + {path: "encoding/json", forbidden: false}, + {path: "github.com/ArdurAI/sith/internal/intent", forbidden: false}, + } + for _, test := range tests { + if got := forbiddenBrainSideEffectImport(test.path); got != test.forbidden { + t.Errorf("forbiddenBrainSideEffectImport(%q) = %t, want %t", test.path, got, test.forbidden) + } + } +} + +func forbiddenBrainSideEffectImport(path string) bool { + for _, forbidden := range []string{"database/sql", "net", "net/http", "os", "os/exec", "google.golang.org/grpc"} { + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + return true } } + return false } diff --git a/internal/brain/cache.go b/internal/brain/cache.go index dde3f31..e42aab5 100644 --- a/internal/brain/cache.go +++ b/internal/brain/cache.go @@ -132,5 +132,8 @@ func coverageReason(snapshot fleetcache.Snapshot) string { if len(snapshot.Coverage.Unreachable) > 0 { return "one or more kubeconfig contexts are unreachable" } + if len(snapshot.Coverage.Truncated) > 0 { + return "one or more kubeconfig contexts returned truncated evidence" + } return "" } diff --git a/internal/brain/cache_test.go b/internal/brain/cache_test.go index a72b568..011c7ba 100644 --- a/internal/brain/cache_test.go +++ b/internal/brain/cache_test.go @@ -3,6 +3,7 @@ package brain import ( + "reflect" "testing" "time" @@ -33,6 +34,55 @@ func TestFromCacheProjectsLiveFactsAndHonestCoverage(t *testing.T) { } } +func TestFromCacheProjectsExactImagePullReasons(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC) + ref := fleet.ResourceRef{SourceKind: "kubeconfig", Scope: "alpha", Kind: "Pod", Namespace: "prod", Name: "payments-0"} + + for _, reason := range []string{"ImagePullBackOff", "ErrImagePull"} { + reason := reason + t.Run(reason, func(t *testing.T) { + t.Parallel() + input := FromCache(fleet.LocalWorkspace, fleetcache.Snapshot{ + Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + Records: []fleetcache.Record{{ + Fact: fleet.Fact{Evidence: fleet.Evidence{ + Ref: ref, + Source: "alpha", + }, Workspace: fleet.LocalWorkspace}, + Workspace: fleet.LocalWorkspace, Kind: "Pod", Cluster: "alpha", Namespace: "prod", Name: "payments-0", + Reasons: []string{reason}, ObservedAt: now, + }}, + }) + if len(input.Observations) != 1 || input.Observations[0].Key != "pod.reason" || input.Observations[0].Value != reason { + t.Fatalf("observations = %#v, want one exact sanitized reason", input.Observations) + } + observation := input.Observations[0] + if !reflect.DeepEqual(observation.Ref, ref) || observation.Lens != fleet.LensLive || observation.Source != "alpha" || + !observation.ObservedAt.Equal(now) { + t.Fatalf("observation metadata = %#v, want exact cached Pod citation metadata", observation) + } + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleImagePull { + t.Fatalf("verdicts = %#v, want R7", result.Verdicts) + } + }) + } +} + +func TestFromCacheExplainsTruncatedLiveCoverage(t *testing.T) { + t.Parallel() + input := FromCache(fleet.LocalWorkspace, fleetcache.Snapshot{ + Coverage: fleet.Coverage{Requested: 2, Reachable: 2, Truncated: []string{"beta"}}, + }) + if got := input.Coverage[fleet.LensLive]; !got.Available || got.Reason != "one or more kubeconfig contexts returned truncated evidence" { + t.Fatalf("live coverage = %#v, want available but explicitly truncated", got) + } +} + func TestFromCacheProjectsOnlyOneProvenRepoDigest(t *testing.T) { t.Parallel() diff --git a/internal/brain/evaluate.go b/internal/brain/evaluate.go index badbd3a..8035da3 100644 --- a/internal/brain/evaluate.go +++ b/internal/brain/evaluate.go @@ -51,7 +51,8 @@ func compose(verdicts []Verdict) []Verdict { } func evaluateRule(candidate rule, observations []Observation, coverage map[fleet.Lens]LensCoverage) (Verdict, bool) { - trigger, ok := match(candidate.trigger, observations) + observations = candidate.applicableObservations(observations) + trigger, ok := match(candidate.trigger, observations, candidate.exactTrigger) if !ok { return Verdict{}, false } @@ -59,11 +60,12 @@ func evaluateRule(candidate rule, observations []Observation, coverage map[fleet verdict := Verdict{ Rule: candidate.id, FailureMode: candidate.failureMode, Status: StatusConfirmed, Hypothesis: candidate.rootCause, Scope: ref.Scope, Ref: ref, Advisory: renderAdvisory(candidate.advisory, ref), + RemediationCandidate: remediationCandidateFor(candidate.remediation), } verdict.Citations = append(verdict.Citations, citation(candidate.trigger, trigger)) verdict.Score += candidate.trigger.weight for _, signal := range candidate.signals { - if observation, found := match(signal, observations); found { + if observation, found := match(signal, observations, false); found { verdict.Citations = append(verdict.Citations, citation(signal, observation)) verdict.Score += signal.weight } @@ -87,7 +89,27 @@ func evaluateRule(candidate rule, observations []Observation, coverage map[fleet return verdict, true } -func match(predicate predicate, observations []Observation) (Observation, bool) { +func (candidate rule) applicableObservations(observations []Observation) []Observation { + if candidate.sourceKind == "" && candidate.resourceKind == "" { + return observations + } + filtered := make([]Observation, 0, len(observations)) + for _, observation := range observations { + if candidate.appliesTo(observation) { + filtered = append(filtered, observation) + } + } + return filtered +} + +func (candidate rule) appliesTo(observation Observation) bool { + if candidate.sourceKind != "" && observation.Ref.SourceKind != candidate.sourceKind { + return false + } + return candidate.resourceKind == "" || observation.Ref.Kind == candidate.resourceKind +} + +func match(predicate predicate, observations []Observation, exact bool) (Observation, bool) { var stale Observation staleMatched := false for _, observation := range observations { @@ -102,7 +124,7 @@ func match(predicate predicate, observations []Observation) (Observation, bool) continue } for _, expected := range predicate.values { - if strings.EqualFold(strings.TrimSpace(observation.Value), expected) { + if observation.Value == expected || (!exact && strings.EqualFold(observation.Value, expected)) { if !observation.Stale { return observation, true } @@ -115,7 +137,7 @@ func match(predicate predicate, observations []Observation) (Observation, bool) func unavailable(lens fleet.Lens, coverage map[fleet.Lens]LensCoverage, observations []Observation) bool { state, declared := coverage[lens] - if declared && (!state.Available || state.Stale) { + if !declared || !state.Available || state.Stale { return true } for _, observation := range observations { @@ -152,10 +174,18 @@ func correlateFleet(verdicts []Verdict, observations map[string][]Observation) [ rule RuleID digest string } - groups := make(map[groupKey][]Verdict) + type groupMember struct { + verdict Verdict + digest Observation + } + groups := make(map[groupKey]map[string]groupMember) for _, verdict := range verdicts { - for _, observation := range observations[entityKey(verdict.Ref)] { - if observation.Key != fleet.OTelContainerImageRepoDigests { + if !supportsFleetCorrelation(verdict.Rule) { + continue + } + entity := entityKey(verdict.Ref) + for _, observation := range observations[entity] { + if observation.Key != fleet.OTelContainerImageRepoDigests || observation.Stale { continue } digest, err := fleet.ImageDigestFromRepoDigest(observation.Value) @@ -163,17 +193,28 @@ func correlateFleet(verdicts []Verdict, observations map[string][]Observation) [ continue } key := groupKey{verdict.Rule, digest} - groups[key] = append(groups[key], verdict) + if groups[key] == nil { + groups[key] = make(map[string]groupMember) + } + current, exists := groups[key][entity] + if !exists || preferCorrelationObservation(observation, current.digest) { + groups[key][entity] = groupMember{verdict: verdict, digest: observation} + } } } result := make([]Verdict, 0) - for key, group := range groups { + for key, members := range groups { + group := make([]Verdict, 0, len(members)) + for _, member := range members { + group = append(group, member.verdict) + } sortVerdicts(group) clusters := uniqueClusters(group) if len(clusters) < 2 { continue } correlated := group[0] + correlated.RemediationCandidate = cloneRemediationCandidate(correlated.RemediationCandidate) correlated.FleetWide = true correlated.Scope = "fleet" correlated.Clusters = clusters @@ -182,6 +223,7 @@ func correlateFleet(verdicts []Verdict, observations map[string][]Observation) [ correlated.Citations = nil for _, verdict := range group { correlated.Citations = append(correlated.Citations, verdict.Citations...) + correlated.Citations = append(correlated.Citations, evidenceCitation(members[entityKey(verdict.Ref)].digest)) correlated.MissingLenses = unionLenses(correlated.MissingLenses, verdict.MissingLenses) if confidenceRank(verdict.Status) < confidenceRank(correlated.Status) { correlated.Status = verdict.Status @@ -193,6 +235,51 @@ func correlateFleet(verdicts []Verdict, observations map[string][]Observation) [ return result } +func supportsFleetCorrelation(ruleID RuleID) bool { + switch ruleID { + case RuleBadDeploy, RuleOOMKilled, RuleCrashLoop, RuleConfigDrift, RuleCertExpiry, RuleNodePressure: + return true + default: + return false + } +} + +func preferCorrelationObservation(candidate, current Observation) bool { + if !candidate.ObservedAt.Equal(current.ObservedAt) { + return candidate.ObservedAt.After(current.ObservedAt) + } + if candidate.Source != current.Source { + return candidate.Source < current.Source + } + if candidate.Value != current.Value { + return candidate.Value < current.Value + } + if candidateRef, currentRef := correlationRefKey(candidate.Ref), correlationRefKey(current.Ref); candidateRef != currentRef { + return candidateRef < currentRef + } + return candidate.Lens < current.Lens +} + +func correlationRefKey(ref fleet.ResourceRef) string { + parts := []string{ref.SourceKind, ref.Scope, ref.Kind, ref.Namespace, ref.Name} + keys := make([]string, 0, len(ref.Attributes)) + for key := range ref.Attributes { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + parts = append(parts, key, ref.Attributes[key]) + } + return strings.Join(parts, "\x00") +} + +func evidenceCitation(observation Observation) Citation { + return Citation{ + Ref: observation.Ref, Lens: observation.Lens, Predicate: observation.Key, Observed: observation.Value, + ObservedAt: observation.ObservedAt, Source: observation.Source, Stale: observation.Stale, + } +} + func sortCitations(citations []Citation) { sort.Slice(citations, func(left, right int) bool { if citations[left].Weight != citations[right].Weight { diff --git a/internal/brain/evaluate_test.go b/internal/brain/evaluate_test.go index f224ec7..89c86d0 100644 --- a/internal/brain/evaluate_test.go +++ b/internal/brain/evaluate_test.go @@ -3,7 +3,9 @@ package brain import ( + "reflect" "slices" + "strings" "testing" "time" @@ -62,6 +64,612 @@ func TestEvaluateCanonicalRules(t *testing.T) { } } +func TestEvaluateImagePullFailureRuleIsBounded(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC) + + for _, reason := range []string{"ImagePullBackOff", "imagepullbackoff", "ImAgEpUlLbAcKoFf", "ErrImagePull", "errimagepull"} { + reason := reason + t.Run(reason, func(t *testing.T) { + t.Parallel() + observation := observe(now, fleet.LensLive, "pod.reason", reason) + observation.Ref.Scope = "dev'; printf injected #" + observation.Ref.Namespace = "payments'; printf injected #" + observation.Ref.Name = "api'; printf injected #" + + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{observation}, + Coverage: covered(fleet.LensLive), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 { + t.Fatalf("verdicts = %#v, want one R7 verdict", result.Verdicts) + } + verdict := result.Verdicts[0] + if verdict.Rule != RuleImagePull || verdict.Status != StatusConfirmed || verdict.FleetWide { + t.Fatalf("verdict = %#v, want confirmed non-fleet R7", verdict) + } + if len(verdict.Citations) != 1 { + t.Fatalf("citations = %#v, want one exact LIVE citation", verdict.Citations) + } + citation := verdict.Citations[0] + if !reflect.DeepEqual(citation.Ref, observation.Ref) || citation.Lens != fleet.LensLive || citation.Predicate != "pod.reason" || + citation.Observed != reason || citation.Source != observation.Source || !citation.ObservedAt.Equal(now) { + t.Fatalf("citation = %#v, want exact source observation", citation) + } + wantCommand := "kubectl --context 'dev'\\''; printf injected #' describe pod 'api'\\''; printf injected #' -n 'payments'\\''; printf injected #'" + if verdict.Advisory.Command != wantCommand || verdict.Advisory.PRDiff != "" || !verdict.Advisory.Sensitive { + t.Fatalf("advisory = %#v, want quoted sensitive read-only describe", verdict.Advisory) + } + if !strings.Contains(verdict.Hypothesis, "does not identify") || strings.Contains(strings.ToLower(verdict.Hypothesis), "authentication failure") { + t.Fatalf("hypothesis = %q, want explicit cause uncertainty", verdict.Hypothesis) + } + }) + } + + for _, reason := range []string{ + "ImagePullBackOff/registry-auth", + "ErrImagePull: unauthorized", + " ImagePullBackOff", + "ImagePullBackOff ", + "\tErrImagePull", + "ImagePull", + "PullBackOff", + } { + reason := reason + t.Run("reject "+reason, func(t *testing.T) { + t.Parallel() + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{observe(now, fleet.LensLive, "pod.reason", reason)}, + Coverage: covered(fleet.LensLive), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 0 { + t.Fatalf("caller-controlled reason %q yielded %#v", reason, result.Verdicts) + } + }) + } +} + +func TestEvaluateImagePullFailureAbstainsOnUnusableLiveEvidence(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC) + + for _, test := range []struct { + name string + observation Observation + coverage map[fleet.Lens]LensCoverage + }{ + {name: "stale observation", observation: func() Observation { + observation := observe(now, fleet.LensLive, "pod.reason", "ImagePullBackOff") + observation.Stale = true + return observation + }(), coverage: covered(fleet.LensLive)}, + {name: "stale coverage", observation: observe(now, fleet.LensLive, "pod.reason", "ImagePullBackOff"), coverage: map[fleet.Lens]LensCoverage{ + fleet.LensLive: {Available: true, Stale: true, Reason: "watch disconnected"}, + }}, + {name: "unavailable coverage", observation: observe(now, fleet.LensLive, "pod.reason", "ErrImagePull"), coverage: map[fleet.Lens]LensCoverage{ + fleet.LensLive: {Available: false, Reason: "context unreachable"}, + }}, + {name: "missing coverage", observation: observe(now, fleet.LensLive, "pod.reason", "ErrImagePull"), coverage: map[fleet.Lens]LensCoverage{}}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{test.observation}, + Coverage: test.coverage, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleImagePull || + result.Verdicts[0].Status != StatusUnconfirmed || + !slices.Equal(result.Verdicts[0].MissingLenses, []fleet.Lens{fleet.LensLive}) { + t.Fatalf("verdicts = %#v, want unconfirmed R7 with LIVE gap", result.Verdicts) + } + }) + } +} + +func TestEvaluateImagePullFailureDoesNotCorrelateFleetWide(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC) + const repoDigest = "registry.example/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + observations := make([]Observation, 0, 4) + for _, cluster := range []string{"alpha", "beta"} { + ref := testRef() + ref.Scope = cluster + observations = append(observations, + Observation{Ref: ref, Lens: fleet.LensLive, Key: "pod.reason", Value: "ImagePullBackOff", ObservedAt: now, Source: "kubeconfig"}, + Observation{Ref: ref, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now, Source: "kubeconfig"}, + ) + } + + result, err := Evaluate(Investigation{Workspace: fleet.LocalWorkspace, Observations: observations, Coverage: covered(fleet.LensLive)}) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 2 { + t.Fatalf("verdicts = %#v, want one R7 verdict per Pod", result.Verdicts) + } + for _, verdict := range result.Verdicts { + if verdict.Rule != RuleImagePull || verdict.FleetWide || len(verdict.Clusters) != 0 { + t.Fatalf("verdict = %#v, R7 must remain entity-local", verdict) + } + } +} + +func TestEvaluateArgoSyncFailureRuleIsBounded(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 30, 0, 0, time.UTC) + observation := Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "argocd", + Scope: "dev'; printf injected #", + Kind: "Application", + Namespace: "argocd'; printf injected #", + Name: "payments'; printf injected #", + }, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "sync-failed", + ObservedAt: now, Source: "fixture", + } + + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: []Observation{observation}, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 { + t.Fatalf("verdicts = %#v, want one R8 verdict", result.Verdicts) + } + verdict := result.Verdicts[0] + if verdict.Rule != RuleArgoSyncFail || verdict.Status != StatusConfirmed || verdict.FleetWide { + t.Fatalf("verdict = %#v, want confirmed entity-local R8", verdict) + } + if len(verdict.Citations) != 1 { + t.Fatalf("citations = %#v, want one exact TIMELINE citation", verdict.Citations) + } + citation := verdict.Citations[0] + if !reflect.DeepEqual(citation.Ref, observation.Ref) || citation.Lens != fleet.LensTimeline || + citation.Predicate != "change.kind" || citation.Observed != "sync-failed" || + citation.Source != observation.Source || !citation.ObservedAt.Equal(now) { + t.Fatalf("citation = %#v, want exact normalized source observation", citation) + } + wantCommand := "kubectl --context 'dev'\\''; printf injected #' describe application.argoproj.io 'payments'\\''; printf injected #' -n 'argocd'\\''; printf injected #'" + if verdict.Advisory.Command != wantCommand || verdict.Advisory.PRDiff != "" || !verdict.Advisory.Sensitive { + t.Fatalf("advisory = %#v, want quoted sensitive read-only Application describe", verdict.Advisory) + } + hypothesis := strings.ToLower(verdict.Hypothesis) + if !strings.Contains(hypothesis, "does not identify") { + t.Fatalf("hypothesis = %q, want explicit cause uncertainty", verdict.Hypothesis) + } + for _, overclaim := range []string{"authentication is", "network is", "rendering is", "hook is"} { + if strings.Contains(hypothesis, overclaim) { + t.Fatalf("hypothesis = %q, overclaims %q", verdict.Hypothesis, overclaim) + } + } +} + +func TestEvaluateArgoSyncFailureRejectsNearMisses(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 30, 0, 0, time.UTC) + for _, value := range []string{ + "OutOfSync", + "Unknown", + "Degraded", + "argocd-sync", + "sync_failed", + "sync-failure", + " sync-failed", + "sync-failed ", + "sync-failed/render-error", + "SYNC-FAILED", + "Sync-Failed", + } { + value := value + t.Run(value, func(t *testing.T) { + t.Parallel() + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{{ + Ref: fleet.ResourceRef{SourceKind: "argocd", Scope: "alpha", Kind: "Application", Namespace: "argocd", Name: "payments"}, + Lens: fleet.LensTimeline, Key: "change.kind", Value: value, ObservedAt: now, Source: "argocd", + }}, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 0 { + t.Fatalf("near-miss value %q yielded %#v", value, result.Verdicts) + } + }) + } +} + +func TestEvaluateArgoSyncFailureRequiresExactArgoApplicationApplicability(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 30, 0, 0, time.UTC) + base := Observation{ + Ref: fleet.ResourceRef{SourceKind: "argocd", Scope: "alpha", Kind: "Application", Namespace: "argocd", Name: "payments"}, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "sync-failed", ObservedAt: now, Source: "alpha", + } + for _, test := range []struct { + name string + sourceKind string + kind string + }{ + {name: "non-Argo source", sourceKind: "kubeconfig", kind: "Application"}, + {name: "source case mismatch", sourceKind: "ArgoCD", kind: "Application"}, + {name: "non-Application resource", sourceKind: "argocd", kind: "Deployment"}, + {name: "resource case mismatch", sourceKind: "argocd", kind: "application"}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + observation := base + observation.Ref.SourceKind = test.sourceKind + observation.Ref.Kind = test.kind + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: []Observation{observation}, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 0 { + t.Fatalf("source_kind=%q kind=%q yielded %#v, want no R8 advisory", test.sourceKind, test.kind, result.Verdicts) + } + }) + } +} + +func TestEvaluateArgoSyncFailureApplicabilityIsInputOrderIndependent(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 30, 0, 0, time.UTC) + valid := Observation{ + Ref: fleet.ResourceRef{SourceKind: "argocd", Scope: "alpha", Kind: "Application", Namespace: "argocd", Name: "payments"}, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "sync-failed", ObservedAt: now, Source: "alpha", + } + nonArgo := valid + nonArgo.Ref.SourceKind = "kubeconfig" + + for _, observations := range [][]Observation{{valid, nonArgo}, {nonArgo, valid}} { + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: observations, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleArgoSyncFail || + result.Verdicts[0].Citations[0].Ref.SourceKind != argoGraphSource { + t.Fatalf("verdicts = %#v, want one R8 verdict cited only to Argo evidence", result.Verdicts) + } + } +} + +func TestEvaluateArgoSyncFailureAbstainsOnUnusableTimelineEvidence(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 30, 0, 0, time.UTC) + base := Observation{ + Ref: fleet.ResourceRef{SourceKind: "argocd", Scope: "alpha", Kind: "Application", Namespace: "argocd", Name: "payments"}, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "sync-failed", ObservedAt: now, Source: "argocd", + } + for _, test := range []struct { + name string + observation Observation + coverage map[fleet.Lens]LensCoverage + }{ + {name: "stale observation", observation: func() Observation { + observation := base + observation.Stale = true + return observation + }(), coverage: covered(fleet.LensTimeline)}, + {name: "stale coverage", observation: base, coverage: map[fleet.Lens]LensCoverage{ + fleet.LensTimeline: {Available: true, Stale: true, Reason: "collector stale"}, + }}, + {name: "unavailable coverage", observation: base, coverage: map[fleet.Lens]LensCoverage{ + fleet.LensTimeline: {Reason: "Argo evidence unavailable"}, + }}, + {name: "missing coverage", observation: base, coverage: map[fleet.Lens]LensCoverage{}}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: []Observation{test.observation}, Coverage: test.coverage, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleArgoSyncFail || + result.Verdicts[0].Status != StatusUnconfirmed || + !slices.Equal(result.Verdicts[0].MissingLenses, []fleet.Lens{fleet.LensTimeline}) { + t.Fatalf("verdicts = %#v, want unconfirmed R8 with TIMELINE gap", result.Verdicts) + } + }) + } +} + +func TestEvaluateArgoSyncFailureDoesNotCorrelateFleetWide(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 30, 0, 0, time.UTC) + const repoDigest = "registry.example/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + observations := make([]Observation, 0, 4) + for _, cluster := range []string{"alpha", "beta"} { + ref := fleet.ResourceRef{SourceKind: "argocd", Scope: cluster, Kind: "Application", Namespace: "argocd", Name: "payments"} + observations = append(observations, + Observation{Ref: ref, Lens: fleet.LensTimeline, Key: "change.kind", Value: "sync-failed", ObservedAt: now, Source: "argocd"}, + Observation{Ref: ref, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now, Source: "fixture"}, + ) + } + + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: observations, + Coverage: covered(fleet.LensLive, fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 2 { + t.Fatalf("verdicts = %#v, want one R8 verdict per Application", result.Verdicts) + } + scopes := map[string]bool{} + for _, verdict := range result.Verdicts { + if verdict.Rule != RuleArgoSyncFail || verdict.FleetWide || len(verdict.Clusters) != 0 { + t.Fatalf("verdict = %#v, R8 must remain entity-local", verdict) + } + scopes[verdict.Ref.Scope] = true + } + if !reflect.DeepEqual(scopes, map[string]bool{"alpha": true, "beta": true}) { + t.Fatalf("verdict scopes = %#v, want alpha and beta", scopes) + } +} + +func TestEvaluateWorkflowRunFailureRuleIsBounded(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + observation := Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI'; printf injected #", Name: "sith#30433642-attempt-2'; printf injected #", + }, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "workflow-run-failed", + ObservedAt: now, Source: "github.com", + } + + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: []Observation{observation}, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 { + t.Fatalf("verdicts = %#v, want one R9 verdict", result.Verdicts) + } + verdict := result.Verdicts[0] + if verdict.Rule != RuleWorkflowFail || verdict.Status != StatusConfirmed || verdict.FleetWide { + t.Fatalf("verdict = %#v, want confirmed entity-local R9", verdict) + } + if len(verdict.Citations) != 1 { + t.Fatalf("citations = %#v, want one exact TIMELINE citation", verdict.Citations) + } + citation := verdict.Citations[0] + if !reflect.DeepEqual(citation.Ref, observation.Ref) || citation.Lens != fleet.LensTimeline || + citation.Predicate != "change.kind" || citation.Observed != "workflow-run-failed" || + citation.Source != observation.Source || !citation.ObservedAt.Equal(now) { + t.Fatalf("citation = %#v, want exact normalized source observation", citation) + } + wantCommand := "inspect GitHub Actions workflow run 'sith#30433642-attempt-2'\\''; printf injected #' owned by 'ArdurAI'\\''; printf injected #' and its failed jobs and logs before considering a rerun" + if verdict.Advisory.Command != wantCommand || verdict.Advisory.PRDiff != "" || !verdict.Advisory.Sensitive { + t.Fatalf("advisory = %#v, want quoted sensitive read-only inspection guidance", verdict.Advisory) + } + hypothesis := strings.ToLower(verdict.Hypothesis) + if !strings.Contains(hypothesis, "does not identify") { + t.Fatalf("hypothesis = %q, want explicit cause uncertainty", verdict.Hypothesis) + } + for _, overclaim := range []string{"code is", "credential is", "permission is", "capacity is", "dependency is"} { + if strings.Contains(hypothesis, overclaim) { + t.Fatalf("hypothesis = %q, overclaims %q", verdict.Hypothesis, overclaim) + } + } +} + +func TestEvaluateWorkflowRunFailureRejectsNearMisses(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + for _, value := range []string{ + "failure", "timed_out", "startup_failure", "pipeline-failed", "workflow-failed", + "workflow_run_failed", " workflow-run-failed", "workflow-run-failed ", + "workflow-run-failed/job", "WORKFLOW-RUN-FAILED", "Workflow-Run-Failed", + } { + value := value + t.Run(value, func(t *testing.T) { + t.Parallel() + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{{ + Ref: fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + }, + Lens: fleet.LensTimeline, Key: "change.kind", Value: value, + ObservedAt: now, Source: "github.com", + }}, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 0 { + t.Fatalf("near-miss value %q yielded %#v", value, result.Verdicts) + } + }) + } +} + +func TestEvaluateWorkflowRunFailureRequiresExactGitHubApplicability(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + base := Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + }, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "workflow-run-failed", + ObservedAt: now, Source: "github.com", + } + for _, test := range []struct { + name string + sourceKind string + kind string + }{ + {name: "non-GitHub source", sourceKind: "kubeconfig", kind: "WorkflowRun"}, + {name: "source case mismatch", sourceKind: "GitHub", kind: "WorkflowRun"}, + {name: "non-workflow resource", sourceKind: "github", kind: "PullRequest"}, + {name: "resource case mismatch", sourceKind: "github", kind: "workflowrun"}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + observation := base + observation.Ref.SourceKind = test.sourceKind + observation.Ref.Kind = test.kind + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: []Observation{observation}, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 0 { + t.Fatalf("source_kind=%q kind=%q yielded %#v, want no R9 advisory", test.sourceKind, test.kind, result.Verdicts) + } + }) + } +} + +func TestEvaluateWorkflowRunFailureApplicabilityIsInputOrderIndependent(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + valid := Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + }, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "workflow-run-failed", + ObservedAt: now, Source: "github.com", + } + unrelated := valid + unrelated.Ref.SourceKind = "kubeconfig" + + for _, observations := range [][]Observation{{valid, unrelated}, {unrelated, valid}} { + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: observations, + Coverage: covered(fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleWorkflowFail || + result.Verdicts[0].Citations[0].Ref.SourceKind != githubGraphSource { + t.Fatalf("verdicts = %#v, want one R9 verdict cited only to GitHub evidence", result.Verdicts) + } + } +} + +func TestEvaluateWorkflowRunFailureAbstainsOnUnusableTimelineEvidence(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + base := Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + }, + Lens: fleet.LensTimeline, Key: "change.kind", Value: "workflow-run-failed", + ObservedAt: now, Source: "github.com", + } + for _, test := range []struct { + name string + observation Observation + coverage map[fleet.Lens]LensCoverage + }{ + {name: "stale observation", observation: func() Observation { + observation := base + observation.Stale = true + return observation + }(), coverage: covered(fleet.LensTimeline)}, + {name: "stale coverage", observation: base, coverage: map[fleet.Lens]LensCoverage{ + fleet.LensTimeline: {Available: true, Stale: true, Reason: "collector stale"}, + }}, + {name: "unavailable coverage", observation: base, coverage: map[fleet.Lens]LensCoverage{ + fleet.LensTimeline: {Reason: "workflow-run evidence unavailable"}, + }}, + {name: "missing coverage", observation: base, coverage: map[fleet.Lens]LensCoverage{}}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: []Observation{test.observation}, Coverage: test.coverage, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleWorkflowFail || + result.Verdicts[0].Status != StatusUnconfirmed || + !slices.Equal(result.Verdicts[0].MissingLenses, []fleet.Lens{fleet.LensTimeline}) { + t.Fatalf("verdicts = %#v, want unconfirmed R9 with TIMELINE gap", result.Verdicts) + } + }) + } +} + +func TestEvaluateWorkflowRunFailureDoesNotCorrelateFleetWide(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + const repoDigest = "registry.example/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + observations := make([]Observation, 0, 4) + for _, host := range []string{"github.com", "github.example.com"} { + ref := fleet.ResourceRef{ + SourceKind: "github", Scope: host, Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + } + observations = append(observations, + Observation{Ref: ref, Lens: fleet.LensTimeline, Key: "change.kind", Value: "workflow-run-failed", ObservedAt: now, Source: host}, + Observation{Ref: ref, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now, Source: "fixture"}, + ) + } + + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, Observations: observations, + Coverage: covered(fleet.LensLive, fleet.LensTimeline), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 2 { + t.Fatalf("verdicts = %#v, want one R9 verdict per workflow run", result.Verdicts) + } + for _, verdict := range result.Verdicts { + if verdict.Rule != RuleWorkflowFail || verdict.FleetWide || len(verdict.Clusters) != 0 { + t.Fatalf("verdict = %#v, R9 must remain source-native and entity-local", verdict) + } + } +} + func TestEvaluateAbstainsWhenRequiredLensIsStale(t *testing.T) { now := time.Now().UTC() result, err := Evaluate(Investigation{ @@ -148,6 +756,81 @@ func TestEvaluateRanksFleetDigestCorrelationFirst(t *testing.T) { if len(result.Verdicts) != 3 || !result.Verdicts[0].FleetWide || !slices.Equal(result.Verdicts[0].Clusters, []string{"alpha", "beta"}) { t.Fatalf("verdicts = %#v, want fleet-wide correlation first", result.Verdicts) } + digestCitations := citationsForPredicate(result.Verdicts[0].Citations, fleet.OTelContainerImageRepoDigests) + if len(digestCitations) != 2 || digestCitations[0].Ref.Scope != "alpha" || digestCitations[1].Ref.Scope != "beta" { + t.Fatalf("digest citations = %#v, want one fresh citation per correlated cluster", digestCitations) + } +} + +func TestEvaluateRejectsStaleFleetDigestCorrelation(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + const repoDigest = "registry.example/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + observations := make([]Observation, 0, 4) + for _, cluster := range []string{"alpha", "beta"} { + ref := testRef() + ref.Scope = cluster + digest := Observation{ + Ref: ref, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, + Value: repoDigest, ObservedAt: now, Source: "kubeconfig", + } + if cluster == "beta" { + digest.Stale = true + } + observations = append(observations, + Observation{Ref: ref, Lens: fleet.LensLive, Key: "pod.failure", Value: "CrashLoopBackOff", ObservedAt: now, Source: "kubeconfig"}, + digest, + ) + } + + result, err := Evaluate(Investigation{Workspace: fleet.LocalWorkspace, Observations: observations, Coverage: covered(fleet.LensLive)}) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + for _, verdict := range result.Verdicts { + if verdict.FleetWide { + t.Fatalf("stale digest yielded fleet-wide verdict %#v", verdict) + } + } +} + +func TestEvaluateFleetDigestCorrelationDeduplicatesDeterministically(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + const repoDigest = "registry.example/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + alpha, beta := testRef(), testRef() + alpha.Scope, beta.Scope = "alpha", "beta" + stale := Observation{Ref: alpha, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now.Add(-2 * time.Hour), Source: "cache", Stale: true} + older := Observation{Ref: alpha, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now.Add(-time.Hour), Source: "watch"} + freshest := Observation{Ref: alpha, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now, Source: "query"} + tiedSource := freshest + tiedSource.Ref.SourceKind = "z-source" + tiedSource.Ref.Attributes = map[string]string{"container": "sidecar"} + observations := []Observation{ + {Ref: alpha, Lens: fleet.LensLive, Key: "pod.failure", Value: "CrashLoopBackOff", ObservedAt: now, Source: "query"}, + stale, older, freshest, tiedSource, + {Ref: beta, Lens: fleet.LensLive, Key: "pod.failure", Value: "CrashLoopBackOff", ObservedAt: now, Source: "query"}, + {Ref: beta, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now, Source: "query"}, + } + reversed := append([]Observation(nil), observations...) + slices.Reverse(reversed) + + first, err := Evaluate(Investigation{Workspace: fleet.LocalWorkspace, Observations: observations, Coverage: covered(fleet.LensLive)}) + if err != nil { + t.Fatalf("Evaluate() first error = %v", err) + } + second, err := Evaluate(Investigation{Workspace: fleet.LocalWorkspace, Observations: reversed, Coverage: covered(fleet.LensLive)}) + if err != nil { + t.Fatalf("Evaluate() reversed error = %v", err) + } + if len(first.Verdicts) == 0 || len(second.Verdicts) == 0 || !reflect.DeepEqual(first.Verdicts[0], second.Verdicts[0]) { + t.Fatalf("top verdict changed with duplicate input order:\nfirst: %#v\nsecond: %#v", first.Verdicts, second.Verdicts) + } + digestCitations := citationsForPredicate(first.Verdicts[0].Citations, fleet.OTelContainerImageRepoDigests) + if len(digestCitations) != 2 || digestCitations[0].Ref.Scope != "alpha" || digestCitations[1].Ref.Scope != "beta" || + digestCitations[0].ObservedAt != freshest.ObservedAt || digestCitations[0].Stale { + t.Fatalf("digest citations = %#v, want deduplicated freshest evidence", digestCitations) + } } func TestEvaluateRejectsUnprovenFleetImageCorrelation(t *testing.T) { @@ -226,6 +909,16 @@ func observe(at time.Time, lens fleet.Lens, key, value string) Observation { return Observation{Ref: testRef(), Lens: lens, Key: key, Value: value, ObservedAt: at, Source: "fixture"} } +func citationsForPredicate(citations []Citation, predicate string) []Citation { + result := make([]Citation, 0) + for _, citation := range citations { + if citation.Predicate == predicate { + result = append(result, citation) + } + } + return result +} + func testRef() fleet.ResourceRef { return fleet.ResourceRef{SourceKind: "kubeconfig", Scope: "alpha", Kind: "Pod", Namespace: "prod", Name: "payments-0"} } diff --git a/internal/brain/graph.go b/internal/brain/graph.go new file mode 100644 index 0000000..632f960 --- /dev/null +++ b/internal/brain/graph.go @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + argoGraphSource = "argocd" + argoGraphProtocolVersion = "1.0.0" + maxArgoChangePayload = 16 << 10 + + githubGraphSource = "github" + githubWorkflowGraphProtocolVersion = "workflow-runs/2026-03-10" + maxGitHubWorkflowChangePayload = 4 << 10 + + elasticsearchGraphSource = "elasticsearch" + elasticsearchGraphProtocolVersion = "search/ecs-v1" + maxElasticsearchLogCausePayload = 4 << 10 + maxElasticsearchCauseCount = 128 + maxElasticsearchCauseWindow = 15 * time.Minute + maxElasticsearchClockSkew = 5 * time.Minute + + maxGraphJSONDepth = 64 +) + +type argoChangePayload struct { + ChangeKind string `json:"change_kind"` + Revision string `json:"revision,omitempty"` + Phase string `json:"phase,omitempty"` + EventAt time.Time `json:"event_at"` + HistoryID string `json:"history_id,omitempty"` + HistoryTruncated bool `json:"history_truncated,omitempty"` +} + +type githubWorkflowChangePayload struct { + RunID int64 `json:"run_id"` + WorkflowID int64 `json:"workflow_id"` + RunAttempt int64 `json:"run_attempt"` + ChangeKind string `json:"change_kind"` + Conclusion string `json:"conclusion"` + EventAt time.Time `json:"event_at"` +} + +type elasticsearchLogCausePayload struct { + Key string `json:"key"` + Value string `json:"value"` + Count int `json:"count"` + FirstEventAt time.Time `json:"first_event_at"` + LastEventAt time.Time `json:"last_event_at"` + Container string `json:"container,omitempty"` +} + +// FromGraphFacts converts reviewed graph facts into normalized brain observations while preserving +// caller-declared coverage. It performs no discovery, I/O, coverage inference, or mutation. +func FromGraphFacts( + workspace string, + facts []fleet.GraphFact, + coverage map[fleet.Lens]LensCoverage, +) (Investigation, error) { + if strings.TrimSpace(workspace) == "" { + return Investigation{}, fmt.Errorf("project graph facts: workspace is required") + } + graph, err := fleet.NewGraph(workspace, facts) + if err != nil { + return Investigation{}, fmt.Errorf("project graph facts: %w", err) + } + + input := Investigation{ + Workspace: workspace, + Coverage: make(map[fleet.Lens]LensCoverage, len(coverage)), + } + for lens, state := range coverage { + if !lens.Valid() { + return Investigation{}, fmt.Errorf("project graph facts: coverage lens %q is invalid", lens) + } + input.Coverage[lens] = state + } + for _, node := range graph.Nodes { + for _, fact := range node.Facts { + observation, present, err := observationFromGraphFact(fact) + if err != nil { + return Investigation{}, err + } + if present { + input.Observations = append(input.Observations, observation) + } + } + } + for _, fact := range graph.Unattached { + observation, present, err := observationFromGraphFact(fact) + if err != nil { + return Investigation{}, err + } + if present { + input.Observations = append(input.Observations, observation) + } + } + return input, nil +} + +func observationFromGraphFact(fact fleet.GraphFact) (Observation, bool, error) { + provenance := fact.Fact.Provenance + ref := fact.Fact.Ref + if provenance.ProtocolV == elasticsearchGraphProtocolVersion { + return observationFromElasticsearchLogCauseGraphFact(fact) + } + if provenance.ProtocolV == githubWorkflowGraphProtocolVersion { + return observationFromGitHubWorkflowGraphFact(fact) + } + if (ref.SourceKind == elasticsearchGraphSource || provenance.Adapter == elasticsearchGraphSource) && + ref.Kind == "LogSignal" { + return observationFromElasticsearchLogCauseGraphFact(fact) + } + if fact.Fact.Kind != fleet.FactChange || fact.Lens != fleet.LensTimeline { + return Observation{}, false, nil + } + if ref.SourceKind == argoGraphSource || provenance.Adapter == argoGraphSource { + return observationFromArgoGraphFact(fact) + } + if (ref.SourceKind == githubGraphSource || provenance.Adapter == githubGraphSource) && + ref.Kind == "WorkflowRun" { + return observationFromGitHubWorkflowGraphFact(fact) + } + return Observation{}, false, nil +} + +func observationFromArgoGraphFact(fact fleet.GraphFact) (Observation, bool, error) { + provenance := fact.Fact.Provenance + ref := fact.Fact.Ref + if ref.SourceKind != argoGraphSource && provenance.Adapter != argoGraphSource { + return Observation{}, false, nil + } + if ref.SourceKind != argoGraphSource || provenance.Adapter != argoGraphSource { + return Observation{}, false, fmt.Errorf("project Argo change fact: source and provenance must both be %q", argoGraphSource) + } + if provenance.ProtocolV != argoGraphProtocolVersion { + return Observation{}, false, fmt.Errorf("project Argo change fact: unsupported protocol version %q", provenance.ProtocolV) + } + if fact.Entity == nil { + return Observation{}, false, fmt.Errorf("project Argo change fact: Application entity is required") + } + entity := *fact.Entity + if ref.Kind != "Application" || ref.Namespace == "" || + entity.Cluster != ref.Scope || entity.Namespace != ref.Namespace || + entity.Kind != ref.Kind || entity.Name != ref.Name || + entity.Pod != "" || entity.Node != "" || entity.ImageDigest != "" { + return Observation{}, false, fmt.Errorf("project Argo change fact: source and entity must identify the same Application") + } + if fact.Fact.Source != ref.Scope { + return Observation{}, false, fmt.Errorf("project Argo change fact: evidence source must match the Application scope") + } + if len(ref.Attributes) != 0 || len(fact.Fact.Display) != 0 { + return Observation{}, false, fmt.Errorf("project Argo change fact: unexpected attributes or display fields") + } + if len(fact.Fact.Observed) == 0 || len(fact.Fact.Observed) > maxArgoChangePayload { + return Observation{}, false, fmt.Errorf("project Argo change fact: payload must be between 1 and %d bytes", maxArgoChangePayload) + } + + payload, err := decodeArgoChangePayload(fact.Fact.Observed) + if err != nil { + return Observation{}, false, err + } + if payload.EventAt.IsZero() || !payload.EventAt.Equal(fact.Fact.ObservedAt) { + return Observation{}, false, fmt.Errorf("project Argo change fact: payload event time must equal the fact observation time") + } + switch payload.ChangeKind { + case "argocd-sync": + switch payload.Phase { + case "", "Succeeded", "Running", "Terminating": + return Observation{}, false, nil + default: + return Observation{}, false, fmt.Errorf("project Argo change fact: phase %q is inconsistent with an Argo sync change", payload.Phase) + } + case "sync-failed": + if payload.Phase != "Failed" && payload.Phase != "Error" { + return Observation{}, false, fmt.Errorf("project Argo change fact: phase %q does not prove a failed sync operation", payload.Phase) + } + if payload.HistoryID != "" || payload.HistoryTruncated { + return Observation{}, false, fmt.Errorf("project Argo change fact: failed operation must not carry history metadata") + } + nativePrefix := ref.Scope + "/" + ref.Namespace + "/" + ref.Name + "#operation/" + operationID, found := strings.CutPrefix(provenance.NativeID, nativePrefix) + if !found || strings.TrimSpace(operationID) == "" { + return Observation{}, false, fmt.Errorf("project Argo change fact: operation provenance is inconsistent with the Application") + } + default: + return Observation{}, false, fmt.Errorf("project Argo change fact: unsupported change kind %q", payload.ChangeKind) + } + + return Observation{ + Ref: ref, + Lens: fleet.LensTimeline, + Key: "change.kind", + Value: "sync-failed", + ObservedAt: fact.Fact.ObservedAt, + Source: fact.Fact.Source, + Stale: fact.Fact.Stale, + }, true, nil +} + +func observationFromElasticsearchLogCauseGraphFact(fact fleet.GraphFact) (Observation, bool, error) { + provenance := fact.Fact.Provenance + ref := fact.Fact.Ref + if fact.Fact.Kind != fleet.FactDerived || fact.Lens != fleet.LensTelemetry { + return Observation{}, false, fmt.Errorf("project Elasticsearch log-cause fact: fact must be derived TELEMETRY") + } + if ref.SourceKind != elasticsearchGraphSource || provenance.Adapter != elasticsearchGraphSource { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: source and provenance must both be %q", + elasticsearchGraphSource, + ) + } + if provenance.ProtocolV != elasticsearchGraphProtocolVersion { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: unsupported protocol version %q", + provenance.ProtocolV, + ) + } + if provenance.DeepLink != "" || provenance.Collector != "" { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: unexpected provenance metadata", + ) + } + if ref.Kind != "LogSignal" || ref.Scope == "" || ref.Namespace == "" { + return Observation{}, false, fmt.Errorf("project Elasticsearch log-cause fact: resource identity is invalid") + } + if fact.Entity == nil { + return Observation{}, false, fmt.Errorf("project Elasticsearch log-cause fact: Pod entity is required") + } + entity := *fact.Entity + if entity.Cluster != ref.Scope || entity.Namespace != ref.Namespace || entity.Pod == "" || + entity.Kind != "" || entity.Name != "" || entity.Node != "" || entity.ImageDigest != "" { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: source and entity must identify one exact Pod", + ) + } + if fact.Fact.Source != ref.Scope { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: evidence source must match the Elasticsearch scope", + ) + } + if len(ref.Attributes) != 0 || len(fact.Fact.Display) != 0 { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: unexpected attributes or display fields", + ) + } + if len(fact.Fact.Observed) == 0 || len(fact.Fact.Observed) > maxElasticsearchLogCausePayload { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: payload must be between 1 and %d bytes", + maxElasticsearchLogCausePayload, + ) + } + + payload, err := decodeElasticsearchLogCausePayload(fact.Fact.Observed) + if err != nil { + return Observation{}, false, err + } + if payload.Key != "logs.cause" { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: unsupported key %q", + payload.Key, + ) + } + switch payload.Value { + case "panic", "missing-config", "dependency-failure": + default: + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: unsupported cause %q", + payload.Value, + ) + } + if payload.Count <= 0 || payload.Count > maxElasticsearchCauseCount { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: count must be between 1 and %d", + maxElasticsearchCauseCount, + ) + } + if payload.FirstEventAt.IsZero() || payload.LastEventAt.IsZero() || + payload.LastEventAt.Before(payload.FirstEventAt) || + payload.LastEventAt.Sub(payload.FirstEventAt) > maxElasticsearchCauseWindow { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: event interval is invalid", + ) + } + if payload.LastEventAt.After(fact.Fact.ObservedAt.Add(maxElasticsearchClockSkew)) { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: last event exceeds collection clock skew", + ) + } + if payload.Container != "" && !validElasticsearchGraphText(payload.Container) { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: container identity is invalid", + ) + } + digest, valid := validElasticsearchNativeID(provenance.NativeID) + if !valid { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: native identity is inconsistent with the log signal", + ) + } + wantDigest, err := elasticsearchLogCauseDigest(fact, payload) + if err != nil || digest != wantDigest || ref.Name != "log-"+wantDigest[:32] { + return Observation{}, false, fmt.Errorf( + "project Elasticsearch log-cause fact: native identity is inconsistent with the log signal", + ) + } + + return Observation{ + Ref: fleet.ResourceRef{ + SourceKind: elasticsearchGraphSource, + Scope: entity.Cluster, + Kind: "Pod", + Namespace: entity.Namespace, + Name: entity.Pod, + }, + Lens: fleet.LensTelemetry, + Key: "logs.cause", + Value: payload.Value, + ObservedAt: payload.LastEventAt, + Source: fact.Fact.Source, + Stale: fact.Fact.Stale, + }, true, nil +} + +func observationFromGitHubWorkflowGraphFact(fact fleet.GraphFact) (Observation, bool, error) { + provenance := fact.Fact.Provenance + ref := fact.Fact.Ref + if fact.Fact.Kind != fleet.FactChange || fact.Lens != fleet.LensTimeline { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: fact must be a TIMELINE change") + } + if ref.SourceKind != githubGraphSource || provenance.Adapter != githubGraphSource { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: source and provenance must both be %q", githubGraphSource) + } + if provenance.ProtocolV != githubWorkflowGraphProtocolVersion { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: unsupported protocol version %q", provenance.ProtocolV) + } + if fact.Entity != nil { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: source-native fact must be unattached") + } + if ref.Kind != "WorkflowRun" || !validGitHubGraphHost(ref.Scope) || + !validGitHubGraphPathComponent(ref.Namespace) { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: resource identity is invalid") + } + if fact.Fact.Source != ref.Scope { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: evidence source must match the GitHub host") + } + if len(ref.Attributes) != 0 || len(fact.Fact.Display) != 0 { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: unexpected attributes or display fields") + } + if len(fact.Fact.Observed) == 0 || len(fact.Fact.Observed) > maxGitHubWorkflowChangePayload { + return Observation{}, false, fmt.Errorf( + "project GitHub workflow-run fact: payload must be between 1 and %d bytes", + maxGitHubWorkflowChangePayload, + ) + } + + payload, err := decodeGitHubWorkflowChangePayload(fact.Fact.Observed) + if err != nil { + return Observation{}, false, err + } + if payload.RunID <= 0 || payload.WorkflowID <= 0 || payload.RunAttempt <= 0 { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: run, workflow, and attempt IDs must be positive") + } + if payload.ChangeKind != "workflow-run-failed" { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: unsupported change kind %q", payload.ChangeKind) + } + switch payload.Conclusion { + case "failure", "timed_out", "startup_failure": + default: + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: conclusion %q does not prove failure", payload.Conclusion) + } + if payload.EventAt.IsZero() || !payload.EventAt.Equal(fact.Fact.ObservedAt) { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: payload event time must equal the fact observation time") + } + + repository, runIdentity, found := strings.Cut(ref.Name, "#") + if !found || strings.Contains(runIdentity, "#") || !validGitHubGraphPathComponent(repository) || + strings.HasSuffix(strings.ToLower(repository), ".git") { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: resource name is invalid") + } + wantRunIdentity := fmt.Sprintf("%d-attempt-%d", payload.RunID, payload.RunAttempt) + if runIdentity != wantRunIdentity || provenance.NativeID != ref.Namespace+"/"+ref.Name { + return Observation{}, false, fmt.Errorf("project GitHub workflow-run fact: native identity is inconsistent with the workflow run") + } + + return Observation{ + Ref: ref, + Lens: fleet.LensTimeline, + Key: "change.kind", + Value: "workflow-run-failed", + ObservedAt: fact.Fact.ObservedAt, + Source: fact.Fact.Source, + Stale: fact.Fact.Stale, + }, true, nil +} + +func decodeArgoChangePayload(raw json.RawMessage) (argoChangePayload, error) { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var payload argoChangePayload + if err := decoder.Decode(&payload); err != nil { + return argoChangePayload{}, fmt.Errorf("project Argo change fact: decode payload: %w", err) + } + var trailer json.RawMessage + if err := decoder.Decode(&trailer); err != io.EOF { + if err == nil { + return argoChangePayload{}, fmt.Errorf("project Argo change fact: payload must contain one JSON value") + } + return argoChangePayload{}, fmt.Errorf("project Argo change fact: decode payload trailer: %w", err) + } + return payload, nil +} + +func decodeGitHubWorkflowChangePayload(raw json.RawMessage) (githubWorkflowChangePayload, error) { + if err := rejectDuplicateGraphJSON(raw); err != nil { + return githubWorkflowChangePayload{}, fmt.Errorf("project GitHub workflow-run fact: decode payload: %w", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return githubWorkflowChangePayload{}, fmt.Errorf("project GitHub workflow-run fact: decode payload fields: %w", err) + } + for field := range fields { + switch field { + case "run_id", "workflow_id", "run_attempt", "change_kind", "conclusion", "event_at": + default: + return githubWorkflowChangePayload{}, fmt.Errorf("project GitHub workflow-run fact: payload field %q is unsupported", field) + } + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var payload githubWorkflowChangePayload + if err := decoder.Decode(&payload); err != nil { + return githubWorkflowChangePayload{}, fmt.Errorf("project GitHub workflow-run fact: decode payload: %w", err) + } + var trailer json.RawMessage + if err := decoder.Decode(&trailer); err != io.EOF { + if err == nil { + return githubWorkflowChangePayload{}, fmt.Errorf("project GitHub workflow-run fact: payload must contain one JSON value") + } + return githubWorkflowChangePayload{}, fmt.Errorf("project GitHub workflow-run fact: decode payload trailer: %w", err) + } + return payload, nil +} + +func decodeElasticsearchLogCausePayload(raw json.RawMessage) (elasticsearchLogCausePayload, error) { + if !utf8.Valid(raw) { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: decode payload: invalid UTF-8", + ) + } + if err := rejectDuplicateGraphJSON(raw); err != nil { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: decode payload: %w", + err, + ) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: decode payload fields: %w", + err, + ) + } + for field := range fields { + switch field { + case "key", "value", "count", "first_event_at", "last_event_at", "container": + default: + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: payload field %q is unsupported", + field, + ) + } + } + if container, present := fields["container"]; present && bytes.Equal(bytes.TrimSpace(container), []byte("null")) { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: container must be a non-null string", + ) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + var payload elasticsearchLogCausePayload + if err := decoder.Decode(&payload); err != nil { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: decode payload: %w", + err, + ) + } + var trailer json.RawMessage + if err := decoder.Decode(&trailer); err != io.EOF { + if err == nil { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: payload must contain one JSON value", + ) + } + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: decode payload trailer: %w", + err, + ) + } + if _, present := fields["container"]; present && payload.Container == "" { + return elasticsearchLogCausePayload{}, fmt.Errorf( + "project Elasticsearch log-cause fact: container must be a non-empty string when present", + ) + } + return payload, nil +} + +func validElasticsearchNativeID(nativeID string) (string, bool) { + digest, found := strings.CutPrefix(nativeID, "sha256:") + if !found || len(digest) != 64 { + return "", false + } + for _, character := range digest { + if (character < '0' || character > '9') && (character < 'a' || character > 'f') { + return "", false + } + } + return digest, true +} + +func elasticsearchLogCauseDigest(fact fleet.GraphFact, payload elasticsearchLogCausePayload) (string, error) { + identity, err := json.Marshal(struct { + Workspace string `json:"workspace"` + Scope string `json:"scope"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` + Container string `json:"container,omitempty"` + Cause string `json:"cause"` + Count int `json:"count"` + FirstEventAt time.Time `json:"first_event_at"` + LastEventAt time.Time `json:"last_event_at"` + ObservedAt time.Time `json:"observed_at"` + }{ + Workspace: fact.Fact.Workspace, Scope: fact.Fact.Ref.Scope, Namespace: fact.Fact.Ref.Namespace, + Pod: fact.Entity.Pod, Container: payload.Container, Cause: payload.Value, Count: payload.Count, + FirstEventAt: payload.FirstEventAt, LastEventAt: payload.LastEventAt, + ObservedAt: fact.Fact.ObservedAt, + }) + if err != nil { + return "", err + } + digest := sha256.Sum256(identity) + return hex.EncodeToString(digest[:]), nil +} + +func validElasticsearchGraphText(value string) bool { + if value == "" || len(value) > 253 || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return false + } + for _, character := range value { + if unicode.IsControl(character) { + return false + } + } + return true +} + +func rejectDuplicateGraphJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueGraphJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueGraphJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxGraphJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxGraphJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueGraphJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueGraphJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || (delimiter == '{' && closing != json.Delim('}')) || + (delimiter == '[' && closing != json.Delim(']')) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func validGitHubGraphHost(value string) bool { + if value == "" || len(value) > 253 || strings.TrimSpace(value) != value || strings.ToLower(value) != value || + strings.HasPrefix(value, ".") || strings.HasSuffix(value, ".") { + return false + } + for _, label := range strings.Split(value, ".") { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for _, character := range label { + if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '-' { + return false + } + } + } + return true +} + +func validGitHubGraphPathComponent(value string) bool { + if value == "" || len(value) > 100 || strings.TrimSpace(value) != value || value == "." || value == ".." { + return false + } + for _, character := range value { + if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && character != '-' && character != '_' && character != '.' { + return false + } + } + return true +} diff --git a/internal/brain/graph_elasticsearch_test.go b/internal/brain/graph_elasticsearch_test.go new file mode 100644 index 0000000..912a32f --- /dev/null +++ b/internal/brain/graph_elasticsearch_test.go @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + connectorelasticsearch "github.com/ArdurAI/sith/internal/connector/elasticsearch" + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestFromGraphFactsProjectsReviewedElasticsearchLogCausesIntoR3(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 123000000, time.UTC) + tests := []struct { + cause string + message string + }{ + {cause: "panic", message: "panic: discard-raw-secret"}, + {cause: "missing-config", message: "missing required environment variable DISCARD_RAW_SECRET"}, + {cause: "dependency-failure", message: "failed to connect to discard-raw-dependency"}, + } + for _, test := range tests { + test := test + t.Run(test.cause, func(t *testing.T) { + t.Parallel() + facts := projectedElasticsearchLogCause(t, test.message, eventAt) + input, err := FromGraphFacts( + fleet.LocalWorkspace, + facts, + covered(fleet.LensLive, fleet.LensTelemetry), + ) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 1 { + t.Fatalf("observations = %#v, want one sanitized log cause", input.Observations) + } + observation := input.Observations[0] + wantRef := fleet.ResourceRef{ + SourceKind: "elasticsearch", Scope: "cluster-a", Kind: "Pod", + Namespace: "payments", Name: "api-0", + } + if !reflect.DeepEqual(observation.Ref, wantRef) || + observation.Lens != fleet.LensTelemetry || + observation.Key != "logs.cause" || + observation.Value != test.cause || + !observation.ObservedAt.Equal(eventAt) || + observation.Source != "cluster-a" || + observation.Stale { + t.Fatalf("observation = %#v, want exact Pod-scoped Elasticsearch cause", observation) + } + + input.Observations = append(input.Observations, crashLoopObservation(eventAt.Add(time.Minute))) + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 { + t.Fatalf("verdicts = %#v, want one R3 verdict", result.Verdicts) + } + verdict := result.Verdicts[0] + if verdict.Rule != RuleCrashLoop || verdict.Status != StatusConfirmed || + verdict.Score != 6 || verdict.FleetWide { + t.Fatalf("verdict = %#v, want confirmed entity-local R3", verdict) + } + citations := citationsForPredicate(verdict.Citations, "logs.cause") + if len(citations) != 1 || citations[0].Observed != test.cause || + !citations[0].ObservedAt.Equal(eventAt) || + citations[0].Source != "cluster-a" || + citations[0].Stale { + t.Fatalf("log citations = %#v, want one fresh closed cause", citations) + } + + encoded, err := json.Marshal(struct { + Input Investigation `json:"input"` + Result Result `json:"result"` + }{Input: input, Result: result}) + if err != nil { + t.Fatalf("marshal projected result: %v", err) + } + for _, discarded := range []string{ + "discard-raw-secret", + "DISCARD_RAW_SECRET", + "discard-raw-dependency", + `"count"`, + `"container"`, + `"first_event_at"`, + `"last_event_at"`, + } { + if strings.Contains(string(encoded), discarded) { + t.Fatalf("brain output retained discarded Elasticsearch data %q: %s", discarded, encoded) + } + } + }) + } +} + +func TestFromGraphFactsPreservesElasticsearchStalenessAndDeclaredCoverage(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 0, time.UTC) + tests := []struct { + name string + stale bool + coverage map[fleet.Lens]LensCoverage + }{ + { + name: "stale fact", + stale: true, + coverage: covered(fleet.LensLive, fleet.LensTelemetry), + }, + { + name: "telemetry coverage omitted", + coverage: covered(fleet.LensLive), + }, + { + name: "telemetry coverage unavailable", + coverage: map[fleet.Lens]LensCoverage{ + fleet.LensLive: {Available: true}, + fleet.LensTelemetry: {Reason: "log connector unavailable"}, + }, + }, + { + name: "telemetry coverage stale", + coverage: map[fleet.Lens]LensCoverage{ + fleet.LensLive: {Available: true}, + fleet.LensTelemetry: {Available: true, Stale: true, Reason: "log search stale"}, + }, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + facts := projectedElasticsearchLogCause(t, "panic: discard-me", eventAt) + facts[0].Fact.Stale = test.stale + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, test.coverage) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + input.Observations = append(input.Observations, crashLoopObservation(eventAt.Add(time.Minute))) + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleCrashLoop || + result.Verdicts[0].Status != StatusDetected || + !reflect.DeepEqual(result.Verdicts[0].MissingLenses, []fleet.Lens{fleet.LensTelemetry}) { + t.Fatalf("verdicts = %#v, want coverage-honest detected R3", result.Verdicts) + } + citations := citationsForPredicate(result.Verdicts[0].Citations, "logs.cause") + if len(citations) != 1 || citations[0].Stale != test.stale { + t.Fatalf("log citations = %#v, stale = %t", citations, test.stale) + } + }) + } + + facts := projectedElasticsearchLogCause(t, "panic: discard-me", eventAt) + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, nil) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Coverage) != 0 { + t.Fatalf("coverage = %#v, fact presence must not infer coverage", input.Coverage) + } +} + +func TestFromGraphFactsFailsClosedOnAmbiguousElasticsearchLogCauseFacts(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 0, time.UTC) + baseFacts := projectedElasticsearchLogCause(t, "panic: discard-me", eventAt) + if len(baseFacts) != 1 { + t.Fatalf("base facts = %#v, want one log-cause fact", baseFacts) + } + base := baseFacts[0] + payload := func(key, value string, count int, first, last time.Time, container, extra string) json.RawMessage { + t.Helper() + raw := `{"key":` + quotedJSON(t, key) + + `,"value":` + quotedJSON(t, value) + + `,"count":` + quotedJSON(t, count) + + `,"first_event_at":` + quotedJSON(t, first.Format(time.RFC3339Nano)) + + `,"last_event_at":` + quotedJSON(t, last.Format(time.RFC3339Nano)) + if container != "" { + raw += `,"container":` + quotedJSON(t, container) + } + return json.RawMessage(raw + extra + `}`) + } + validPayload := func(extra string) json.RawMessage { + t.Helper() + return payload("logs.cause", "panic", 1, eventAt, eventAt, "api", extra) + } + + tests := []struct { + name string + mutate func(*fleet.GraphFact) + }{ + {name: "workspace mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Workspace = "other" + }}, + {name: "both source fields mismatch with exact protocol", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.SourceKind = "other" + fact.Fact.Provenance.Adapter = "other" + }}, + {name: "source kind mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.SourceKind = "other" }}, + {name: "provenance adapter mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Provenance.Adapter = "other" }}, + {name: "exact protocol on non-telemetry fact", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Kind = fleet.FactHealth + fact.Lens = fleet.LensLive + }}, + {name: "protocol mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.ProtocolV = "search/ecs-v2" + }}, + {name: "unexpected provenance deep link", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.DeepLink = "https://private.example.invalid/logs" + }}, + {name: "unexpected provenance collector", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.Collector = "unreviewed-collector" + }}, + {name: "resource kind mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Kind = "OtherSignal" }}, + {name: "resource scope missing", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Scope = "" + fact.Fact.Source = "" + fact.Entity.Cluster = "" + }}, + {name: "resource namespace missing", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Namespace = "" + fact.Entity.Namespace = "" + }}, + {name: "Pod entity missing", mutate: func(fact *fleet.GraphFact) { fact.Entity = nil }}, + {name: "entity cluster mismatch", mutate: func(fact *fleet.GraphFact) { fact.Entity.Cluster = "cluster-b" }}, + {name: "entity namespace mismatch", mutate: func(fact *fleet.GraphFact) { fact.Entity.Namespace = "other" }}, + {name: "entity Pod missing", mutate: func(fact *fleet.GraphFact) { fact.Entity.Pod = "" }}, + {name: "entity Pod retargeted", mutate: func(fact *fleet.GraphFact) { fact.Entity.Pod = "api-1" }}, + {name: "source scope retargeted consistently", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Scope = "cluster-b" + fact.Fact.Source = "cluster-b" + fact.Entity.Cluster = "cluster-b" + }}, + {name: "namespace retargeted consistently", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Namespace = "other" + fact.Entity.Namespace = "other" + }}, + {name: "entity carries kind", mutate: func(fact *fleet.GraphFact) { + fact.Entity.Kind = "Pod" + fact.Entity.Name = "api-0" + }}, + {name: "entity carries node", mutate: func(fact *fleet.GraphFact) { fact.Entity.Node = "worker-a" }}, + {name: "entity carries image digest", mutate: func(fact *fleet.GraphFact) { + fact.Entity.ImageDigest = "sha256:" + strings.Repeat("a", 64) + }}, + {name: "evidence source mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Source = "cluster-b" }}, + {name: "unexpected reference attributes", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Attributes = map[string]string{"message": "discard-me"} + }}, + {name: "unexpected display field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Display = []fleet.DisplayField{{Name: "message", Value: "discard-me"}} + }}, + {name: "native prefix mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.NativeID = "md5:" + strings.Repeat("a", 64) + }}, + {name: "native digest length mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.NativeID = "sha256:" + strings.Repeat("a", 63) + }}, + {name: "native digest case mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.NativeID = "sha256:" + strings.Repeat("A", 64) + }}, + {name: "resource digest prefix mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Name = "log-" + strings.Repeat("b", 32) + }}, + {name: "unknown payload field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = validPayload(`,"message":"discard-me"`) + }}, + {name: "duplicate payload field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = validPayload(`,"value":"panic"`) + }}, + {name: "mixed-case payload alias", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = validPayload(`,"VALUE":"panic"`) + }}, + {name: "unsupported key", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.message", "panic", 1, eventAt, eventAt, "api", "") + }}, + {name: "unsupported cause", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "credential-failure", 1, eventAt, eventAt, "api", "") + }}, + {name: "supported cause retargeted without native identity", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "missing-config", 1, eventAt, eventAt, "api", "") + }}, + {name: "valid count retargeted without native identity", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 2, eventAt, eventAt, "api", "") + }}, + {name: "valid event interval retargeted without native identity", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 1, eventAt.Add(-time.Second), eventAt, "api", "") + }}, + {name: "valid container retargeted without native identity", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 1, eventAt, eventAt, "other", "") + }}, + {name: "collection time retargeted without native identity", mutate: func(fact *fleet.GraphFact) { + fact.Fact.ObservedAt = fact.Fact.ObservedAt.Add(time.Second) + }}, + {name: "missing key", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `"key":"logs.cause",`, "", 1)) + }}, + {name: "missing value", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `,"value":"panic"`, "", 1)) + }}, + {name: "missing count", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `,"count":1`, "", 1)) + }}, + {name: "missing first event", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace( + string(validPayload("")), + `,"first_event_at":"`+eventAt.Format(time.RFC3339Nano)+`"`, + "", + 1, + )) + }}, + {name: "missing last event", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace( + string(validPayload("")), + `,"last_event_at":"`+eventAt.Format(time.RFC3339Nano)+`"`, + "", + 1, + )) + }}, + {name: "zero count", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 0, eventAt, eventAt, "api", "") + }}, + {name: "negative count", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", -1, eventAt, eventAt, "api", "") + }}, + {name: "count above projector bound", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", maxElasticsearchCauseCount+1, eventAt, eventAt, "api", "") + }}, + {name: "count has wrong type", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `"count":1`, `"count":"1"`, 1)) + }}, + {name: "event order reversed", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 1, eventAt, eventAt.Add(-time.Second), "api", "") + }}, + {name: "event interval above projector bound", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload( + "logs.cause", "panic", 1, + eventAt.Add(-maxElasticsearchCauseWindow-time.Nanosecond), eventAt, "api", "", + ) + }}, + {name: "last event exceeds clock skew", mutate: func(fact *fleet.GraphFact) { + future := fact.Fact.ObservedAt.Add(maxElasticsearchClockSkew + time.Nanosecond) + fact.Fact.Observed = payload("logs.cause", "panic", 1, future, future, "api", "") + }}, + {name: "null event time", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace( + string(validPayload("")), + `"first_event_at":"`+eventAt.Format(time.RFC3339Nano)+`"`, + `"first_event_at":null`, + 1, + )) + }}, + {name: "container null", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `"container":"api"`, `"container":null`, 1)) + }}, + {name: "container empty", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `"container":"api"`, `"container":""`, 1)) + }}, + {name: "container invalid control", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 1, eventAt, eventAt, "api\nother", "") + }}, + {name: "container has surrounding whitespace", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 1, eventAt, eventAt, " api ", "") + }}, + {name: "container oversized", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("logs.cause", "panic", 1, eventAt, eventAt, strings.Repeat("a", 254), "") + }}, + {name: "container wrong type", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Replace(string(validPayload("")), `"container":"api"`, `"container":7`, 1)) + }}, + {name: "empty payload", mutate: func(fact *fleet.GraphFact) { fact.Fact.Observed = nil }}, + {name: "oversized payload", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Repeat(" ", maxElasticsearchLogCausePayload+1)) + }}, + {name: "multiple JSON values", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = append(append(json.RawMessage(nil), fact.Fact.Observed...), []byte(` {}`)...) + }}, + {name: "malformed JSON", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(`{"key":`) + }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + fact := cloneBrainGraphFact(base) + test.mutate(&fact) + if _, err := FromGraphFacts( + fleet.LocalWorkspace, + []fleet.GraphFact{fact}, + covered(fleet.LensTelemetry), + ); err == nil { + t.Fatalf("FromGraphFacts() error = nil for %#v", fact) + } + }) + } +} + +func TestDecodeElasticsearchLogCausePayloadRejectsInvalidUTF8(t *testing.T) { + t.Parallel() + raw := append([]byte(`{"key":"logs.cause","value":"panic","count":1,"first_event_at":"2026-07-18T23:30:00Z","last_event_at":"2026-07-18T23:30:00Z","container":"`), 0xff) + raw = append(raw, []byte(`"}`)...) + + if _, err := decodeElasticsearchLogCausePayload(raw); err == nil || !strings.Contains(err.Error(), "invalid UTF-8") { + t.Fatalf("decodeElasticsearchLogCausePayload() error = %v, want invalid UTF-8", err) + } +} + +func TestFromGraphFactsDoesNotCorrelateElasticsearchCauseAcrossPods(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 0, time.UTC) + facts := projectedElasticsearchLogCause(t, "panic: discard-me", eventAt) + input, err := FromGraphFacts( + fleet.LocalWorkspace, + facts, + covered(fleet.LensLive, fleet.LensTelemetry), + ) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + crashLoop := crashLoopObservation(eventAt.Add(time.Minute)) + crashLoop.Ref.Name = "api-1" + input.Observations = append(input.Observations, crashLoop) + + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleCrashLoop || + result.Verdicts[0].Status != StatusDetected || result.Verdicts[0].Score != 3 || + len(citationsForPredicate(result.Verdicts[0].Citations, "logs.cause")) != 0 { + t.Fatalf("verdicts = %#v, cross-Pod log evidence must not strengthen R3", result.Verdicts) + } +} + +func TestFromGraphFactsIgnoresUnrelatedDerivedTelemetryFacts(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 0, time.UTC) + facts := projectedElasticsearchLogCause(t, "panic: discard-me", eventAt) + facts[0].Fact.Ref.SourceKind = "other" + facts[0].Fact.Ref.Kind = "OtherSignal" + facts[0].Fact.Provenance.Adapter = "other" + facts[0].Fact.Provenance.ProtocolV = "other/v1" + + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, covered(fleet.LensTelemetry)) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 0 { + t.Fatalf("observations = %#v, unrelated derived fact must be ignored", input.Observations) + } +} + +func FuzzFromGraphFactsElasticsearchLogCausePayload(f *testing.F) { + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 0, time.UTC) + base := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: "elasticsearch", Scope: "cluster-a", Kind: "LogSignal", + Namespace: "payments", Name: "log-" + strings.Repeat("0", 32), + }, + Kind: fleet.FactDerived, + ObservedAt: eventAt.Add(time.Minute), + Source: "cluster-a", + Provenance: fleet.Provenance{ + Adapter: "elasticsearch", ProtocolV: "search/ecs-v1", + NativeID: "sha256:" + strings.Repeat("0", 64), + }, + }, + Workspace: fleet.LocalWorkspace, + }, + Lens: fleet.LensTelemetry, + Entity: &fleet.EntityRef{ + Cluster: "cluster-a", Namespace: "payments", Pod: "api-0", + }, + } + for _, seed := range [][]byte{ + []byte(`{"key":"logs.cause","value":"panic","count":1,"first_event_at":"2026-07-18T23:30:00Z","last_event_at":"2026-07-18T23:30:00Z","container":"api"}`), + []byte(`{"key":"logs.cause","value":"dependency-failure","count":2,"first_event_at":"2026-07-18T23:29:00Z","last_event_at":"2026-07-18T23:30:00Z"}`), + []byte(`{"KEY":"logs.cause"}`), + []byte(`null`), + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, payload []byte) { + fact := cloneBrainGraphFact(base) + fact.Fact.Observed = append(json.RawMessage(nil), payload...) + decoded, decodeErr := decodeElasticsearchLogCausePayload(fact.Fact.Observed) + if decodeErr == nil { + digest, digestErr := elasticsearchLogCauseDigest(fact, decoded) + if digestErr != nil { + t.Fatalf("elasticsearchLogCauseDigest() error = %v", digestErr) + } + fact.Fact.Provenance.NativeID = "sha256:" + digest + fact.Fact.Ref.Name = "log-" + digest[:32] + } + input, err := FromGraphFacts( + fleet.LocalWorkspace, + []fleet.GraphFact{fact}, + covered(fleet.LensTelemetry), + ) + if err != nil { + return + } + if len(input.Observations) != 1 { + t.Fatalf("successful projection observations = %#v", input.Observations) + } + observation := input.Observations[0] + if observation.Ref.SourceKind != "elasticsearch" || + observation.Ref.Scope != "cluster-a" || + observation.Ref.Kind != "Pod" || + observation.Ref.Namespace != "payments" || + observation.Ref.Name != "api-0" || + observation.Lens != fleet.LensTelemetry || + observation.Key != "logs.cause" || + (observation.Value != "panic" && + observation.Value != "missing-config" && + observation.Value != "dependency-failure") || + observation.Source != "cluster-a" { + t.Fatalf("successful projection escaped closed contract: %#v", observation) + } + }) +} + +func projectedElasticsearchLogCause(t *testing.T, message string, eventAt time.Time) []fleet.GraphFact { + t.Helper() + response, err := json.Marshal(map[string]any{ + "timed_out": false, + "_shards": map[string]any{ + "total": 1, "successful": 1, "skipped": 0, "failed": 0, + }, + "hits": map[string]any{ + "hits": []any{ + map[string]any{ + "fields": map[string]any{ + "@timestamp": []string{eventAt.Format(time.RFC3339Nano)}, + "message": []string{message}, + "orchestrator.cluster.name": []string{"cluster-a"}, + "kubernetes.namespace": []string{"payments"}, + "kubernetes.pod.name": []string{"api-0"}, + "kubernetes.container.name": []string{"api"}, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("marshal Elasticsearch response: %v", err) + } + facts, err := connectorelasticsearch.ProjectLogCauses(connectorelasticsearch.Projection{ + Workspace: fleet.LocalWorkspace, + Scope: "cluster-a", + Namespace: "payments", + Pod: "api-0", + Container: "api", + WindowStart: eventAt.Add(-time.Minute), + WindowEnd: eventAt, + ObservedAt: eventAt.Add(time.Minute), + Response: response, + }) + if err != nil { + t.Fatalf("ProjectLogCauses() error = %v", err) + } + return facts +} + +func crashLoopObservation(observedAt time.Time) Observation { + return Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "kubeconfig", Scope: "cluster-a", Kind: "Pod", + Namespace: "payments", Name: "api-0", + }, + Lens: fleet.LensLive, + Key: "pod.failure", + Value: "CrashLoopBackOff", + ObservedAt: observedAt, + Source: "cluster-a", + } +} diff --git a/internal/brain/graph_test.go b/internal/brain/graph_test.go new file mode 100644 index 0000000..3523338 --- /dev/null +++ b/internal/brain/graph_test.go @@ -0,0 +1,507 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ArdurAI/sith/internal/connector/argocd" + connectorgithub "github.com/ArdurAI/sith/internal/connector/github" + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestFromGraphFactsProjectsReviewedArgoSyncFailures(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 15, 45, 0, 0, time.UTC) + for _, phase := range []string{"Failed", "Error"} { + phase := phase + t.Run(phase, func(t *testing.T) { + t.Parallel() + facts := projectedArgoOperation(t, phase, eventAt) + coverage := covered(fleet.LensTimeline) + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, coverage) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + coverage[fleet.LensTimeline] = LensCoverage{} + if !input.Coverage[fleet.LensTimeline].Available { + t.Fatalf("coverage alias mutated projected input: %#v", input.Coverage) + } + if len(input.Observations) != 1 { + t.Fatalf("observations = %#v, want one canonical sync failure", input.Observations) + } + observation := input.Observations[0] + wantRef := fleet.ResourceRef{ + SourceKind: "argocd", Scope: "cluster-a", Kind: "Application", Namespace: "argocd", Name: "payments", + } + if !reflect.DeepEqual(observation.Ref, wantRef) || observation.Lens != fleet.LensTimeline || + observation.Key != "change.kind" || observation.Value != "sync-failed" || + !observation.ObservedAt.Equal(eventAt) || observation.Source != "cluster-a" || observation.Stale { + t.Fatalf("observation = %#v, want exact sanitized Argo citation metadata", observation) + } + + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleArgoSyncFail || + result.Verdicts[0].Status != StatusConfirmed { + t.Fatalf("verdicts = %#v, want confirmed R8", result.Verdicts) + } + encoded, err := json.Marshal(struct { + Input Investigation `json:"input"` + Result Result `json:"result"` + }{Input: input, Result: result}) + if err != nil { + t.Fatalf("marshal projected result: %v", err) + } + for _, discarded := range []string{"discard-this-revision", phase} { + if strings.Contains(string(encoded), discarded) { + t.Fatalf("brain output retained discarded Argo payload %q: %s", discarded, encoded) + } + } + }) + } +} + +func TestFromGraphFactsDoesNotTreatDriftOrNonFailedOperationsAsR8(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 15, 45, 0, 0, time.UTC) + for _, phase := range []string{"Succeeded", "Running", "Terminating"} { + phase := phase + t.Run(phase, func(t *testing.T) { + t.Parallel() + input, err := FromGraphFacts( + fleet.LocalWorkspace, + projectedArgoOperation(t, phase, eventAt), + covered(fleet.LensTimeline), + ) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 0 { + t.Fatalf("phase %q observations = %#v, want no R8 signal", phase, input.Observations) + } + }) + } + + for _, syncStatus := range []string{"OutOfSync", "Unknown", "Synced"} { + syncStatus := syncStatus + t.Run(syncStatus, func(t *testing.T) { + t.Parallel() + facts, err := argocd.ProjectApplication(argocd.Projection{ + Workspace: fleet.LocalWorkspace, + Scope: "cluster-a", + ObservedAt: eventAt.Add(time.Minute), + Application: argoApplication(map[string]any{ + "sync": map[string]any{"status": syncStatus, "revision": "discard-this-revision"}, + }), + }) + if err != nil { + t.Fatalf("ProjectApplication() error = %v", err) + } + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, covered(fleet.LensDesired)) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 0 { + t.Fatalf("sync status %q observations = %#v, want no R8 signal", syncStatus, input.Observations) + } + }) + } +} + +func TestFromGraphFactsFailsClosedOnAmbiguousArgoChangeFacts(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 15, 45, 0, 0, time.UTC) + baseFacts := projectedArgoOperation(t, "Failed", eventAt) + if len(baseFacts) != 1 { + t.Fatalf("base facts = %#v, want one operation change", baseFacts) + } + base := baseFacts[0] + payload := func(changeKind, phase string, at time.Time, extra string) json.RawMessage { + t.Helper() + raw := `{"change_kind":` + quotedJSON(t, changeKind) + + `,"revision":"discard-this-revision","phase":` + quotedJSON(t, phase) + + `,"event_at":` + quotedJSON(t, at.Format(time.RFC3339Nano)) + extra + `}` + return json.RawMessage(raw) + } + + tests := []struct { + name string + mutate func(*fleet.GraphFact) + }{ + {name: "source kind mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.SourceKind = "kubeconfig" }}, + {name: "provenance adapter mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Provenance.Adapter = "other" }}, + {name: "protocol mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Provenance.ProtocolV = "2.0.0" }}, + {name: "evidence source mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Source = "cluster-b" }}, + {name: "resource kind mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Kind = "Deployment" }}, + {name: "entity name mismatch", mutate: func(fact *fleet.GraphFact) { fact.Entity.Name = "other" }}, + {name: "entity carries Pod identity", mutate: func(fact *fleet.GraphFact) { fact.Entity.Pod = "payments-0" }}, + {name: "unexpected reference attributes", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Attributes = map[string]string{"untrusted": "value"} + }}, + {name: "unexpected display field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Display = []fleet.DisplayField{{Name: "message", Value: "raw failure"}} + }}, + {name: "unknown payload field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("sync-failed", "Failed", eventAt, `,"message":"do not retain"`) + }}, + {name: "successful phase with failure kind", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("sync-failed", "Succeeded", eventAt, "") + }}, + {name: "failed phase with successful kind", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("argocd-sync", "Failed", eventAt, "") + }}, + {name: "unsupported change kind", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("pipeline-failed", "Failed", eventAt, "") + }}, + {name: "event time mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("sync-failed", "Failed", eventAt.Add(time.Second), "") + }}, + {name: "history metadata on operation", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload("sync-failed", "Failed", eventAt, `,"history_id":"7"`) + }}, + {name: "native operation identity mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.NativeID = "cluster-a/argocd/other#operation/id" + }}, + {name: "native operation identity missing", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Provenance.NativeID = "cluster-a/argocd/payments#operation/" + }}, + {name: "unattached fact", mutate: func(fact *fleet.GraphFact) { fact.Entity = nil }}, + {name: "oversized payload", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(`{"change_kind":"sync-failed","phase":"Failed","event_at":"` + + eventAt.Format(time.RFC3339Nano) + `","revision":"` + strings.Repeat("x", maxArgoChangePayload) + `"}`) + }}, + {name: "multiple JSON values", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = append(append(json.RawMessage(nil), fact.Fact.Observed...), []byte(` {}`)...) + }}, + {name: "malformed JSON", mutate: func(fact *fleet.GraphFact) { fact.Fact.Observed = json.RawMessage(`{"change_kind":`) }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + fact := cloneBrainGraphFact(base) + test.mutate(&fact) + if _, err := FromGraphFacts(fleet.LocalWorkspace, []fleet.GraphFact{fact}, covered(fleet.LensTimeline)); err == nil { + t.Fatalf("FromGraphFacts() error = nil for %#v", fact) + } + }) + } + + if _, err := FromGraphFacts("other-workspace", baseFacts, covered(fleet.LensTimeline)); err == nil { + t.Fatal("FromGraphFacts() workspace mismatch error = nil") + } + if _, err := FromGraphFacts(fleet.LocalWorkspace, baseFacts, map[fleet.Lens]LensCoverage{"unknown": {Available: true}}); err == nil { + t.Fatal("FromGraphFacts() unknown coverage lens error = nil") + } +} + +func TestFromGraphFactsPreservesStalenessWithoutInferringCoverage(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 15, 45, 0, 0, time.UTC) + facts := projectedArgoOperation(t, "Failed", eventAt) + facts[0].Fact.Stale = true + + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, nil) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 1 || !input.Observations[0].Stale { + t.Fatalf("observations = %#v, want one stale R8 signal", input.Observations) + } + if len(input.Coverage) != 0 { + t.Fatalf("coverage = %#v, projection must not infer coverage", input.Coverage) + } + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleArgoSyncFail || + result.Verdicts[0].Status != StatusUnconfirmed || + !result.Verdicts[0].Citations[0].Stale { + t.Fatalf("verdicts = %#v, want cited stale unconfirmed R8", result.Verdicts) + } +} + +func TestFromGraphFactsProjectsReviewedGitHubWorkflowFailures(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + for _, conclusion := range []string{"failure", "timed_out", "startup_failure"} { + conclusion := conclusion + t.Run(conclusion, func(t *testing.T) { + t.Parallel() + facts := projectedGitHubWorkflowRun(t, conclusion, eventAt) + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, covered(fleet.LensTimeline)) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 1 { + t.Fatalf("observations = %#v, want one canonical workflow-run failure", input.Observations) + } + observation := input.Observations[0] + wantRef := fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + } + if !reflect.DeepEqual(observation.Ref, wantRef) || observation.Lens != fleet.LensTimeline || + observation.Key != "change.kind" || observation.Value != "workflow-run-failed" || + !observation.ObservedAt.Equal(eventAt) || observation.Source != "github.com" || observation.Stale { + t.Fatalf("observation = %#v, want exact sanitized GitHub citation metadata", observation) + } + + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleWorkflowFail || + result.Verdicts[0].Status != StatusConfirmed || result.Verdicts[0].FleetWide { + t.Fatalf("verdicts = %#v, want confirmed entity-local R9", result.Verdicts) + } + encoded, err := json.Marshal(struct { + Input Investigation `json:"input"` + Result Result `json:"result"` + }{Input: input, Result: result}) + if err != nil { + t.Fatalf("marshal projected result: %v", err) + } + for _, discarded := range []string{`"workflow_id":`, `"run_attempt":`, `"conclusion":`, "raw-workflow-secret"} { + if strings.Contains(string(encoded), discarded) { + t.Fatalf("brain output retained discarded workflow-run payload %q: %s", discarded, encoded) + } + } + }) + } +} + +func TestFromGraphFactsIgnoresUnrelatedGitHubTimelineFacts(t *testing.T) { + t.Parallel() + mergedAt := time.Date(2026, 7, 18, 18, 30, 0, 0, time.UTC) + response, err := json.Marshal(map[string]any{ + "number": 42, "state": "closed", "draft": false, "merged": true, + "merged_at": mergedAt.Format(time.RFC3339Nano), + "merge_commit_sha": strings.Repeat("c", 40), + "head": map[string]any{"sha": strings.Repeat("a", 40)}, + "base": map[string]any{"sha": strings.Repeat("b", 40)}, + }) + if err != nil { + t.Fatalf("marshal pull response: %v", err) + } + facts, err := connectorgithub.ProjectMergedPullRequest(connectorgithub.Projection{ + Workspace: fleet.LocalWorkspace, Host: "github.com", Owner: "ArdurAI", Repository: "sith", + PullNumber: 42, ObservedAt: mergedAt.Add(time.Minute), Response: response, + }) + if err != nil { + t.Fatalf("ProjectMergedPullRequest() error = %v", err) + } + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, covered(fleet.LensTimeline)) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 0 { + t.Fatalf("observations = %#v, merged pull request must not prove R9", input.Observations) + } +} + +func TestFromGraphFactsFailsClosedOnAmbiguousGitHubWorkflowFacts(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + baseFacts := projectedGitHubWorkflowRun(t, "failure", eventAt) + if len(baseFacts) != 1 { + t.Fatalf("base facts = %#v, want one workflow-run change", baseFacts) + } + base := baseFacts[0] + payload := func(runID, workflowID, attempt int64, changeKind, conclusion string, at time.Time, extra string) json.RawMessage { + t.Helper() + raw := `{"run_id":` + quotedJSON(t, runID) + + `,"workflow_id":` + quotedJSON(t, workflowID) + + `,"run_attempt":` + quotedJSON(t, attempt) + + `,"change_kind":` + quotedJSON(t, changeKind) + + `,"conclusion":` + quotedJSON(t, conclusion) + + `,"event_at":` + quotedJSON(t, at.Format(time.RFC3339Nano)) + extra + `}` + return json.RawMessage(raw) + } + + tests := []struct { + name string + mutate func(*fleet.GraphFact) + }{ + {name: "source kind mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.SourceKind = "kubeconfig" }}, + {name: "provenance adapter mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Provenance.Adapter = "other" }}, + {name: "both source fields mismatch with workflow protocol", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.SourceKind = "kubeconfig" + fact.Fact.Provenance.Adapter = "other" + }}, + {name: "workflow protocol on non-timeline fact", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Kind = fleet.FactHealth + fact.Lens = fleet.LensLive + }}, + {name: "protocol mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Provenance.ProtocolV = connectorgithub.ProtocolVersion }}, + {name: "attached fact", mutate: func(fact *fleet.GraphFact) { fact.Entity = &fleet.EntityRef{Cluster: "alpha"} }}, + {name: "resource kind mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Kind = "PullRequest" }}, + {name: "invalid host", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Scope = "https://github.com" }}, + {name: "invalid owner", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Namespace = "ArdurAI/other" }}, + {name: "evidence source mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Source = "github.example.com" }}, + {name: "unexpected reference attributes", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Ref.Attributes = map[string]string{"untrusted": "value"} + }}, + {name: "unexpected display field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Display = []fleet.DisplayField{{Name: "message", Value: "raw failure"}} + }}, + {name: "unknown payload field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 2, "workflow-run-failed", "failure", eventAt, `,"message":"do not retain"`) + }}, + {name: "duplicate payload field", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 2, "workflow-run-failed", "failure", eventAt, `,"run_id":30433642`) + }}, + {name: "mixed-case payload alias", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 2, "workflow-run-failed", "failure", eventAt, `,"RUN_ID":30433642`) + }}, + {name: "zero run ID", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(0, 159038, 2, "workflow-run-failed", "failure", eventAt, "") + }}, + {name: "zero workflow ID", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 0, 2, "workflow-run-failed", "failure", eventAt, "") + }}, + {name: "zero attempt", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 0, "workflow-run-failed", "failure", eventAt, "") + }}, + {name: "unsupported change kind", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 2, "pipeline-failed", "failure", eventAt, "") + }}, + {name: "nonfailure conclusion", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 2, "workflow-run-failed", "success", eventAt, "") + }}, + {name: "event time mismatch", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = payload(30433642, 159038, 2, "workflow-run-failed", "failure", eventAt.Add(time.Second), "") + }}, + {name: "resource run ID mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Name = "sith#7-attempt-2" }}, + {name: "resource attempt mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Name = "sith#30433642-attempt-3" }}, + {name: "invalid repository identity", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Name = "other/repo#30433642-attempt-2" }}, + {name: "git-suffixed repository identity", mutate: func(fact *fleet.GraphFact) { fact.Fact.Ref.Name = "sith.git#30433642-attempt-2" }}, + {name: "native identity mismatch", mutate: func(fact *fleet.GraphFact) { fact.Fact.Provenance.NativeID = "ArdurAI/other#30433642-attempt-2" }}, + {name: "oversized payload", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = json.RawMessage(strings.Repeat(" ", maxGitHubWorkflowChangePayload+1)) + }}, + {name: "multiple JSON values", mutate: func(fact *fleet.GraphFact) { + fact.Fact.Observed = append(append(json.RawMessage(nil), fact.Fact.Observed...), []byte(` {}`)...) + }}, + {name: "malformed JSON", mutate: func(fact *fleet.GraphFact) { fact.Fact.Observed = json.RawMessage(`{"run_id":`) }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + fact := cloneBrainGraphFact(base) + test.mutate(&fact) + if _, err := FromGraphFacts(fleet.LocalWorkspace, []fleet.GraphFact{fact}, covered(fleet.LensTimeline)); err == nil { + t.Fatalf("FromGraphFacts() error = nil for %#v", fact) + } + }) + } +} + +func TestFromGraphFactsPreservesGitHubWorkflowStalenessWithoutInferringCoverage(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + facts := projectedGitHubWorkflowRun(t, "failure", eventAt) + facts[0].Fact.Stale = true + + input, err := FromGraphFacts(fleet.LocalWorkspace, facts, nil) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + if len(input.Observations) != 1 || !input.Observations[0].Stale || len(input.Coverage) != 0 { + t.Fatalf("input = %#v, want one stale R9 signal and no inferred coverage", input) + } + result, err := Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleWorkflowFail || + result.Verdicts[0].Status != StatusUnconfirmed || !result.Verdicts[0].Citations[0].Stale { + t.Fatalf("verdicts = %#v, want cited stale unconfirmed R9", result.Verdicts) + } +} + +func projectedArgoOperation(t *testing.T, phase string, eventAt time.Time) []fleet.GraphFact { + t.Helper() + facts, err := argocd.ProjectApplication(argocd.Projection{ + Workspace: fleet.LocalWorkspace, + Scope: "cluster-a", + ObservedAt: eventAt.Add(time.Minute), + Application: argoApplication(map[string]any{ + "operationState": map[string]any{ + "phase": phase, "finishedAt": eventAt.Format(time.RFC3339Nano), + "syncResult": map[string]any{"revision": "discard-this-revision"}, + }, + }), + }) + if err != nil { + t.Fatalf("ProjectApplication() error = %v", err) + } + return facts +} + +func projectedGitHubWorkflowRun(t *testing.T, conclusion string, eventAt time.Time) []fleet.GraphFact { + t.Helper() + response, err := json.Marshal(map[string]any{ + "id": 30433642, "workflow_id": 159038, "run_attempt": 2, + "status": "completed", "conclusion": conclusion, "updated_at": eventAt.Format(time.RFC3339Nano), + "repository": map[string]any{"full_name": "ArdurAI/sith"}, + "display_title": "raw-workflow-secret", "jobs_url": "https://attacker.example/jobs", + }) + if err != nil { + t.Fatalf("marshal workflow-run response: %v", err) + } + facts, err := connectorgithub.ProjectFailedWorkflowRun(connectorgithub.WorkflowRunProjection{ + Workspace: fleet.LocalWorkspace, Host: "github.com", Owner: "ArdurAI", Repository: "sith", + RunID: 30433642, ObservedAt: eventAt.Add(time.Minute), Response: response, + }) + if err != nil { + t.Fatalf("ProjectFailedWorkflowRun() error = %v", err) + } + return facts +} + +func argoApplication(status map[string]any) unstructured.Unstructured { + return unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "argoproj.io/v1alpha1", + "kind": "Application", + "metadata": map[string]any{"name": "payments", "namespace": "argocd"}, + "status": status, + }} +} + +func quotedJSON(t *testing.T, value any) string { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal JSON string: %v", err) + } + return string(encoded) +} + +func cloneBrainGraphFact(fact fleet.GraphFact) fleet.GraphFact { + cloned := fact + cloned.Fact.Observed = append(json.RawMessage(nil), fact.Fact.Observed...) + cloned.Fact.Display = append([]fleet.DisplayField(nil), fact.Fact.Display...) + if fact.Fact.Ref.Attributes != nil { + cloned.Fact.Ref.Attributes = make(map[string]string, len(fact.Fact.Ref.Attributes)) + for key, value := range fact.Fact.Ref.Attributes { + cloned.Fact.Ref.Attributes[key] = value + } + } + if fact.Entity != nil { + entity := *fact.Entity + cloned.Entity = &entity + } + return cloned +} diff --git a/internal/brain/model.go b/internal/brain/model.go index 4f2597f..b930fd4 100644 --- a/internal/brain/model.go +++ b/internal/brain/model.go @@ -15,7 +15,7 @@ import ( // RuleID identifies one stable hypothesis rule. type RuleID string -// Canonical rule identifiers. +// Stable rule identifiers. const ( RuleBadDeploy RuleID = "R1" RuleOOMKilled RuleID = "R2" @@ -23,6 +23,9 @@ const ( RuleConfigDrift RuleID = "R4" RuleCertExpiry RuleID = "R5" RuleNodePressure RuleID = "R6" + RuleImagePull RuleID = "R7" + RuleArgoSyncFail RuleID = "R8" + RuleWorkflowFail RuleID = "R9" ) // Status is the confidence state of a verdict. @@ -81,19 +84,20 @@ type Advisory struct { // Verdict is one ranked and coverage-honest hypothesis. type Verdict struct { - Rule RuleID `json:"rule"` - FailureMode string `json:"failure_mode"` - Status Status `json:"status"` - Hypothesis string `json:"hypothesis"` - Scope string `json:"scope"` - Ref fleet.ResourceRef `json:"ref"` - Score int `json:"score"` - FleetWide bool `json:"fleet_wide"` - Clusters []string `json:"clusters,omitempty"` - CauseOf []RuleID `json:"cause_of,omitempty"` - MissingLenses []fleet.Lens `json:"missing_lenses,omitempty"` - Citations []Citation `json:"citations"` - Advisory Advisory `json:"advisory"` + Rule RuleID `json:"rule"` + FailureMode string `json:"failure_mode"` + Status Status `json:"status"` + Hypothesis string `json:"hypothesis"` + Scope string `json:"scope"` + Ref fleet.ResourceRef `json:"ref"` + Score int `json:"score"` + FleetWide bool `json:"fleet_wide"` + Clusters []string `json:"clusters,omitempty"` + CauseOf []RuleID `json:"cause_of,omitempty"` + MissingLenses []fleet.Lens `json:"missing_lenses,omitempty"` + Citations []Citation `json:"citations"` + Advisory Advisory `json:"advisory"` + RemediationCandidate *RemediationCandidate `json:"remediation_candidate,omitempty"` } // Result is a deterministic ranked investigation answer. diff --git a/internal/brain/remediation.go b/internal/brain/remediation.go new file mode 100644 index 0000000..2a8ed2e --- /dev/null +++ b/internal/brain/remediation.go @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "slices" + + "github.com/ArdurAI/sith/internal/intent" +) + +// ProvenanceRequirement names one authoritative fact that a later governed-plan renderer must +// resolve before it may construct handler-owned arguments. These values are requirement +// identifiers, not graph predicates or caller-provided data. +type ProvenanceRequirement string + +// Reviewed provenance requirements for the first structured remediation candidates. +const ( + ProvenanceArgoApplicationTarget ProvenanceRequirement = "argocd.application.target" + ProvenanceArgoRevision ProvenanceRequirement = "argocd.revision" + ProvenanceGitRepository ProvenanceRequirement = "git.repository" + ProvenanceGitBaseRef ProvenanceRequirement = "git.base-ref" + ProvenanceGitBaseCommit ProvenanceRequirement = "git.base-commit" + ProvenanceGitFilePath ProvenanceRequirement = "git.file-path" + ProvenanceGitObservedBlob ProvenanceRequirement = "git.observed-blob" + ProvenanceGitDesiredContent ProvenanceRequirement = "git.desired-content" +) + +// RemediationCandidate is an inert rule-owned requirement template. It is not a resolved target, +// handler argument document, policy proposal, approval, or execution capability. +type RemediationCandidate struct { + Verb intent.Verb `json:"verb"` + RequiredProvenance []ProvenanceRequirement `json:"required_provenance"` +} + +type remediationCandidateWire struct { + Verb intent.Verb `json:"verb"` + RequiredProvenance []ProvenanceRequirement `json:"required_provenance"` +} + +// Validate accepts only an exact reviewed verb-to-requirements mapping. Requirement order is part +// of the deterministic contract. +func (candidate RemediationCandidate) Validate() error { + expected, reviewed := canonicalRemediationRequirementsFor(candidate.Verb) + if !reviewed || !candidate.Verb.Valid() || !slices.Equal(candidate.RequiredProvenance, expected) { + return fmt.Errorf("remediation candidate is not canonical") + } + return nil +} + +// MarshalJSON prevents a mutated or forged candidate from crossing an output boundary. +func (candidate RemediationCandidate) MarshalJSON() ([]byte, error) { + if err := candidate.Validate(); err != nil { + return nil, fmt.Errorf("encode remediation candidate: %w", err) + } + return json.Marshal(remediationCandidateWire{ + Verb: candidate.Verb, RequiredProvenance: slices.Clone(candidate.RequiredProvenance), + }) +} + +// UnmarshalJSON accepts only the exact closed candidate vocabulary and rejects shape drift. +func (candidate *RemediationCandidate) UnmarshalJSON(payload []byte) error { + if candidate == nil { + return fmt.Errorf("decode remediation candidate: destination is nil") + } + if err := rejectDuplicateRemediationCandidateMembers(payload); err != nil { + return fmt.Errorf("decode remediation candidate: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + var wire remediationCandidateWire + if err := decoder.Decode(&wire); err != nil { + return fmt.Errorf("decode remediation candidate: %w", err) + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return fmt.Errorf("decode remediation candidate: expected one JSON value") + } + return fmt.Errorf("decode remediation candidate: %w", err) + } + decoded := RemediationCandidate{ + Verb: wire.Verb, RequiredProvenance: slices.Clone(wire.RequiredProvenance), + } + if err := decoded.Validate(); err != nil { + return fmt.Errorf("decode remediation candidate: %w", err) + } + *candidate = decoded + return nil +} + +func rejectDuplicateRemediationCandidateMembers(payload []byte) error { + decoder := json.NewDecoder(bytes.NewReader(payload)) + opening, err := decoder.Token() + if err != nil { + return err + } + if opening != json.Delim('{') { + return nil + } + seen := make(map[string]struct{}, 2) + for decoder.More() { + member, err := decoder.Token() + if err != nil { + return err + } + name, ok := member.(string) + if !ok { + return fmt.Errorf("JSON object contains an invalid member") + } + if name != "verb" && name != "required_provenance" { + return fmt.Errorf("JSON object contains an unknown member") + } + if _, duplicate := seen[name]; duplicate { + return fmt.Errorf("JSON object contains a duplicate member") + } + seen[name] = struct{}{} + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return err + } + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim('}') { + return fmt.Errorf("JSON object has an invalid closing delimiter") + } + return nil +} + +func remediationCandidateFor(verb intent.Verb) *RemediationCandidate { + requirements, reviewed := canonicalRemediationRequirementsFor(verb) + if !reviewed { + return nil + } + return &RemediationCandidate{Verb: verb, RequiredProvenance: slices.Clone(requirements)} +} + +func cloneRemediationCandidate(candidate *RemediationCandidate) *RemediationCandidate { + if candidate == nil { + return nil + } + return &RemediationCandidate{ + Verb: candidate.Verb, + RequiredProvenance: slices.Clone(candidate.RequiredProvenance), + } +} + +func canonicalRemediationRequirementsFor(verb intent.Verb) ([]ProvenanceRequirement, bool) { + switch verb { + case intent.VerbArgoCDRollback: + return []ProvenanceRequirement{ + ProvenanceArgoApplicationTarget, + ProvenanceArgoRevision, + }, true + case intent.VerbGitOpsOpenPR: + return []ProvenanceRequirement{ + ProvenanceGitRepository, + ProvenanceGitBaseRef, + ProvenanceGitBaseCommit, + ProvenanceGitFilePath, + ProvenanceGitObservedBlob, + ProvenanceGitDesiredContent, + }, true + default: + return nil, false + } +} diff --git a/internal/brain/remediation_test.go b/internal/brain/remediation_test.go new file mode 100644 index 0000000..f9e68d5 --- /dev/null +++ b/internal/brain/remediation_test.go @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "encoding/json" + "reflect" + "slices" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" +) + +func TestRemediationCandidatePublicShapeIsNonAuthorizing(t *testing.T) { + t.Parallel() + typeOfCandidate := reflect.TypeFor[RemediationCandidate]() + if typeOfCandidate.NumField() != 2 || typeOfCandidate.Field(0).Name != "Verb" || + typeOfCandidate.Field(1).Name != "RequiredProvenance" { + t.Fatalf("RemediationCandidate fields = %#v, want only Verb and RequiredProvenance", typeOfCandidate) + } + if typeOfCandidate.Field(0).Type != reflect.TypeFor[intent.Verb]() || + typeOfCandidate.Field(1).Type != reflect.TypeFor[[]ProvenanceRequirement]() { + t.Fatalf("RemediationCandidate field types are not the closed contract: %#v", typeOfCandidate) + } +} + +func TestCatalogUsesOnlyReviewedRemediationMappings(t *testing.T) { + t.Parallel() + want := map[RuleID]intent.Verb{ + RuleBadDeploy: intent.VerbArgoCDRollback, + RuleOOMKilled: intent.VerbGitOpsOpenPR, + RuleConfigDrift: intent.VerbGitOpsOpenPR, + } + for _, candidateRule := range catalog { + candidate := remediationCandidateFor(candidateRule.remediation) + verb, expected := want[candidateRule.id] + if !expected { + if candidateRule.remediation != "" || candidate != nil { + t.Fatalf("rule %s has unreviewed remediation %#v", candidateRule.id, candidate) + } + continue + } + if candidate == nil || candidate.Verb != verb { + t.Fatalf("rule %s candidate = %#v, want verb %s", candidateRule.id, candidate, verb) + } + if err := candidate.Validate(); err != nil { + t.Fatalf("rule %s candidate is invalid: %v", candidateRule.id, err) + } + } +} + +func TestRemediationCandidateWireBoundaryIsCanonical(t *testing.T) { + t.Parallel() + want := remediationCandidateFor(intent.VerbGitOpsOpenPR) + payload, err := json.Marshal(want) + if err != nil { + t.Fatalf("Marshal() error = %v", err) + } + var decoded RemediationCandidate + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + if decoded.Verb != want.Verb || !slices.Equal(decoded.RequiredProvenance, want.RequiredProvenance) { + t.Fatalf("decoded candidate = %#v, want %#v", decoded, want) + } + + invalidCandidates := []RemediationCandidate{ + {}, + {Verb: intent.VerbDeploymentRestart, RequiredProvenance: []ProvenanceRequirement{ProvenanceArgoRevision}}, + {Verb: intent.VerbArgoCDRollback, RequiredProvenance: []ProvenanceRequirement{ProvenanceArgoApplicationTarget}}, + {Verb: intent.VerbArgoCDRollback, RequiredProvenance: []ProvenanceRequirement{ProvenanceArgoRevision, ProvenanceArgoApplicationTarget}}, + {Verb: intent.VerbArgoCDRollback, RequiredProvenance: []ProvenanceRequirement{ProvenanceArgoApplicationTarget, ProvenanceArgoRevision, ProvenanceArgoRevision}}, + } + for index, candidate := range invalidCandidates { + if _, err := json.Marshal(candidate); err == nil { + t.Fatalf("json.Marshal(invalid candidate %d) error = nil", index) + } + } + + for _, payload := range []string{ + `{"verb":"argocd.rollback","required_provenance":["argocd.application.target"]}`, + `{"verb":"argocd.rollback","required_provenance":["argocd.revision","argocd.application.target"]}`, + `{"verb":"argocd.rollback","required_provenance":["argocd.application.target","argocd.revision"],"target":"forged"}`, + `{"verb":"argocd.rollback","verb":"argocd.rollback","required_provenance":["argocd.application.target","argocd.revision"]}`, + `{"Verb":"argocd.rollback","required_provenance":["argocd.application.target","argocd.revision"]}`, + `{"verb":"argocd.rollback","Required_Provenance":["argocd.application.target","argocd.revision"]}`, + `{"verb":"shell.exec","required_provenance":["argocd.application.target","argocd.revision"]}`, + `{"verb":"argocd.rollback","required_provenance":["argocd.application.target","argocd.revision"]}{}`, + } { + if err := json.Unmarshal([]byte(payload), &decoded); err == nil { + t.Fatalf("json.Unmarshal(%s) error = nil", payload) + } + } +} + +func TestEvaluateReturnsMutationIsolatedRemediationCandidate(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) + input := Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{ + observe(now, fleet.LensLive, "pod.reason", "OOMKilled"), + observe(now, fleet.LensTelemetry, "memory.variant", "near-limit"), + }, + Coverage: covered(fleet.LensLive, fleet.LensTelemetry), + } + first, err := Evaluate(input) + if err != nil { + t.Fatalf("first Evaluate() error = %v", err) + } + if len(first.Verdicts) != 1 || first.Verdicts[0].RemediationCandidate == nil { + t.Fatalf("first verdicts = %#v, want R2 candidate", first.Verdicts) + } + first.Verdicts[0].RemediationCandidate.RequiredProvenance[0] = ProvenanceArgoRevision + + second, err := Evaluate(input) + if err != nil { + t.Fatalf("second Evaluate() error = %v", err) + } + want := remediationCandidateFor(intent.VerbGitOpsOpenPR) + if len(second.Verdicts) != 1 || second.Verdicts[0].RemediationCandidate == nil || + second.Verdicts[0].RemediationCandidate.Verb != want.Verb || + !slices.Equal(second.Verdicts[0].RemediationCandidate.RequiredProvenance, want.RequiredProvenance) { + t.Fatalf("second candidate = %#v, want mutation-isolated %#v", second.Verdicts[0].RemediationCandidate, want) + } +} + +func TestFleetCandidateDoesNotAliasEntityCandidate(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) + const repoDigest = "registry.example/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + observations := make([]Observation, 0, 6) + for _, cluster := range []string{"alpha", "beta"} { + ref := testRef() + ref.Scope = cluster + observations = append(observations, + Observation{Ref: ref, Lens: fleet.LensLive, Key: "pod.reason", Value: "OOMKilled", ObservedAt: now, Source: "kubeconfig"}, + Observation{Ref: ref, Lens: fleet.LensTelemetry, Key: "memory.variant", Value: "near-limit", ObservedAt: now, Source: "prometheus"}, + Observation{Ref: ref, Lens: fleet.LensLive, Key: fleet.OTelContainerImageRepoDigests, Value: repoDigest, ObservedAt: now, Source: "kubeconfig"}, + ) + } + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: observations, + Coverage: covered(fleet.LensLive, fleet.LensTelemetry), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 3 || !result.Verdicts[0].FleetWide || result.Verdicts[0].RemediationCandidate == nil { + t.Fatalf("verdicts = %#v, want fleet candidate followed by two entity candidates", result.Verdicts) + } + result.Verdicts[0].RemediationCandidate.RequiredProvenance[0] = ProvenanceArgoRevision + want := remediationCandidateFor(intent.VerbGitOpsOpenPR) + for index, verdict := range result.Verdicts[1:] { + if verdict.RemediationCandidate == nil || + !slices.Equal(verdict.RemediationCandidate.RequiredProvenance, want.RequiredProvenance) { + t.Fatalf("entity candidate %d = %#v, want independent %#v", index, verdict.RemediationCandidate, want) + } + } +} + +func TestCoverageUnconfirmedVerdictKeepsAdvisoryAndInertCandidate(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{ + observe(now, fleet.LensLive, "workload.status", "Degraded"), + observe(now, fleet.LensTimeline, "change.kind", "deploy"), + }, + Coverage: covered(fleet.LensLive), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 { + t.Fatalf("verdicts = %#v, want one R1 verdict", result.Verdicts) + } + verdict := result.Verdicts[0] + if verdict.Rule != RuleBadDeploy || verdict.Status != StatusUnconfirmed || verdict.Advisory.Command == "" || + verdict.RemediationCandidate == nil || verdict.RemediationCandidate.Verb != intent.VerbArgoCDRollback { + t.Fatalf("verdict = %#v, want unconfirmed advisory plus inert R1 candidate", verdict) + } +} + +func TestAdvisoryOnlyRuleHasNoRemediationCandidate(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) + result, err := Evaluate(Investigation{ + Workspace: fleet.LocalWorkspace, + Observations: []Observation{observe(now, fleet.LensLive, "pod.failure", "CrashLoopBackOff")}, + Coverage: covered(fleet.LensLive), + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != RuleCrashLoop || + result.Verdicts[0].Advisory.Command == "" || result.Verdicts[0].RemediationCandidate != nil { + t.Fatalf("verdicts = %#v, want advisory-only R3", result.Verdicts) + } +} diff --git a/internal/brain/replay_test.go b/internal/brain/replay_test.go index de499c1..6b36e35 100644 --- a/internal/brain/replay_test.go +++ b/internal/brain/replay_test.go @@ -28,15 +28,16 @@ type replayFixture struct { } type replayExpectation struct { - TopRule RuleID `json:"top_rule"` - Status Status `json:"status"` - Scope string `json:"scope"` - FleetWide bool `json:"fleet_wide"` - Clusters []string `json:"clusters"` - CauseOf []RuleID `json:"cause_of"` - CitationEvidence []replayCitationEvidence `json:"citation_evidence"` - MissingLenses []fleet.Lens `json:"missing_lenses"` - Advisory replayAdvisoryExpectation `json:"advisory"` + TopRule RuleID `json:"top_rule"` + Status Status `json:"status"` + Scope string `json:"scope"` + FleetWide bool `json:"fleet_wide"` + Clusters []string `json:"clusters"` + CauseOf []RuleID `json:"cause_of"` + CitationEvidence []replayCitationEvidence `json:"citation_evidence"` + MissingLenses []fleet.Lens `json:"missing_lenses"` + Advisory replayAdvisoryExpectation `json:"advisory"` + RemediationCandidate *RemediationCandidate `json:"remediation_candidate"` } type replayCitationEvidence struct { @@ -93,6 +94,7 @@ func TestIncidentReplayFixturesAreDeterministic(t *testing.T) { func TestIncidentReplayCorpusCoversRequiredSafetyCases(t *testing.T) { seenRules := make(map[RuleID]struct{}) + seenRemediationRules := make(map[RuleID]struct{}) var hasCauseChain, hasFleetWide, hasNonFleet, hasUnconfirmed bool for _, path := range replayFixturePaths(t, "testdata/replays") { fixture := loadReplayFixture(t, path) @@ -101,8 +103,11 @@ func TestIncidentReplayCorpusCoversRequiredSafetyCases(t *testing.T) { hasFleetWide = hasFleetWide || fixture.Expect.FleetWide hasNonFleet = hasNonFleet || !fixture.Expect.FleetWide hasUnconfirmed = hasUnconfirmed || fixture.Expect.Status == StatusUnconfirmed + if fixture.Expect.RemediationCandidate != nil { + seenRemediationRules[fixture.Expect.TopRule] = struct{}{} + } } - for _, ruleID := range []RuleID{RuleBadDeploy, RuleOOMKilled, RuleCrashLoop, RuleConfigDrift, RuleCertExpiry, RuleNodePressure} { + for _, ruleID := range []RuleID{RuleBadDeploy, RuleOOMKilled, RuleCrashLoop, RuleConfigDrift, RuleCertExpiry, RuleNodePressure, RuleImagePull, RuleArgoSyncFail, RuleWorkflowFail} { if _, found := seenRules[ruleID]; !found { t.Fatalf("replay corpus does not cover %s", ruleID) } @@ -110,6 +115,14 @@ func TestIncidentReplayCorpusCoversRequiredSafetyCases(t *testing.T) { if !hasCauseChain || !hasFleetWide || !hasNonFleet || !hasUnconfirmed { t.Fatalf("replay corpus lacks required safety cases: cause_chain=%t fleet_wide=%t non_fleet=%t unconfirmed=%t", hasCauseChain, hasFleetWide, hasNonFleet, hasUnconfirmed) } + for _, ruleID := range []RuleID{RuleBadDeploy, RuleOOMKilled, RuleConfigDrift} { + if _, found := seenRemediationRules[ruleID]; !found { + t.Fatalf("replay corpus does not cover %s remediation candidate", ruleID) + } + } + if len(seenRemediationRules) != 3 { + t.Fatalf("replay corpus names unreviewed remediation rules: %#v", seenRemediationRules) + } } func TestIncidentReplayFixturesRejectMalformedInput(t *testing.T) { @@ -225,6 +238,11 @@ func (fixture replayFixture) Validate() error { if fixture.Expect.Advisory.Command == nil || fixture.Expect.Advisory.PRDiff == nil || fixture.Expect.Advisory.Sensitive == nil { return fmt.Errorf("replay expectation must declare every advisory shape field") } + if fixture.Expect.RemediationCandidate != nil { + if err := fixture.Expect.RemediationCandidate.Validate(); err != nil { + return fmt.Errorf("replay remediation candidate is invalid: %w", err) + } + } if fixture.Expect.FleetWide { if fixture.Expect.Scope != "fleet" || len(fixture.Expect.Clusters) < 2 { return fmt.Errorf("fleet-wide replay expectation requires fleet scope and two clusters") @@ -354,6 +372,14 @@ func assertReplayExpectation(t *testing.T, fixture replayFixture, result Result) if top.Advisory.Sensitive != *want.Advisory.Sensitive { t.Fatalf("advisory sensitive = %t, want %t", top.Advisory.Sensitive, *want.Advisory.Sensitive) } + if (top.RemediationCandidate == nil) != (want.RemediationCandidate == nil) { + t.Fatalf("remediation candidate = %#v, want %#v", top.RemediationCandidate, want.RemediationCandidate) + } + if top.RemediationCandidate != nil && + (top.RemediationCandidate.Verb != want.RemediationCandidate.Verb || + !slices.Equal(top.RemediationCandidate.RequiredProvenance, want.RemediationCandidate.RequiredProvenance)) { + t.Fatalf("remediation candidate = %#v, want %#v", top.RemediationCandidate, want.RemediationCandidate) + } } func sortedCitationEvidence(citations []Citation) []replayCitationEvidence { diff --git a/internal/brain/rules.go b/internal/brain/rules.go index 41b1446..ffef596 100644 --- a/internal/brain/rules.go +++ b/internal/brain/rules.go @@ -2,7 +2,10 @@ package brain -import "github.com/ArdurAI/sith/internal/fleet" +import ( + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" +) type predicate struct { lens fleet.Lens @@ -15,11 +18,15 @@ type rule struct { id RuleID failureMode string rootCause string + sourceKind string + resourceKind string + exactTrigger bool trigger predicate signals []predicate required []fleet.Lens strengthener []fleet.Lens advisory Advisory + remediation intent.Verb } var catalog = []rule{ @@ -28,14 +35,14 @@ var catalog = []rule{ trigger: predicate{fleet.LensLive, "workload.status", []string{"degraded", "progressing"}, 2}, signals: []predicate{{fleet.LensTimeline, "change.kind", []string{"deploy", "rollout", "argocd-sync", "image-change"}, 3}, {fleet.LensDesired, "desired.changed", []string{"true"}, 1}}, required: []fleet.Lens{fleet.LensLive, fleet.LensTimeline}, strengthener: []fleet.Lens{fleet.LensDesired, fleet.LensTelemetry}, - advisory: Advisory{Command: "kubectl --context {context} rollout undo {kind}/{name} -n {namespace}"}, + advisory: Advisory{Command: "kubectl --context {context} rollout undo {kind}/{name} -n {namespace}"}, remediation: intent.VerbArgoCDRollback, }, { id: RuleOOMKilled, failureMode: "OOMKilled", rootCause: "the container was terminated for exceeding available memory", trigger: predicate{fleet.LensLive, "pod.reason", []string{"oomkilled"}, 3}, signals: []predicate{{fleet.LensLive, "pod.restarts", nil, 1}, {fleet.LensTelemetry, "memory.variant", nil, 2}, {fleet.LensTimeline, "change.kind", nil, 1}}, required: []fleet.Lens{fleet.LensLive}, strengthener: []fleet.Lens{fleet.LensTelemetry}, - advisory: Advisory{PRDiff: "increase spec.template.spec.containers[].resources.limits.memory after validating measured usage"}, + advisory: Advisory{PRDiff: "increase spec.template.spec.containers[].resources.limits.memory after validating measured usage"}, remediation: intent.VerbGitOpsOpenPR, }, { id: RuleCrashLoop, failureMode: "CrashLoopBackOff", rootCause: "the container repeatedly exits and Kubernetes is backing off restarts", @@ -49,7 +56,7 @@ var catalog = []rule{ trigger: predicate{fleet.LensDesired, "desired.drift", []string{"true", "outofsync"}, 3}, signals: []predicate{{fleet.LensTimeline, "change.kind", []string{"kubectl-edit", "kubectl-patch", "sync-failed"}, 2}}, required: []fleet.Lens{fleet.LensLive, fleet.LensDesired}, strengthener: []fleet.Lens{fleet.LensTimeline}, - advisory: Advisory{PRDiff: "reconcile the cited field delta in Git, or revert the out-of-band live mutation"}, + advisory: Advisory{PRDiff: "reconcile the cited field delta in Git, or revert the out-of-band live mutation"}, remediation: intent.VerbGitOpsOpenPR, }, { id: RuleCertExpiry, failureMode: "certificate expiry", rootCause: "a certificate is expired or inside the renewal safety window", @@ -64,4 +71,26 @@ var catalog = []rule{ required: []fleet.Lens{fleet.LensLive}, strengthener: []fleet.Lens{fleet.LensTelemetry}, advisory: Advisory{Command: "inspect capacity and autoscaler state for node {name} before any cordon or drain", Sensitive: true}, }, + { + id: RuleImagePull, failureMode: "image pull failure", rootCause: "Kubernetes reports that it cannot pull a Pod image; the waiting reason does not identify the underlying registry, reference, network, rate-limit, or platform cause", + trigger: predicate{fleet.LensLive, "pod.reason", []string{"imagepullbackoff", "errimagepull"}, 3}, + required: []fleet.Lens{fleet.LensLive}, + advisory: Advisory{Command: "kubectl --context {context} describe pod {name} -n {namespace}", Sensitive: true}, + }, + { + id: RuleArgoSyncFail, failureMode: "Argo CD sync failure", rootCause: "Argo CD reports that the Application sync operation failed; the normalized operation phase does not identify whether the underlying cause is rendering, validation, authorization, a hook, network access, the Kubernetes API, a resource, or another failure", + sourceKind: argoGraphSource, resourceKind: "Application", + exactTrigger: true, + trigger: predicate{fleet.LensTimeline, "change.kind", []string{"sync-failed"}, 3}, + required: []fleet.Lens{fleet.LensTimeline}, + advisory: Advisory{Command: "kubectl --context {context} describe application.argoproj.io {name} -n {namespace}", Sensitive: true}, + }, + { + id: RuleWorkflowFail, failureMode: "GitHub Actions workflow-run failure", rootCause: "GitHub reports that the completed workflow run failed; the normalized conclusion does not identify which job or step failed, or whether the underlying cause is code, configuration, credentials, permissions, capacity, a dependency, or another failure", + sourceKind: githubGraphSource, resourceKind: "WorkflowRun", + exactTrigger: true, + trigger: predicate{fleet.LensTimeline, "change.kind", []string{"workflow-run-failed"}, 3}, + required: []fleet.Lens{fleet.LensTimeline}, + advisory: Advisory{Command: "inspect GitHub Actions workflow run {name} owned by {namespace} and its failed jobs and logs before considering a rerun", Sensitive: true}, + }, } diff --git a/internal/brain/testdata/replays/README.md b/internal/brain/testdata/replays/README.md index 8ab39f3..e60a557 100644 --- a/internal/brain/testdata/replays/README.md +++ b/internal/brain/testdata/replays/README.md @@ -5,6 +5,11 @@ they must not contain real cluster names, endpoint URLs, credentials, secrets, r customer incident data. The test-only harness rejects unknown JSON fields so every expected verdict shape remains explicit as the rule catalog evolves. +Connector-backed replay observations contain only the connector's reviewed normalized values. In +particular, the Elasticsearch R3 fixture stores the closed `logs.cause` classification and exact +Pod identity, never a raw log message, index or document identifier, query, label, URL, credential, +or user data. + Each fixture is versioned through its required `version` field. A fixture asserts the top verdict's rule, confidence, scope, cited lens/predicate/value evidence, coverage gaps, fleet-wide flag, and advisory shape. The harness evaluates every fixture twice and fails if the serialized result differs. diff --git a/internal/brain/testdata/replays/r1-bad-deploy-causes-crashloop.json b/internal/brain/testdata/replays/r1-bad-deploy-causes-crashloop.json index b1d4e56..71c6a58 100644 --- a/internal/brain/testdata/replays/r1-bad-deploy-causes-crashloop.json +++ b/internal/brain/testdata/replays/r1-bad-deploy-causes-crashloop.json @@ -30,6 +30,10 @@ {"lens": "desired", "predicate": "desired.changed", "observed": "true"} ], "missing_lenses": [], - "advisory": {"command": true, "pr_diff": false, "sensitive": false} + "advisory": {"command": true, "pr_diff": false, "sensitive": false}, + "remediation_candidate": { + "verb": "argocd.rollback", + "required_provenance": ["argocd.application.target", "argocd.revision"] + } } } diff --git a/internal/brain/testdata/replays/r1-stale-timeline-unconfirmed.json b/internal/brain/testdata/replays/r1-stale-timeline-unconfirmed.json index ca747cc..18b360e 100644 --- a/internal/brain/testdata/replays/r1-stale-timeline-unconfirmed.json +++ b/internal/brain/testdata/replays/r1-stale-timeline-unconfirmed.json @@ -24,6 +24,10 @@ {"lens": "timeline", "predicate": "change.kind", "observed": "deploy"} ], "missing_lenses": ["timeline"], - "advisory": {"command": true, "pr_diff": false, "sensitive": false} + "advisory": {"command": true, "pr_diff": false, "sensitive": false}, + "remediation_candidate": { + "verb": "argocd.rollback", + "required_provenance": ["argocd.application.target", "argocd.revision"] + } } } diff --git a/internal/brain/testdata/replays/r2-oomkilled.json b/internal/brain/testdata/replays/r2-oomkilled.json index 89959c3..d717d07 100644 --- a/internal/brain/testdata/replays/r2-oomkilled.json +++ b/internal/brain/testdata/replays/r2-oomkilled.json @@ -26,6 +26,17 @@ {"lens": "telemetry", "predicate": "memory.variant", "observed": "near-limit"} ], "missing_lenses": [], - "advisory": {"command": false, "pr_diff": true, "sensitive": false} + "advisory": {"command": false, "pr_diff": true, "sensitive": false}, + "remediation_candidate": { + "verb": "gitops.open-pr", + "required_provenance": [ + "git.repository", + "git.base-ref", + "git.base-commit", + "git.file-path", + "git.observed-blob", + "git.desired-content" + ] + } } } diff --git a/internal/brain/testdata/replays/r3-elasticsearch-log-cause.json b/internal/brain/testdata/replays/r3-elasticsearch-log-cause.json new file mode 100644 index 0000000..f7528f1 --- /dev/null +++ b/internal/brain/testdata/replays/r3-elasticsearch-log-cause.json @@ -0,0 +1,29 @@ +{ + "version": 1, + "name": "r3-elasticsearch-log-cause", + "sanitized": true, + "investigation": { + "workspace": "local", + "coverage": { + "live": {"available": true, "stale": false}, + "telemetry": {"available": true, "stale": false} + }, + "observations": [ + {"ref": {"source_kind": "kubeconfig", "scope": "alpha", "kind": "Pod", "namespace": "synthetic", "name": "payments-0"}, "lens": "live", "key": "pod.failure", "value": "CrashLoopBackOff", "observed_at": "2026-07-18T23:31:00Z", "source": "alpha", "stale": false}, + {"ref": {"source_kind": "elasticsearch", "scope": "alpha", "kind": "Pod", "namespace": "synthetic", "name": "payments-0"}, "lens": "telemetry", "key": "logs.cause", "value": "missing-config", "observed_at": "2026-07-18T23:30:00Z", "source": "alpha", "stale": false} + ] + }, + "expect": { + "top_rule": "R3", + "status": "confirmed", + "scope": "alpha", + "fleet_wide": false, + "cause_of": [], + "citation_evidence": [ + {"lens": "live", "predicate": "pod.failure", "observed": "CrashLoopBackOff"}, + {"lens": "telemetry", "predicate": "logs.cause", "observed": "missing-config"} + ], + "missing_lenses": [], + "advisory": {"command": true, "pr_diff": false, "sensitive": false} + } +} diff --git a/internal/brain/testdata/replays/r3-fleet-image-correlation.json b/internal/brain/testdata/replays/r3-fleet-image-correlation.json index c138ddd..0d42ee0 100644 --- a/internal/brain/testdata/replays/r3-fleet-image-correlation.json +++ b/internal/brain/testdata/replays/r3-fleet-image-correlation.json @@ -22,6 +22,8 @@ "clusters": ["alpha", "beta"], "cause_of": [], "citation_evidence": [ + {"lens": "live", "predicate": "container.image.repo_digests", "observed": "registry.synthetic/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, + {"lens": "live", "predicate": "container.image.repo_digests", "observed": "registry.synthetic/payments@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}, {"lens": "live", "predicate": "pod.failure", "observed": "CrashLoopBackOff"}, {"lens": "live", "predicate": "pod.failure", "observed": "CrashLoopBackOff"} ], diff --git a/internal/brain/testdata/replays/r4-config-drift.json b/internal/brain/testdata/replays/r4-config-drift.json index 61a4652..88c7e41 100644 --- a/internal/brain/testdata/replays/r4-config-drift.json +++ b/internal/brain/testdata/replays/r4-config-drift.json @@ -26,6 +26,17 @@ {"lens": "timeline", "predicate": "change.kind", "observed": "kubectl-edit"} ], "missing_lenses": [], - "advisory": {"command": false, "pr_diff": true, "sensitive": false} + "advisory": {"command": false, "pr_diff": true, "sensitive": false}, + "remediation_candidate": { + "verb": "gitops.open-pr", + "required_provenance": [ + "git.repository", + "git.base-ref", + "git.base-commit", + "git.file-path", + "git.observed-blob", + "git.desired-content" + ] + } } } diff --git a/internal/brain/testdata/replays/r7-image-pull-backoff.json b/internal/brain/testdata/replays/r7-image-pull-backoff.json new file mode 100644 index 0000000..e5cf4ee --- /dev/null +++ b/internal/brain/testdata/replays/r7-image-pull-backoff.json @@ -0,0 +1,19 @@ +{ + "version": 1, + "name": "r7-image-pull-backoff", + "sanitized": true, + "investigation": { + "workspace": "local", + "coverage": {"live": {"available": true, "stale": false}}, + "observations": [ + {"ref": {"source_kind": "fixture", "scope": "alpha", "kind": "Pod", "namespace": "payments", "name": "api-abc"}, "lens": "live", "key": "pod.reason", "value": "ImagePullBackOff", "observed_at": "2026-07-18T00:00:00Z", "source": "synthetic", "stale": false} + ] + }, + "expect": { + "top_rule": "R7", "status": "confirmed", "scope": "alpha", "fleet_wide": false, + "cause_of": [], + "citation_evidence": [{"lens": "live", "predicate": "pod.reason", "observed": "ImagePullBackOff"}], + "missing_lenses": [], + "advisory": {"command": true, "pr_diff": false, "sensitive": true} + } +} diff --git a/internal/brain/testdata/replays/r8-argocd-sync-failed.json b/internal/brain/testdata/replays/r8-argocd-sync-failed.json new file mode 100644 index 0000000..acb4302 --- /dev/null +++ b/internal/brain/testdata/replays/r8-argocd-sync-failed.json @@ -0,0 +1,19 @@ +{ + "version": 1, + "name": "r8-argocd-sync-failed", + "sanitized": true, + "investigation": { + "workspace": "local", + "coverage": {"timeline": {"available": true, "stale": false}}, + "observations": [ + {"ref": {"source_kind": "argocd", "scope": "alpha", "kind": "Application", "namespace": "argocd", "name": "payments"}, "lens": "timeline", "key": "change.kind", "value": "sync-failed", "observed_at": "2026-07-18T00:00:00Z", "source": "synthetic", "stale": false} + ] + }, + "expect": { + "top_rule": "R8", "status": "confirmed", "scope": "alpha", "fleet_wide": false, + "cause_of": [], + "citation_evidence": [{"lens": "timeline", "predicate": "change.kind", "observed": "sync-failed"}], + "missing_lenses": [], + "advisory": {"command": true, "pr_diff": false, "sensitive": true} + } +} diff --git a/internal/brain/testdata/replays/r9-github-actions-workflow-failed.json b/internal/brain/testdata/replays/r9-github-actions-workflow-failed.json new file mode 100644 index 0000000..6e1c064 --- /dev/null +++ b/internal/brain/testdata/replays/r9-github-actions-workflow-failed.json @@ -0,0 +1,19 @@ +{ + "version": 1, + "name": "r9-github-actions-workflow-failed", + "sanitized": true, + "investigation": { + "workspace": "local", + "coverage": {"timeline": {"available": true, "stale": false}}, + "observations": [ + {"ref": {"source_kind": "github", "scope": "github.example", "kind": "WorkflowRun", "namespace": "example-org", "name": "example-repo#30433642-attempt-2"}, "lens": "timeline", "key": "change.kind", "value": "workflow-run-failed", "observed_at": "2026-07-18T19:55:00Z", "source": "github.example", "stale": false} + ] + }, + "expect": { + "top_rule": "R9", "status": "confirmed", "scope": "github.example", "fleet_wide": false, + "cause_of": [], + "citation_evidence": [{"lens": "timeline", "predicate": "change.kind", "observed": "workflow-run-failed"}], + "missing_lenses": [], + "advisory": {"command": true, "pr_diff": false, "sensitive": true} + } +} diff --git a/internal/cli/audit.go b/internal/cli/audit.go new file mode 100644 index 0000000..8b2569a --- /dev/null +++ b/internal/cli/audit.go @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "encoding/json" + "fmt" + "io" + "os" + "unicode/utf8" + + "github.com/spf13/cobra" + strictjson "sigs.k8s.io/json" + + "github.com/ArdurAI/sith/internal/auditrecord" +) + +const auditIntegrityInternal = "internally-consistent" + +type auditVerifyResult struct { + Integrity string `json:"integrity"` + ExternallyAnchored bool `json:"externally_anchored"` + Schema string `json:"schema"` + WorkspaceID string `json:"workspace_id"` + Entries int `json:"entries"` + HeadSequence int64 `json:"head_sequence"` + HeadHash string `json:"head_hash"` +} + +type auditPageVerifyResult struct { + Integrity string `json:"integrity"` + ExternallyAnchored bool `json:"externally_anchored"` + Schema string `json:"schema"` + WorkspaceID string `json:"workspace_id"` + Pages int `json:"pages"` + Entries int64 `json:"entries"` + HeadSequence int64 `json:"head_sequence"` + HeadHash string `json:"head_hash"` +} + +func newAuditCommand(options *rootOptions) *cobra.Command { + command := &cobra.Command{ + Use: "audit", + Short: "Inspect portable audit records", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + return command.Help() + }, + } + command.AddCommand(newAuditVerifyCommand(options), newAuditVerifyPagesCommand(options)) + return command +} + +func newAuditVerifyCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{ + Use: "verify ", + Short: "Verify the internal integrity of one portable audit export", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + exported, err := readAuditExportFile(args[0]) + if err != nil { + return err + } + result := auditVerifyResult{ + Integrity: auditIntegrityInternal, ExternallyAnchored: false, + Schema: exported.Schema, WorkspaceID: exported.WorkspaceID, Entries: len(exported.Entries), + HeadSequence: exported.Chain.HeadSequence, HeadHash: exported.Chain.HeadHash, + } + return writeAuditVerifyResult(command, options.output, result) + }, + } +} + +func newAuditVerifyPagesCommand(options *rootOptions) *cobra.Command { + return &cobra.Command{ + Use: "verify-pages [page.json...]", + Short: "Verify one complete ordered audit page sequence", + Args: cobra.MinimumNArgs(1), + RunE: func(command *cobra.Command, args []string) error { + var verifier auditrecord.PageSequenceVerifier + for _, path := range args { + page, err := readAuditPageFile(path) + if err != nil { + return err + } + if err := verifier.Add(page); err != nil { + return fmt.Errorf("audit page sequence integrity is invalid: %w", err) + } + } + verified, err := verifier.Finish() + if err != nil { + return fmt.Errorf("audit page sequence integrity is invalid: %w", err) + } + result := auditPageVerifyResult{ + Integrity: auditIntegrityInternal, ExternallyAnchored: false, + Schema: verified.Schema, WorkspaceID: verified.WorkspaceID, Pages: verified.Pages, + Entries: verified.Entries, HeadSequence: verified.Chain.HeadSequence, + HeadHash: verified.Chain.HeadHash, + } + return writeAuditPageVerifyResult(command, options.output, result) + }, + } +} + +func readAuditExportFile(path string) (auditrecord.Export, error) { + payload, err := readBoundedAuditDocument(path, "audit export") + if err != nil { + return auditrecord.Export{}, err + } + + var exported auditrecord.Export + strictErrors, decodeErr := strictjson.UnmarshalStrict(payload, &exported) + if decodeErr != nil || len(strictErrors) != 0 { + return auditrecord.Export{}, fmt.Errorf("audit export JSON is invalid") + } + if err := exported.Verify(); err != nil { + return auditrecord.Export{}, fmt.Errorf("audit export integrity is invalid: %w", err) + } + return exported, nil +} + +func readAuditPageFile(path string) (auditrecord.Page, error) { + payload, err := readBoundedAuditDocument(path, "audit page") + if err != nil { + return auditrecord.Page{}, err + } + var page auditrecord.Page + strictErrors, decodeErr := strictjson.UnmarshalStrict(payload, &page) + if decodeErr != nil || len(strictErrors) != 0 { + return auditrecord.Page{}, fmt.Errorf("audit page JSON is invalid") + } + if err := page.Verify(); err != nil { + return auditrecord.Page{}, fmt.Errorf("audit page integrity is invalid: %w", err) + } + return page, nil +} + +func readBoundedAuditDocument(path, name string) ([]byte, error) { + before, err := os.Lstat(path) + if err != nil { + return nil, fmt.Errorf("%s file is unavailable", name) + } + if !before.Mode().IsRegular() { + return nil, fmt.Errorf("%s must be a regular file", name) + } + if before.Size() < 0 || before.Size() > auditrecord.MaxDocumentBytes { + return nil, fmt.Errorf("%s exceeds the %d-byte limit", name, auditrecord.MaxDocumentBytes) + } + + // #nosec G304 -- the path is the command's explicit local argument; Lstat, regular-file, + // same-file, and bounded-read checks constrain it before any document content is trusted. + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("%s file is unavailable", name) + } + after, statErr := file.Stat() + if statErr != nil || !after.Mode().IsRegular() || !os.SameFile(before, after) { + _ = file.Close() + return nil, fmt.Errorf("%s must be a stable regular file", name) + } + payload, readErr := io.ReadAll(io.LimitReader(file, auditrecord.MaxDocumentBytes+1)) + closeErr := file.Close() + if readErr != nil || closeErr != nil { + return nil, fmt.Errorf("%s file could not be read", name) + } + if len(payload) > auditrecord.MaxDocumentBytes { + return nil, fmt.Errorf("%s exceeds the %d-byte limit", name, auditrecord.MaxDocumentBytes) + } + if !utf8.Valid(payload) { + return nil, fmt.Errorf("%s JSON is invalid", name) + } + return payload, nil +} + +func writeAuditVerifyResult(command *cobra.Command, format string, result auditVerifyResult) error { + switch format { + case "json": + if err := json.NewEncoder(command.OutOrStdout()).Encode(result); err != nil { + return fmt.Errorf("write audit verification JSON: %w", err) + } + return nil + case "yaml": + return writeYAML(command.OutOrStdout(), result, "audit verification") + default: + _, err := fmt.Fprintf(command.OutOrStdout(), + "Audit export integrity: %s (not externally anchored)\nSchema: %s\nWorkspace: %s\nEntries: %d\nHead: %d %s\n", + result.Integrity, result.Schema, result.WorkspaceID, result.Entries, result.HeadSequence, result.HeadHash, + ) + if err != nil { + return fmt.Errorf("write audit verification output: %w", err) + } + return nil + } +} + +func writeAuditPageVerifyResult(command *cobra.Command, format string, result auditPageVerifyResult) error { + switch format { + case "json": + if err := json.NewEncoder(command.OutOrStdout()).Encode(result); err != nil { + return fmt.Errorf("write audit page verification JSON: %w", err) + } + return nil + case "yaml": + return writeYAML(command.OutOrStdout(), result, "audit page verification") + default: + _, err := fmt.Fprintf(command.OutOrStdout(), + "Audit page sequence integrity: %s (not externally anchored)\nSchema: %s\nWorkspace: %s\nPages: %d\nEntries: %d\nHead: %d %s\n", + result.Integrity, result.Schema, result.WorkspaceID, result.Pages, result.Entries, + result.HeadSequence, result.HeadHash, + ) + if err != nil { + return fmt.Errorf("write audit page verification output: %w", err) + } + return nil + } +} diff --git a/internal/cli/audit_test.go b/internal/cli/audit_test.go new file mode 100644 index 0000000..abc9406 --- /dev/null +++ b/internal/cli/audit_test.go @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/auditrecord" + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestAuditVerifyReportsBoundedInternalIntegritySummary(t *testing.T) { + path := writeAuditExportFile(t, validCLIAuditExport()) + stdout, stderr, exitCode := runCLI(t, []string{"audit", "verify", path}, fleet.StubSource{}) + if exitCode != 0 || stderr != "" { + t.Fatalf("exit/stderr = %d/%q", exitCode, stderr) + } + for _, want := range []string{ + "Audit export integrity: internally-consistent (not externally anchored)", + "Schema: sith.policy-audit-export/v1", "Workspace: workspace-a", "Entries: 1", "Head: 1 sha256:", + } { + if !strings.Contains(stdout, want) { + t.Fatalf("stdout = %q, want %q", stdout, want) + } + } +} + +func TestAuditVerifyJSONDoesNotEchoEntries(t *testing.T) { + path := writeAuditExportFile(t, validCLIAuditExport()) + stdout, stderr, exitCode := runCLI(t, []string{"audit", "verify", path, "-o", "json"}, fleet.StubSource{}) + if exitCode != 0 || stderr != "" { + t.Fatalf("exit/stderr = %d/%q", exitCode, stderr) + } + var result auditVerifyResult + if err := json.Unmarshal([]byte(stdout), &result); err != nil { + t.Fatalf("unmarshal output: %v", err) + } + if result.Integrity != auditIntegrityInternal || result.ExternallyAnchored || result.Entries != 1 || + result.WorkspaceID != "workspace-a" || strings.Contains(stdout, "user:alice") { + t.Fatalf("verification result leaked or overstated integrity: %#v / %q", result, stdout) + } +} + +func TestAuditVerifyPagesReportsCompleteBoundedSequence(t *testing.T) { + pages := validCLIAuditPages(t, auditrecord.MaxEntries+1) + paths := writeAuditPageFiles(t, pages) + args := append([]string{"audit", "verify-pages"}, paths...) + stdout, stderr, exitCode := runCLI(t, args, fleet.StubSource{}) + if exitCode != 0 || stderr != "" { + t.Fatalf("exit/stderr = %d/%q", exitCode, stderr) + } + for _, want := range []string{ + "Audit page sequence integrity: internally-consistent (not externally anchored)", + "Schema: sith.policy-audit-page/v1", "Workspace: workspace-a", "Pages: 2", "Entries: 513", "Head: 513 sha256:", + } { + if !strings.Contains(stdout, want) { + t.Fatalf("stdout = %q, want %q", stdout, want) + } + } +} + +func TestAuditVerifyPagesJSONDoesNotEchoEntries(t *testing.T) { + paths := writeAuditPageFiles(t, validCLIAuditPages(t, 1)) + stdout, stderr, exitCode := runCLI(t, []string{"audit", "verify-pages", paths[0], "-o", "json"}, fleet.StubSource{}) + if exitCode != 0 || stderr != "" { + t.Fatalf("exit/stderr = %d/%q", exitCode, stderr) + } + var result auditPageVerifyResult + if err := json.Unmarshal([]byte(stdout), &result); err != nil { + t.Fatal(err) + } + if result.Integrity != auditIntegrityInternal || result.ExternallyAnchored || result.Pages != 1 || + result.Entries != 1 || result.WorkspaceID != "workspace-a" || strings.Contains(stdout, "user:alice") { + t.Fatalf("page verification leaked or overstated integrity: %#v / %q", result, stdout) + } +} + +func TestAuditVerifyPagesRejectsMissingReorderedAndTamperedPages(t *testing.T) { + pages := validCLIAuditPages(t, auditrecord.MaxEntries+1) + for name, candidate := range map[string][]auditrecord.Page{ + "missing final": {pages[0]}, + "reordered": {pages[1], pages[0]}, + "replayed": {pages[0], pages[0]}, + "tampered": func() []auditrecord.Page { + changed := pages[1] + changed.Entries = append([]auditrecord.Entry(nil), changed.Entries...) + changed.Entries[0].Actor = "user:mallory" + return []auditrecord.Page{pages[0], changed} + }(), + } { + t.Run(name, func(t *testing.T) { + paths := writeAuditPageFiles(t, candidate) + args := append([]string{"audit", "verify-pages"}, paths...) + _, stderr, exitCode := runCLI(t, args, fleet.StubSource{}) + if exitCode == 0 || !strings.Contains(stderr, "integrity is invalid") { + t.Fatalf("exit/stderr = %d/%q", exitCode, stderr) + } + }) + } +} + +func TestReadAuditPageFileRejectsStrictJSONAndNonRegularInput(t *testing.T) { + page := validCLIAuditPages(t, 1)[0] + payload, err := json.Marshal(page) + if err != nil { + t.Fatal(err) + } + unknown := bytes.Replace(payload, []byte(`{"schema":`), []byte(`{"unknown":"value","schema":`), 1) + path := filepath.Join(t.TempDir(), "audit-page.json") + if err := os.WriteFile(path, unknown, 0o600); err != nil { + t.Fatal(err) + } + if _, err := readAuditPageFile(path); err == nil || !strings.Contains(err.Error(), "JSON is invalid") { + t.Fatalf("strict page error = %v", err) + } + target := writeAuditPageFiles(t, []auditrecord.Page{page})[0] + symlink := filepath.Join(t.TempDir(), "audit-page-link.json") + if err := os.Symlink(target, symlink); err != nil { + t.Fatal(err) + } + if _, err := readAuditPageFile(symlink); err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("page symlink error = %v", err) + } +} + +func TestReadAuditExportFileRejectsStrictJSONAndTampering(t *testing.T) { + validPayload, err := json.Marshal(validCLIAuditExport()) + if err != nil { + t.Fatal(err) + } + tests := map[string][]byte{ + "unknown field": bytes.Replace(validPayload, []byte(`{"schema":`), []byte(`{"unknown":"value","schema":`), 1), + "duplicate field": bytes.Replace(validPayload, []byte(`{"schema":`), []byte(`{"schema":"duplicate","schema":`), 1), + "case mismatch": bytes.Replace(validPayload, []byte(`"schema"`), []byte(`"Schema"`), 1), + "trailing JSON": append(append([]byte{}, validPayload...), []byte(` {}`)...), + "malformed": []byte(`{"schema":`), + "invalid UTF-8": bytes.Replace(validPayload, []byte("user:alice"), []byte{'u', 's', 'e', 'r', ':', 0xff}, 1), + } + for name, payload := range tests { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.json") + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatal(err) + } + if _, err := readAuditExportFile(path); err == nil || !strings.Contains(err.Error(), "JSON is invalid") { + t.Fatalf("readAuditExportFile() error = %v", err) + } + }) + } + + tampered := validCLIAuditExport() + tampered.Entries[0].Actor = "user:mallory" + path := writeAuditExportFile(t, tampered) + if _, err := readAuditExportFile(path); err == nil || !strings.Contains(err.Error(), "integrity is invalid") { + t.Fatalf("tampered readAuditExportFile() error = %v", err) + } +} + +func TestReadAuditExportFileRejectsOversizedAndNonRegularInputs(t *testing.T) { + directory := t.TempDir() + if _, err := readAuditExportFile(directory); err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("directory error = %v", err) + } + + target := writeAuditExportFile(t, validCLIAuditExport()) + symlink := filepath.Join(t.TempDir(), "audit-link.json") + if err := os.Symlink(target, symlink); err != nil { + t.Fatal(err) + } + if _, err := readAuditExportFile(symlink); err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("symlink error = %v", err) + } + + oversized := filepath.Join(t.TempDir(), "oversized.json") + if err := os.WriteFile(oversized, bytes.Repeat([]byte{' '}, auditrecord.MaxDocumentBytes+1), 0o600); err != nil { + t.Fatal(err) + } + if _, err := readAuditExportFile(oversized); err == nil || !strings.Contains(err.Error(), "byte limit") { + t.Fatalf("oversized error = %v", err) + } +} + +func validCLIAuditExport() auditrecord.Export { + entry := auditrecord.Entry{ + Sequence: 1, FormatVersion: 1, RecordedAt: time.Date(2026, time.July, 18, 10, 0, 0, 0, time.UTC), + TraceID: strings.Repeat("1", 32), Actor: "user:alice", Role: "admin", Action: "export-audit", + Verb: "audit.export", Verdict: "allow", ReasonCode: "phase-1-audit-export", + EventKind: "policy-decision", PreviousHash: "sha256:" + strings.Repeat("0", 64), + } + entryHash, err := auditrecord.RecomputeEntryHash("workspace-a", entry) + if err != nil { + panic(err) + } + entry.EntryHash = entryHash + return auditrecord.Export{ + Schema: auditrecord.SchemaV1, WorkspaceID: "workspace-a", + Chain: auditrecord.Chain{HashAlgorithm: auditrecord.HashAlgorithm, HeadSequence: 1, HeadHash: entryHash}, + Entries: []auditrecord.Entry{entry}, + } +} + +func writeAuditExportFile(t *testing.T, exported auditrecord.Export) string { + t.Helper() + payload, err := json.Marshal(exported) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "audit.json") + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func validCLIAuditPages(t *testing.T, count int) []auditrecord.Page { + t.Helper() + entries := make([]auditrecord.Entry, count) + previous := "sha256:" + strings.Repeat("0", 64) + for index := range entries { + entry := auditrecord.Entry{ + Sequence: int64(index + 1), FormatVersion: 1, + RecordedAt: time.Date(2026, time.July, 18, 10, 0, 0, index*1000, time.UTC).Truncate(time.Microsecond), + TraceID: strings.Repeat("1", 32), Actor: "user:alice", Role: "admin", Action: "export-audit", + Verb: "audit.export", Verdict: "allow", ReasonCode: "phase-1-audit-export", + EventKind: "policy-decision", PreviousHash: previous, + } + hash, err := auditrecord.RecomputeEntryHash("workspace-a", entry) + if err != nil { + t.Fatal(err) + } + entry.EntryHash = hash + entries[index] = entry + previous = hash + } + snapshot := auditrecord.Chain{HashAlgorithm: auditrecord.HashAlgorithm, HeadSequence: int64(count), HeadHash: previous} + pages := make([]auditrecord.Page, 0, (count+auditrecord.MaxEntries-1)/auditrecord.MaxEntries) + for start := 0; start < count; start += auditrecord.MaxEntries { + end := start + auditrecord.MaxEntries + if end > count { + end = count + } + page := auditrecord.Page{ + Schema: auditrecord.PageSchemaV1, WorkspaceID: "workspace-a", Snapshot: snapshot, + StartSequence: int64(start + 1), PreviousHash: entries[start].PreviousHash, + Entries: append([]auditrecord.Entry(nil), entries[start:end]...), + } + if end < count { + cursor, err := auditrecord.EncodePageCursor( + "workspace-a", int64(count), previous, int64(end+1), entries[end-1].EntryHash, + ) + if err != nil { + t.Fatal(err) + } + page.NextCursor = cursor + } + pages = append(pages, page) + } + return pages +} + +func writeAuditPageFiles(t *testing.T, pages []auditrecord.Page) []string { + t.Helper() + directory := t.TempDir() + paths := make([]string, len(pages)) + for index, page := range pages { + payload, err := json.Marshal(page) + if err != nil { + t.Fatal(err) + } + paths[index] = filepath.Join(directory, fmt.Sprintf("audit-page-%04d.json", index)) + if err := os.WriteFile(paths[index], payload, 0o600); err != nil { + t.Fatal(err) + } + } + return paths +} diff --git a/internal/cli/audit_unix_test.go b/internal/cli/audit_unix_test.go new file mode 100644 index 0000000..4d3e83e --- /dev/null +++ b/internal/cli/audit_unix_test.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package cli + +import ( + "path/filepath" + "strings" + "syscall" + "testing" +) + +func TestReadAuditExportFileRejectsNamedPipeBeforeOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.pipe") + if err := syscall.Mkfifo(path, 0o600); err != nil { + t.Fatal(err) + } + if _, err := readAuditExportFile(path); err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatalf("named pipe error = %v", err) + } +} diff --git a/internal/cli/cached_test.go b/internal/cli/cached_test.go index 76d3b24..429a92c 100644 --- a/internal/cli/cached_test.go +++ b/internal/cli/cached_test.go @@ -145,7 +145,8 @@ func (*cacheReader) Capabilities() []connector.Capability { func (reader *cacheReader) Descriptor() connector.Descriptor { return connector.Descriptor{ - Kind: reader.Kind(), ConnKind: connector.KindReadAdapter, ProtocolV: "1.0.0", Owner: "test", + Kind: reader.Kind(), ConnKind: connector.KindReadAdapter, + WireVersions: []connector.WireVersion{connector.CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", Capabilities: reader.Capabilities(), } } diff --git a/internal/cli/desktop.go b/internal/cli/desktop.go index 9264e2a..2119136 100644 --- a/internal/cli/desktop.go +++ b/internal/cli/desktop.go @@ -13,6 +13,7 @@ import ( "github.com/ArdurAI/sith/internal/connector" "github.com/ArdurAI/sith/internal/connector/kubeconfig" + "github.com/ArdurAI/sith/internal/fleet" "github.com/ArdurAI/sith/internal/fleetcache" "github.com/ArdurAI/sith/internal/hydrate" "github.com/ArdurAI/sith/internal/localops" @@ -77,7 +78,7 @@ func runDesktopHydration(ctx context.Context, store *fleetcache.Store, run func( if err := run(ctx); err != nil && ctx.Err() == nil { // The cache/API exposes only a closed operational category. Raw watch // errors can carry cluster-specific details and do not cross this boundary. - store.EndSync(errors.New(desktopHydrationStopped)) + _ = store.EndSync(fleet.LocalWorkspace, errors.New(desktopHydrationStopped)) } } diff --git a/internal/cli/investigate_test.go b/internal/cli/investigate_test.go index 1955542..b9a561b 100644 --- a/internal/cli/investigate_test.go +++ b/internal/cli/investigate_test.go @@ -5,6 +5,7 @@ package cli import ( "bytes" "encoding/json" + "reflect" "strings" "testing" "time" @@ -12,6 +13,7 @@ import ( "github.com/spf13/cobra" "github.com/ArdurAI/sith/internal/brain" + connectorelasticsearch "github.com/ArdurAI/sith/internal/connector/elasticsearch" "github.com/ArdurAI/sith/internal/fleet" ) @@ -41,6 +43,93 @@ func TestInvestigateJSONUsesStableVerdictSchema(t *testing.T) { } } +func TestInvestigationSurfaceRendersSanitizedElasticsearchR3Evidence(t *testing.T) { + t.Parallel() + eventAt := time.Date(2026, 7, 18, 23, 30, 0, 0, time.UTC) + rawMarker := "DISCARD_RAW_PASSWORD=https://user:pass@example.invalid" + containerMarker := "discard-container-metadata" + response, err := json.Marshal(map[string]any{ + "timed_out": false, + "_shards": map[string]any{ + "total": 1, "successful": 1, "skipped": 0, "failed": 0, + }, + "hits": map[string]any{ + "hits": []any{map[string]any{ + "fields": map[string]any{ + "@timestamp": []string{eventAt.Format(time.RFC3339Nano)}, + "message": []string{"missing required environment variable " + rawMarker}, + "orchestrator.cluster.name": []string{"alpha"}, + "kubernetes.namespace": []string{"synthetic"}, + "kubernetes.pod.name": []string{"payments-0"}, + "kubernetes.container.name": []string{containerMarker}, + }, + }}, + }, + }) + if err != nil { + t.Fatalf("marshal Elasticsearch response: %v", err) + } + facts, err := connectorelasticsearch.ProjectLogCauses(connectorelasticsearch.Projection{ + Workspace: fleet.LocalWorkspace, Scope: "alpha", Namespace: "synthetic", Pod: "payments-0", + Container: containerMarker, WindowStart: eventAt.Add(-time.Minute), WindowEnd: eventAt, + ObservedAt: eventAt.Add(time.Minute), Response: response, + }) + if err != nil { + t.Fatalf("ProjectLogCauses() error = %v", err) + } + input, err := brain.FromGraphFacts( + fleet.LocalWorkspace, + facts, + map[fleet.Lens]brain.LensCoverage{ + fleet.LensLive: {Available: true}, + fleet.LensTelemetry: {Available: true}, + }, + ) + if err != nil { + t.Fatalf("FromGraphFacts() error = %v", err) + } + input.Observations = append(input.Observations, brain.Observation{ + Ref: fleet.ResourceRef{ + SourceKind: "kubeconfig", Scope: "alpha", Kind: "Pod", + Namespace: "synthetic", Name: "payments-0", + }, + Lens: fleet.LensLive, Key: "pod.failure", Value: "CrashLoopBackOff", + ObservedAt: eventAt.Add(time.Minute), Source: "alpha", + }) + result, err := brain.Evaluate(input) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + if len(result.Verdicts) != 1 || result.Verdicts[0].Rule != brain.RuleCrashLoop || + result.Verdicts[0].Status != brain.StatusConfirmed || result.Verdicts[0].FleetWide { + t.Fatalf("result = %#v, want confirmed entity-local R3", result) + } + + for _, format := range []string{"text", "json"} { + format := format + t.Run(format, func(t *testing.T) { + t.Parallel() + var output bytes.Buffer + command := &cobra.Command{} + command.SetOut(&output) + if err := writeInvestigation(command, format, result); err != nil { + t.Fatalf("writeInvestigation(%s) error = %v", format, err) + } + encoded := output.String() + for _, expected := range []string{"R3", "logs.cause", "missing-config"} { + if !strings.Contains(encoded, expected) { + t.Errorf("output = %q, want %q", encoded, expected) + } + } + for _, discarded := range []string{rawMarker, containerMarker, "example.invalid", `"count"`, `"first_event_at"`, `"last_event_at"`} { + if strings.Contains(encoded, discarded) { + t.Errorf("output retained discarded Elasticsearch data %q: %s", discarded, encoded) + } + } + }) + } +} + func TestInvestigateUnknownContextFailsClosed(t *testing.T) { stdout, stderr, exitCode := runCLIWithReader(t, []string{"investigate", "--context", "missing", "-o", "json"}, &cacheReader{}) if exitCode == 0 || !strings.Contains(stderr, "reached 0/1 contexts") { @@ -86,3 +175,156 @@ func TestInvestigationSurfaceRendersR1ThroughR4Fixtures(t *testing.T) { }) } } + +func TestInvestigationSurfaceRendersBoundedImagePullFailure(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 12, 0, 0, 0, time.UTC) + ref := fleet.ResourceRef{SourceKind: "fixture", Scope: "alpha", Kind: "Pod", Namespace: "prod", Name: "payments-0"} + result, err := brain.Evaluate(brain.Investigation{ + Workspace: fleet.LocalWorkspace, + Coverage: map[fleet.Lens]brain.LensCoverage{fleet.LensLive: {Available: true}}, + Observations: []brain.Observation{{ + Ref: ref, Lens: fleet.LensLive, Key: "pod.reason", Value: "ErrImagePull", ObservedAt: now, Source: "fixture", + }}, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + + var textOutput bytes.Buffer + textCommand := &cobra.Command{} + textCommand.SetOut(&textOutput) + if err := writeInvestigation(textCommand, "text", result); err != nil { + t.Fatalf("writeInvestigation(text) error = %v", err) + } + for _, expected := range []string{ + "R7 image pull failure [confirmed]", + "evidence: live pod.reason=ErrImagePull", + "suggested: kubectl --context 'alpha' describe pod 'payments-0' -n 'prod'", + "sensitive: human review required", + } { + if !strings.Contains(textOutput.String(), expected) { + t.Errorf("text output = %q, want %q", textOutput.String(), expected) + } + } + + var jsonOutput bytes.Buffer + jsonCommand := &cobra.Command{} + jsonCommand.SetOut(&jsonOutput) + if err := writeInvestigation(jsonCommand, "json", result); err != nil { + t.Fatalf("writeInvestigation(json) error = %v", err) + } + var decoded brain.Result + if err := json.Unmarshal(jsonOutput.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal R7 JSON: %v", err) + } + if !reflect.DeepEqual(decoded, result) { + t.Fatalf("decoded JSON = %#v, want %#v", decoded, result) + } +} + +func TestInvestigationSurfaceRendersBoundedArgoSyncFailure(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 15, 45, 0, 0, time.UTC) + ref := fleet.ResourceRef{SourceKind: "argocd", Scope: "alpha", Kind: "Application", Namespace: "argocd", Name: "payments"} + result, err := brain.Evaluate(brain.Investigation{ + Workspace: fleet.LocalWorkspace, + Coverage: map[fleet.Lens]brain.LensCoverage{fleet.LensTimeline: {Available: true}}, + Observations: []brain.Observation{{ + Ref: ref, Lens: fleet.LensTimeline, Key: "change.kind", Value: "sync-failed", ObservedAt: now, Source: "argocd", + }}, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + + var textOutput bytes.Buffer + textCommand := &cobra.Command{} + textCommand.SetOut(&textOutput) + if err := writeInvestigation(textCommand, "text", result); err != nil { + t.Fatalf("writeInvestigation(text) error = %v", err) + } + for _, expected := range []string{ + "R8 Argo CD sync failure [confirmed]", + "evidence: timeline change.kind=sync-failed", + "suggested: kubectl --context 'alpha' describe application.argoproj.io 'payments' -n 'argocd'", + "sensitive: human review required", + } { + if !strings.Contains(textOutput.String(), expected) { + t.Errorf("text output = %q, want %q", textOutput.String(), expected) + } + } + + var jsonOutput bytes.Buffer + jsonCommand := &cobra.Command{} + jsonCommand.SetOut(&jsonOutput) + if err := writeInvestigation(jsonCommand, "json", result); err != nil { + t.Fatalf("writeInvestigation(json) error = %v", err) + } + var decoded brain.Result + if err := json.Unmarshal(jsonOutput.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal R8 JSON: %v", err) + } + if !reflect.DeepEqual(decoded, result) { + t.Fatalf("decoded JSON = %#v, want %#v", decoded, result) + } + if strings.Contains(jsonOutput.String(), "operationState") || strings.Contains(jsonOutput.String(), "revision") { + t.Fatalf("R8 JSON exposed discarded Argo payload fields: %s", jsonOutput.String()) + } +} + +func TestInvestigationSurfaceRendersBoundedWorkflowRunFailure(t *testing.T) { + t.Parallel() + now := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + ref := fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "WorkflowRun", + Namespace: "ArdurAI", Name: "sith#30433642-attempt-2", + } + result, err := brain.Evaluate(brain.Investigation{ + Workspace: fleet.LocalWorkspace, + Coverage: map[fleet.Lens]brain.LensCoverage{fleet.LensTimeline: {Available: true}}, + Observations: []brain.Observation{{ + Ref: ref, Lens: fleet.LensTimeline, Key: "change.kind", Value: "workflow-run-failed", + ObservedAt: now, Source: "github.com", + }}, + }) + if err != nil { + t.Fatalf("Evaluate() error = %v", err) + } + + var textOutput bytes.Buffer + textCommand := &cobra.Command{} + textCommand.SetOut(&textOutput) + if err := writeInvestigation(textCommand, "text", result); err != nil { + t.Fatalf("writeInvestigation(text) error = %v", err) + } + for _, expected := range []string{ + "R9 GitHub Actions workflow-run failure [confirmed]", + "evidence: timeline change.kind=workflow-run-failed", + "suggested: inspect GitHub Actions workflow run 'sith#30433642-attempt-2' owned by 'ArdurAI' and its failed jobs and logs before considering a rerun", + "sensitive: human review required", + } { + if !strings.Contains(textOutput.String(), expected) { + t.Errorf("text output = %q, want %q", textOutput.String(), expected) + } + } + + var jsonOutput bytes.Buffer + jsonCommand := &cobra.Command{} + jsonCommand.SetOut(&jsonOutput) + if err := writeInvestigation(jsonCommand, "json", result); err != nil { + t.Fatalf("writeInvestigation(json) error = %v", err) + } + var decoded brain.Result + if err := json.Unmarshal(jsonOutput.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal R9 JSON: %v", err) + } + if !reflect.DeepEqual(decoded, result) { + t.Fatalf("decoded JSON = %#v, want %#v", decoded, result) + } + for _, discarded := range []string{`"workflow_id":`, `"run_attempt":`, `"conclusion":`, "jobs_url", "logs_url"} { + if strings.Contains(jsonOutput.String(), discarded) { + t.Fatalf("R9 JSON exposed discarded workflow-run payload field %q: %s", discarded, jsonOutput.String()) + } + } +} diff --git a/internal/cli/launch.go b/internal/cli/launch.go new file mode 100644 index 0000000..ab93408 --- /dev/null +++ b/internal/cli/launch.go @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "fmt" + "runtime" + + "github.com/spf13/cobra" + + "github.com/ArdurAI/sith/internal/connector" + "github.com/ArdurAI/sith/internal/localops" +) + +type launchMode string + +const ( + launchModeAuto launchMode = "auto" + launchModeDesktop launchMode = "desktop" + launchModeUI launchMode = "ui" +) + +type launchOptions struct { + mode string + ui uiOptions +} + +func newLaunchCommand(reader connector.Reader, local localops.Client) *cobra.Command { + options := &launchOptions{ + mode: string(launchModeAuto), + ui: uiOptions{address: "127.0.0.1"}, + } + command := &cobra.Command{ + Use: "launch", + Short: "Open the local fleet IDE", + Long: "Open the local fleet IDE using the native desktop on macOS and the loopback-only UI " + + "on other supported platforms. Listener and browser flags require --mode ui.", + Example: " sith launch\n" + + " sith launch --kubeconfig-dir /path/to/kubeconfigs\n" + + " sith launch --mode ui --no-open", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, _ []string) error { + mode, err := resolveLaunchMode(options.mode, runtime.GOOS) + if err != nil { + return err + } + switch mode { + case launchModeDesktop: + if err := rejectDesktopUIFlags(command); err != nil { + return err + } + if err := validateDesktopDependencies(reader, local, options.ui.kubeconfigDir); err != nil { + return err + } + return runDesktop(command.Context(), reader, local, options.ui.kubeconfigDir) + case launchModeUI: + selectedReader, selectedLocal, err := selectUIBackend(reader, local, options.ui.kubeconfigDir) + if err != nil { + return err + } + return runWebUI(command.Context(), command, selectedReader, selectedLocal, &options.ui) + default: + return fmt.Errorf("launch mode %q is unsupported", mode) + } + }, + } + command.Flags().StringVar(&options.mode, "mode", options.mode, "launch mode: auto, desktop, or ui") + bindUIFlags(command, &options.ui) + return command +} + +func resolveLaunchMode(requested, goos string) (launchMode, error) { + switch launchMode(requested) { + case launchModeAuto: + if goos == "darwin" { + return launchModeDesktop, nil + } + return launchModeUI, nil + case launchModeDesktop: + if goos != "darwin" { + return "", fmt.Errorf("launch mode %q is available only on macOS", requested) + } + return launchModeDesktop, nil + case launchModeUI: + return launchModeUI, nil + default: + return "", fmt.Errorf("invalid launch mode %q: expected auto, desktop, or ui", requested) + } +} + +func rejectDesktopUIFlags(command *cobra.Command) error { + for _, name := range []string{"address", "port", "no-open"} { + if command.Flags().Changed(name) { + return fmt.Errorf("--%s requires --mode ui", name) + } + } + return nil +} diff --git a/internal/cli/launch_test.go b/internal/cli/launch_test.go new file mode 100644 index 0000000..e36a307 --- /dev/null +++ b/internal/cli/launch_test.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestResolveLaunchMode(t *testing.T) { + t.Parallel() + tests := []struct { + name string + requested string + goos string + want launchMode + wantError bool + }{ + {name: "macOS auto", requested: "auto", goos: "darwin", want: launchModeDesktop}, + {name: "Linux auto", requested: "auto", goos: "linux", want: launchModeUI}, + {name: "Windows auto", requested: "auto", goos: "windows", want: launchModeUI}, + {name: "macOS desktop", requested: "desktop", goos: "darwin", want: launchModeDesktop}, + {name: "Linux desktop", requested: "desktop", goos: "linux", wantError: true}, + {name: "explicit UI", requested: "ui", goos: "darwin", want: launchModeUI}, + {name: "unknown", requested: "browser", goos: "darwin", wantError: true}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := resolveLaunchMode(test.requested, test.goos) + if test.wantError { + if err == nil { + t.Fatalf("resolveLaunchMode(%q, %q) = %q, want error", test.requested, test.goos, got) + } + return + } + if err != nil || got != test.want { + t.Fatalf("resolveLaunchMode(%q, %q) = %q, %v, want %q", test.requested, test.goos, got, err, test.want) + } + }) + } +} + +func TestLaunchCommandIsRegistered(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, []string{"--help"}, fleet.StubSource{}) + if exitCode != 0 || stderr != "" || !strings.Contains(stdout, "launch") || + !strings.Contains(stdout, "Open the local fleet IDE") { + t.Fatalf("root help exit/stdout/stderr = %d/%q/%q", exitCode, stdout, stderr) + } +} + +func TestLaunchRejectsUnknownModeBeforeBackendSelection(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, []string{"launch", "--mode", "browser"}, fleet.StubSource{}) + if exitCode == 0 || stdout != "" || !strings.Contains(stderr, "invalid launch mode") { + t.Fatalf("launch exit/stdout/stderr = %d/%q/%q", exitCode, stdout, stderr) + } +} + +func TestLaunchUIRequiresLocalBackend(t *testing.T) { + stdout, stderr, exitCode := runCLI(t, []string{"launch", "--mode", "ui", "--no-open"}, fleet.StubSource{}) + if exitCode == 0 || stdout != "" || !strings.Contains(stderr, "requires a Kubernetes reader") { + t.Fatalf("launch exit/stdout/stderr = %d/%q/%q", exitCode, stdout, stderr) + } +} + +func TestLaunchUIStartsOnLoopback(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stdout, stderr, exitCode := runUICLI(ctx, t, []string{ + "launch", "--mode", "ui", "--port", "0", "--no-open", + }, &cacheReader{}, &fakeLocalClient{}) + if exitCode != 0 || stderr != "" || !strings.Contains(stdout, "sith ui listening on http://127.0.0.1:") { + t.Fatalf("launch exit/stdout/stderr = %d/%q/%q", exitCode, stdout, stderr) + } +} + +func TestSelectUIBackendAcceptsExplicitDirectoryWithoutDefaultBackend(t *testing.T) { + directory := writeLaunchKubeconfig(t) + reader, local, err := selectUIBackend(nil, nil, directory) + if err != nil || reader == nil || local == nil { + t.Fatalf("selectUIBackend() = %#v, %#v, %v, want explicit directory backend", reader, local, err) + } +} + +func TestLaunchUIStartsFromExplicitDirectoryWithoutDefaultBackend(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("SITH_KUBECONFIG", "") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var stdout, stderr bytes.Buffer + exitCode := executeBackendContext(ctx, []string{ + "launch", "--mode", "ui", "--port", "0", "--no-open", + "--kubeconfig-dir", writeLaunchKubeconfig(t), + }, backend{source: fleet.StubSource{}}, &stdout, &stderr) + if exitCode != 0 || stderr.Len() != 0 || + !strings.Contains(stdout.String(), "sith ui listening on http://127.0.0.1:") { + t.Fatalf("launch exit/stdout/stderr = %d/%q/%q", exitCode, stdout.String(), stderr.String()) + } +} + +func writeLaunchKubeconfig(t *testing.T) string { + t.Helper() + directory := t.TempDir() + config := []byte(`apiVersion: v1 +kind: Config +clusters: + - name: demo + cluster: + server: https://demo.invalid +users: + - name: demo + user: {} +contexts: + - name: demo + context: + cluster: demo + user: demo +current-context: demo +`) + if err := os.WriteFile(filepath.Join(directory, "config.yaml"), config, 0o600); err != nil { + t.Fatal(err) + } + return directory +} + +func TestDesktopLaunchRejectsUIOnlyFlags(t *testing.T) { + command := &cobra.Command{Use: "launch"} + options := &uiOptions{address: "127.0.0.1"} + bindUIFlags(command, options) + if err := command.ParseFlags([]string{"--no-open"}); err != nil { + t.Fatal(err) + } + if err := rejectDesktopUIFlags(command); err == nil || !strings.Contains(err.Error(), "--mode ui") { + t.Fatalf("rejectDesktopUIFlags() error = %v, want explicit UI-mode guidance", err) + } +} + +func TestDesktopLaunchAcceptsSharedKubeconfigDirectoryFlag(t *testing.T) { + command := &cobra.Command{Use: "launch"} + options := &uiOptions{address: "127.0.0.1"} + bindUIFlags(command, options) + if err := command.ParseFlags([]string{"--kubeconfig-dir", t.TempDir()}); err != nil { + t.Fatal(err) + } + if err := rejectDesktopUIFlags(command); err != nil { + t.Fatalf("rejectDesktopUIFlags() error = %v, want shared directory flag accepted", err) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index a8c93fa..ef83c7d 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/term" + "github.com/ArdurAI/sith/internal/auditdelivery" "github.com/ArdurAI/sith/internal/config" "github.com/ArdurAI/sith/internal/connector" "github.com/ArdurAI/sith/internal/connector/kubeconfig" @@ -50,6 +51,12 @@ type backend struct { // Execute builds and runs the command tree, returning a process exit code. func Execute() int { + if len(os.Args) == 2 && os.Args[1] == auditdelivery.ChildArgument { + if auditdelivery.RunChild(os.Stderr) != nil { + return 1 + } + return 0 + } adapter := kubeconfig.Default() ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() @@ -142,9 +149,11 @@ func newRootCommand(runtime backend, stdout, stderr io.Writer) *cobra.Command { commands := []*cobra.Command{ newVersionCommand(options), + newAuditCommand(options), newClustersCommand(options, runtime.source), newUICommand(runtime.reader, runtime.local), newDesktopCommand(runtime.reader, runtime.local), + newLaunchCommand(runtime.reader, runtime.local), newHubCommand(), } if runtime.reader != nil { diff --git a/internal/cli/ui.go b/internal/cli/ui.go index edfcabb..ec5dcd7 100644 --- a/internal/cli/ui.go +++ b/internal/cli/ui.go @@ -37,24 +37,40 @@ func newUICommand(reader connector.Reader, local localops.Client) *cobra.Command Short: "Start the loopback-only local fleet IDE", Args: cobra.NoArgs, RunE: func(command *cobra.Command, _ []string) error { - if reader == nil || local == nil { - return fmt.Errorf("local fleet UI requires a Kubernetes reader and local operations client") + selectedReader, selectedLocal, err := selectUIBackend(reader, local, options.kubeconfigDir) + if err != nil { + return err } - if options.kubeconfigDir != "" { - adapter, err := kubeconfig.New(kubeconfig.WithDirectory(options.kubeconfigDir)) - if err != nil { - return fmt.Errorf("import kubeconfig directory: %w", err) - } - reader, local = adapter, adapter - } - return runWebUI(command.Context(), command, reader, local, options) + return runWebUI(command.Context(), command, selectedReader, selectedLocal, options) }, } + bindUIFlags(command, options) + return command +} + +func bindUIFlags(command *cobra.Command, options *uiOptions) { command.Flags().StringVar(&options.address, "address", options.address, "loopback listen address") command.Flags().IntVar(&options.port, "port", 0, "loopback listen port; 0 selects an available port") command.Flags().BoolVar(&options.noOpen, "no-open", false, "do not open the system browser") command.Flags().StringVar(&options.kubeconfigDir, "kubeconfig-dir", "", "import kubeconfig files from this directory for this local UI session") - return command +} + +func selectUIBackend( + reader connector.Reader, + local localops.Client, + kubeconfigDirectory string, +) (connector.Reader, localops.Client, error) { + if kubeconfigDirectory != "" { + adapter, err := kubeconfig.New(kubeconfig.WithDirectory(kubeconfigDirectory)) + if err != nil { + return nil, nil, fmt.Errorf("import kubeconfig directory: %w", err) + } + return adapter, adapter, nil + } + if reader == nil || local == nil { + return nil, nil, fmt.Errorf("local fleet UI requires a Kubernetes reader and local operations client") + } + return reader, local, nil } func runWebUI( diff --git a/internal/connector/argocd/boundary_test.go b/internal/connector/argocd/boundary_test.go new file mode 100644 index 0000000..4080f40 --- /dev/null +++ b/internal/connector/argocd/boundary_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package argocd + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestProjectorHasNoNetworkOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read argocd package: %v", err) + } + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + switch path { + case "net/http", "os/exec", "k8s.io/client-go/dynamic", "k8s.io/client-go/rest": + t.Fatalf("projector imports network or execution package %q", path) + } + } + ast.Inspect(file, func(node ast.Node) bool { + declaration, ok := node.(*ast.FuncDecl) + if !ok { + return true + } + switch declaration.Name.Name { + case "Plan", "Execute", "Verify", "Sync", "Rollback": + t.Errorf("projector declares forbidden mutation seam %s", declaration.Name.Name) + } + return true + }) + } +} diff --git a/internal/connector/argocd/project.go b/internal/connector/argocd/project.go new file mode 100644 index 0000000..1faf3e8 --- /dev/null +++ b/internal/connector/argocd/project.go @@ -0,0 +1,670 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package argocd normalizes read-only Argo CD evidence for Sith's operational graph. +package argocd + +import ( + "encoding/json" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for Argo CD read evidence. + Kind = "argocd" + // ProtocolVersion identifies the normalized fact contract emitted by this projector. + ProtocolVersion = "1.0.0" + + maxApplicationSources = 16 + maxApplicationHistory = 32 + maxApplicationFacts = 3 + maxApplicationHistory + 1 + maxFactPayloadBytes = 16 << 10 + maxTextBytes = 2 << 10 + maxRevisionBytes = 512 +) + +// Projection supplies one already-authorized Application observation. ProjectApplication does +// not perform discovery, network access, credential loading, or mutation. +type Projection struct { + Workspace string + Scope string + ObservedAt time.Time + Application unstructured.Unstructured +} + +type desiredObservation struct { + Project string `json:"project,omitempty"` + Sources []applicationSource `json:"sources,omitempty"` + Destination applicationTarget `json:"destination,omitempty"` +} + +type applicationSource struct { + Repository string `json:"repository"` + Path string `json:"path,omitempty"` + Chart string `json:"chart,omitempty"` + TargetRevision string `json:"target_revision,omitempty"` +} + +type applicationTarget struct { + Name string `json:"name,omitempty"` + Server string `json:"server,omitempty"` + Namespace string `json:"namespace,omitempty"` +} + +type healthObservation struct { + Health string `json:"health"` +} + +type driftObservation struct { + SyncStatus string `json:"sync_status"` + Revision string `json:"revision,omitempty"` + Drifted *bool `json:"drifted,omitempty"` +} + +type changeObservation struct { + ChangeKind string `json:"change_kind"` + Revision string `json:"revision,omitempty"` + Phase string `json:"phase,omitempty"` + EventAt time.Time `json:"event_at"` + HistoryID string `json:"history_id,omitempty"` + HistoryTruncated bool `json:"history_truncated,omitempty"` +} + +type projectedChange struct { + observation changeObservation + nativeID string +} + +// ProjectApplication returns deterministic, bounded, allowlisted graph facts for one Argo CD +// Application. Missing status fields produce no corresponding fact; the projector never invents +// healthy, synced, or change evidence. +func ProjectApplication(input Projection) ([]fleet.GraphFact, error) { + application := input.Application + if err := validateProjection(input); err != nil { + return nil, err + } + observedAt := input.ObservedAt.UTC() + entity := fleet.EntityRef{ + Cluster: input.Scope, Namespace: application.GetNamespace(), Kind: "Application", Name: application.GetName(), + } + baseNativeID := input.Scope + "/" + application.GetNamespace() + "/" + application.GetName() + + facts := make([]fleet.GraphFact, 0, 3+maxApplicationHistory+1) + desired, present, err := projectDesired(application.Object) + if err != nil { + return nil, fmt.Errorf("project Argo CD Application desired state: %w", err) + } + if present { + fact, err := graphFact(input, entity, fleet.FactDesired, fleet.LensDesired, observedAt, baseNativeID+"#desired", desired) + if err != nil { + return nil, err + } + facts = append(facts, fact) + } + + status, err := optionalObjectMap(application.Object, "status") + if err != nil { + return nil, fmt.Errorf("project Argo CD Application status: %w", err) + } + if status == nil { + return facts, nil + } + if health, present, err := projectHealth(status); err != nil { + return nil, err + } else if present { + fact, factErr := graphFact(input, entity, fleet.FactHealth, fleet.LensLive, observedAt, baseNativeID+"#health", health) + if factErr != nil { + return nil, factErr + } + facts = append(facts, fact) + } + if drift, present, err := projectDrift(status); err != nil { + return nil, err + } else if present { + fact, factErr := graphFact(input, entity, fleet.FactDrift, fleet.LensDesired, observedAt, baseNativeID+"#drift", drift) + if factErr != nil { + return nil, factErr + } + facts = append(facts, fact) + } + + changes, err := projectChanges(status) + if err != nil { + return nil, err + } + for _, change := range changes { + fact, factErr := graphFact( + input, entity, fleet.FactChange, fleet.LensTimeline, change.observation.EventAt, + baseNativeID+"#"+change.nativeID, change.observation, + ) + if factErr != nil { + return nil, factErr + } + facts = append(facts, fact) + } + if len(facts) > maxApplicationFacts { + return nil, fmt.Errorf("argo CD application fact count exceeds %d", maxApplicationFacts) + } + return facts, nil +} + +func validateProjection(input Projection) error { + if err := validateText("workspace", input.Workspace, 253); err != nil { + return err + } + if err := validateText("scope", input.Scope, 253); err != nil { + return err + } + if input.ObservedAt.IsZero() { + return fmt.Errorf("projection observation time is required") + } + application := input.Application + groupVersion, err := schema.ParseGroupVersion(application.GetAPIVersion()) + if err != nil || groupVersion.Group != "argoproj.io" || groupVersion.Version == "" || application.GetKind() != "Application" { + return fmt.Errorf("projection object must be an argoproj.io Application") + } + if err := validateText("Application name", application.GetName(), 253); err != nil { + return err + } + if err := validateText("Application namespace", application.GetNamespace(), 253); err != nil { + return err + } + return nil +} + +func projectDesired(object map[string]any) (desiredObservation, bool, error) { + spec, err := optionalObjectMap(object, "spec") + if err != nil { + return desiredObservation{}, false, err + } + if spec == nil { + return desiredObservation{}, false, nil + } + project, err := optionalText(spec, "project", 253) + if err != nil { + return desiredObservation{}, false, err + } + destination, err := projectDestination(spec) + if err != nil { + return desiredObservation{}, false, err + } + sources, err := projectSources(spec) + if err != nil { + return desiredObservation{}, false, err + } + present := project != "" || len(sources) != 0 || destination != (applicationTarget{}) + return desiredObservation{Project: project, Sources: sources, Destination: destination}, present, nil +} + +func projectSources(spec map[string]any) ([]applicationSource, error) { + single, singlePresent, err := optionalMap(spec, "source") + if err != nil { + return nil, err + } + multiple, multiplePresent, err := optionalSlice(spec, "sources") + if err != nil { + return nil, err + } + if singlePresent && multiplePresent && len(multiple) != 0 { + return nil, fmt.Errorf("application source and sources are mutually exclusive") + } + if singlePresent { + multiple = []any{single} + } + if len(multiple) > maxApplicationSources { + return nil, fmt.Errorf("application source count exceeds %d", maxApplicationSources) + } + result := make([]applicationSource, 0, len(multiple)) + for index, raw := range multiple { + source, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("application source %d must be an object", index) + } + repository, err := requiredSanitizedURL(source, "repoURL") + if err != nil { + return nil, fmt.Errorf("application source %d: %w", index, err) + } + path, err := optionalText(source, "path", maxTextBytes) + if err != nil { + return nil, fmt.Errorf("application source %d: %w", index, err) + } + chart, err := optionalText(source, "chart", maxTextBytes) + if err != nil { + return nil, fmt.Errorf("application source %d: %w", index, err) + } + revision, err := optionalText(source, "targetRevision", maxRevisionBytes) + if err != nil { + return nil, fmt.Errorf("application source %d: %w", index, err) + } + result = append(result, applicationSource{ + Repository: repository, Path: path, Chart: chart, TargetRevision: revision, + }) + } + return result, nil +} + +func projectDestination(spec map[string]any) (applicationTarget, error) { + destination, present, err := optionalMap(spec, "destination") + if err != nil || !present { + return applicationTarget{}, err + } + name, err := optionalText(destination, "name", 253) + if err != nil { + return applicationTarget{}, err + } + server, err := optionalText(destination, "server", maxTextBytes) + if err != nil { + return applicationTarget{}, err + } + if name != "" && server != "" { + return applicationTarget{}, fmt.Errorf("application destination name and server are mutually exclusive") + } + if server != "" { + server, err = sanitizeAbsoluteURL(server, map[string]bool{"http": true, "https": true}) + if err != nil { + return applicationTarget{}, fmt.Errorf("application destination server: %w", err) + } + } + namespace, err := optionalText(destination, "namespace", 253) + if err != nil { + return applicationTarget{}, err + } + return applicationTarget{Name: name, Server: server, Namespace: namespace}, nil +} + +func projectHealth(status map[string]any) (healthObservation, bool, error) { + health, present, err := optionalMap(status, "health") + if err != nil || !present { + return healthObservation{}, false, err + } + value, err := optionalText(health, "status", 64) + if err != nil || value == "" { + return healthObservation{}, false, err + } + switch value { + case "Healthy", "Progressing", "Degraded", "Suspended", "Missing", "Unknown": + return healthObservation{Health: value}, true, nil + default: + return healthObservation{}, false, fmt.Errorf("application health status %q is unsupported", value) + } +} + +func projectDrift(status map[string]any) (driftObservation, bool, error) { + syncStatus, present, err := optionalMap(status, "sync") + if err != nil || !present { + return driftObservation{}, false, err + } + value, err := optionalText(syncStatus, "status", 64) + if err != nil || value == "" { + return driftObservation{}, false, err + } + revision, err := optionalText(syncStatus, "revision", maxRevisionBytes) + if err != nil { + return driftObservation{}, false, err + } + result := driftObservation{SyncStatus: value, Revision: revision} + switch value { + case "Synced": + value := false + result.Drifted = &value + case "OutOfSync": + value := true + result.Drifted = &value + case "Unknown": + default: + return driftObservation{}, false, fmt.Errorf("application sync status %q is unsupported", value) + } + return result, true, nil +} + +func projectChanges(status map[string]any) ([]projectedChange, error) { + changes := make([]projectedChange, 0, maxApplicationHistory+1) + history, present, err := optionalSlice(status, "history") + if err != nil { + return nil, fmt.Errorf("application history: %w", err) + } + truncated := false + if present && len(history) > maxApplicationHistory { + history = history[len(history)-maxApplicationHistory:] + truncated = true + } + for index, raw := range history { + entry, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("application history entry %d must be an object", index) + } + at, present, err := optionalTime(entry, "deployedAt") + if err != nil { + return nil, fmt.Errorf("application history entry %d: %w", index, err) + } + if !present { + continue + } + revision, err := optionalText(entry, "revision", maxRevisionBytes) + if err != nil { + return nil, fmt.Errorf("application history entry %d: %w", index, err) + } + id, err := optionalScalarID(entry, "id") + if err != nil { + return nil, fmt.Errorf("application history entry %d: %w", index, err) + } + changes = append(changes, projectedChange{ + observation: changeObservation{ + ChangeKind: "argocd-sync", Revision: revision, EventAt: at, HistoryID: id, + HistoryTruncated: truncated && index == 0, + }, + nativeID: "history/" + stableChangeID(id, revision, at), + }) + } + + operation, operationPresent, err := optionalMap(status, "operationState") + if err != nil { + return nil, fmt.Errorf("application operation state: %w", err) + } + if operationPresent { + change, present, err := projectOperation(operation) + if err != nil { + return nil, err + } + if present { + changes = append(changes, change) + } + } + + deduplicated := make(map[string]projectedChange, len(changes)) + for _, change := range changes { + key := change.observation.ChangeKind + "\x00" + change.observation.Revision + "\x00" + change.observation.EventAt.Format(time.RFC3339Nano) + if previous, exists := deduplicated[key]; exists { + previous.observation.HistoryTruncated = previous.observation.HistoryTruncated || change.observation.HistoryTruncated + if previous.observation.Phase == "" { + previous.observation.Phase = change.observation.Phase + } + deduplicated[key] = previous + continue + } + deduplicated[key] = change + } + result := make([]projectedChange, 0, len(deduplicated)) + for _, change := range deduplicated { + result = append(result, change) + } + sort.Slice(result, func(left, right int) bool { + leftAt, rightAt := result[left].observation.EventAt, result[right].observation.EventAt + if !leftAt.Equal(rightAt) { + return leftAt.Before(rightAt) + } + return result[left].nativeID < result[right].nativeID + }) + return result, nil +} + +func projectOperation(operation map[string]any) (projectedChange, bool, error) { + phase, err := optionalText(operation, "phase", 64) + if err != nil || phase == "" { + return projectedChange{}, false, err + } + var changeKind string + switch phase { + case "Succeeded": + changeKind = "argocd-sync" + case "Failed", "Error": + changeKind = "sync-failed" + case "Running", "Terminating": + changeKind = "argocd-sync" + default: + return projectedChange{}, false, fmt.Errorf("application operation phase %q is unsupported", phase) + } + at, present, err := optionalTime(operation, "finishedAt") + if err != nil { + return projectedChange{}, false, err + } + if !present { + at, present, err = optionalTime(operation, "startedAt") + if err != nil || !present { + return projectedChange{}, false, err + } + } + revision := "" + if syncResult, ok, err := optionalMap(operation, "syncResult"); err != nil { + return projectedChange{}, false, err + } else if ok { + revision, err = optionalText(syncResult, "revision", maxRevisionBytes) + if err != nil { + return projectedChange{}, false, err + } + } + return projectedChange{ + observation: changeObservation{ChangeKind: changeKind, Revision: revision, Phase: phase, EventAt: at}, + nativeID: "operation/" + stableChangeID(phase, revision, at), + }, true, nil +} + +func graphFact( + input Projection, + entity fleet.EntityRef, + kind fleet.FactKind, + lens fleet.Lens, + observedAt time.Time, + nativeID string, + payload any, +) (fleet.GraphFact, error) { + encoded, err := json.Marshal(payload) + if err != nil { + return fleet.GraphFact{}, fmt.Errorf("encode Argo CD Application %s fact: %w", kind, err) + } + if len(encoded) > maxFactPayloadBytes { + return fleet.GraphFact{}, fmt.Errorf("argo CD application %s fact exceeds %d encoded bytes", kind, maxFactPayloadBytes) + } + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, Scope: input.Scope, Kind: "Application", + Namespace: input.Application.GetNamespace(), Name: input.Application.GetName(), + }, + Kind: kind, Observed: encoded, ObservedAt: observedAt.UTC(), Source: input.Scope, + Provenance: fleet.Provenance{Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID}, + }, + Workspace: input.Workspace, + }, + Lens: lens, Entity: &entity, + } + if err := fact.Validate(input.Workspace); err != nil { + return fleet.GraphFact{}, fmt.Errorf("validate Argo CD Application %s fact: %w", kind, err) + } + return fact, nil +} + +func requiredSanitizedURL(object map[string]any, key string) (string, error) { + value, err := optionalText(object, key, maxTextBytes) + if err != nil { + return "", err + } + if value == "" { + return "", fmt.Errorf("%s is required", key) + } + result, err := sanitizeRepositoryURL(value) + if err != nil { + return "", fmt.Errorf("%s: %w", key, err) + } + return result, nil +} + +func sanitizeRepositoryURL(value string) (string, error) { + if err := validateText("URL", value, maxTextBytes); err != nil { + return "", err + } + if strings.Contains(value, "://") { + return sanitizeAbsoluteURL(value, map[string]bool{ + "git": true, "http": true, "https": true, "oci": true, "ssh": true, + }) + } + if strings.ContainsAny(value, "?#") { + return "", fmt.Errorf("scheme-less URL must not contain query or fragment data") + } + if at := strings.LastIndex(value, "@"); at >= 0 { + remainder := value[at+1:] + if !strings.Contains(remainder, ":") { + return "", fmt.Errorf("scheme-less URL user information requires host:path form") + } + value = remainder + } + host := value + if boundary := strings.IndexAny(host, ":/"); boundary >= 0 { + host = host[:boundary] + } + if host == "" || host == "." || host == ".." || strings.ContainsAny(value, "\\ ") { + return "", fmt.Errorf("scheme-less URL must include a repository host") + } + return value, nil +} + +func sanitizeAbsoluteURL(value string, allowedSchemes map[string]bool) (string, error) { + if err := validateText("URL", value, maxTextBytes); err != nil { + return "", err + } + parsed, err := url.Parse(value) + if err != nil || parsed.Scheme == "" || parsed.Host == "" { + return "", fmt.Errorf("url must include a scheme and host") + } + if !allowedSchemes[strings.ToLower(parsed.Scheme)] { + return "", fmt.Errorf("url scheme %q is unsupported", parsed.Scheme) + } + parsed.User = nil + parsed.RawQuery = "" + parsed.ForceQuery = false + parsed.Fragment = "" + return parsed.String(), nil +} + +func optionalObjectMap(object map[string]any, key string) (map[string]any, error) { + value, exists := object[key] + if !exists || value == nil { + return nil, nil + } + result, ok := value.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s must be an object", key) + } + return result, nil +} + +func optionalMap(object map[string]any, key string) (map[string]any, bool, error) { + value, exists := object[key] + if !exists || value == nil { + return nil, false, nil + } + result, ok := value.(map[string]any) + if !ok { + return nil, false, fmt.Errorf("%s must be an object", key) + } + return result, true, nil +} + +func optionalSlice(object map[string]any, key string) ([]any, bool, error) { + value, exists := object[key] + if !exists || value == nil { + return nil, false, nil + } + result, ok := value.([]any) + if !ok { + return nil, false, fmt.Errorf("%s must be an array", key) + } + return result, true, nil +} + +func optionalText(object map[string]any, key string, maximum int) (string, error) { + value, exists := object[key] + if !exists || value == nil { + return "", nil + } + result, ok := value.(string) + if !ok { + return "", fmt.Errorf("%s must be a string", key) + } + if result == "" { + return "", nil + } + if err := validateText(key, result, maximum); err != nil { + return "", err + } + return result, nil +} + +func optionalTime(object map[string]any, key string) (time.Time, bool, error) { + value, err := optionalText(object, key, 64) + if err != nil || value == "" { + return time.Time{}, false, err + } + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, false, fmt.Errorf("%s must be an RFC3339 timestamp", key) + } + return parsed.UTC(), true, nil +} + +func optionalScalarID(object map[string]any, key string) (string, error) { + value, exists := object[key] + if !exists || value == nil { + return "", nil + } + var result string + switch typed := value.(type) { + case string: + result = typed + case int64: + result = strconv.FormatInt(typed, 10) + case int32: + result = strconv.FormatInt(int64(typed), 10) + case int: + result = strconv.Itoa(typed) + case float64: + if typed != float64(int64(typed)) { + return "", fmt.Errorf("%s must be an integer or string", key) + } + result = strconv.FormatInt(int64(typed), 10) + case json.Number: + if _, err := typed.Int64(); err != nil { + return "", fmt.Errorf("%s must be an integer or string", key) + } + result = typed.String() + default: + return "", fmt.Errorf("%s must be an integer or string", key) + } + if result == "" { + return "", nil + } + if err := validateText(key, result, 128); err != nil { + return "", err + } + return result, nil +} + +func validateText(label, value string, maximum int) error { + if value == "" || strings.TrimSpace(value) != value || len(value) > maximum || strings.ContainsAny(value, "\x00\r\n") { + return fmt.Errorf("%s is invalid", label) + } + return nil +} + +func stableChangeID(parts ...any) string { + values := make([]string, 0, len(parts)) + for _, part := range parts { + switch typed := part.(type) { + case time.Time: + values = append(values, typed.UTC().Format(time.RFC3339Nano)) + default: + values = append(values, fmt.Sprint(part)) + } + } + return strings.Join(values, "/") +} diff --git a/internal/connector/argocd/project_test.go b/internal/connector/argocd/project_test.go new file mode 100644 index 0000000..1a40f71 --- /dev/null +++ b/internal/connector/argocd/project_test.go @@ -0,0 +1,339 @@ +// SPDX-License-Identifier: Apache-2.0 + +package argocd + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestProjectApplicationEmitsSanitizedFourLensFacts(t *testing.T) { + t.Parallel() + input := Projection{ + Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Date(2026, 7, 16, 21, 0, 0, 0, time.FixedZone("CDT", -5*60*60)), + Application: completeApplication(), + } + facts, err := ProjectApplication(input) + if err != nil { + t.Fatalf("ProjectApplication() error = %v", err) + } + if len(facts) != 5 { + t.Fatalf("fact count = %d, want 5: %#v", len(facts), facts) + } + wantKinds := []fleet.FactKind{fleet.FactDesired, fleet.FactHealth, fleet.FactDrift, fleet.FactChange, fleet.FactChange} + wantLenses := []fleet.Lens{fleet.LensDesired, fleet.LensLive, fleet.LensDesired, fleet.LensTimeline, fleet.LensTimeline} + for index, fact := range facts { + if fact.Fact.Kind != wantKinds[index] || fact.Lens != wantLenses[index] { + t.Fatalf("fact %d kind/lens = %s/%s, want %s/%s", index, fact.Fact.Kind, fact.Lens, wantKinds[index], wantLenses[index]) + } + if fact.Entity == nil || fact.Entity.Cluster != "cluster-a" || fact.Entity.Namespace != "argocd" || + fact.Entity.Kind != "Application" || fact.Entity.Name != "payments" { + t.Fatalf("fact %d entity = %#v", index, fact.Entity) + } + if fact.Fact.Ref.SourceKind != Kind || fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion { + t.Fatalf("fact %d provenance = %#v", index, fact.Fact.Provenance) + } + } + + var desired desiredObservation + if err := json.Unmarshal(facts[0].Fact.Observed, &desired); err != nil { + t.Fatalf("decode desired fact: %v", err) + } + if len(desired.Sources) != 1 || desired.Sources[0].Repository != "https://github.example/ardur/payments.git" || + desired.Destination.Server != "https://kubernetes.default.svc" || desired.Destination.Namespace != "payments" { + t.Fatalf("desired fact = %#v", desired) + } + var drift driftObservation + if err := json.Unmarshal(facts[2].Fact.Observed, &drift); err != nil { + t.Fatalf("decode drift fact: %v", err) + } + if drift.SyncStatus != "OutOfSync" || drift.Drifted == nil || !*drift.Drifted || drift.Revision != "abc123" { + t.Fatalf("drift fact = %#v", drift) + } + var firstChange, secondChange changeObservation + if err := json.Unmarshal(facts[3].Fact.Observed, &firstChange); err != nil { + t.Fatalf("decode first change: %v", err) + } + if err := json.Unmarshal(facts[4].Fact.Observed, &secondChange); err != nil { + t.Fatalf("decode second change: %v", err) + } + if !firstChange.EventAt.Before(secondChange.EventAt) || secondChange.Phase != "Succeeded" { + t.Fatalf("changes = %#v then %#v", firstChange, secondChange) + } + + encoded, err := json.Marshal(facts) + if err != nil { + t.Fatalf("marshal facts: %v", err) + } + for _, secret := range []string{"git-token", "git-password", "access_token", "cluster-password", "cluster-token", "raw-helm-secret"} { + if strings.Contains(string(encoded), secret) { + t.Fatalf("projected facts retained secret marker %q: %s", secret, encoded) + } + } + graph, err := fleet.NewGraph(input.Workspace, facts) + if err != nil { + t.Fatalf("NewGraph() error = %v", err) + } + if len(graph.Nodes) != 1 || len(graph.Nodes[0].Facts) != len(facts) || len(graph.Unattached) != 0 { + t.Fatalf("graph = %#v", graph) + } +} + +func TestProjectApplicationAbstainsWhenEvidenceIsMissing(t *testing.T) { + t.Parallel() + application := completeApplication() + delete(application.Object, "status") + facts, err := ProjectApplication(Projection{ + Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Application: application, + }) + if err != nil { + t.Fatalf("ProjectApplication() error = %v", err) + } + if len(facts) != 1 || facts[0].Fact.Kind != fleet.FactDesired || facts[0].Lens != fleet.LensDesired { + t.Fatalf("facts = %#v, want only desired evidence", facts) + } + + delete(application.Object, "spec") + facts, err = ProjectApplication(Projection{ + Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Application: application, + }) + if err != nil { + t.Fatalf("ProjectApplication(no spec/status) error = %v", err) + } + if len(facts) != 0 { + t.Fatalf("facts = %#v, want no fabricated evidence", facts) + } +} + +func TestProjectApplicationBoundsHistoryAndMarksTruncation(t *testing.T) { + t.Parallel() + application := minimalApplication() + history := make([]any, maxApplicationHistory+8) + for index := range history { + history[index] = map[string]any{ + "id": int64(index), "revision": fmt.Sprintf("revision-%02d", index), + "deployedAt": time.Date(2026, 7, 1, 0, index, 0, 0, time.UTC).Format(time.RFC3339), + } + } + application.Object["status"] = map[string]any{"history": history} + input := Projection{Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Application: application} + first, err := ProjectApplication(input) + if err != nil { + t.Fatalf("ProjectApplication() error = %v", err) + } + second, err := ProjectApplication(input) + if err != nil { + t.Fatalf("second ProjectApplication() error = %v", err) + } + if len(first) != maxApplicationHistory+1 { + t.Fatalf("fact count = %d, want desired + %d history", len(first), maxApplicationHistory) + } + firstJSON, _ := json.Marshal(first) + secondJSON, _ := json.Marshal(second) + if !slices.Equal(firstJSON, secondJSON) { + t.Fatalf("projection is not deterministic\nfirst: %s\nsecond: %s", firstJSON, secondJSON) + } + var oldest, newest changeObservation + if err := json.Unmarshal(first[1].Fact.Observed, &oldest); err != nil { + t.Fatalf("decode oldest change: %v", err) + } + if err := json.Unmarshal(first[len(first)-1].Fact.Observed, &newest); err != nil { + t.Fatalf("decode newest change: %v", err) + } + if oldest.HistoryID != "8" || !oldest.HistoryTruncated || newest.HistoryID != "39" || newest.HistoryTruncated { + t.Fatalf("oldest/newest = %#v / %#v", oldest, newest) + } +} + +func TestProjectApplicationRejectsMalformedOrAmbiguousEvidence(t *testing.T) { + t.Parallel() + valid := Projection{Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Application: completeApplication()} + tests := []struct { + name string + mutate func(*Projection) + }{ + {"missing workspace", func(input *Projection) { input.Workspace = "" }}, + {"invalid scope", func(input *Projection) { input.Scope = " cluster-a" }}, + {"zero observation time", func(input *Projection) { input.ObservedAt = time.Time{} }}, + {"wrong API group", func(input *Projection) { input.Application.SetAPIVersion("example.io/v1") }}, + {"wrong kind", func(input *Projection) { input.Application.SetKind("Deployment") }}, + {"missing name", func(input *Projection) { input.Application.SetName("") }}, + {"missing namespace", func(input *Projection) { input.Application.SetNamespace("") }}, + {"source and sources", func(input *Projection) { + input.Application.Object["spec"].(map[string]any)["sources"] = []any{map[string]any{"repoURL": "https://github.example/other/repo.git"}} + }}, + {"too many sources", func(input *Projection) { + spec := input.Application.Object["spec"].(map[string]any) + delete(spec, "source") + sources := make([]any, maxApplicationSources+1) + for index := range sources { + sources[index] = map[string]any{"repoURL": fmt.Sprintf("https://github.example/owner/repo-%d.git", index)} + } + spec["sources"] = sources + }}, + {"ambiguous destination", func(input *Projection) { + input.Application.Object["spec"].(map[string]any)["destination"] = map[string]any{ + "name": "in-cluster", "server": "https://kubernetes.default.svc", + } + }}, + {"unsupported health", func(input *Projection) { + input.Application.Object["status"].(map[string]any)["health"] = map[string]any{"status": "DefinitelyFine"} + }}, + {"unsupported sync", func(input *Projection) { + input.Application.Object["status"].(map[string]any)["sync"] = map[string]any{"status": "Maybe"} + }}, + {"malformed history", func(input *Projection) { + input.Application.Object["status"].(map[string]any)["history"] = []any{"not-an-object"} + }}, + {"invalid history time", func(input *Projection) { + input.Application.Object["status"].(map[string]any)["history"] = []any{map[string]any{"deployedAt": "yesterday"}} + }}, + {"oversized revision", func(input *Projection) { + input.Application.Object["spec"].(map[string]any)["source"].(map[string]any)["targetRevision"] = strings.Repeat("a", maxRevisionBytes+1) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := cloneProjection(valid) + test.mutate(&input) + if facts, err := ProjectApplication(input); err == nil { + t.Fatalf("ProjectApplication() facts = %#v, error = nil", facts) + } + }) + } +} + +func TestProjectApplicationEnforcesEncodedPayloadBudget(t *testing.T) { + t.Parallel() + application := minimalApplication() + spec := application.Object["spec"].(map[string]any) + delete(spec, "source") + sources := make([]any, maxApplicationSources) + for index := range sources { + sources[index] = map[string]any{ + "repoURL": fmt.Sprintf("https://github.example/owner/repo-%d.git", index), + "path": strings.Repeat("a", maxTextBytes-1), + } + } + spec["sources"] = sources + _, err := ProjectApplication(Projection{ + Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Application: application, + }) + if err == nil || !strings.Contains(err.Error(), "encoded bytes") { + t.Fatalf("ProjectApplication() error = %v, want encoded payload budget rejection", err) + } +} + +func TestSanitizeRepositoryURLSupportsDocumentedArgoForms(t *testing.T) { + t.Parallel() + tests := []struct { + name string + input string + want string + }{ + { + name: "HTTPS credentials", + input: "https://user:password@git.example/owner/repo.git?token=secret#credential", + want: "https://git.example/owner/repo.git", + }, + {name: "SCP-like SSH", input: "git@git.example:owner/repo.git", want: "git.example:owner/repo.git"}, + { + name: "scheme-less Helm OCI registry", + input: "registry-1.docker.io/bitnamicharts", + want: "registry-1.docker.io/bitnamicharts", + }, + { + name: "OCI Application source", + input: "oci://user:password@registry.example/owner/artifact?token=secret#credential", + want: "oci://registry.example/owner/artifact", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := sanitizeRepositoryURL(test.input) + if err != nil { + t.Fatalf("sanitizeRepositoryURL() error = %v", err) + } + if got != test.want { + t.Fatalf("sanitizeRepositoryURL() = %q, want %q", got, test.want) + } + }) + } +} + +func TestSanitizeRepositoryURLRejectsAmbiguousOrUnsupportedForms(t *testing.T) { + t.Parallel() + for _, value := range []string{ + "file://localhost/tmp/repository", + "registry.example/charts?token=secret", + "user@registry.example/charts", + "./local/repository", + "registry.example\\charts", + } { + t.Run(value, func(t *testing.T) { + if got, err := sanitizeRepositoryURL(value); err == nil { + t.Fatalf("sanitizeRepositoryURL(%q) = %q, error = nil", value, got) + } + }) + } +} + +func completeApplication() unstructured.Unstructured { + application := minimalApplication() + application.Object["spec"] = map[string]any{ + "project": "default", + "source": map[string]any{ + "repoURL": "https://git-token:git-password@github.example/ardur/payments.git?access_token=cluster-token#credential-fragment", + "path": "clusters/prod/payments", + "targetRevision": "main", + "helm": map[string]any{ + "parameters": []any{map[string]any{"name": "database.password", "value": "raw-helm-secret"}}, + }, + }, + "destination": map[string]any{ + "server": "https://admin:cluster-password@kubernetes.default.svc?token=cluster-token#credentials", + "namespace": "payments", + }, + } + application.Object["status"] = map[string]any{ + "health": map[string]any{"status": "Healthy", "message": "must not be retained"}, + "sync": map[string]any{"status": "OutOfSync", "revision": "abc123"}, + "history": []any{ + map[string]any{"id": int64(1), "revision": "old123", "deployedAt": "2026-07-15T20:00:00Z"}, + map[string]any{"id": int64(2), "revision": "abc123", "deployedAt": "2026-07-16T20:00:00Z"}, + }, + "operationState": map[string]any{ + "phase": "Succeeded", "startedAt": "2026-07-16T19:59:00Z", "finishedAt": "2026-07-16T20:00:00Z", + "message": "credential-shaped status text must not be retained", + "syncResult": map[string]any{"revision": "abc123"}, + }, + } + return application +} + +func minimalApplication() unstructured.Unstructured { + return unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "argoproj.io/v1alpha1", + "kind": "Application", + "metadata": map[string]any{ + "name": "payments", "namespace": "argocd", + }, + "spec": map[string]any{ + "source": map[string]any{"repoURL": "https://github.example/ardur/payments.git"}, + "destination": map[string]any{"name": "in-cluster", "namespace": "payments"}, + }, + }} +} + +func cloneProjection(input Projection) Projection { + input.Application = *input.Application.DeepCopy() + return input +} diff --git a/internal/connector/awseks/boundary_test.go b/internal/connector/awseks/boundary_test.go new file mode 100644 index 0000000..c68ec43 --- /dev/null +++ b/internal/connector/awseks/boundary_test.go @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: Apache-2.0 + +package awseks + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var allowedProductionImports = map[string]bool{ + "bytes": true, + "crypto/sha256": true, + "encoding/hex": true, + "encoding/json": true, + "fmt": true, + "io": true, + "sort": true, + "strings": true, + "time": true, + "unicode": true, + "unicode/utf8": true, + "github.com/ArdurAI/sith/internal/fleet": true, +} + +var allowedProductionFiles = map[string]string{ + "project.go": "5ba3f13177ed2d4eb5ef316c767bb70199de5540b1500b5f3d4e92bfac51bbb6", +} + +var allowedProductionDeclarations = map[string]bool{ + "func:ProjectNodegroupFacts": true, + "func:buildFact": true, + "func:consumeUniqueJSON": true, + "func:matchingDelimiter": true, + "func:rejectDuplicateJSON": true, + "func:validAccount": true, + "func:validCapacityType": true, + "func:validEKSName": true, + "func:validHealthIssueCode": true, + "func:validNodegroupStatus": true, + "func:validPartition": true, + "func:validRegion": true, + "func:validUUID": true, + "func:validateNodegroup": true, + "func:validateNodegroupARN": true, + "func:validateProjection": true, + "func:validateText": true, + "type:Projection": true, + "type:describeNodegroupResponse": true, + "type:healthObservation": true, + "type:inventoryObservation": true, + "type:nodegroup": true, + "type:nodegroupHealth": true, + "type:nodegroupIssue": true, + "type:scalingConfig": true, + "value:Kind": true, + "value:ProtocolVersion": true, + "value:maxARNBytes": true, + "value:maxClusterNameBytes": true, + "value:maxFactPayloadBytes": true, + "value:maxHealthIssues": true, + "value:maxJSONDepth": true, + "value:maxNodegroupBytes": true, + "value:maxResponseBytes": true, + "value:maxScalingSize": true, + "value:maxScopeBytes": true, + "value:maxWorkspaceBytes": true, +} + +func functionDeclarationKey(declaration *ast.FuncDecl) string { + if declaration.Recv == nil { + return "func:" + declaration.Name.Name + } + if len(declaration.Recv.List) != 1 { + return "method:." + declaration.Name.Name + } + receiver := declaration.Recv.List[0].Type + if pointer, ok := receiver.(*ast.StarExpr); ok { + receiver = pointer.X + } + identifier, ok := receiver.(*ast.Ident) + if !ok { + return "method:." + declaration.Name.Name + } + return "method:" + identifier.Name + "." + declaration.Name.Name +} + +func projectNodegroupFactsHasExpectedSignature(declaration *ast.FuncDecl) bool { + if declaration.Recv != nil || declaration.Type.TypeParams != nil || declaration.Type.Params == nil || + len(declaration.Type.Params.List) != 1 || declaration.Type.Results == nil || + len(declaration.Type.Results.List) != 2 { + return false + } + parameter := declaration.Type.Params.List[0] + if len(parameter.Names) != 1 || parameter.Names[0].Name != "input" || !isIdentifier(parameter.Type, "Projection") { + return false + } + firstResult, ok := declaration.Type.Results.List[0].Type.(*ast.ArrayType) + return ok && firstResult.Len == nil && isSelector(firstResult.Elt, "fleet", "GraphFact") && + isIdentifier(declaration.Type.Results.List[1].Type, "error") +} + +func isIdentifier(expression ast.Expr, name string) bool { + identifier, ok := expression.(*ast.Ident) + return ok && identifier.Name == name +} + +func isSelector(expression ast.Expr, packageName, selectedName string) bool { + selector, ok := expression.(*ast.SelectorExpr) + return ok && isIdentifier(selector.X, packageName) && selector.Sel.Name == selectedName +} + +func projectionHasExpectedFields(specification *ast.TypeSpec) bool { + structure, ok := specification.Type.(*ast.StructType) + if !ok || structure.Fields == nil { + return false + } + expected := []struct { + name string + identifier string + packageName string + selected string + slice bool + }{ + {name: "Workspace", identifier: "string"}, + {name: "Scope", identifier: "string"}, + {name: "Partition", identifier: "string"}, + {name: "Account", identifier: "string"}, + {name: "Region", identifier: "string"}, + {name: "ClusterName", identifier: "string"}, + {name: "NodegroupName", identifier: "string"}, + {name: "ObservedAt", packageName: "time", selected: "Time"}, + {name: "Response", identifier: "byte", slice: true}, + } + if len(structure.Fields.List) != len(expected) { + return false + } + for index, field := range structure.Fields.List { + want := expected[index] + if len(field.Names) != 1 || field.Names[0].Name != want.name || field.Tag != nil { + return false + } + if want.slice { + slice, ok := field.Type.(*ast.ArrayType) + if !ok || slice.Len != nil || !isIdentifier(slice.Elt, want.identifier) { + return false + } + continue + } + if want.packageName != "" { + if !isSelector(field.Type, want.packageName, want.selected) { + return false + } + continue + } + if !isIdentifier(field.Type, want.identifier) { + return false + } + } + return true +} + +func productionStructureFingerprint(fileSet *token.FileSet, file *ast.File) (string, error) { + var document bytes.Buffer + if err := format.Node(&document, fileSet, file); err != nil { + return "", err + } + digest := sha256.Sum256(document.Bytes()) + return hex.EncodeToString(digest[:]), nil +} + +func TestProjectorHasNoIOCredentialPersistenceOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read awseks package: %v", err) + } + seenFiles := make(map[string]bool, len(allowedProductionFiles)) + seenDeclarations := make(map[string]bool, len(allowedProductionDeclarations)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + expectedFingerprint, allowed := allowedProductionFiles[entry.Name()] + if !allowed { + t.Errorf("projector contains unreviewed production file %s", entry.Name()) + } + seenFiles[entry.Name()] = true + fileSet := token.NewFileSet() + file, err := parser.ParseFile(fileSet, entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + fingerprint, err := productionStructureFingerprint(fileSet, file) + if err != nil { + t.Fatalf("fingerprint %s: %v", entry.Name(), err) + } + if fingerprint != expectedFingerprint { + t.Errorf("projector production structure changed for %s: got %s", entry.Name(), fingerprint) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if imported.Name != nil { + t.Errorf("projector aliases production import %q as %q", path, imported.Name.Name) + } + if !allowedProductionImports[path] { + t.Fatalf("projector imports unreviewed package %q", path) + } + } + for _, node := range file.Decls { + switch declaration := node.(type) { + case *ast.FuncDecl: + key := functionDeclarationKey(declaration) + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed function or method %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector repeats reviewed declaration %s", key) + } + seenDeclarations[key] = true + if key == "func:ProjectNodegroupFacts" && !projectNodegroupFactsHasExpectedSignature(declaration) { + t.Errorf("ProjectNodegroupFacts must retain the reviewed pure-projector signature") + } + case *ast.GenDecl: + for _, specification := range declaration.Specs { + switch typed := specification.(type) { + case *ast.TypeSpec: + key := "type:" + typed.Name.Name + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed type %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector repeats reviewed declaration %s", key) + } + seenDeclarations[key] = true + if key == "type:Projection" && !projectionHasExpectedFields(typed) { + t.Errorf("Projection must retain the reviewed value-only fields") + } + case *ast.ValueSpec: + for _, name := range typed.Names { + key := "value:" + name.Name + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed value %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector repeats reviewed declaration %s", key) + } + seenDeclarations[key] = true + } + } + } + } + } + ast.Inspect(file, func(node ast.Node) bool { + switch typed := node.(type) { + case *ast.InterfaceType: + t.Errorf("projector declares an injected interface seam") + case *ast.SelectorExpr: + if isIdentifier(typed.X, "io") && typed.Sel.Name != "EOF" { + t.Errorf("projector uses disallowed io capability io.%s", typed.Sel.Name) + } + } + return true + }) + } + for name := range allowedProductionFiles { + if !seenFiles[name] { + t.Errorf("projector is missing reviewed production file %s", name) + } + } + for key := range allowedProductionDeclarations { + if !seenDeclarations[key] { + t.Errorf("projector is missing reviewed declaration %s", key) + } + } +} + +func TestProjectNodegroupFactsSignatureRejectsCapabilitySeams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + signature string + want bool + }{ + {name: "reviewed", signature: `func ProjectNodegroupFacts(input Projection) ([]fleet.GraphFact, error)`, want: true}, + {name: "reader parameter", signature: `func ProjectNodegroupFacts(input Projection, reader io.Reader) ([]fleet.GraphFact, error)`}, + {name: "reader result", signature: `func ProjectNodegroupFacts(input Projection) ([]fleet.GraphFact, io.Reader)`}, + {name: "interface input", signature: `func ProjectNodegroupFacts(input any) ([]fleet.GraphFact, error)`}, + {name: "method", signature: `func (Projection) ProjectNodegroupFacts(input Projection) ([]fleet.GraphFact, error)`}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "signature.go", "package example\n"+test.signature+" {}", 0) + if err != nil { + t.Fatalf("parse signature: %v", err) + } + declaration := file.Decls[0].(*ast.FuncDecl) + if got := projectNodegroupFactsHasExpectedSignature(declaration); got != test.want { + t.Fatalf("projectNodegroupFactsHasExpectedSignature() = %t; want %t", got, test.want) + } + }) + } +} + +func TestProjectionFieldsRejectCapabilitySeams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + fields string + want bool + }{ + { + name: "reviewed", + fields: `Workspace string +Scope string +Partition string +Account string +Region string +ClusterName string +NodegroupName string +ObservedAt time.Time +Response []byte`, + want: true, + }, + {name: "callback hook", fields: `Workspace string; Hook func()`}, + {name: "reader", fields: `Workspace string; Reader io.Reader`}, + {name: "credential", fields: `Workspace string; AccessKey string; Response []byte`}, + {name: "extra response", fields: `Workspace string; Response []byte; Extra []byte`}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + source := "package example\ntype Projection struct {\n" + test.fields + "\n}" + file, err := parser.ParseFile(token.NewFileSet(), "projection.go", source, 0) + if err != nil { + t.Fatalf("parse Projection: %v", err) + } + specification := file.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec) + if got := projectionHasExpectedFields(specification); got != test.want { + t.Fatalf("projectionHasExpectedFields() = %t; want %t", got, test.want) + } + }) + } +} + +func TestIOCapabilityGuardAllowsOnlyEOFSentinel(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "io.go", `package example +import "io" +var end = io.EOF +type seam struct { reader io.Reader } +`, 0) + if err != nil { + t.Fatalf("parse io declarations: %v", err) + } + var rejected []string + ast.Inspect(file, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if ok && isIdentifier(selector.X, "io") && selector.Sel.Name != "EOF" { + rejected = append(rejected, selector.Sel.Name) + } + return true + }) + if len(rejected) != 1 || rejected[0] != "Reader" { + t.Fatalf("io capability guard rejected %v; want [Reader]", rejected) + } +} diff --git a/internal/connector/awseks/project.go b/internal/connector/awseks/project.go new file mode 100644 index 0000000..f1498b3 --- /dev/null +++ b/internal/connector/awseks/project.go @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package awseks normalizes bounded Amazon EKS managed-nodegroup evidence for +// Sith's operational graph. +package awseks + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for Amazon EKS read evidence. + Kind = "aws-eks" + // ProtocolVersion identifies the normalized DescribeNodegroup fact contract. + ProtocolVersion = "describe-nodegroup/v1" + + maxResponseBytes = 512 << 10 + maxFactPayloadBytes = 4 << 10 + maxJSONDepth = 64 + maxHealthIssues = 64 + maxScalingSize = 100_000 + maxWorkspaceBytes = 253 + maxScopeBytes = 253 + maxClusterNameBytes = 100 + maxNodegroupBytes = 63 + maxARNBytes = 512 +) + +// Projection supplies one already-authorized EKS DescribeNodegroup response. The trusted caller +// owns regional enumeration, pagination, authorization, request identity, and collection time. +// ProjectNodegroupFacts performs no discovery, network access, credential loading, persistence, +// process execution, or mutation. +type Projection struct { + Workspace string + Scope string + Partition string + Account string + Region string + ClusterName string + NodegroupName string + ObservedAt time.Time + Response []byte +} + +type describeNodegroupResponse struct { + Nodegroup *nodegroup `json:"nodegroup"` +} + +type nodegroup struct { + ARN *string `json:"nodegroupArn"` + ClusterName *string `json:"clusterName"` + Name *string `json:"nodegroupName"` + Status *string `json:"status"` + CapacityType *string `json:"capacityType"` + Scaling *scalingConfig `json:"scalingConfig"` + Health *nodegroupHealth `json:"health"` +} + +type scalingConfig struct { + MinSize *int64 `json:"minSize"` + MaxSize *int64 `json:"maxSize"` + DesiredSize *int64 `json:"desiredSize"` +} + +type nodegroupHealth struct { + Issues *[]nodegroupIssue `json:"issues"` +} + +type nodegroupIssue struct { + Code *string `json:"code"` +} + +type inventoryObservation struct { + NodegroupName string `json:"nodegroup_name"` + CapacityType string `json:"capacity_type"` + MinSize int64 `json:"min_size"` + DesiredSize int64 `json:"desired_size"` + MaxSize int64 `json:"max_size"` +} + +type healthObservation struct { + NodegroupName string `json:"nodegroup_name"` + ProviderStatus string `json:"provider_status"` + IssueCodes []string `json:"issue_codes"` +} + +// ProjectNodegroupFacts returns one inventory fact and one provider-health fact for a managed +// nodegroup. The health fact reports only EKS status and closed issue codes; it never claims that +// Kubernetes nodes or workloads are healthy. +func ProjectNodegroupFacts(input Projection) ([]fleet.GraphFact, error) { + if err := validateProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode EKS DescribeNodegroup response") + } + + var response describeNodegroupResponse + if err := json.Unmarshal(input.Response, &response); err != nil { + return nil, fmt.Errorf("decode EKS DescribeNodegroup response") + } + issueCodes, err := validateNodegroup(input, response.Nodegroup) + if err != nil { + return nil, err + } + nodegroup := response.Nodegroup + + inventory := inventoryObservation{ + NodegroupName: input.NodegroupName, + CapacityType: *nodegroup.CapacityType, + MinSize: *nodegroup.Scaling.MinSize, + DesiredSize: *nodegroup.Scaling.DesiredSize, + MaxSize: *nodegroup.Scaling.MaxSize, + } + health := healthObservation{ + NodegroupName: input.NodegroupName, + ProviderStatus: *nodegroup.Status, + IssueCodes: issueCodes, + } + + inventoryFact, err := buildFact(input, *nodegroup.ARN, fleet.FactInventory, inventory) + if err != nil { + return nil, err + } + healthFact, err := buildFact(input, *nodegroup.ARN, fleet.FactHealth, health) + if err != nil { + return nil, err + } + return []fleet.GraphFact{inventoryFact, healthFact}, nil +} + +func validateProjection(input Projection) error { + if err := validateText("workspace", input.Workspace, maxWorkspaceBytes); err != nil { + return err + } + if err := validateText("scope", input.Scope, maxScopeBytes); err != nil || strings.Contains(input.Scope, "/") { + return fmt.Errorf("scope is invalid") + } + if !validPartition(input.Partition) { + return fmt.Errorf("AWS partition is invalid") + } + if !validAccount(input.Account) { + return fmt.Errorf("AWS account is invalid") + } + if !validRegion(input.Partition, input.Region) { + return fmt.Errorf("AWS region is invalid for partition") + } + if !validEKSName(input.ClusterName, maxClusterNameBytes) { + return fmt.Errorf("EKS cluster name is invalid") + } + if !validEKSName(input.NodegroupName, maxNodegroupBytes) { + return fmt.Errorf("EKS nodegroup name is invalid") + } + if input.ObservedAt.IsZero() || input.ObservedAt.Year() < 2000 || input.ObservedAt.Year() > 9999 { + return fmt.Errorf("observation time is invalid") + } + if len(input.Response) == 0 { + return fmt.Errorf("EKS DescribeNodegroup response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("EKS DescribeNodegroup response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("EKS DescribeNodegroup response must be valid UTF-8") + } + return nil +} + +func validateNodegroup(input Projection, value *nodegroup) ([]string, error) { + if value == nil || value.ARN == nil || value.ClusterName == nil || value.Name == nil || + value.Status == nil || value.CapacityType == nil || value.Scaling == nil || value.Health == nil { + return nil, fmt.Errorf("EKS nodegroup identity, status, capacity, scaling, and health are required") + } + if *value.ClusterName != input.ClusterName || *value.Name != input.NodegroupName { + return nil, fmt.Errorf("EKS nodegroup identity does not match trusted caller identity") + } + if err := validateNodegroupARN(input, *value.ARN); err != nil { + return nil, err + } + if !validNodegroupStatus(*value.Status) { + return nil, fmt.Errorf("EKS nodegroup status is invalid") + } + if !validCapacityType(*value.CapacityType) { + return nil, fmt.Errorf("EKS nodegroup capacity type is invalid") + } + if value.Scaling.MinSize == nil || value.Scaling.MaxSize == nil || value.Scaling.DesiredSize == nil { + return nil, fmt.Errorf("EKS nodegroup scaling values are required") + } + minimum := *value.Scaling.MinSize + maximum := *value.Scaling.MaxSize + desired := *value.Scaling.DesiredSize + if minimum < 0 || maximum < 1 || desired < 0 || minimum > maxScalingSize || + maximum > maxScalingSize || desired > maxScalingSize || minimum > desired || desired > maximum { + return nil, fmt.Errorf("EKS nodegroup scaling values are invalid") + } + if value.Health.Issues == nil { + return nil, fmt.Errorf("EKS nodegroup health issues are required") + } + issues := *value.Health.Issues + if len(issues) > maxHealthIssues { + return nil, fmt.Errorf("EKS nodegroup health issue count exceeds %d", maxHealthIssues) + } + codes := make([]string, 0, len(issues)) + seen := make(map[string]bool, len(issues)) + for _, issue := range issues { + if issue.Code == nil || !validHealthIssueCode(*issue.Code) { + return nil, fmt.Errorf("EKS nodegroup health issue code is invalid") + } + if seen[*issue.Code] { + return nil, fmt.Errorf("EKS nodegroup health issue codes must be unique") + } + seen[*issue.Code] = true + codes = append(codes, *issue.Code) + } + sort.Strings(codes) + return codes, nil +} + +func validateNodegroupARN(input Projection, arn string) error { + if err := validateText("EKS nodegroup ARN", arn, maxARNBytes); err != nil { + return err + } + prefix := "arn:" + input.Partition + ":eks:" + input.Region + ":" + input.Account + + ":nodegroup/" + input.ClusterName + "/" + input.NodegroupName + "/" + if !strings.HasPrefix(arn, prefix) || !validUUID(strings.TrimPrefix(arn, prefix)) { + return fmt.Errorf("EKS nodegroup ARN does not match trusted caller identity") + } + return nil +} + +func validPartition(value string) bool { + switch value { + case "aws", "aws-us-gov", "aws-cn": + return true + default: + return false + } +} + +func validAccount(value string) bool { + if len(value) != 12 { + return false + } + for _, character := range value { + if character < '0' || character > '9' { + return false + } + } + return true +} + +func validRegion(partition, value string) bool { + switch partition { + case "aws": + switch value { + case "us-east-1", "us-east-2", "us-west-1", "us-west-2", + "af-south-1", + "ap-east-1", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", + "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", + "ca-central-1", "ca-west-1", + "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", + "eu-west-1", "eu-west-2", "eu-west-3", + "il-central-1", "me-central-1", "me-south-1", "mx-central-1", "sa-east-1": + return true + default: + return false + } + case "aws-us-gov": + return value == "us-gov-east-1" || value == "us-gov-west-1" + case "aws-cn": + return value == "cn-north-1" || value == "cn-northwest-1" + default: + return false + } +} + +func validEKSName(value string, maximum int) bool { + if len(value) == 0 || len(value) > maximum { + return false + } + for index, character := range value { + if (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z') || + (character >= '0' && character <= '9') || (index > 0 && (character == '-' || character == '_')) { + continue + } + return false + } + return true +} + +func validUUID(value string) bool { + if len(value) != 36 || value[8] != '-' || value[13] != '-' || value[18] != '-' || value[23] != '-' { + return false + } + for index, character := range value { + if index == 8 || index == 13 || index == 18 || index == 23 { + continue + } + if (character < '0' || character > '9') && (character < 'a' || character > 'f') && (character < 'A' || character > 'F') { + return false + } + } + return true +} + +func validNodegroupStatus(value string) bool { + switch value { + case "CREATING", "ACTIVE", "UPDATING", "DELETING", "CREATE_FAILED", "DELETE_FAILED", "DEGRADED": + return true + default: + return false + } +} + +func validCapacityType(value string) bool { + switch value { + case "ON_DEMAND", "SPOT", "CAPACITY_BLOCK": + return true + default: + return false + } +} + +func validHealthIssueCode(value string) bool { + switch value { + case "AutoScalingGroupNotFound", "AutoScalingGroupInvalidConfiguration", + "Ec2SecurityGroupNotFound", "Ec2SecurityGroupDeletionFailure", + "Ec2LaunchTemplateNotFound", "Ec2LaunchTemplateVersionMismatch", + "Ec2SubnetNotFound", "Ec2SubnetInvalidConfiguration", "IamInstanceProfileNotFound", + "Ec2SubnetMissingIpv6Assignment", "IamLimitExceeded", "IamNodeRoleNotFound", + "NodeCreationFailure", "AsgInstanceLaunchFailures", "InstanceLimitExceeded", + "InsufficientFreeAddresses", "AccessDenied", "InternalFailure", "ClusterUnreachable", + "AmiIdNotFound", "AutoScalingGroupOptInRequired", "AutoScalingGroupRateLimitExceeded", + "Ec2LaunchTemplateDeletionFailure", "Ec2LaunchTemplateInvalidConfiguration", + "Ec2LaunchTemplateMaxLimitExceeded", "Ec2SubnetListTooLong", "IamThrottling", + "NodeTerminationFailure", "PodEvictionFailure", "SourceEc2LaunchTemplateNotFound", + "LimitExceeded", "Unknown", "AutoScalingGroupInstanceRefreshActive", + "KubernetesLabelInvalid", "Ec2LaunchTemplateVersionMaxLimitExceeded", + "Ec2InstanceTypeDoesNotExist": + return true + default: + return false + } +} + +func buildFact(input Projection, arn string, kind fleet.FactKind, observation any) (fleet.GraphFact, error) { + encoded, err := json.Marshal(observation) + if err != nil { + return fleet.GraphFact{}, fmt.Errorf("encode EKS nodegroup fact") + } + if len(encoded) > maxFactPayloadBytes { + return fleet.GraphFact{}, fmt.Errorf("EKS nodegroup fact exceeds %d encoded bytes", maxFactPayloadBytes) + } + digest := sha256.Sum256([]byte(arn + "#" + string(kind))) + nativeID := "sha256:" + hex.EncodeToString(digest[:]) + entity := fleet.EntityRef{Cluster: input.Scope} + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, + Scope: input.Scope, + Kind: "ManagedNodeGroup", + Name: input.NodegroupName, + }, + Kind: kind, + Observed: encoded, + ObservedAt: input.ObservedAt.UTC(), + Source: input.Scope, + Provenance: fleet.Provenance{Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID}, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensLive, + Entity: &entity, + } + if err := fact.Validate(input.Workspace); err != nil { + return fleet.GraphFact{}, fmt.Errorf("validate EKS nodegroup fact: %w", err) + } + return fact, nil +} + +func validateText(label, value string, maximum int) error { + if value == "" || len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("%s is invalid", label) + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func rejectDuplicateJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || closing != matchingDelimiter(delimiter) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func matchingDelimiter(open json.Delim) json.Delim { + if open == '{' { + return '}' + } + return ']' +} diff --git a/internal/connector/awseks/project_test.go b/internal/connector/awseks/project_test.go new file mode 100644 index 0000000..5f6818e --- /dev/null +++ b/internal/connector/awseks/project_test.go @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: Apache-2.0 + +package awseks + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +var testObservedAt = time.Date(2026, 7, 17, 8, 0, 0, 0, time.UTC) + +func TestProjectNodegroupFactsEmitsSanitizedLiveFacts(t *testing.T) { + t.Parallel() + secret := "arn:aws:iam::123456789012:role/private-node-role" + input := validProjection(t) + input.Response = responseJSON(t, input, map[string]any{ + "status": "DEGRADED", + "capacityType": "SPOT", + "health": map[string]any{"issues": []any{ + map[string]any{"code": "NodeCreationFailure", "message": "credential=" + secret, "resourceIds": []string{"i-secret"}}, + map[string]any{"code": "AccessDenied", "message": "do not retain"}, + }}, + "nodeRole": secret, + "resources": map[string]any{"autoScalingGroups": []any{map[string]any{"name": "secret-asg"}}, "remoteAccessSecurityGroup": "sg-secret"}, + "subnets": []string{"subnet-secret"}, + "labels": map[string]any{"password": "secret"}, + "taints": []any{map[string]any{"key": "secret", "value": "secret", "effect": "NO_SCHEDULE"}}, + "tags": map[string]any{"token": "secret"}, + "launchTemplate": map[string]any{"id": "lt-secret", "name": "secret", "version": "7"}, + "remoteAccess": map[string]any{"ec2SshKey": "secret", "sourceSecurityGroups": []string{"sg-secret"}}, + "version": "1.99-secret", + "releaseVersion": "ami-secret", + "updateConfig": map[string]any{"maxUnavailable": 1}, + "instanceTypes": []string{"m7i.private"}, + "diskSize": 100, + "amiType": "AL2023_x86_64_STANDARD", + }) + + facts, err := ProjectNodegroupFacts(input) + if err != nil { + t.Fatalf("ProjectNodegroupFacts() error = %v", err) + } + if len(facts) != 2 { + t.Fatalf("ProjectNodegroupFacts() returned %d facts; want 2", len(facts)) + } + + for index, fact := range facts { + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("fact %d invalid: %v", index, err) + } + if fact.Lens != fleet.LensLive || fact.Fact.Ref.SourceKind != Kind || fact.Fact.Ref.Scope != input.Scope || + fact.Fact.Ref.Kind != "ManagedNodeGroup" || fact.Fact.Ref.Name != input.NodegroupName || + fact.Fact.Source != input.Scope || fact.Fact.ObservedAt != input.ObservedAt || + fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion || + !strings.HasPrefix(fact.Fact.Provenance.NativeID, "sha256:") { + t.Fatalf("fact %d has unexpected envelope: %#v", index, fact) + } + if fact.Entity == nil || *fact.Entity != (fleet.EntityRef{Cluster: input.Scope}) { + t.Fatalf("fact %d entity = %#v; want trusted cluster root", index, fact.Entity) + } + } + if facts[0].Fact.Kind != fleet.FactInventory || facts[1].Fact.Kind != fleet.FactHealth { + t.Fatalf("fact kinds = %q, %q; want inventory, health", facts[0].Fact.Kind, facts[1].Fact.Kind) + } + + var inventory inventoryObservation + if err := json.Unmarshal(facts[0].Fact.Observed, &inventory); err != nil { + t.Fatalf("decode inventory observation: %v", err) + } + if inventory != (inventoryObservation{ + NodegroupName: input.NodegroupName, CapacityType: "SPOT", MinSize: 1, DesiredSize: 3, MaxSize: 5, + }) { + t.Fatalf("inventory observation = %#v", inventory) + } + var health healthObservation + if err := json.Unmarshal(facts[1].Fact.Observed, &health); err != nil { + t.Fatalf("decode health observation: %v", err) + } + if health.ProviderStatus != "DEGRADED" || !reflect.DeepEqual(health.IssueCodes, []string{"AccessDenied", "NodeCreationFailure"}) { + t.Fatalf("health observation = %#v", health) + } + + encoded, err := json.Marshal(facts) + if err != nil { + t.Fatalf("encode facts: %v", err) + } + for _, forbidden := range []string{ + secret, input.Account, input.Region, "nodegroupArn", "credential=", "i-secret", "secret-asg", + "subnet-secret", "sg-secret", "lt-secret", "password", "ami-secret", "m7i.private", + } { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("facts retain forbidden provider data %q: %s", forbidden, encoded) + } + } + if bytes.Contains(facts[1].Fact.Observed, []byte(`"health":"Healthy"`)) { + t.Fatalf("provider status must not become a Kubernetes health claim: %s", facts[1].Fact.Observed) + } +} + +func TestProjectNodegroupFactsIsDeterministicAcrossDiscardedDataAndIssueOrder(t *testing.T) { + t.Parallel() + first := validProjection(t) + first.Response = responseJSON(t, first, map[string]any{ + "health": map[string]any{"issues": []any{ + map[string]any{"code": "PodEvictionFailure", "message": "first"}, + map[string]any{"code": "AccessDenied", "resourceIds": []string{"i-first"}}, + }}, + "tags": map[string]any{"first": "secret"}, + }) + second := first + second.Response = responseJSON(t, second, map[string]any{ + "health": map[string]any{"issues": []any{ + map[string]any{"code": "AccessDenied", "message": "changed"}, + map[string]any{"code": "PodEvictionFailure", "resourceIds": []string{"i-second"}}, + }}, + "tags": map[string]any{"second": "different"}, + }) + + firstFacts, err := ProjectNodegroupFacts(first) + if err != nil { + t.Fatalf("first projection: %v", err) + } + secondFacts, err := ProjectNodegroupFacts(second) + if err != nil { + t.Fatalf("second projection: %v", err) + } + if !reflect.DeepEqual(firstFacts, secondFacts) { + t.Fatalf("discarded provider data changed facts:\nfirst=%#v\nsecond=%#v", firstFacts, secondFacts) + } +} + +func TestProjectNodegroupFactsSupportsReviewedPartitions(t *testing.T) { + t.Parallel() + tests := []struct { + name string + partition string + region string + }{ + {name: "commercial", partition: "aws", region: "us-west-2"}, + {name: "govcloud", partition: "aws-us-gov", region: "us-gov-west-1"}, + {name: "china", partition: "aws-cn", region: "cn-north-1"}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := validProjection(t) + input.Partition = test.partition + input.Region = test.region + input.Response = responseJSON(t, input, nil) + if _, err := ProjectNodegroupFacts(input); err != nil { + t.Fatalf("ProjectNodegroupFacts() error = %v", err) + } + }) + } +} + +func TestProjectNodegroupFactsRejectsInvalidTrustedProjection(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(*Projection) + }{ + {name: "empty workspace", mutate: func(input *Projection) { input.Workspace = "" }}, + {name: "scope separator", mutate: func(input *Projection) { input.Scope = "cluster/other" }}, + {name: "partition", mutate: func(input *Projection) { input.Partition = "aws-iso" }}, + {name: "account", mutate: func(input *Projection) { input.Account = "1234-secret" }}, + {name: "commercial china region", mutate: func(input *Projection) { input.Region = "cn-north-1" }}, + {name: "gov commercial region", mutate: func(input *Projection) { input.Partition = "aws-us-gov" }}, + {name: "malformed region", mutate: func(input *Projection) { input.Region = "US WEST 2" }}, + {name: "cluster starts separator", mutate: func(input *Projection) { input.ClusterName = "-cluster" }}, + {name: "nodegroup slash", mutate: func(input *Projection) { input.NodegroupName = "ng/other" }}, + {name: "zero observation", mutate: func(input *Projection) { input.ObservedAt = time.Time{} }}, + {name: "pre-contract observation", mutate: func(input *Projection) { input.ObservedAt = time.Date(1999, 1, 1, 0, 0, 0, 0, time.UTC) }}, + {name: "empty response", mutate: func(input *Projection) { input.Response = nil }}, + {name: "invalid utf8", mutate: func(input *Projection) { input.Response = []byte{0xff} }}, + {name: "oversized response", mutate: func(input *Projection) { input.Response = bytes.Repeat([]byte(" "), maxResponseBytes+1) }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := validProjection(t) + test.mutate(&input) + if _, err := ProjectNodegroupFacts(input); err == nil { + t.Fatalf("ProjectNodegroupFacts() succeeded") + } + }) + } +} + +func TestProjectNodegroupFactsRejectsMismatchedOrUnsafeResponse(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(map[string]any) + }{ + {name: "missing nodegroup", mutate: func(response map[string]any) { delete(response, "nodegroup") }}, + {name: "null nodegroup", mutate: func(response map[string]any) { response["nodegroup"] = nil }}, + {name: "cluster mismatch", mutate: mutateNodegroup("clusterName", "other")}, + {name: "name mismatch", mutate: mutateNodegroup("nodegroupName", "other")}, + {name: "partition ARN mismatch", mutate: mutateNodegroup("nodegroupArn", "arn:aws-cn:eks:us-west-2:123456789012:nodegroup/prod/workers/a8c75f2f-df78-a72f-4063-4b69af3de5b1")}, + {name: "ARN UUID invalid", mutate: mutateNodegroup("nodegroupArn", "arn:aws:eks:us-west-2:123456789012:nodegroup/prod/workers/not-a-uuid")}, + {name: "unknown status", mutate: mutateNodegroup("status", "READY")}, + {name: "null status", mutate: mutateNodegroup("status", nil)}, + {name: "unknown capacity", mutate: mutateNodegroup("capacityType", "FREE")}, + {name: "missing scaling", mutate: func(response map[string]any) { delete(response["nodegroup"].(map[string]any), "scalingConfig") }}, + {name: "negative minimum", mutate: mutateScaling("minSize", -1)}, + {name: "zero maximum", mutate: mutateScaling("maxSize", 0)}, + {name: "minimum above desired", mutate: mutateScaling("minSize", 4)}, + {name: "desired above maximum", mutate: mutateScaling("desiredSize", 6)}, + {name: "oversized maximum", mutate: mutateScaling("maxSize", maxScalingSize+1)}, + {name: "fractional scaling", mutate: mutateScaling("desiredSize", 1.5)}, + {name: "missing health", mutate: func(response map[string]any) { delete(response["nodegroup"].(map[string]any), "health") }}, + {name: "missing issues", mutate: mutateNodegroup("health", map[string]any{})}, + {name: "null issues", mutate: mutateNodegroup("health", map[string]any{"issues": nil})}, + {name: "unknown issue", mutate: mutateNodegroup("health", map[string]any{"issues": []any{map[string]any{"code": "NewUnreviewedCode"}}})}, + {name: "null issue", mutate: mutateNodegroup("health", map[string]any{"issues": []any{map[string]any{"code": nil}}})}, + {name: "duplicate issue", mutate: mutateNodegroup("health", map[string]any{"issues": []any{map[string]any{"code": "AccessDenied"}, map[string]any{"code": "AccessDenied"}}})}, + {name: "too many issues", mutate: mutateNodegroup("health", healthWithIssueCount(maxHealthIssues+1))}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := validProjection(t) + response := responseMap(input) + test.mutate(response) + input.Response = marshalJSON(t, response) + if _, err := ProjectNodegroupFacts(input); err == nil { + t.Fatalf("ProjectNodegroupFacts() succeeded") + } + }) + } +} + +func TestProjectNodegroupFactsRejectsJSONAttacksWithoutEchoingValues(t *testing.T) { + t.Parallel() + input := validProjection(t) + secret := "do-not-echo-credential" + tests := []struct { + name string + response []byte + }{ + {name: "duplicate", response: []byte(`{"nodegroup":{"status":"ACTIVE","status":"` + secret + `"}}`)}, + {name: "trailing", response: append(input.Response, []byte(` {"token":"`+secret+`"}`)...)}, + {name: "malformed", response: []byte(`{"nodegroup":"` + secret)}, + {name: "deep", response: []byte(strings.Repeat("[", maxJSONDepth+1) + `"` + secret + `"` + strings.Repeat("]", maxJSONDepth+1))}, + {name: "untrusted enum", response: responseJSON(t, input, map[string]any{"status": secret})}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + candidate := input + candidate.Response = test.response + _, err := ProjectNodegroupFacts(candidate) + if err == nil { + t.Fatalf("ProjectNodegroupFacts() succeeded") + } + if strings.Contains(err.Error(), secret) { + t.Fatalf("error echoed attacker value: %v", err) + } + }) + } +} + +func TestClosedAWSNodegroupTaxonomies(t *testing.T) { + t.Parallel() + for _, status := range []string{"CREATING", "ACTIVE", "UPDATING", "DELETING", "CREATE_FAILED", "DELETE_FAILED", "DEGRADED"} { + if !validNodegroupStatus(status) { + t.Errorf("reviewed status %q rejected", status) + } + } + for _, capacity := range []string{"ON_DEMAND", "SPOT", "CAPACITY_BLOCK"} { + if !validCapacityType(capacity) { + t.Errorf("reviewed capacity type %q rejected", capacity) + } + } + for _, code := range []string{"AutoScalingGroupNotFound", "Ec2InstanceTypeDoesNotExist", "Unknown"} { + if !validHealthIssueCode(code) { + t.Errorf("reviewed issue code %q rejected", code) + } + } + for _, value := range []string{"", "HEALTHY", "unknown", "AccessDenied\nsecret"} { + if validNodegroupStatus(value) || validCapacityType(value) || validHealthIssueCode(value) { + t.Errorf("unreviewed taxonomy value %q accepted", value) + } + } +} + +func TestAWSIdentityValidatorsRejectAmbiguousForms(t *testing.T) { + t.Parallel() + for _, test := range []struct { + account string + want bool + }{ + {account: "123456789012", want: true}, + {account: "123"}, + {account: "12345678901x"}, + } { + if got := validAccount(test.account); got != test.want { + t.Errorf("validAccount(%q) = %t; want %t", test.account, got, test.want) + } + } + for _, test := range []struct { + partition string + region string + want bool + }{ + {partition: "aws", region: "us-west-2", want: true}, + {partition: "aws-cn", region: "cn-north-1", want: true}, + {partition: "aws-us-gov", region: "us-gov-west-1", want: true}, + {partition: "aws", region: "short"}, + {partition: "aws", region: "foo-bar-1"}, + {partition: "aws", region: "-us-west-2"}, + {partition: "aws", region: "us-west-2-"}, + {partition: "aws", region: "us_West-2"}, + {partition: "aws", region: "us-2"}, + {partition: "aws", region: "us-west-final"}, + {partition: "aws", region: "us--west-2"}, + {partition: "aws", region: "cn-north-1"}, + {partition: "aws", region: "us-gov-west-1"}, + {partition: "aws", region: "us-iso-east-1"}, + {partition: "aws", region: "us-isob-east-1"}, + {partition: "aws", region: "us-isoe-east-1"}, + {partition: "aws", region: "eu-isoe-west-1"}, + {partition: "aws", region: "us-isof-south-1"}, + {partition: "aws-cn", region: "us-west-2"}, + {partition: "aws-us-gov", region: "us-west-2"}, + {partition: "unreviewed", region: "us-west-2"}, + } { + if got := validRegion(test.partition, test.region); got != test.want { + t.Errorf("validRegion(%q, %q) = %t; want %t", test.partition, test.region, got, test.want) + } + } + for _, test := range []struct { + value string + max int + want bool + }{ + {value: "A_0-name", max: 100, want: true}, + {value: "", max: 100}, + {value: "abcd", max: 3}, + {value: "-leading", max: 100}, + {value: "bad/name", max: 100}, + {value: "é", max: 100}, + } { + if got := validEKSName(test.value, test.max); got != test.want { + t.Errorf("validEKSName(%q) = %t; want %t", test.value, got, test.want) + } + } + for _, test := range []struct { + value string + want bool + }{ + {value: "A8C75F2F-DF78-A72F-4063-4B69AF3DE5B1", want: true}, + {value: "short"}, + {value: "a8c75f2f_df78-a72f-4063-4b69af3de5b1"}, + {value: "g8c75f2f-df78-a72f-4063-4b69af3de5b1"}, + } { + if got := validUUID(test.value); got != test.want { + t.Errorf("validUUID(%q) = %t; want %t", test.value, got, test.want) + } + } +} + +func TestValidRegionPinsCurrentEKSEndpoints(t *testing.T) { + t.Parallel() + tests := []struct { + partition string + regions []string + }{ + {partition: "aws", regions: []string{ + "us-east-1", "us-east-2", "us-west-1", "us-west-2", + "af-south-1", + "ap-east-1", "ap-east-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", "ap-south-2", "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", + "ap-southeast-4", "ap-southeast-5", "ap-southeast-6", "ap-southeast-7", + "ca-central-1", "ca-west-1", + "eu-central-1", "eu-central-2", "eu-north-1", "eu-south-1", "eu-south-2", + "eu-west-1", "eu-west-2", "eu-west-3", + "il-central-1", "me-central-1", "me-south-1", "mx-central-1", "sa-east-1", + }}, + {partition: "aws-us-gov", regions: []string{"us-gov-east-1", "us-gov-west-1"}}, + {partition: "aws-cn", regions: []string{"cn-north-1", "cn-northwest-1"}}, + } + for _, test := range tests { + for _, region := range test.regions { + if !validRegion(test.partition, region) { + t.Errorf("validRegion(%q, %q) rejected a pinned EKS endpoint", test.partition, region) + } + } + } + for _, region := range []string{"foo-bar-1", "us-isoz-east-1", "ap-southeast-99"} { + if validRegion("aws", region) { + t.Errorf("validRegion(aws, %q) accepted an unreviewed endpoint", region) + } + } +} + +func TestInternalFactAndTextGuardsFailClosed(t *testing.T) { + t.Parallel() + input := validProjection(t) + arn := responseMap(input)["nodegroup"].(map[string]any)["nodegroupArn"].(string) + for _, test := range []struct { + name string + input Projection + kind fleet.FactKind + observation any + }{ + {name: "unencodable observation", input: input, kind: fleet.FactHealth, observation: make(chan int)}, + {name: "oversized observation", input: input, kind: fleet.FactHealth, observation: strings.Repeat("x", maxFactPayloadBytes+1)}, + {name: "invalid graph scope", input: func() Projection { candidate := input; candidate.Scope = "bad/scope"; return candidate }(), kind: fleet.FactHealth, observation: healthObservation{}}, + {name: "invalid fact kind", input: input, kind: fleet.FactKind("unreviewed"), observation: healthObservation{}}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if _, err := buildFact(test.input, arn, test.kind, test.observation); err == nil { + t.Fatalf("buildFact() succeeded") + } + }) + } + for _, test := range []struct { + name string + value string + max int + want bool + }{ + {name: "valid", value: "safe", max: 4, want: true}, + {name: "empty", value: "", max: 4}, + {name: "too long", value: "longer", max: 4}, + {name: "invalid utf8", value: string([]byte{0xff}), max: 4}, + {name: "whitespace", value: " safe", max: 8}, + {name: "control", value: "bad\n", max: 8}, + } { + err := validateText(test.name, test.value, test.max) + if (err == nil) != test.want { + t.Errorf("validateText(%q) error = %v; want valid=%t", test.value, err, test.want) + } + } + + candidate := input + candidate.Response = responseJSON(t, candidate, map[string]any{"nodegroupArn": ""}) + if _, err := ProjectNodegroupFacts(candidate); err == nil { + t.Fatalf("ProjectNodegroupFacts() accepted empty ARN") + } +} + +func FuzzProjectNodegroupFacts(f *testing.F) { + seed := validProjection(f) + f.Add(seed.Response) + f.Add([]byte(`{"nodegroup":null}`)) + f.Add([]byte(`{"nodegroup":{"status":"ACTIVE","status":"DEGRADED"}}`)) + f.Fuzz(func(t *testing.T, response []byte) { + input := seed + input.Response = response + facts, err := ProjectNodegroupFacts(input) + if err != nil { + return + } + if len(facts) != 2 { + t.Fatalf("successful projection returned %d facts", len(facts)) + } + for _, fact := range facts { + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("successful projection emitted invalid fact: %v", err) + } + if len(fact.Fact.Observed) > maxFactPayloadBytes { + t.Fatalf("successful projection emitted oversized fact") + } + } + }) +} + +type testingHelper interface { + Helper() + Fatalf(string, ...any) +} + +func validProjection(t testingHelper) Projection { + t.Helper() + input := Projection{ + Workspace: "workspace-a", + Scope: "prod-cluster", + Partition: "aws", + Account: "123456789012", + Region: "us-west-2", + ClusterName: "prod", + NodegroupName: "workers", + ObservedAt: testObservedAt, + } + input.Response = responseJSON(t, input, nil) + return input +} + +func responseMap(input Projection) map[string]any { + arn := fmt.Sprintf("arn:%s:eks:%s:%s:nodegroup/%s/%s/a8c75f2f-df78-a72f-4063-4b69af3de5b1", + input.Partition, input.Region, input.Account, input.ClusterName, input.NodegroupName) + return map[string]any{"nodegroup": map[string]any{ + "nodegroupArn": arn, + "clusterName": input.ClusterName, + "nodegroupName": input.NodegroupName, + "status": "ACTIVE", + "capacityType": "ON_DEMAND", + "scalingConfig": map[string]any{"minSize": 1, "desiredSize": 3, "maxSize": 5}, + "health": map[string]any{"issues": []any{}}, + }} +} + +func responseJSON(t testingHelper, input Projection, overrides map[string]any) []byte { + t.Helper() + response := responseMap(input) + nodegroup := response["nodegroup"].(map[string]any) + for name, value := range overrides { + nodegroup[name] = value + } + return marshalJSON(t, response) +} + +func marshalJSON(t testingHelper, value any) []byte { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal response: %v", err) + } + return encoded +} + +func mutateNodegroup(name string, value any) func(map[string]any) { + return func(response map[string]any) { + response["nodegroup"].(map[string]any)[name] = value + } +} + +func mutateScaling(name string, value any) func(map[string]any) { + return func(response map[string]any) { + response["nodegroup"].(map[string]any)["scalingConfig"].(map[string]any)[name] = value + } +} + +func healthWithIssueCount(count int) map[string]any { + issues := make([]any, count) + for index := range issues { + issues[index] = map[string]any{"code": "Unknown"} + } + return map[string]any{"issues": issues} +} diff --git a/internal/connector/contract.go b/internal/connector/contract.go index 0ec7f75..6866aaf 100644 --- a/internal/connector/contract.go +++ b/internal/connector/contract.go @@ -9,6 +9,7 @@ import ( "time" "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" ) // Connector identifies one canonical integration and its declared capabilities. @@ -20,12 +21,17 @@ type Connector interface { // Descriptor is static registry, taxonomy, ownership, and version metadata. type Descriptor struct { - Kind string `json:"kind"` - ConnKind ConnectorKind `json:"connector_kind"` - ProtocolV string `json:"protocol_version"` - Owner string `json:"owner"` - Capabilities []Capability `json:"capabilities"` - Verbs []string `json:"verbs,omitempty"` + // Kind is the canonical target-tool identity; only one connector may own it. + Kind string `json:"kind"` + ConnKind ConnectorKind `json:"connector_kind"` + // WireVersions explicitly lists framework transport versions this connector can speak. + WireVersions []WireVersion `json:"wire_versions"` + // AdapterVersion is an opaque evidence and behavior contract identifier, not transport semver. + AdapterVersion string `json:"adapter_version"` + Owner string `json:"owner"` + Capabilities []Capability `json:"capabilities"` + Verbs []intent.Verb `json:"verbs,omitempty"` + ArgSchemas map[intent.Verb]json.RawMessage `json:"arg_schemas,omitempty"` } // ConnectorKind is the closed integration taxonomy. @@ -101,9 +107,10 @@ const ( WatchError WatchEventType = "error" ) -// WatchEvent carries a source-stamped, per-scope reconciliation delta. +// WatchEvent carries an explicitly workspace-bound, source-stamped reconciliation delta. type WatchEvent struct { Type WatchEventType + Workspace string Kind string Scope string Facts []fleet.Fact @@ -166,7 +173,7 @@ type Intent struct { ID string `json:"id"` Workspace string `json:"workspace"` Actor string `json:"actor"` - Verb string `json:"verb"` + Verb intent.Verb `json:"verb"` Target fleet.ResourceRef `json:"target"` Args json.RawMessage `json:"args"` Justification string `json:"justification"` @@ -177,7 +184,7 @@ type Intent struct { // ActionPlan is the non-mutating, inspectable result of planning an intent. type ActionPlan struct { IntentID string `json:"intent_id"` - Verb string `json:"verb"` + Verb intent.Verb `json:"verb"` Target fleet.ResourceRef `json:"target"` Diff fleet.Diff `json:"diff"` Steps []PlanStep `json:"steps"` @@ -223,16 +230,3 @@ type DiffRequest struct { Target fleet.ResourceRef `json:"target"` Desired json.RawMessage `json:"desired,omitempty"` } - -// ValidVerb reports whether a verb belongs to the reviewed initial action vocabulary. -func ValidVerb(verb string) bool { - switch verb { - case "argocd.sync", "argocd.rollback", - "rollout.promote", "rollout.abort", - "deployment.scale", "deployment.restart", - "gitops.open-pr": - return true - default: - return false - } -} diff --git a/internal/connector/dcgm/boundary_test.go b/internal/connector/dcgm/boundary_test.go new file mode 100644 index 0000000..0359a33 --- /dev/null +++ b/internal/connector/dcgm/boundary_test.go @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: Apache-2.0 + +package dcgm + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var allowedProductionImports = map[string]bool{ + "bytes": true, + "crypto/sha256": true, + "encoding/hex": true, + "encoding/json": true, + "fmt": true, + "io": true, + "sort": true, + "strconv": true, + "strings": true, + "time": true, + "unicode": true, + "unicode/utf8": true, + "github.com/ArdurAI/sith/internal/fleet": true, + "k8s.io/apimachinery/pkg/util/validation": true, +} + +var allowedProductionFiles = map[string]string{ + "project.go": "282d08a75f4d8fa5109e055806e12804f307977cb67e0097600c345d5b01729e", +} + +var allowedProductionDeclarations = map[string]bool{ + "func:ProjectGPUUtilization": true, + "func:canonicalPercent": true, + "func:consumeUniqueJSON": true, + "func:matchingDelimiter": true, + "func:parseUnixTimestamp": true, + "func:projectSeries": true, + "func:rejectDuplicateJSON": true, + "func:requiredLabel": true, + "func:validGPUUUID": true, + "func:validIndex": true, + "func:validLabelName": true, + "func:validSafeToken": true, + "func:validateCanonicalTime": true, + "func:validateLabelValue": true, + "func:validateProjection": true, + "func:validateResponse": true, + "func:validateText": true, + "func:workloadIdentity": true, + "type:InstantQuery": true, + "type:Projection": true, + "type:gpuUtilizationObservation": true, + "type:projectedSeries": true, + "type:queryData": true, + "type:queryEnvelope": true, + "type:seriesIdentity": true, + "type:sourceSeries": true, + "value:GPUUtilizationMetric": true, + "value:Kind": true, + "value:ProtocolVersion": true, + "value:attributionMIGInstance": true, + "value:attributionPhysicalGPU": true, + "value:attributionWorkload": true, + "value:maxClockSkew": true, + "value:maxFactPayloadBytes": true, + "value:maxIdentityText": true, + "value:maxJSONDepth": true, + "value:maxLabelNameBytes": true, + "value:maxLabelValueBytes": true, + "value:maxLabelsPerSeries": true, + "value:maxMIGProfileBytes": true, + "value:maxModelNameBytes": true, + "value:maxPercentLiteralBytes": true, + "value:maxResponseBytes": true, + "value:maxSeries": true, + "value:maxTimestampLiteralSize": true, +} + +func TestProjectorHasNoIOCredentialPersistenceOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read dcgm package: %v", err) + } + seenFiles := make(map[string]bool, len(allowedProductionFiles)) + seenDeclarations := make(map[string]bool, len(allowedProductionDeclarations)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + expectedFingerprint, allowed := allowedProductionFiles[entry.Name()] + if !allowed { + t.Errorf("projector contains unreviewed production file %s", entry.Name()) + } + seenFiles[entry.Name()] = true + fileSet := token.NewFileSet() + file, err := parser.ParseFile(fileSet, entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + fingerprint, err := productionStructureFingerprint(fileSet, file) + if err != nil { + t.Fatalf("fingerprint %s: %v", entry.Name(), err) + } + if fingerprint != expectedFingerprint { + t.Errorf("projector production structure changed for %s: got %s", entry.Name(), fingerprint) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if imported.Name != nil { + t.Errorf("projector aliases production import %q as %q", path, imported.Name.Name) + } + if !allowedProductionImports[path] { + t.Errorf("projector imports unreviewed package %q", path) + } + } + for _, declaration := range file.Decls { + for _, key := range productionDeclarationKeys(declaration) { + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed symbol %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector declaration %s is duplicated", key) + } + seenDeclarations[key] = true + } + } + ast.Inspect(file, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if ok && isIdentifier(selector.X, "io") && selector.Sel.Name != "EOF" { + t.Errorf("projector uses unreviewed io capability io.%s", selector.Sel.Name) + } + function, ok := node.(*ast.FuncDecl) + if !ok { + return true + } + switch function.Name.Name { + case "Plan", "Execute", "Verify", "Write", "Delete", "Update", "Create", "Reload", "Dial", "Fetch": + t.Errorf("projector declares forbidden capability seam %s", function.Name.Name) + } + return true + }) + } + for name := range allowedProductionFiles { + if !seenFiles[name] { + t.Errorf("reviewed production file %s is missing", name) + } + } + for key := range allowedProductionDeclarations { + if !seenDeclarations[key] { + t.Errorf("reviewed production declaration %s is missing", key) + } + } +} + +func TestPublicProjectorBoundaryIsValueOnlyAndExact(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "project.go", nil, 0) + if err != nil { + t.Fatalf("parse project.go: %v", err) + } + var projector *ast.FuncDecl + var projection, query *ast.TypeSpec + for _, declaration := range file.Decls { + switch value := declaration.(type) { + case *ast.FuncDecl: + if value.Name.Name == "ProjectGPUUtilization" { + projector = value + } + case *ast.GenDecl: + for _, specification := range value.Specs { + typeSpecification, ok := specification.(*ast.TypeSpec) + if !ok { + continue + } + switch typeSpecification.Name.Name { + case "Projection": + projection = typeSpecification + case "InstantQuery": + query = typeSpecification + } + } + } + } + if projector == nil || !projectorHasExpectedSignature(projector) { + t.Fatal("ProjectGPUUtilization must keep the exact value-only Projection to []fleet.GraphFact,error signature") + } + if projection == nil || !structHasExpectedFields(projection, []fieldShape{ + {name: "Workspace", identifier: "string"}, + {name: "Scope", identifier: "string"}, + {name: "Query", identifier: "InstantQuery"}, + {name: "CollectedAt", packageName: "time", selected: "Time"}, + {name: "Response", identifier: "byte", slice: true}, + }) { + t.Fatal("Projection boundary changed or gained a capability-bearing field") + } + if query == nil || !structHasExpectedFields(query, []fieldShape{ + {name: "Expression", identifier: "string"}, + {name: "EvaluatedAt", packageName: "time", selected: "Time"}, + {name: "Limit", identifier: "int"}, + {name: "LookbackDelta", packageName: "time", selected: "Duration"}, + }) { + t.Fatal("InstantQuery boundary changed or gained an unreviewed field") + } +} + +type fieldShape struct { + name string + identifier string + packageName string + selected string + slice bool +} + +func structHasExpectedFields(specification *ast.TypeSpec, expected []fieldShape) bool { + structure, ok := specification.Type.(*ast.StructType) + if !ok || structure.Fields == nil || len(structure.Fields.List) != len(expected) { + return false + } + for index, field := range structure.Fields.List { + want := expected[index] + if len(field.Names) != 1 || field.Names[0].Name != want.name || field.Tag != nil { + return false + } + if want.slice { + slice, ok := field.Type.(*ast.ArrayType) + if !ok || slice.Len != nil || !isIdentifier(slice.Elt, want.identifier) { + return false + } + continue + } + if want.packageName != "" { + if !isSelector(field.Type, want.packageName, want.selected) { + return false + } + continue + } + if !isIdentifier(field.Type, want.identifier) { + return false + } + } + return true +} + +func projectorHasExpectedSignature(declaration *ast.FuncDecl) bool { + if declaration.Recv != nil || declaration.Type.TypeParams != nil || declaration.Type.Params == nil || + len(declaration.Type.Params.List) != 1 || declaration.Type.Results == nil || + len(declaration.Type.Results.List) != 2 { + return false + } + parameter := declaration.Type.Params.List[0] + if len(parameter.Names) != 1 || parameter.Names[0].Name != "input" || !isIdentifier(parameter.Type, "Projection") { + return false + } + firstResult, ok := declaration.Type.Results.List[0].Type.(*ast.ArrayType) + return ok && firstResult.Len == nil && isSelector(firstResult.Elt, "fleet", "GraphFact") && + isIdentifier(declaration.Type.Results.List[1].Type, "error") +} + +func productionDeclarationKeys(declaration ast.Decl) []string { + switch value := declaration.(type) { + case *ast.FuncDecl: + return []string{functionDeclarationKey(value)} + case *ast.GenDecl: + if value.Tok == token.IMPORT { + return nil + } + keys := make([]string, 0, len(value.Specs)) + for _, specification := range value.Specs { + switch typed := specification.(type) { + case *ast.TypeSpec: + keys = append(keys, "type:"+typed.Name.Name) + case *ast.ValueSpec: + for _, name := range typed.Names { + keys = append(keys, "value:"+name.Name) + } + } + } + return keys + default: + return []string{"declaration:"} + } +} + +func functionDeclarationKey(declaration *ast.FuncDecl) string { + if declaration.Recv == nil { + return "func:" + declaration.Name.Name + } + if len(declaration.Recv.List) != 1 { + return "method:." + declaration.Name.Name + } + receiver := declaration.Recv.List[0].Type + if pointer, ok := receiver.(*ast.StarExpr); ok { + receiver = pointer.X + } + identifier, ok := receiver.(*ast.Ident) + if !ok { + return "method:." + declaration.Name.Name + } + return "method:" + identifier.Name + "." + declaration.Name.Name +} + +func productionStructureFingerprint(fileSet *token.FileSet, file *ast.File) (string, error) { + var document bytes.Buffer + if err := format.Node(&document, fileSet, file); err != nil { + return "", err + } + digest := sha256.Sum256(document.Bytes()) + return hex.EncodeToString(digest[:]), nil +} + +func isIdentifier(expression ast.Expr, name string) bool { + identifier, ok := expression.(*ast.Ident) + return ok && identifier.Name == name +} + +func isSelector(expression ast.Expr, packageName, selectedName string) bool { + selector, ok := expression.(*ast.SelectorExpr) + return ok && isIdentifier(selector.X, packageName) && selector.Sel.Name == selectedName +} diff --git a/internal/connector/dcgm/project.go b/internal/connector/dcgm/project.go new file mode 100644 index 0000000..d847dd3 --- /dev/null +++ b/internal/connector/dcgm/project.go @@ -0,0 +1,652 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package dcgm normalizes bounded NVIDIA dcgm-exporter GPU utilization evidence +// from an already-authorized Prometheus instant query. +package dcgm + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "k8s.io/apimachinery/pkg/util/validation" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for dcgm-exporter evidence. + Kind = "dcgm" + // ProtocolVersion identifies the normalized Prometheus GPU-utilization vector contract. + ProtocolVersion = "prometheus/gpu-utilization-vector-v1" + // GPUUtilizationMetric is the exact dcgm-exporter metric accepted by this projector. + GPUUtilizationMetric = "DCGM_FI_DEV_GPU_UTIL" + + maxResponseBytes = 2 << 20 + maxSeries = 4_096 + maxLabelsPerSeries = 64 + maxLabelNameBytes = 128 + maxLabelValueBytes = 2 << 10 + maxIdentityText = 253 + maxModelNameBytes = 256 + maxMIGProfileBytes = 64 + maxPercentLiteralBytes = 32 + maxTimestampLiteralSize = 32 + maxFactPayloadBytes = 2 << 10 + maxJSONDepth = 64 + maxClockSkew = 5 * time.Minute + + attributionPhysicalGPU = "physical_gpu" + attributionMIGInstance = "mig_instance" + attributionWorkload = "workload_best_effort" +) + +// InstantQuery records the exact Prometheus query contract used to produce Response. +type InstantQuery struct { + Expression string + EvaluatedAt time.Time + Limit int + LookbackDelta time.Duration +} + +// Projection supplies one already-authorized Prometheus /api/v1/query response. The trusted +// caller owns source selection, authorization, request execution, and response collection. +// ProjectGPUUtilization performs no discovery, network access, credential loading, persistence, +// billing, optimization, process execution, or mutation. +type Projection struct { + Workspace string + Scope string + Query InstantQuery + CollectedAt time.Time + Response []byte +} + +type queryEnvelope struct { + Status string `json:"status"` + Data *queryData `json:"data"` + ErrorType string `json:"errorType"` + Error string `json:"error"` + Warnings []string `json:"warnings"` + Infos []string `json:"infos"` +} + +type queryData struct { + ResultType string `json:"resultType"` + Result *[]sourceSeries `json:"result"` +} + +type sourceSeries struct { + Metric map[string]string `json:"metric"` + Value []json.RawMessage `json:"value"` +} + +type gpuUtilizationObservation struct { + UtilizationPercent string `json:"utilization_percent"` + Attribution string `json:"attribution"` + DeviceScope string `json:"device_scope"` + ModelName string `json:"model_name"` + MIGInstanceID string `json:"mig_instance_id,omitempty"` + MIGProfile string `json:"mig_profile,omitempty"` + Namespace string `json:"namespace,omitempty"` + Pod string `json:"pod,omitempty"` + Container string `json:"container,omitempty"` +} + +type seriesIdentity struct { + Metric string `json:"metric"` + GPU string `json:"gpu"` + UUID string `json:"uuid"` + Device string `json:"device"` + Hostname string `json:"hostname"` + MIGInstanceID string `json:"mig_instance_id,omitempty"` + MIGProfile string `json:"mig_profile,omitempty"` + Namespace string `json:"namespace,omitempty"` + Pod string `json:"pod,omitempty"` + Container string `json:"container,omitempty"` +} + +type projectedSeries struct { + fact fleet.GraphFact + nativeID string +} + +// ProjectGPUUtilization returns deterministic, bounded TELEMETRY facts from one successful +// Prometheus instant vector. Unknown labels are validated but not retained. Whole-input failure is +// atomic, and complete workload labels are always marked best-effort rather than exact accounting. +func ProjectGPUUtilization(input Projection) ([]fleet.GraphFact, error) { + if err := validateProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode DCGM Prometheus response: %w", err) + } + + var envelope queryEnvelope + if err := json.Unmarshal(input.Response, &envelope); err != nil { + return nil, fmt.Errorf("decode DCGM Prometheus response") + } + series, err := validateResponse(envelope) + if err != nil { + return nil, err + } + + projected := make([]projectedSeries, 0, len(series)) + seen := make(map[string]bool, len(series)) + for index, source := range series { + fact, nativeID, err := projectSeries(input, source) + if err != nil { + return nil, fmt.Errorf("project DCGM GPU utilization series %d: %w", index, err) + } + if seen[nativeID] { + return nil, fmt.Errorf("project DCGM GPU utilization series %d: duplicate projected identity", index) + } + seen[nativeID] = true + projected = append(projected, projectedSeries{fact: fact, nativeID: nativeID}) + } + + sort.Slice(projected, func(left, right int) bool { + return projected[left].nativeID < projected[right].nativeID + }) + facts := make([]fleet.GraphFact, len(projected)) + for index := range projected { + facts[index] = projected[index].fact + } + return facts, nil +} + +func validateProjection(input Projection) error { + if err := validateText("workspace", input.Workspace, maxIdentityText, false); err != nil { + return err + } + if err := validateText("scope", input.Scope, maxIdentityText, false); err != nil { + return err + } + if strings.Contains(input.Scope, "/") { + return fmt.Errorf("scope is invalid") + } + if input.Query.Expression != GPUUtilizationMetric { + return fmt.Errorf("DCGM Prometheus expression must equal %s", GPUUtilizationMetric) + } + if input.Query.Limit != 0 { + return fmt.Errorf("DCGM Prometheus query limit must be disabled") + } + if input.Query.LookbackDelta != 0 { + return fmt.Errorf("DCGM Prometheus query lookback override must be disabled") + } + if err := validateCanonicalTime("query evaluation time", input.Query.EvaluatedAt); err != nil { + return err + } + if err := validateCanonicalTime("collection time", input.CollectedAt); err != nil { + return err + } + if input.Query.EvaluatedAt.After(input.CollectedAt.Add(maxClockSkew)) { + return fmt.Errorf("DCGM Prometheus query evaluation time is after collection time") + } + if len(input.Response) == 0 { + return fmt.Errorf("DCGM Prometheus response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("DCGM Prometheus response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("DCGM Prometheus response must be valid UTF-8") + } + return nil +} + +func validateResponse(envelope queryEnvelope) ([]sourceSeries, error) { + if envelope.Status != "success" { + return nil, fmt.Errorf("DCGM Prometheus response status must be success") + } + if envelope.ErrorType != "" || envelope.Error != "" { + return nil, fmt.Errorf("successful DCGM Prometheus response must not contain an error") + } + if len(envelope.Warnings) != 0 || len(envelope.Infos) != 0 { + return nil, fmt.Errorf("DCGM Prometheus response must not contain warnings or infos") + } + if envelope.Data == nil || envelope.Data.Result == nil { + return nil, fmt.Errorf("DCGM Prometheus response data.result is required") + } + if envelope.Data.ResultType != "vector" { + return nil, fmt.Errorf("DCGM Prometheus response resultType must be vector") + } + series := *envelope.Data.Result + if len(series) > maxSeries { + return nil, fmt.Errorf("DCGM Prometheus series count exceeds %d", maxSeries) + } + return series, nil +} + +func projectSeries(input Projection, source sourceSeries) (fleet.GraphFact, string, error) { + if len(source.Metric) == 0 { + return fleet.GraphFact{}, "", fmt.Errorf("metric labels are required") + } + if len(source.Metric) > maxLabelsPerSeries { + return fleet.GraphFact{}, "", fmt.Errorf("metric label count exceeds %d", maxLabelsPerSeries) + } + for name, value := range source.Metric { + if !validLabelName(name) { + return fleet.GraphFact{}, "", fmt.Errorf("metric label name is invalid") + } + if err := validateLabelValue(value); err != nil { + return fleet.GraphFact{}, "", err + } + } + + metric, err := requiredLabel(source.Metric, "__name__", maxLabelValueBytes) + if err != nil || metric != GPUUtilizationMetric { + return fleet.GraphFact{}, "", fmt.Errorf("metric name must equal %s", GPUUtilizationMetric) + } + gpu, err := requiredLabel(source.Metric, "gpu", 10) + if err != nil || !validIndex(gpu) { + return fleet.GraphFact{}, "", fmt.Errorf("gpu label is invalid") + } + uuid, err := requiredLabel(source.Metric, "UUID", 128) + if err != nil || !validGPUUUID(uuid) { + return fleet.GraphFact{}, "", fmt.Errorf("UUID label is invalid") + } + device, err := requiredLabel(source.Metric, "device", 32) + if err != nil || device != "nvidia"+gpu { + return fleet.GraphFact{}, "", fmt.Errorf("device label is invalid") + } + modelName, err := requiredLabel(source.Metric, "modelName", maxModelNameBytes) + if err != nil { + return fleet.GraphFact{}, "", fmt.Errorf("modelName label is invalid") + } + hostname, err := requiredLabel(source.Metric, "hostname", maxIdentityText) + if err != nil || !validSafeToken(hostname, ".-_") { + return fleet.GraphFact{}, "", fmt.Errorf("hostname label is invalid") + } + + migInstanceID, hasMIGID := source.Metric["GPU_I_ID"] + migProfile, hasMIGProfile := source.Metric["GPU_I_PROFILE"] + if hasMIGID != hasMIGProfile { + return fleet.GraphFact{}, "", fmt.Errorf("MIG identity labels must be supplied together") + } + deviceScope := attributionPhysicalGPU + if hasMIGID { + if !validIndex(migInstanceID) { + return fleet.GraphFact{}, "", fmt.Errorf("GPU_I_ID label is invalid") + } + if err := validateText("GPU_I_PROFILE label", migProfile, maxMIGProfileBytes, false); err != nil || + !validSafeToken(migProfile, ".-+_") { + return fleet.GraphFact{}, "", fmt.Errorf("GPU_I_PROFILE label is invalid") + } + deviceScope = attributionMIGInstance + } + + namespace, pod, container, hasWorkload, err := workloadIdentity(source.Metric) + if err != nil { + return fleet.GraphFact{}, "", err + } + attribution := deviceScope + if hasWorkload { + attribution = attributionWorkload + } + + if len(source.Value) != 2 { + return fleet.GraphFact{}, "", fmt.Errorf("instant-vector value must contain timestamp and sample") + } + sampleTime, err := parseUnixTimestamp(source.Value[0]) + if err != nil { + return fleet.GraphFact{}, "", err + } + if !sampleTime.Equal(input.Query.EvaluatedAt) { + return fleet.GraphFact{}, "", fmt.Errorf("sample timestamp must equal query evaluation time") + } + var sourcePercent string + if err := json.Unmarshal(source.Value[1], &sourcePercent); err != nil { + return fleet.GraphFact{}, "", fmt.Errorf("GPU utilization sample must be a string") + } + percent, err := canonicalPercent(sourcePercent) + if err != nil { + return fleet.GraphFact{}, "", err + } + + identity := seriesIdentity{ + Metric: metric, GPU: gpu, UUID: uuid, Device: device, Hostname: hostname, + MIGInstanceID: migInstanceID, MIGProfile: migProfile, + Namespace: namespace, Pod: pod, Container: container, + } + identityJSON, err := json.Marshal(identity) + if err != nil { + return fleet.GraphFact{}, "", fmt.Errorf("encode DCGM series identity: %w", err) + } + digest := sha256.Sum256(identityJSON) + hexDigest := hex.EncodeToString(digest[:]) + nativeID := "sha256:" + hexDigest + + observation := gpuUtilizationObservation{ + UtilizationPercent: percent, + Attribution: attribution, + DeviceScope: deviceScope, + ModelName: modelName, + MIGInstanceID: migInstanceID, + MIGProfile: migProfile, + Namespace: namespace, + Pod: pod, + Container: container, + } + canonical, err := json.Marshal(observation) + if err != nil { + return fleet.GraphFact{}, "", fmt.Errorf("encode DCGM GPU utilization fact: %w", err) + } + if len(canonical) > maxFactPayloadBytes { + return fleet.GraphFact{}, "", fmt.Errorf("DCGM GPU utilization fact exceeds %d bytes", maxFactPayloadBytes) + } + + entity := &fleet.EntityRef{Cluster: input.Scope} + if hasWorkload { + entity.Namespace = namespace + entity.Pod = pod + } + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, Scope: input.Scope, Kind: "GPUUtilization", + Namespace: namespace, Name: "gpu-utilization-" + hexDigest, + }, + Kind: fleet.FactDerived, Observed: canonical, ObservedAt: sampleTime, + Source: input.Scope, + Provenance: fleet.Provenance{ + Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID, + }, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensTelemetry, Entity: entity, + } + if err := fact.Validate(input.Workspace); err != nil { + return fleet.GraphFact{}, "", fmt.Errorf("validate DCGM GPU utilization fact: %w", err) + } + return fact, nativeID, nil +} + +func requiredLabel(labels map[string]string, name string, maximum int) (string, error) { + value, exists := labels[name] + if !exists { + return "", fmt.Errorf("required metric label is missing") + } + if err := validateText("metric label", value, maximum, false); err != nil { + return "", err + } + return value, nil +} + +func workloadIdentity(labels map[string]string) (string, string, string, bool, error) { + namespace, hasNamespace := labels["namespace"] + pod, hasPod := labels["pod"] + container, hasContainer := labels["container"] + count := 0 + for _, present := range []bool{hasNamespace, hasPod, hasContainer} { + if present { + count++ + } + } + if count == 0 { + return "", "", "", false, nil + } + if count != 3 { + return "", "", "", false, fmt.Errorf("workload labels must be supplied together") + } + if len(validation.IsDNS1123Label(namespace)) != 0 { + return "", "", "", false, fmt.Errorf("namespace label is invalid") + } + if len(validation.IsDNS1123Subdomain(pod)) != 0 { + return "", "", "", false, fmt.Errorf("pod label is invalid") + } + if len(validation.IsDNS1123Label(container)) != 0 { + return "", "", "", false, fmt.Errorf("container label is invalid") + } + return namespace, pod, container, true, nil +} + +func validLabelName(value string) bool { + if value == "" || len(value) > maxLabelNameBytes { + return false + } + for index := 0; index < len(value); index++ { + character := value[index] + if (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || character == '_' || + (index > 0 && character >= '0' && character <= '9') { + continue + } + return false + } + return true +} + +func validateLabelValue(value string) error { + if len(value) > maxLabelValueBytes || !utf8.ValidString(value) { + return fmt.Errorf("metric label value is invalid") + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("metric label value is invalid") + } + } + return nil +} + +func validIndex(value string) bool { + if value == "" || len(value) > 10 || (len(value) > 1 && value[0] == '0') { + return false + } + for index := range value { + if value[index] < '0' || value[index] > '9' { + return false + } + } + return true +} + +func validGPUUUID(value string) bool { + if !strings.HasPrefix(value, "GPU-") { + return false + } + identifier := value[len("GPU-"):] + if len(identifier) != 36 { + return false + } + for index := range identifier { + if index == 8 || index == 13 || index == 18 || index == 23 { + if identifier[index] != '-' { + return false + } + continue + } + character := identifier[index] + if (character < '0' || character > '9') && (character < 'a' || character > 'f') && + (character < 'A' || character > 'F') { + return false + } + } + return true +} + +func validSafeToken(value, extra string) bool { + if value == "" { + return false + } + for index := range value { + character := value[index] + if (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') || + (character >= '0' && character <= '9') || strings.ContainsRune(extra, rune(character)) { + continue + } + return false + } + return true +} + +func canonicalPercent(value string) (string, error) { + if value == "" || len(value) > maxPercentLiteralBytes { + return "", fmt.Errorf("GPU utilization sample is invalid") + } + integer, fraction, hasFraction := strings.Cut(value, ".") + if integer == "" || (len(integer) > 1 && integer[0] == '0') || len(integer) > 3 { + return "", fmt.Errorf("GPU utilization sample is invalid") + } + for index := range integer { + if integer[index] < '0' || integer[index] > '9' { + return "", fmt.Errorf("GPU utilization sample is invalid") + } + } + if hasFraction { + if fraction == "" || len(fraction) > 18 { + return "", fmt.Errorf("GPU utilization sample is invalid") + } + for index := range fraction { + if fraction[index] < '0' || fraction[index] > '9' { + return "", fmt.Errorf("GPU utilization sample is invalid") + } + } + } + integerValue, err := strconv.Atoi(integer) + if err != nil || integerValue > 100 || (integerValue == 100 && strings.Trim(fraction, "0") != "") { + return "", fmt.Errorf("GPU utilization sample must be between 0 and 100 percent") + } + fraction = strings.TrimRight(fraction, "0") + if fraction == "" { + return strconv.Itoa(integerValue), nil + } + return strconv.Itoa(integerValue) + "." + fraction, nil +} + +func parseUnixTimestamp(document json.RawMessage) (time.Time, error) { + value := string(document) + if value == "" || len(value) > maxTimestampLiteralSize { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + secondsLiteral, fraction, hasFraction := strings.Cut(value, ".") + if secondsLiteral == "" || (len(secondsLiteral) > 1 && secondsLiteral[0] == '0') { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + for index := range secondsLiteral { + if secondsLiteral[index] < '0' || secondsLiteral[index] > '9' { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + } + if hasFraction { + if fraction == "" || len(fraction) > 9 { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + for index := range fraction { + if fraction[index] < '0' || fraction[index] > '9' { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + } + } + seconds, err := strconv.ParseInt(secondsLiteral, 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + for len(fraction) < 9 { + fraction += "0" + } + nanoseconds := int64(0) + if fraction != "" { + nanoseconds, err = strconv.ParseInt(fraction, 10, 32) + if err != nil { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + } + parsed := time.Unix(seconds, nanoseconds).UTC() + if parsed.Year() < 2000 || parsed.Year() > 9999 { + return time.Time{}, fmt.Errorf("sample timestamp is invalid") + } + return parsed, nil +} + +func validateCanonicalTime(label string, value time.Time) error { + if value.IsZero() || value.Year() < 2000 || value.Year() > 9999 || value.Location() != time.UTC { + return fmt.Errorf("%s is invalid", label) + } + return nil +} + +func validateText(label, value string, maximum int, allowEmpty bool) error { + if (!allowEmpty && value == "") || len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("%s is invalid", label) + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func rejectDuplicateJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || closing != matchingDelimiter(delimiter) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func matchingDelimiter(open json.Delim) json.Delim { + if open == '{' { + return '}' + } + return ']' +} diff --git a/internal/connector/dcgm/project_test.go b/internal/connector/dcgm/project_test.go new file mode 100644 index 0000000..ccb1a62 --- /dev/null +++ b/internal/connector/dcgm/project_test.go @@ -0,0 +1,493 @@ +// SPDX-License-Identifier: Apache-2.0 + +package dcgm + +import ( + "bytes" + "encoding/json" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +type fixtureSeries struct { + Metric map[string]string `json:"metric"` + Value []any `json:"value"` +} + +func TestProjectGPUUtilizationProjectsBoundedAttribution(t *testing.T) { + t.Parallel() + input := validProjection(t, + validSeries("3", "GPU-dddddddd-1111-2222-3333-444444444444", "node-c", "25.2500", map[string]string{ + "GPU_I_ID": "2", "GPU_I_PROFILE": "1g.10gb", + "namespace": "ml", "pod": "trainer-0", "container": "trainer", + }), + validSeries("1", "GPU-bbbbbbbb-1111-2222-3333-444444444444", "node-a", "37.5000", map[string]string{ + "GPU_I_ID": "1", "GPU_I_PROFILE": "2g.20gb", + }), + validSeries("2", "GPU-cccccccc-1111-2222-3333-444444444444", "node-b", "100.000", map[string]string{ + "namespace": "inference", "pod": "server-6d8f8f7d9f-abcde", "container": "server", + "pod_label_team": "sensitive-team", "job": "dcgm-exporter", "instance": "10.0.0.8:9400", + }), + validSeries("0", "GPU-aaaaaaaa-1111-2222-3333-444444444444", "node-a", "0.000", nil), + ) + + facts, err := ProjectGPUUtilization(input) + if err != nil { + t.Fatalf("project GPU utilization: %v", err) + } + if len(facts) != 4 { + t.Fatalf("fact count = %d, want 4", len(facts)) + } + + seen := make(map[string]gpuUtilizationObservation, len(facts)) + for index, fact := range facts { + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("fact %d invalid: %v", index, err) + } + if fact.Fact.Kind != fleet.FactDerived || fact.Lens != fleet.LensTelemetry { + t.Fatalf("fact %d has kind/lens %q/%q", index, fact.Fact.Kind, fact.Lens) + } + if fact.Fact.Ref.SourceKind != Kind || fact.Fact.Ref.Kind != "GPUUtilization" || + fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion { + t.Fatalf("fact %d has unexpected source contract: %#v", index, fact) + } + if !fact.Fact.ObservedAt.Equal(input.Query.EvaluatedAt) || fact.Fact.Source != input.Scope { + t.Fatalf("fact %d has unexpected time/source", index) + } + if !strings.HasPrefix(fact.Fact.Provenance.NativeID, "sha256:") || + !strings.HasSuffix(fact.Fact.Ref.Name, strings.TrimPrefix(fact.Fact.Provenance.NativeID, "sha256:")) { + t.Fatalf("fact %d does not bind its hashed native identity", index) + } + if index > 0 && facts[index-1].Fact.Provenance.NativeID >= fact.Fact.Provenance.NativeID { + t.Fatalf("facts are not sorted by native identity") + } + var observation gpuUtilizationObservation + if err := json.Unmarshal(fact.Fact.Observed, &observation); err != nil { + t.Fatalf("decode observation %d: %v", index, err) + } + key := observation.Attribution + "/" + observation.DeviceScope + seen[key] = observation + if observation.Attribution == attributionWorkload { + if fact.Entity == nil || fact.Entity.Namespace != observation.Namespace || fact.Entity.Pod != observation.Pod || + fact.Fact.Ref.Namespace != observation.Namespace { + t.Fatalf("workload fact %d has mismatched graph identity", index) + } + } else if fact.Entity == nil || fact.Entity.Key() != "cluster/cluster-a" || fact.Fact.Ref.Namespace != "" { + t.Fatalf("device fact %d was attached beyond the cluster root", index) + } + } + + assertObservation(t, seen[attributionPhysicalGPU+"/"+attributionPhysicalGPU], gpuUtilizationObservation{ + UtilizationPercent: "0", Attribution: attributionPhysicalGPU, + DeviceScope: attributionPhysicalGPU, ModelName: "NVIDIA H100 80GB HBM3", + }) + assertObservation(t, seen[attributionMIGInstance+"/"+attributionMIGInstance], gpuUtilizationObservation{ + UtilizationPercent: "37.5", Attribution: attributionMIGInstance, + DeviceScope: attributionMIGInstance, ModelName: "NVIDIA H100 80GB HBM3", + MIGInstanceID: "1", MIGProfile: "2g.20gb", + }) + assertObservation(t, seen[attributionWorkload+"/"+attributionPhysicalGPU], gpuUtilizationObservation{ + UtilizationPercent: "100", Attribution: attributionWorkload, + DeviceScope: attributionPhysicalGPU, ModelName: "NVIDIA H100 80GB HBM3", + Namespace: "inference", Pod: "server-6d8f8f7d9f-abcde", Container: "server", + }) + assertObservation(t, seen[attributionWorkload+"/"+attributionMIGInstance], gpuUtilizationObservation{ + UtilizationPercent: "25.25", Attribution: attributionWorkload, + DeviceScope: attributionMIGInstance, ModelName: "NVIDIA H100 80GB HBM3", + MIGInstanceID: "2", MIGProfile: "1g.10gb", Namespace: "ml", Pod: "trainer-0", Container: "trainer", + }) + + encoded, err := json.Marshal(facts) + if err != nil { + t.Fatalf("encode facts: %v", err) + } + for _, secret := range []string{ + "GPU-aaaaaaaa-1111-2222-3333-444444444444", "GPU-bbbbbbbb-1111-2222-3333-444444444444", + "GPU-cccccccc-1111-2222-3333-444444444444", "GPU-dddddddd-1111-2222-3333-444444444444", + "node-a", "node-b", "node-c", "10.0.0.8:9400", "sensitive-team", "pod_label_team", "pci_bus_id", + } { + if bytes.Contains(encoded, []byte(secret)) { + t.Fatalf("projected facts retain private source identity %q", secret) + } + } +} + +func TestProjectGPUUtilizationIsDeterministicAndDropsUnknownLabels(t *testing.T) { + t.Parallel() + first := validSeries("0", "GPU-aaaaaaaa-1111-2222-3333-444444444444", "node-a", "42.500", map[string]string{ + "job": "first", "external_label": "private-a", + }) + second := validSeries("1", "GPU-bbbbbbbb-1111-2222-3333-444444444444", "node-b", "7", map[string]string{ + "GPU_I_ID": "0", "GPU_I_PROFILE": "1g.10gb", "instance": "private-b", + }) + forward := validProjection(t, first, second) + reverse := validProjection(t, second, first) + forwardFacts, err := ProjectGPUUtilization(forward) + if err != nil { + t.Fatalf("project forward: %v", err) + } + reverseFacts, err := ProjectGPUUtilization(reverse) + if err != nil { + t.Fatalf("project reverse: %v", err) + } + if !reflect.DeepEqual(forwardFacts, reverseFacts) { + t.Fatalf("input permutation changed projected facts") + } + + changedUnknown := validSeries("0", "GPU-aaaaaaaa-1111-2222-3333-444444444444", "node-a", "42.500", map[string]string{ + "job": "changed", "external_label": "private-c", + }) + changedFacts, err := ProjectGPUUtilization(validProjection(t, changedUnknown)) + if err != nil { + t.Fatalf("project changed unknown labels: %v", err) + } + index := projectedIndex(forwardFacts, "42.5") + if index < 0 { + t.Fatal("projected physical-GPU fact is missing") + } + if !reflect.DeepEqual(forwardFacts[index], changedFacts[0]) { + t.Fatalf("unknown labels changed retained identity or payload") + } +} + +func TestProjectGPUUtilizationSuccessfulEmptyAbstains(t *testing.T) { + t.Parallel() + facts, err := ProjectGPUUtilization(validProjection(t)) + if err != nil { + t.Fatalf("project empty vector: %v", err) + } + if facts == nil || len(facts) != 0 { + t.Fatalf("empty vector facts = %#v, want non-nil empty slice", facts) + } +} + +func TestProjectGPUUtilizationRejectsInvalidInputAtomically(t *testing.T) { + t.Parallel() + evaluation := fixtureEvaluationTime() + valid := validSeries("0", "GPU-aaaaaaaa-1111-2222-3333-444444444444", "node-a", "50", nil) + + tests := []struct { + name string + mutate func(*Projection) + }{ + {name: "workspace missing", mutate: func(input *Projection) { input.Workspace = "" }}, + {name: "workspace control", mutate: func(input *Projection) { input.Workspace = "local\nother" }}, + {name: "scope slash", mutate: func(input *Projection) { input.Scope = "cluster/a" }}, + {name: "wrong expression", mutate: func(input *Projection) { input.Query.Expression = "avg(DCGM_FI_DEV_GPU_UTIL)" }}, + {name: "series limit", mutate: func(input *Projection) { input.Query.Limit = 4_096 }}, + {name: "lookback override", mutate: func(input *Projection) { input.Query.LookbackDelta = time.Minute }}, + {name: "zero evaluation", mutate: func(input *Projection) { input.Query.EvaluatedAt = time.Time{} }}, + {name: "non UTC evaluation", mutate: func(input *Projection) { + input.Query.EvaluatedAt = input.Query.EvaluatedAt.In(time.FixedZone("offset", 3600)) + }}, + {name: "zero collection", mutate: func(input *Projection) { input.CollectedAt = time.Time{} }}, + {name: "evaluation too far ahead", mutate: func(input *Projection) { + input.Query.EvaluatedAt = input.CollectedAt.Add(maxClockSkew + time.Nanosecond) + }}, + {name: "response missing", mutate: func(input *Projection) { input.Response = nil }}, + {name: "response oversized", mutate: func(input *Projection) { input.Response = bytes.Repeat([]byte(" "), maxResponseBytes+1) }}, + {name: "response invalid utf8", mutate: func(input *Projection) { input.Response = []byte{'{', 0xff, '}'} }}, + {name: "duplicate json", mutate: func(input *Projection) { + input.Response = []byte(`{"status":"success","status":"success","data":{"resultType":"vector","result":[]}}`) + }}, + {name: "trailing json", mutate: func(input *Projection) { input.Response = append(input.Response, []byte(` {}`)...) }}, + {name: "deep json", mutate: func(input *Projection) { + input.Response = []byte(`{"status":"success","deep":` + strings.Repeat("[", maxJSONDepth+1) + `0` + strings.Repeat("]", maxJSONDepth+1) + `}`) + }}, + {name: "malformed json", mutate: func(input *Projection) { input.Response = []byte(`{"status":`) }}, + {name: "error status", mutate: func(input *Projection) { + input.Response = responseDocument(t, "error", "vector", []fixtureSeries{}, nil, nil, "execution", "failed") + }}, + {name: "success with error", mutate: func(input *Projection) { + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{}, nil, nil, "execution", "failed") + }}, + {name: "missing data", mutate: func(input *Projection) { input.Response = []byte(`{"status":"success"}`) }}, + {name: "null result", mutate: func(input *Projection) { + input.Response = []byte(`{"status":"success","data":{"resultType":"vector","result":null}}`) + }}, + {name: "wrong result type", mutate: func(input *Projection) { + input.Response = responseDocument(t, "success", "matrix", []fixtureSeries{}, nil, nil, "", "") + }}, + {name: "warning", mutate: func(input *Projection) { + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{}, []string{"partial"}, nil, "", "") + }}, + {name: "info", mutate: func(input *Projection) { + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{}, nil, []string{"annotation"}, "", "") + }}, + {name: "too many series", mutate: func(input *Projection) { + series := make([]fixtureSeries, maxSeries+1) + for index := range series { + series[index] = validSeries(stringIndex(index), "GPU-series-"+stringIndex(index), "node-a", "1", nil) + } + input.Response = responseDocument(t, "success", "vector", series, nil, nil, "", "") + }}, + {name: "labels missing", mutate: mutateOnlySeries(t, fixtureSeries{Metric: map[string]string{}, Value: valid.Value})}, + {name: "too many labels", mutate: func(input *Projection) { + series := cloneFixtureSeries(valid) + for index := 0; index <= maxLabelsPerSeries; index++ { + series.Metric["extra_"+stringIndex(index)] = "value" + } + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{series}, nil, nil, "", "") + }}, + {name: "invalid label name", mutate: mutateSeriesLabel(t, valid, "bad-label", "value")}, + {name: "invalid label value", mutate: mutateSeriesLabel(t, valid, "job", "bad\nvalue")}, + {name: "missing metric name", mutate: deleteSeriesLabel(t, valid, "__name__")}, + {name: "wrong metric name", mutate: mutateSeriesLabel(t, valid, "__name__", "up")}, + {name: "invalid gpu", mutate: mutateSeriesLabel(t, valid, "gpu", "00")}, + {name: "invalid uuid", mutate: mutateSeriesLabel(t, valid, "UUID", "MIG-private")}, + {name: "mismatched device", mutate: mutateSeriesLabel(t, valid, "device", "nvidia1")}, + {name: "empty model", mutate: mutateSeriesLabel(t, valid, "modelName", "")}, + {name: "invalid hostname", mutate: mutateSeriesLabel(t, valid, "hostname", "node/a")}, + {name: "partial mig id", mutate: mutateSeriesLabel(t, valid, "GPU_I_ID", "0")}, + {name: "partial mig profile", mutate: mutateSeriesLabel(t, valid, "GPU_I_PROFILE", "1g.10gb")}, + {name: "invalid mig id", mutate: mutateSeriesLabels(t, valid, map[string]string{"GPU_I_ID": "01", "GPU_I_PROFILE": "1g.10gb"})}, + {name: "invalid mig profile", mutate: mutateSeriesLabels(t, valid, map[string]string{"GPU_I_ID": "0", "GPU_I_PROFILE": "1g/10gb"})}, + {name: "partial workload", mutate: mutateSeriesLabel(t, valid, "namespace", "ml")}, + {name: "invalid namespace", mutate: mutateSeriesLabels(t, valid, map[string]string{"namespace": "ML", "pod": "trainer-0", "container": "trainer"})}, + {name: "invalid pod", mutate: mutateSeriesLabels(t, valid, map[string]string{"namespace": "ml", "pod": "trainer/0", "container": "trainer"})}, + {name: "invalid container", mutate: mutateSeriesLabels(t, valid, map[string]string{"namespace": "ml", "pod": "trainer-0", "container": "Trainer"})}, + {name: "value arity", mutate: func(input *Projection) { + series := cloneFixtureSeries(valid) + series.Value = []any{json.Number(unixTimestampLiteral(evaluation))} + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{series}, nil, nil, "", "") + }}, + {name: "timestamp string", mutate: mutateSeriesValue(t, valid, []any{unixTimestampLiteral(evaluation), "50"})}, + {name: "timestamp exponent", mutate: mutateSeriesValue(t, valid, []any{json.Number("1.721322e9"), "50"})}, + {name: "timestamp precision", mutate: mutateSeriesValue(t, valid, []any{json.Number("1721323200.1234567890"), "50"})}, + {name: "timestamp mismatch", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation.Add(time.Second))), "50"})}, + {name: "sample number", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), json.Number("50")})}, + {name: "sample nan", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), "NaN"})}, + {name: "sample infinity", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), "+Inf"})}, + {name: "sample negative", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), "-0"})}, + {name: "sample above range", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), "100.0001"})}, + {name: "sample leading zero", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), "01"})}, + {name: "sample exponent", mutate: mutateSeriesValue(t, valid, []any{json.Number(unixTimestampLiteral(evaluation)), "1e2"})}, + {name: "duplicate projected identity", mutate: func(input *Projection) { + other := cloneFixtureSeries(valid) + other.Metric["job"] = "other" + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{valid, other}, nil, nil, "", "") + }}, + {name: "late invalid series", mutate: func(input *Projection) { + late := cloneFixtureSeries(valid) + delete(late.Metric, "UUID") + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{valid, late}, nil, nil, "", "") + }}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := validProjection(t, valid) + test.mutate(&input) + facts, err := ProjectGPUUtilization(input) + if err == nil { + t.Fatalf("ProjectGPUUtilization accepted invalid input; facts=%#v", facts) + } + if facts != nil { + t.Fatalf("ProjectGPUUtilization returned partial facts on error: %#v", facts) + } + }) + } +} + +func TestCanonicalPercent(t *testing.T) { + t.Parallel() + for _, test := range []struct { + input string + want string + ok bool + }{ + {input: "0", want: "0", ok: true}, + {input: "0.000", want: "0", ok: true}, + {input: "7.5000", want: "7.5", ok: true}, + {input: "99.999999999999999999", want: "99.999999999999999999", ok: true}, + {input: "100.000", want: "100", ok: true}, + {input: "", ok: false}, + {input: ".5", ok: false}, + {input: "5.", ok: false}, + {input: "00", ok: false}, + {input: "-1", ok: false}, + {input: "100.1", ok: false}, + {input: "101", ok: false}, + {input: "NaN", ok: false}, + {input: "1e2", ok: false}, + } { + got, err := canonicalPercent(test.input) + if (err == nil) != test.ok || got != test.want { + t.Errorf("canonicalPercent(%q) = %q, %v; want %q, ok=%v", test.input, got, err, test.want, test.ok) + } + } +} + +func FuzzProjectGPUUtilization(f *testing.F) { + input := validProjection(f, validSeries("0", "GPU-aaaaaaaa-1111-2222-3333-444444444444", "node-a", "50", nil)) + f.Add(input.Response) + f.Add(responseDocument(f, "success", "vector", []fixtureSeries{}, nil, nil, "", "")) + f.Add([]byte(`{"status":"success","data":{"resultType":"vector","result":null}}`)) + f.Add([]byte(`{"status":"success","status":"error"}`)) + f.Fuzz(func(t *testing.T, response []byte) { + projection := input + projection.Response = append([]byte(nil), response...) + facts, err := ProjectGPUUtilization(projection) + if err != nil { + if facts != nil { + t.Fatalf("error returned with partial facts: %#v", facts) + } + return + } + if len(facts) > maxSeries { + t.Fatalf("successful projection exceeded series bound: %d", len(facts)) + } + for index, fact := range facts { + if err := fact.Validate(projection.Workspace); err != nil { + t.Fatalf("fact %d invalid: %v", index, err) + } + if !fact.Fact.ObservedAt.Equal(projection.Query.EvaluatedAt) { + t.Fatalf("fact %d changed observation time", index) + } + var observation map[string]any + if err := json.Unmarshal(fact.Fact.Observed, &observation); err != nil { + t.Fatalf("fact %d payload invalid: %v", index, err) + } + for _, forbidden := range []string{"UUID", "uuid", "hostname", "job", "instance", "pci_bus_id"} { + if _, exists := observation[forbidden]; exists { + t.Fatalf("fact %d retained forbidden field %q", index, forbidden) + } + } + } + }) +} + +func validProjection(t testing.TB, series ...fixtureSeries) Projection { + t.Helper() + evaluation := fixtureEvaluationTime() + return Projection{ + Workspace: "local", Scope: "cluster-a", + Query: InstantQuery{Expression: GPUUtilizationMetric, EvaluatedAt: evaluation}, + CollectedAt: evaluation.Add(2 * time.Second), + Response: responseDocument(t, "success", "vector", series, nil, nil, "", ""), + } +} + +func validSeries(gpu, uuid, hostname, percent string, extra map[string]string) fixtureSeries { + labels := map[string]string{ + "__name__": GPUUtilizationMetric, + "gpu": gpu, "UUID": uuid, "device": "nvidia" + gpu, + "modelName": "NVIDIA H100 80GB HBM3", "hostname": hostname, + "pci_bus_id": "00000000:01:00.0", + } + for name, value := range extra { + labels[name] = value + } + return fixtureSeries{ + Metric: labels, + Value: []any{json.Number(unixTimestampLiteral(fixtureEvaluationTime())), percent}, + } +} + +func responseDocument(t testing.TB, status, resultType string, series []fixtureSeries, warnings, infos []string, errorType, sourceError string) []byte { + t.Helper() + if series == nil { + series = make([]fixtureSeries, 0) + } + document := map[string]any{ + "status": status, + "data": map[string]any{"resultType": resultType, "result": series}, + } + if warnings != nil { + document["warnings"] = warnings + } + if infos != nil { + document["infos"] = infos + } + if errorType != "" { + document["errorType"] = errorType + } + if sourceError != "" { + document["error"] = sourceError + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("encode fixture: %v", err) + } + return encoded +} + +func fixtureEvaluationTime() time.Time { + return time.Date(2026, 7, 18, 20, 0, 0, 123456000, time.UTC) +} + +func unixTimestampLiteral(value time.Time) string { + seconds := value.Unix() + nanoseconds := value.Nanosecond() + if nanoseconds == 0 { + return stringIndex64(seconds) + } + fraction := strings.TrimRight(stringIndex64(int64(nanoseconds) + 1_000_000_000)[1:], "0") + return stringIndex64(seconds) + "." + fraction +} + +func stringIndex(value int) string { return stringIndex64(int64(value)) } + +func stringIndex64(value int64) string { return strconv.FormatInt(value, 10) } + +func cloneFixtureSeries(source fixtureSeries) fixtureSeries { + clone := fixtureSeries{Metric: make(map[string]string, len(source.Metric)), Value: append([]any(nil), source.Value...)} + for name, value := range source.Metric { + clone.Metric[name] = value + } + return clone +} + +func mutateOnlySeries(t testing.TB, series fixtureSeries) func(*Projection) { + return func(input *Projection) { + input.Response = responseDocument(t, "success", "vector", []fixtureSeries{series}, nil, nil, "", "") + } +} + +func mutateSeriesLabel(t testing.TB, source fixtureSeries, name, value string) func(*Projection) { + return mutateSeriesLabels(t, source, map[string]string{name: value}) +} + +func mutateSeriesLabels(t testing.TB, source fixtureSeries, labels map[string]string) func(*Projection) { + series := cloneFixtureSeries(source) + for name, value := range labels { + series.Metric[name] = value + } + return mutateOnlySeries(t, series) +} + +func deleteSeriesLabel(t testing.TB, source fixtureSeries, name string) func(*Projection) { + series := cloneFixtureSeries(source) + delete(series.Metric, name) + return mutateOnlySeries(t, series) +} + +func mutateSeriesValue(t testing.TB, source fixtureSeries, value []any) func(*Projection) { + series := cloneFixtureSeries(source) + series.Value = value + return mutateOnlySeries(t, series) +} + +func projectedIndex(facts []fleet.GraphFact, percent string) int { + for index, fact := range facts { + var observation gpuUtilizationObservation + if json.Unmarshal(fact.Fact.Observed, &observation) == nil && observation.UtilizationPercent == percent { + return index + } + } + return -1 +} + +func assertObservation(t testing.TB, got, want gpuUtilizationObservation) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("observation = %#v, want %#v", got, want) + } +} diff --git a/internal/connector/elasticsearch/boundary_test.go b/internal/connector/elasticsearch/boundary_test.go new file mode 100644 index 0000000..2315839 --- /dev/null +++ b/internal/connector/elasticsearch/boundary_test.go @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 + +package elasticsearch + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var allowedProductionImports = map[string]bool{ + "bytes": true, + "crypto/sha256": true, + "encoding/hex": true, + "encoding/json": true, + "fmt": true, + "io": true, + "sort": true, + "strings": true, + "time": true, + "unicode": true, + "unicode/utf8": true, + "github.com/ArdurAI/sith/internal/fleet": true, +} + +var allowedProductionFiles = map[string]string{ + "project.go": "965301c6aff18bb9dd8fd5415f8c7781b3cb3557c21d1564eb268860ad2340c1", +} + +var allowedProductionDeclarations = map[string]bool{ + "func:ProjectLogCauses": true, + "func:buildFact": true, + "func:classifyMessage": true, + "func:consumeUniqueJSON": true, + "func:containsAny": true, + "func:decodeOptionalField": true, + "func:indicatesDependencyFailure": true, + "func:indicatesMissingConfig": true, + "func:indicatesPanic": true, + "func:matchingDelimiter": true, + "func:objectFields": true, + "func:optionalSingleString": true, + "func:rejectDuplicateJSON": true, + "func:requiredSingleMessage": true, + "func:requiredSingleString": true, + "func:validateHit": true, + "func:validateProjection": true, + "func:validateSearchResponse": true, + "func:validateText": true, + "method:hitEnvelope.UnmarshalJSON": true, + "method:searchHit.UnmarshalJSON": true, + "method:searchResponse.UnmarshalJSON": true, + "method:shardSummary.UnmarshalJSON": true, + "type:Projection": true, + "type:causeAggregate": true, + "type:causeObservation": true, + "type:hitEnvelope": true, + "type:searchHit": true, + "type:searchResponse": true, + "type:shardSummary": true, + "value:Kind": true, + "value:ProtocolVersion": true, + "value:allowedHitFields": true, + "value:clusterField": true, + "value:containerField": true, + "value:maxCauseFacts": true, + "value:maxClockSkew": true, + "value:maxFactPayloadBytes": true, + "value:maxHits": true, + "value:maxIdentityText": true, + "value:maxJSONDepth": true, + "value:maxMessageBytes": true, + "value:maxQueryWindow": true, + "value:maxResponseBytes": true, + "value:messageField": true, + "value:namespaceField": true, + "value:podField": true, + "value:timestampField": true, +} + +func functionDeclarationKey(declaration *ast.FuncDecl) string { + if declaration.Recv == nil { + return "func:" + declaration.Name.Name + } + if len(declaration.Recv.List) != 1 { + return "method:." + declaration.Name.Name + } + receiver := declaration.Recv.List[0].Type + if pointer, ok := receiver.(*ast.StarExpr); ok { + receiver = pointer.X + } + identifier, ok := receiver.(*ast.Ident) + if !ok { + return "method:." + declaration.Name.Name + } + return "method:" + identifier.Name + "." + declaration.Name.Name +} + +func projectLogCausesHasExpectedSignature(declaration *ast.FuncDecl) bool { + if declaration.Recv != nil || declaration.Type.TypeParams != nil || declaration.Type.Params == nil || + len(declaration.Type.Params.List) != 1 || declaration.Type.Results == nil || + len(declaration.Type.Results.List) != 2 { + return false + } + parameter := declaration.Type.Params.List[0] + if len(parameter.Names) != 1 || parameter.Names[0].Name != "input" || !isIdentifier(parameter.Type, "Projection") { + return false + } + firstResult, ok := declaration.Type.Results.List[0].Type.(*ast.ArrayType) + return ok && firstResult.Len == nil && isSelector(firstResult.Elt, "fleet", "GraphFact") && + isIdentifier(declaration.Type.Results.List[1].Type, "error") +} + +func isIdentifier(expression ast.Expr, name string) bool { + identifier, ok := expression.(*ast.Ident) + return ok && identifier.Name == name +} + +func isSelector(expression ast.Expr, packageName, selectedName string) bool { + selector, ok := expression.(*ast.SelectorExpr) + return ok && isIdentifier(selector.X, packageName) && selector.Sel.Name == selectedName +} + +func projectionHasExpectedFields(specification *ast.TypeSpec) bool { + structure, ok := specification.Type.(*ast.StructType) + if !ok || structure.Fields == nil { + return false + } + expected := []struct { + name string + identifier string + packageName string + selected string + slice bool + }{ + {name: "Workspace", identifier: "string"}, + {name: "Scope", identifier: "string"}, + {name: "Namespace", identifier: "string"}, + {name: "Pod", identifier: "string"}, + {name: "Container", identifier: "string"}, + {name: "WindowStart", packageName: "time", selected: "Time"}, + {name: "WindowEnd", packageName: "time", selected: "Time"}, + {name: "ObservedAt", packageName: "time", selected: "Time"}, + {name: "Response", identifier: "byte", slice: true}, + } + if len(structure.Fields.List) != len(expected) { + return false + } + for index, field := range structure.Fields.List { + want := expected[index] + if len(field.Names) != 1 || field.Names[0].Name != want.name || field.Tag != nil { + return false + } + if want.slice { + slice, ok := field.Type.(*ast.ArrayType) + if !ok || slice.Len != nil || !isIdentifier(slice.Elt, want.identifier) { + return false + } + continue + } + if want.packageName != "" { + if !isSelector(field.Type, want.packageName, want.selected) { + return false + } + continue + } + if !isIdentifier(field.Type, want.identifier) { + return false + } + } + return true +} + +func productionStructureFingerprint(fileSet *token.FileSet, file *ast.File) (string, error) { + var document bytes.Buffer + if err := format.Node(&document, fileSet, file); err != nil { + return "", err + } + digest := sha256.Sum256(document.Bytes()) + return hex.EncodeToString(digest[:]), nil +} + +func TestProjectorHasNoIOCredentialPersistenceOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read elasticsearch package: %v", err) + } + seenFiles := make(map[string]bool, len(allowedProductionFiles)) + seenDeclarations := make(map[string]bool, len(allowedProductionDeclarations)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + expectedFingerprint, allowed := allowedProductionFiles[entry.Name()] + if !allowed { + t.Errorf("projector contains unreviewed production file %s", entry.Name()) + } + seenFiles[entry.Name()] = true + fileSet := token.NewFileSet() + file, err := parser.ParseFile(fileSet, entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + fingerprint, err := productionStructureFingerprint(fileSet, file) + if err != nil { + t.Fatalf("fingerprint %s: %v", entry.Name(), err) + } + if fingerprint != expectedFingerprint { + t.Errorf("projector production structure changed for %s: got %s", entry.Name(), fingerprint) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if imported.Name != nil { + t.Errorf("projector aliases production import %q as %q", path, imported.Name.Name) + } + if !allowedProductionImports[path] { + t.Fatalf("projector imports unreviewed package %q", path) + } + } + for _, node := range file.Decls { + switch declaration := node.(type) { + case *ast.FuncDecl: + key := functionDeclarationKey(declaration) + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed function or method %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector repeats reviewed declaration %s", key) + } + seenDeclarations[key] = true + if key == "func:ProjectLogCauses" && !projectLogCausesHasExpectedSignature(declaration) { + t.Errorf("ProjectLogCauses must retain the reviewed pure-projector signature") + } + case *ast.GenDecl: + for _, specification := range declaration.Specs { + switch typed := specification.(type) { + case *ast.TypeSpec: + key := "type:" + typed.Name.Name + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed type %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector repeats reviewed declaration %s", key) + } + seenDeclarations[key] = true + if key == "type:Projection" && !projectionHasExpectedFields(typed) { + t.Errorf("Projection must retain the reviewed value-only fields") + } + case *ast.ValueSpec: + for _, name := range typed.Names { + key := "value:" + name.Name + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed value %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector repeats reviewed declaration %s", key) + } + seenDeclarations[key] = true + } + } + } + } + } + ast.Inspect(file, func(node ast.Node) bool { + switch typed := node.(type) { + case *ast.InterfaceType: + t.Errorf("projector declares an injected interface seam") + case *ast.SelectorExpr: + if isIdentifier(typed.X, "io") && typed.Sel.Name != "EOF" { + t.Errorf("projector uses disallowed io capability io.%s", typed.Sel.Name) + } + } + return true + }) + } + for name := range allowedProductionFiles { + if !seenFiles[name] { + t.Errorf("projector is missing reviewed production file %s", name) + } + } + for key := range allowedProductionDeclarations { + if !seenDeclarations[key] { + t.Errorf("projector is missing reviewed declaration %s", key) + } + } +} + +func TestFunctionDeclarationKeyQualifiesReceiverType(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "methods.go", `package example +func Project() {} +func (*searchResponse) UnmarshalJSON([]byte) error { return nil } +func (Projection) UnmarshalJSON([]byte) error { return nil } +`, 0) + if err != nil { + t.Fatalf("parse method declarations: %v", err) + } + + var keys []string + for _, declaration := range file.Decls { + if function, ok := declaration.(*ast.FuncDecl); ok { + keys = append(keys, functionDeclarationKey(function)) + } + } + want := []string{ + "func:Project", + "method:searchResponse.UnmarshalJSON", + "method:Projection.UnmarshalJSON", + } + if len(keys) != len(want) { + t.Fatalf("functionDeclarationKey() produced %v; want %v", keys, want) + } + for index := range want { + if keys[index] != want[index] { + t.Fatalf("functionDeclarationKey() key %d = %q; want %q", index, keys[index], want[index]) + } + } +} + +func TestProjectLogCausesSignatureRejectsCapabilitySeams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + signature string + want bool + }{ + {name: "reviewed", signature: `func ProjectLogCauses(input Projection) ([]fleet.GraphFact, error)`, want: true}, + {name: "reader parameter", signature: `func ProjectLogCauses(input Projection, reader io.Reader) ([]fleet.GraphFact, error)`}, + {name: "reader result", signature: `func ProjectLogCauses(input Projection) ([]fleet.GraphFact, io.Reader)`}, + {name: "interface input", signature: `func ProjectLogCauses(input any) ([]fleet.GraphFact, error)`}, + {name: "method", signature: `func (Projection) ProjectLogCauses(input Projection) ([]fleet.GraphFact, error)`}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "signature.go", "package example\n"+test.signature+" {}", 0) + if err != nil { + t.Fatalf("parse signature: %v", err) + } + declaration := file.Decls[0].(*ast.FuncDecl) + if got := projectLogCausesHasExpectedSignature(declaration); got != test.want { + t.Fatalf("projectLogCausesHasExpectedSignature() = %t; want %t", got, test.want) + } + }) + } +} + +func TestProjectionFieldsRejectCapabilitySeams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + fields string + want bool + }{ + { + name: "reviewed", + fields: `Workspace string +Scope string +Namespace string +Pod string +Container string +WindowStart time.Time +WindowEnd time.Time +ObservedAt time.Time +Response []byte`, + want: true, + }, + {name: "callback hook", fields: `Workspace string; Hook func()`}, + {name: "reader", fields: `Workspace string; Reader io.Reader`}, + {name: "extra response", fields: `Workspace string; Response []byte; Extra []byte`}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + source := "package example\ntype Projection struct {\n" + test.fields + "\n}" + file, err := parser.ParseFile(token.NewFileSet(), "projection.go", source, 0) + if err != nil { + t.Fatalf("parse Projection: %v", err) + } + specification := file.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec) + if got := projectionHasExpectedFields(specification); got != test.want { + t.Fatalf("projectionHasExpectedFields() = %t; want %t", got, test.want) + } + }) + } +} + +func TestIOCapabilityGuardAllowsOnlyEOFSentinel(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "io.go", `package example +import "io" +var end = io.EOF +type seam struct { reader io.Reader } +`, 0) + if err != nil { + t.Fatalf("parse io declarations: %v", err) + } + var rejected []string + ast.Inspect(file, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if ok && isIdentifier(selector.X, "io") && selector.Sel.Name != "EOF" { + rejected = append(rejected, selector.Sel.Name) + } + return true + }) + if len(rejected) != 1 || rejected[0] != "Reader" { + t.Fatalf("io capability guard rejected %v; want [Reader]", rejected) + } +} diff --git a/internal/connector/elasticsearch/project.go b/internal/connector/elasticsearch/project.go new file mode 100644 index 0000000..5a3dc63 --- /dev/null +++ b/internal/connector/elasticsearch/project.go @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package elasticsearch normalizes bounded Elasticsearch log-search evidence for Sith's +// operational graph. +package elasticsearch + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for Elasticsearch read evidence. + Kind = "elasticsearch" + // ProtocolVersion identifies the normalized Elasticsearch Search API ECS fact contract. + ProtocolVersion = "search/ecs-v1" + + maxResponseBytes = 1 << 20 + maxHits = 128 + maxMessageBytes = 16 << 10 + maxIdentityText = 253 + maxFactPayloadBytes = 4 << 10 + maxJSONDepth = 64 + maxQueryWindow = 15 * time.Minute + maxClockSkew = 5 * time.Minute + maxCauseFacts = 3 +) + +const ( + timestampField = "@timestamp" + messageField = "message" + clusterField = "orchestrator.cluster.name" + namespaceField = "kubernetes.namespace" + podField = "kubernetes.pod.name" + containerField = "kubernetes.container.name" +) + +var allowedHitFields = map[string]bool{ + timestampField: true, + messageField: true, + clusterField: true, + namespaceField: true, + podField: true, + containerField: true, +} + +// Projection supplies one already-authorized Elasticsearch Search API response. The trusted +// caller owns the query target, authorization, and exact query contract. ProjectLogCauses does +// not perform discovery, network access, credential loading, persistence, or mutation. +type Projection struct { + Workspace string + Scope string + Namespace string + Pod string + Container string + WindowStart time.Time + WindowEnd time.Time + ObservedAt time.Time + Response []byte +} + +type searchResponse struct { + TimedOut *bool + TerminatedEarly *bool + Shards *shardSummary + Hits *hitEnvelope +} + +type shardSummary struct { + Total *int64 + Successful *int64 + Skipped *int64 + Failed *int64 +} + +type hitEnvelope struct { + Hits *[]searchHit +} + +type searchHit struct { + Fields *map[string]json.RawMessage + SourcePresent bool + IgnoredPresent bool + IgnoredValuesPresent bool + HighlightPresent bool + InnerHitsPresent bool +} + +type causeObservation struct { + Key string `json:"key"` + Value string `json:"value"` + Count int `json:"count"` + FirstEventAt time.Time `json:"first_event_at"` + LastEventAt time.Time `json:"last_event_at"` + Container string `json:"container,omitempty"` +} + +type causeAggregate struct { + Count int + First time.Time + Last time.Time +} + +func (response *searchResponse) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "search response") + if err != nil { + return err + } + if err := decodeOptionalField(fields, "timed_out", &response.TimedOut); err != nil { + return err + } + if err := decodeOptionalField(fields, "terminated_early", &response.TerminatedEarly); err != nil { + return err + } + if err := decodeOptionalField(fields, "_shards", &response.Shards); err != nil { + return err + } + return decodeOptionalField(fields, "hits", &response.Hits) +} + +func (summary *shardSummary) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "shard summary") + if err != nil { + return err + } + for _, field := range []struct { + name string + target **int64 + }{ + {name: "total", target: &summary.Total}, + {name: "successful", target: &summary.Successful}, + {name: "skipped", target: &summary.Skipped}, + {name: "failed", target: &summary.Failed}, + } { + if err := decodeOptionalField(fields, field.name, field.target); err != nil { + return err + } + } + return nil +} + +func (envelope *hitEnvelope) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "hits envelope") + if err != nil { + return err + } + return decodeOptionalField(fields, "hits", &envelope.Hits) +} + +func (hit *searchHit) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "search hit") + if err != nil { + return err + } + if err := decodeOptionalField(fields, "fields", &hit.Fields); err != nil { + return err + } + _, hit.SourcePresent = fields["_source"] + _, hit.IgnoredPresent = fields["_ignored"] + _, hit.IgnoredValuesPresent = fields["ignored_field_values"] + _, hit.HighlightPresent = fields["highlight"] + _, hit.InnerHitsPresent = fields["inner_hits"] + return nil +} + +func objectFields(document []byte, label string) (map[string]json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(document, &fields); err != nil || fields == nil { + return nil, fmt.Errorf("%s must be a JSON object", label) + } + return fields, nil +} + +func decodeOptionalField(fields map[string]json.RawMessage, name string, target any) error { + value, exists := fields[name] + if !exists { + return nil + } + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return fmt.Errorf("search response field %s is invalid", name) + } + if err := json.Unmarshal(value, target); err != nil { + return fmt.Errorf("search response field %s is invalid", name) + } + return nil +} + +// ProjectLogCauses returns deterministic, bounded TELEMETRY facts for conservative R3 cause +// signatures. Raw log messages are classified in memory and discarded before fact construction. +// A successful response with no classified messages abstains with zero facts. +func ProjectLogCauses(input Projection) ([]fleet.GraphFact, error) { + if err := validateProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode Elasticsearch search response: %w", err) + } + + var response searchResponse + decoder := json.NewDecoder(bytes.NewReader(input.Response)) + if err := decoder.Decode(&response); err != nil { + return nil, fmt.Errorf("decode Elasticsearch search response") + } + if err := validateSearchResponse(response); err != nil { + return nil, err + } + + aggregates := make(map[string]*causeAggregate, maxCauseFacts) + for index, hit := range *response.Hits.Hits { + message, eventAt, err := validateHit(input, hit) + if err != nil { + return nil, fmt.Errorf("project Elasticsearch search hit %d: %w", index, err) + } + cause := classifyMessage(message) + if cause == "" { + continue + } + aggregate := aggregates[cause] + if aggregate == nil { + aggregate = &causeAggregate{First: eventAt, Last: eventAt} + aggregates[cause] = aggregate + } + aggregate.Count++ + if eventAt.Before(aggregate.First) { + aggregate.First = eventAt + } + if eventAt.After(aggregate.Last) { + aggregate.Last = eventAt + } + } + + causes := make([]string, 0, len(aggregates)) + for cause := range aggregates { + causes = append(causes, cause) + } + sort.Strings(causes) + if len(causes) > maxCauseFacts { + return nil, fmt.Errorf("elasticsearch cause fact count exceeds %d", maxCauseFacts) + } + + facts := make([]fleet.GraphFact, 0, len(causes)) + for _, cause := range causes { + fact, err := buildFact(input, cause, *aggregates[cause]) + if err != nil { + return nil, err + } + facts = append(facts, fact) + } + return facts, nil +} + +func validateProjection(input Projection) error { + for _, field := range []struct { + label string + value string + optional bool + }{ + {label: "workspace", value: input.Workspace}, + {label: "scope", value: input.Scope}, + {label: "namespace", value: input.Namespace}, + {label: "pod", value: input.Pod}, + {label: "container", value: input.Container, optional: true}, + } { + if err := validateText(field.label, field.value, maxIdentityText, field.optional); err != nil { + return err + } + } + if strings.Contains(input.Scope, "/") { + return fmt.Errorf("scope is invalid") + } + if input.WindowStart.IsZero() || input.WindowEnd.IsZero() || input.ObservedAt.IsZero() { + return fmt.Errorf("query window and observation time are required") + } + windowStart := input.WindowStart.UTC() + windowEnd := input.WindowEnd.UTC() + if !windowStart.Before(windowEnd) { + return fmt.Errorf("query window start must be before end") + } + if windowEnd.Sub(windowStart) > maxQueryWindow { + return fmt.Errorf("query window exceeds %s", maxQueryWindow) + } + if windowEnd.After(input.ObservedAt.UTC().Add(maxClockSkew)) { + return fmt.Errorf("query window exceeds allowed collection clock skew") + } + if len(input.Response) == 0 { + return fmt.Errorf("elasticsearch search response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("elasticsearch search response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("elasticsearch search response must be valid UTF-8") + } + return nil +} + +func validateSearchResponse(response searchResponse) error { + if response.TimedOut == nil { + return fmt.Errorf("elasticsearch search response timed_out is required") + } + if *response.TimedOut { + return fmt.Errorf("elasticsearch search response must not be timed out") + } + if response.TerminatedEarly != nil && *response.TerminatedEarly { + return fmt.Errorf("elasticsearch search response must not be terminated early") + } + if response.Shards == nil { + return fmt.Errorf("elasticsearch search response _shards is required") + } + counts := []*int64{response.Shards.Total, response.Shards.Successful, response.Shards.Skipped, response.Shards.Failed} + for _, count := range counts { + if count == nil || *count < 0 { + return fmt.Errorf("elasticsearch search response shard counts must be present and non-negative") + } + } + if *response.Shards.Total == 0 { + return fmt.Errorf("elasticsearch search response must cover at least one shard") + } + if *response.Shards.Failed != 0 || *response.Shards.Successful+*response.Shards.Skipped != *response.Shards.Total { + return fmt.Errorf("elasticsearch search response must be complete with no failed shards") + } + if response.Hits == nil || response.Hits.Hits == nil { + return fmt.Errorf("elasticsearch search response hits.hits is required") + } + if len(*response.Hits.Hits) > maxHits { + return fmt.Errorf("elasticsearch search hit count exceeds %d", maxHits) + } + return nil +} + +func validateHit(input Projection, hit searchHit) (string, time.Time, error) { + if hit.SourcePresent { + return "", time.Time{}, fmt.Errorf("_source must not be returned") + } + if hit.IgnoredPresent || hit.IgnoredValuesPresent { + return "", time.Time{}, fmt.Errorf("ignored raw field values must not be returned") + } + if hit.HighlightPresent || hit.InnerHitsPresent { + return "", time.Time{}, fmt.Errorf("expanded or highlighted log content must not be returned") + } + if hit.Fields == nil { + return "", time.Time{}, fmt.Errorf("fields are required") + } + fields := *hit.Fields + for name := range fields { + if !allowedHitFields[name] { + return "", time.Time{}, fmt.Errorf("returned field is not allowlisted") + } + } + + timestamp, err := requiredSingleString(fields, timestampField, 64) + if err != nil { + return "", time.Time{}, err + } + message, err := requiredSingleMessage(fields) + if err != nil { + return "", time.Time{}, err + } + cluster, err := requiredSingleString(fields, clusterField, maxIdentityText) + if err != nil { + return "", time.Time{}, err + } + namespace, err := requiredSingleString(fields, namespaceField, maxIdentityText) + if err != nil { + return "", time.Time{}, err + } + pod, err := requiredSingleString(fields, podField, maxIdentityText) + if err != nil { + return "", time.Time{}, err + } + container, containerPresent, err := optionalSingleString(fields, containerField, maxIdentityText) + if err != nil { + return "", time.Time{}, err + } + if cluster != input.Scope || namespace != input.Namespace || pod != input.Pod { + return "", time.Time{}, fmt.Errorf("kubernetes identity does not match trusted caller identity") + } + if input.Container != "" && (!containerPresent || container != input.Container) { + return "", time.Time{}, fmt.Errorf("container identity does not match trusted caller identity") + } + + eventAt, err := time.Parse(time.RFC3339Nano, timestamp) + if err != nil || eventAt.IsZero() { + return "", time.Time{}, fmt.Errorf("@timestamp must be a non-zero RFC3339 timestamp") + } + eventAt = eventAt.UTC() + if eventAt.Before(input.WindowStart.UTC()) || eventAt.After(input.WindowEnd.UTC()) { + return "", time.Time{}, fmt.Errorf("log event timestamp is outside the trusted query window") + } + return message, eventAt, nil +} + +func requiredSingleString(fields map[string]json.RawMessage, name string, maximum int) (string, error) { + value, present, err := optionalSingleString(fields, name, maximum) + if err != nil { + return "", err + } + if !present { + return "", fmt.Errorf("field %s is required", name) + } + return value, nil +} + +func optionalSingleString(fields map[string]json.RawMessage, name string, maximum int) (string, bool, error) { + raw, present := fields[name] + if !present { + return "", false, nil + } + var values []string + if err := json.Unmarshal(raw, &values); err != nil || len(values) != 1 { + return "", false, fmt.Errorf("field %s must contain exactly one string", name) + } + if err := validateText(name, values[0], maximum, false); err != nil { + return "", false, err + } + return values[0], true, nil +} + +func requiredSingleMessage(fields map[string]json.RawMessage) (string, error) { + raw, present := fields[messageField] + if !present { + return "", fmt.Errorf("field %s is required", messageField) + } + var values []string + if err := json.Unmarshal(raw, &values); err != nil || len(values) != 1 { + return "", fmt.Errorf("field %s must contain exactly one string", messageField) + } + if len(values[0]) > maxMessageBytes || !utf8.ValidString(values[0]) { + return "", fmt.Errorf("field %s exceeds the safe message contract", messageField) + } + return values[0], nil +} + +func classifyMessage(message string) string { + normalized := strings.ToLower(message) + if indicatesMissingConfig(normalized) { + return "missing-config" + } + if indicatesDependencyFailure(normalized) { + return "dependency-failure" + } + if indicatesPanic(normalized) { + return "panic" + } + return "" +} + +func indicatesMissingConfig(message string) bool { + missing := containsAny(message, "missing required", "not set", "is required", "not found", "does not exist", "no such file") + if !missing { + return false + } + return containsAny(message, "environment variable", "env var", "configuration", "config", "configmap", "secret") +} + +func indicatesDependencyFailure(message string) bool { + return containsAny(message, + "connection refused", + "connection reset by peer", + "connection timed out", + "context deadline exceeded", + "dial tcp", + "failed to connect", + "i/o timeout", + "no such host", + "tls handshake timeout", + "upstream connect error", + ) +} + +func indicatesPanic(message string) bool { + for _, line := range strings.Split(message, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "panic:") || + strings.HasPrefix(trimmed, "fatal error:") || + strings.HasPrefix(trimmed, "uncaught exception") || + strings.HasPrefix(trimmed, "unhandled exception") || + strings.HasPrefix(trimmed, "exception in thread") || + strings.HasPrefix(trimmed, "traceback (most recent call last):") { + return true + } + } + return false +} + +func containsAny(value string, candidates ...string) bool { + for _, candidate := range candidates { + if strings.Contains(value, candidate) { + return true + } + } + return false +} + +func buildFact(input Projection, cause string, aggregate causeAggregate) (fleet.GraphFact, error) { + observation := causeObservation{ + Key: "logs.cause", + Value: cause, + Count: aggregate.Count, + FirstEventAt: aggregate.First.UTC(), + LastEventAt: aggregate.Last.UTC(), + Container: input.Container, + } + encoded, err := json.Marshal(observation) + if err != nil { + return fleet.GraphFact{}, fmt.Errorf("encode Elasticsearch log-cause fact: %w", err) + } + if len(encoded) > maxFactPayloadBytes { + return fleet.GraphFact{}, fmt.Errorf("elasticsearch log-cause fact exceeds %d encoded bytes", maxFactPayloadBytes) + } + + // Bind the source identity only to fields that survive sanitization so downstream graph + // consumers can independently revalidate the exact workspace, Pod, aggregate, and collection. + identity, err := json.Marshal(struct { + Workspace string `json:"workspace"` + Scope string `json:"scope"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` + Container string `json:"container,omitempty"` + Cause string `json:"cause"` + Count int `json:"count"` + FirstEventAt time.Time `json:"first_event_at"` + LastEventAt time.Time `json:"last_event_at"` + ObservedAt time.Time `json:"observed_at"` + }{ + Workspace: input.Workspace, Scope: input.Scope, Namespace: input.Namespace, Pod: input.Pod, + Container: input.Container, Cause: cause, Count: aggregate.Count, + FirstEventAt: aggregate.First.UTC(), LastEventAt: aggregate.Last.UTC(), + ObservedAt: input.ObservedAt.UTC(), + }) + if err != nil { + return fleet.GraphFact{}, fmt.Errorf("encode Elasticsearch log-cause identity: %w", err) + } + digest := sha256.Sum256(identity) + nativeID := "sha256:" + hex.EncodeToString(digest[:]) + resourceName := "log-" + hex.EncodeToString(digest[:16]) + entity := fleet.EntityRef{Cluster: input.Scope, Namespace: input.Namespace, Pod: input.Pod} + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, + Scope: input.Scope, + Kind: "LogSignal", + Namespace: input.Namespace, + Name: resourceName, + }, + Kind: fleet.FactDerived, + Observed: encoded, + ObservedAt: input.ObservedAt.UTC(), + Source: input.Scope, + Provenance: fleet.Provenance{Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID}, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensTelemetry, + Entity: &entity, + } + if err := fact.Validate(input.Workspace); err != nil { + return fleet.GraphFact{}, fmt.Errorf("validate Elasticsearch log-cause fact: %w", err) + } + return fact, nil +} + +func validateText(label, value string, maximum int, optional bool) error { + if value == "" { + if optional { + return nil + } + return fmt.Errorf("%s is invalid", label) + } + if len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("%s is invalid", label) + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func rejectDuplicateJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || closing != matchingDelimiter(delimiter) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func matchingDelimiter(open json.Delim) json.Delim { + if open == '{' { + return '}' + } + return ']' +} diff --git a/internal/connector/elasticsearch/project_test.go b/internal/connector/elasticsearch/project_test.go new file mode 100644 index 0000000..2397cb6 --- /dev/null +++ b/internal/connector/elasticsearch/project_test.go @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: Apache-2.0 + +package elasticsearch + +import ( + "bytes" + "encoding/json" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +var testObservedAt = time.Date(2026, 7, 17, 6, 0, 0, 0, time.UTC) + +func TestProjectLogCausesAggregatesClosedTaxonomyWithoutRawLogs(t *testing.T) { + t.Parallel() + secret := "postgres://admin:do-not-retain@example.internal/db" + input := testProjection(t, + testHit("panic: worker crashed", testObservedAt.Add(-8*time.Minute)), + testHit("panic: missing required environment variable DATABASE_URL="+secret, testObservedAt.Add(-7*time.Minute)), + testHit("dial tcp 10.0.0.7:5432: connection refused", testObservedAt.Add(-6*time.Minute)), + testHit("request completed successfully", testObservedAt.Add(-5*time.Minute)), + testHit("connection refused while contacting dependency", testObservedAt.Add(-4*time.Minute)), + ) + + facts, err := ProjectLogCauses(input) + if err != nil { + t.Fatalf("ProjectLogCauses() error = %v", err) + } + if len(facts) != 3 { + t.Fatalf("fact count = %d, want 3", len(facts)) + } + + wantCauses := []string{"dependency-failure", "missing-config", "panic"} + wantCounts := []int{2, 1, 1} + for index, fact := range facts { + if fact.Lens != fleet.LensTelemetry || fact.Fact.Kind != fleet.FactDerived { + t.Fatalf("fact %d taxonomy = %s/%s", index, fact.Lens, fact.Fact.Kind) + } + if fact.Fact.Ref.SourceKind != Kind || fact.Fact.Ref.Scope != input.Scope || + fact.Fact.Ref.Kind != "LogSignal" || fact.Fact.Ref.Namespace != input.Namespace { + t.Fatalf("fact %d ref = %#v", index, fact.Fact.Ref) + } + if fact.Entity == nil || fact.Entity.Cluster != input.Scope || fact.Entity.Namespace != input.Namespace || + fact.Entity.Pod != input.Pod || fact.Entity.Kind != "" || fact.Entity.Name != "" { + t.Fatalf("fact %d entity = %#v", index, fact.Entity) + } + if fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion || + !strings.HasPrefix(fact.Fact.Provenance.NativeID, "sha256:") { + t.Fatalf("fact %d provenance = %#v", index, fact.Fact.Provenance) + } + var observed causeObservation + if err := json.Unmarshal(fact.Fact.Observed, &observed); err != nil { + t.Fatalf("decode fact %d: %v", index, err) + } + if observed.Key != "logs.cause" || observed.Value != wantCauses[index] || observed.Count != wantCounts[index] || + observed.Container != input.Container { + t.Fatalf("fact %d observed = %#v", index, observed) + } + encoded, err := json.Marshal(fact) + if err != nil { + t.Fatalf("marshal fact %d: %v", index, err) + } + for _, forbidden := range []string{secret, "worker crashed", "10.0.0.7", "example.internal", "DATABASE_URL"} { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("fact %d retained forbidden raw log fragment %q", index, forbidden) + } + } + } +} + +func TestProjectLogCausesIsDeterministicAcrossHitOrder(t *testing.T) { + t.Parallel() + hits := []map[string]any{ + testHit("panic: one", testObservedAt.Add(-8*time.Minute)), + testHit("panic: two", testObservedAt.Add(-2*time.Minute)), + testHit("configuration file not found", testObservedAt.Add(-6*time.Minute)), + } + forward := testProjection(t, hits...) + reverseHits := slices.Clone(hits) + slices.Reverse(reverseHits) + reverse := testProjection(t, reverseHits...) + + left, err := ProjectLogCauses(forward) + if err != nil { + t.Fatalf("forward projection: %v", err) + } + right, err := ProjectLogCauses(reverse) + if err != nil { + t.Fatalf("reverse projection: %v", err) + } + if !reflect.DeepEqual(left, right) { + t.Fatalf("hit order changed facts:\nforward = %#v\nreverse = %#v", left, right) + } +} + +func TestLogCauseNativeIdentityBindsRetainedAggregateAndPod(t *testing.T) { + t.Parallel() + input := testProjection(t) + aggregate := causeAggregate{ + Count: 2, + First: testObservedAt.Add(-8 * time.Minute), + Last: testObservedAt.Add(-2 * time.Minute), + } + base, err := buildFact(input, "panic", aggregate) + if err != nil { + t.Fatalf("buildFact() error = %v", err) + } + workspaceChanged := input + workspaceChanged.Workspace = "workspace-b" + scopeChanged := input + scopeChanged.Scope = "beta" + namespaceChanged := input + namespaceChanged.Namespace = "other" + podChanged := input + podChanged.Pod = "api-other" + containerChanged := input + containerChanged.Container = "worker" + observedAtChanged := input + observedAtChanged.ObservedAt = observedAtChanged.ObservedAt.Add(time.Second) + tests := []struct { + name string + input Projection + cause string + aggregate causeAggregate + }{ + {name: "workspace", input: workspaceChanged, cause: "panic", aggregate: aggregate}, + {name: "scope", input: scopeChanged, cause: "panic", aggregate: aggregate}, + {name: "namespace", input: namespaceChanged, cause: "panic", aggregate: aggregate}, + {name: "Pod", input: podChanged, cause: "panic", aggregate: aggregate}, + {name: "container", input: containerChanged, cause: "panic", aggregate: aggregate}, + {name: "cause", input: input, cause: "missing-config", aggregate: aggregate}, + {name: "count", input: input, cause: "panic", aggregate: causeAggregate{Count: 3, First: aggregate.First, Last: aggregate.Last}}, + {name: "first event", input: input, cause: "panic", aggregate: causeAggregate{Count: aggregate.Count, First: aggregate.First.Add(time.Second), Last: aggregate.Last}}, + {name: "last event", input: input, cause: "panic", aggregate: causeAggregate{Count: aggregate.Count, First: aggregate.First, Last: aggregate.Last.Add(time.Second)}}, + {name: "observed at", input: observedAtChanged, cause: "panic", aggregate: aggregate}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + changed, err := buildFact(test.input, test.cause, test.aggregate) + if err != nil { + t.Fatalf("buildFact() error = %v", err) + } + if changed.Fact.Provenance.NativeID == base.Fact.Provenance.NativeID || + changed.Fact.Ref.Name == base.Fact.Ref.Name { + t.Fatalf("identity did not bind %s: base=%#v changed=%#v", test.name, base.Fact, changed.Fact) + } + }) + } +} + +func TestProjectLogCausesAbstainsForUnclassifiedAndEmptySuccess(t *testing.T) { + t.Parallel() + for _, hits := range [][]map[string]any{ + {}, + {testHit("service started", testObservedAt.Add(-time.Minute))}, + } { + input := testProjection(t, hits...) + facts, err := ProjectLogCauses(input) + if err != nil { + t.Fatalf("ProjectLogCauses() error = %v", err) + } + if len(facts) != 0 { + t.Fatalf("facts = %#v, want abstention", facts) + } + } +} + +func TestClassifyMessageIsConservativeAndSpecific(t *testing.T) { + t.Parallel() + tests := []struct { + message string + want string + }{ + {message: "panic: runtime failure", want: "panic"}, + {message: "INFO\nTraceback (most recent call last):\nValueError", want: "panic"}, + {message: "panic: required config not found", want: "missing-config"}, + {message: "environment variable API_KEY is not set", want: "missing-config"}, + {message: "configmap payments not found", want: "missing-config"}, + {message: "dial tcp: no such host", want: "dependency-failure"}, + {message: "upstream connect error", want: "dependency-failure"}, + {message: "documentation: avoid panic: by validating input", want: ""}, + {message: "optional config not loaded", want: ""}, + {message: "retrying dependency", want: ""}, + } + for _, test := range tests { + test := test + t.Run(test.message, func(t *testing.T) { + t.Parallel() + if got := classifyMessage(test.message); got != test.want { + t.Fatalf("classifyMessage() = %q, want %q", got, test.want) + } + }) + } +} + +func TestProjectLogCausesRejectsInvalidProjection(t *testing.T) { + t.Parallel() + valid := testProjection(t) + tests := []struct { + name string + mutate func(*Projection) + }{ + {name: "workspace", mutate: func(input *Projection) { input.Workspace = "" }}, + {name: "scope", mutate: func(input *Projection) { input.Scope = "alpha/beta" }}, + {name: "namespace", mutate: func(input *Projection) { input.Namespace = "" }}, + {name: "pod", mutate: func(input *Projection) { input.Pod = " bad " }}, + {name: "container control", mutate: func(input *Projection) { input.Container = "api\nother" }}, + {name: "missing start", mutate: func(input *Projection) { input.WindowStart = time.Time{} }}, + {name: "reversed window", mutate: func(input *Projection) { input.WindowStart = input.WindowEnd }}, + {name: "wide window", mutate: func(input *Projection) { input.WindowStart = input.WindowEnd.Add(-maxQueryWindow - time.Nanosecond) }}, + {name: "future window", mutate: func(input *Projection) { input.WindowEnd = input.ObservedAt.Add(maxClockSkew + time.Nanosecond) }}, + {name: "missing observed", mutate: func(input *Projection) { input.ObservedAt = time.Time{} }}, + {name: "empty response", mutate: func(input *Projection) { input.Response = nil }}, + {name: "oversized response", mutate: func(input *Projection) { input.Response = bytes.Repeat([]byte{' '}, maxResponseBytes+1) }}, + {name: "invalid UTF-8", mutate: func(input *Projection) { input.Response = []byte{0xff} }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := valid + input.Response = slices.Clone(valid.Response) + test.mutate(&input) + if facts, err := ProjectLogCauses(input); err == nil || facts != nil { + t.Fatalf("ProjectLogCauses() = %#v, %v; want nil error result", facts, err) + } + }) + } +} + +func TestProjectLogCausesRejectsIncompleteSearchResponses(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(map[string]any) + }{ + {name: "missing timed out", mutate: func(response map[string]any) { delete(response, "timed_out") }}, + {name: "timed out", mutate: func(response map[string]any) { response["timed_out"] = true }}, + {name: "terminated early", mutate: func(response map[string]any) { response["terminated_early"] = true }}, + {name: "missing shards", mutate: func(response map[string]any) { delete(response, "_shards") }}, + {name: "zero shards", mutate: func(response map[string]any) { + response["_shards"].(map[string]any)["total"] = 0 + response["_shards"].(map[string]any)["successful"] = 0 + }}, + {name: "failed shard", mutate: func(response map[string]any) { response["_shards"].(map[string]any)["failed"] = 1 }}, + {name: "inconsistent shards", mutate: func(response map[string]any) { response["_shards"].(map[string]any)["total"] = 2 }}, + {name: "negative shard count", mutate: func(response map[string]any) { response["_shards"].(map[string]any)["skipped"] = -1 }}, + {name: "missing hits", mutate: func(response map[string]any) { delete(response, "hits") }}, + {name: "missing hit list", mutate: func(response map[string]any) { response["hits"] = map[string]any{} }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + response := testResponseMap(nil) + test.mutate(response) + input := testProjectionWithResponse(t, response) + if facts, err := ProjectLogCauses(input); err == nil || facts != nil { + t.Fatalf("ProjectLogCauses() = %#v, %v; want rejection", facts, err) + } + }) + } +} + +func TestProjectLogCausesRejectsMalformedResponseFieldTypes(t *testing.T) { + t.Parallel() + valid := testProjection(t) + cases := [][]byte{ + []byte(`[]`), + []byte(`{"timed_out":null,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[]}}`), + []byte(`{"timed_out":{},"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"terminated_early":null,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"terminated_early":{},"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"_shards":null,"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"_shards":[],"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"_shards":{"total":null,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"_shards":{"total":"one","successful":1,"skipped":0,"failed":0},"hits":{"hits":[]}}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":null}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":[]}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":null}}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":{}}}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[[]]}}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[{"fields":null}]}}`), + []byte(`{"timed_out":false,"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},"hits":{"hits":[{"fields":[]}]}}`), + } + for index, document := range cases { + input := valid + input.Response = document + facts, err := ProjectLogCauses(input) + if err == nil || facts != nil { + t.Fatalf("case %d = %#v, %v; want rejection", index, facts, err) + } + if len(err.Error()) > 512 { + t.Fatalf("case %d produced attacker-sized error", index) + } + } +} + +func TestProjectLogCausesRejectsUnsafeOrAmbiguousHits(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(map[string]any) + }{ + {name: "source", mutate: func(hit map[string]any) { hit["_source"] = map[string]any{"secret": "raw"} }}, + {name: "ignored", mutate: func(hit map[string]any) { hit["_ignored"] = []string{"message"} }}, + {name: "ignored values", mutate: func(hit map[string]any) { hit["ignored_field_values"] = map[string]any{"message": []string{"raw"}} }}, + {name: "highlight", mutate: func(hit map[string]any) { hit["highlight"] = map[string]any{"message": []string{"raw"}} }}, + {name: "inner hits", mutate: func(hit map[string]any) { hit["inner_hits"] = map[string]any{} }}, + {name: "missing fields", mutate: func(hit map[string]any) { delete(hit, "fields") }}, + {name: "unknown returned field", mutate: func(hit map[string]any) { + hit["fields"].(map[string]any)["user.email"] = []string{"private@example.com"} + }}, + {name: "missing timestamp", mutate: func(hit map[string]any) { delete(hit["fields"].(map[string]any), timestampField) }}, + {name: "missing message", mutate: func(hit map[string]any) { delete(hit["fields"].(map[string]any), messageField) }}, + {name: "ambiguous message", mutate: func(hit map[string]any) { + hit["fields"].(map[string]any)[messageField] = []string{"panic: one", "panic: two"} + }}, + {name: "non-string timestamp", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[timestampField] = []int{42} }}, + {name: "bad timestamp", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[timestampField] = []string{"yesterday"} }}, + {name: "out of window", mutate: func(hit map[string]any) { + hit["fields"].(map[string]any)[timestampField] = []string{testObservedAt.Add(time.Nanosecond).Format(time.RFC3339Nano)} + }}, + {name: "missing cluster", mutate: func(hit map[string]any) { delete(hit["fields"].(map[string]any), clusterField) }}, + {name: "invalid cluster", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[clusterField] = []string{" alpha "} }}, + {name: "cluster mismatch", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[clusterField] = []string{"beta"} }}, + {name: "missing namespace", mutate: func(hit map[string]any) { delete(hit["fields"].(map[string]any), namespaceField) }}, + {name: "namespace mismatch", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[namespaceField] = []string{"other"} }}, + {name: "missing pod", mutate: func(hit map[string]any) { delete(hit["fields"].(map[string]any), podField) }}, + {name: "pod mismatch", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[podField] = []string{"other"} }}, + {name: "container missing", mutate: func(hit map[string]any) { delete(hit["fields"].(map[string]any), containerField) }}, + {name: "container mismatch", mutate: func(hit map[string]any) { hit["fields"].(map[string]any)[containerField] = []string{"sidecar"} }}, + {name: "oversized message", mutate: func(hit map[string]any) { + hit["fields"].(map[string]any)[messageField] = []string{strings.Repeat("x", maxMessageBytes+1)} + }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + hit := testHit("panic: boom", testObservedAt.Add(-time.Minute)) + test.mutate(hit) + input := testProjection(t, hit) + if facts, err := ProjectLogCauses(input); err == nil || facts != nil { + t.Fatalf("ProjectLogCauses() = %#v, %v; want rejection", facts, err) + } + }) + } +} + +func TestProjectLogCausesAcceptsInclusiveWindowAndOptionalContainer(t *testing.T) { + t.Parallel() + startHit := testHit("panic: start", testObservedAt.Add(-10*time.Minute)) + endHit := testHit("panic: end", testObservedAt) + delete(startHit["fields"].(map[string]any), containerField) + delete(endHit["fields"].(map[string]any), containerField) + input := testProjection(t, startHit, endHit) + input.Container = "" + facts, err := ProjectLogCauses(input) + if err != nil { + t.Fatalf("ProjectLogCauses() error = %v", err) + } + if len(facts) != 1 { + t.Fatalf("facts = %#v", facts) + } + var observed causeObservation + if err := json.Unmarshal(facts[0].Fact.Observed, &observed); err != nil { + t.Fatal(err) + } + if observed.Count != 2 || !observed.FirstEventAt.Equal(input.WindowStart) || !observed.LastEventAt.Equal(input.WindowEnd) || observed.Container != "" { + t.Fatalf("observed = %#v", observed) + } +} + +func TestProjectLogCausesRejectsHitCountAndJSONAttacks(t *testing.T) { + t.Parallel() + hits := make([]map[string]any, maxHits+1) + for index := range hits { + hits[index] = testHit("service started", testObservedAt.Add(-time.Minute)) + } + overCount := testProjection(t, hits...) + if facts, err := ProjectLogCauses(overCount); err == nil || facts != nil { + t.Fatalf("over-count projection = %#v, %v", facts, err) + } + + valid := testProjection(t) + attacks := [][]byte{ + []byte(`{"timed_out":false,"timed_out":false}`), + append(slices.Clone(valid.Response), []byte(` {}`)...), + []byte(`{"timed_out":false,"deep":` + strings.Repeat("[", maxJSONDepth+1) + `0` + strings.Repeat("]", maxJSONDepth+1) + `}`), + []byte(`{"timed_out":` + strings.Repeat("9", 64<<10) + `}`), + } + for index, attack := range attacks { + input := valid + input.Response = attack + facts, err := ProjectLogCauses(input) + if err == nil || facts != nil { + t.Fatalf("attack %d = %#v, %v; want rejection", index, facts, err) + } + if len(err.Error()) > 512 { + t.Fatalf("attack %d produced attacker-sized error (%d bytes)", index, len(err.Error())) + } + } +} + +func FuzzProjectLogCauses(f *testing.F) { + valid := testProjection(f, + testHit("panic: boom", testObservedAt.Add(-time.Minute)), + ) + f.Add(valid.Response) + f.Add([]byte(`{"timed_out":false}`)) + f.Add([]byte(`{"timed_out":false,"timed_out":true}`)) + f.Fuzz(func(t *testing.T, response []byte) { + input := valid + input.Response = response + facts, err := ProjectLogCauses(input) + if err != nil { + if facts != nil { + t.Fatalf("error returned non-nil facts: %#v", facts) + } + return + } + if len(facts) > maxCauseFacts { + t.Fatalf("fact count = %d", len(facts)) + } + for _, fact := range facts { + if fact.Lens != fleet.LensTelemetry || fact.Fact.Kind != fleet.FactDerived { + t.Fatalf("invalid fuzz fact taxonomy: %#v", fact) + } + } + }) +} + +type testingHelper interface { + Helper() + Fatalf(string, ...any) +} + +func testProjection(t testingHelper, hits ...map[string]any) Projection { + t.Helper() + return testProjectionWithResponse(t, testResponseMap(hits)) +} + +func testProjectionWithResponse(t testingHelper, response map[string]any) Projection { + t.Helper() + document, err := json.Marshal(response) + if err != nil { + t.Fatalf("marshal test response: %v", err) + } + return Projection{ + Workspace: "workspace-a", + Scope: "alpha", + Namespace: "payments", + Pod: "api-7d9f", + Container: "api", + WindowStart: testObservedAt.Add(-10 * time.Minute), + WindowEnd: testObservedAt, + ObservedAt: testObservedAt, + Response: document, + } +} + +func testResponseMap(hits []map[string]any) map[string]any { + items := make([]any, len(hits)) + for index := range hits { + items[index] = hits[index] + } + return map[string]any{ + "took": 2, + "timed_out": false, + "_shards": map[string]any{ + "total": 1, "successful": 1, "skipped": 0, "failed": 0, + }, + "hits": map[string]any{ + "total": map[string]any{"value": len(hits), "relation": "eq"}, + "hits": items, + }, + } +} + +func testHit(message string, eventAt time.Time) map[string]any { + return map[string]any{ + "_index": "logs-kubernetes.container_logs-default", + "_id": "discarded-document-id", + "_score": nil, + "fields": map[string]any{ + timestampField: []string{eventAt.Format(time.RFC3339Nano)}, + messageField: []string{message}, + clusterField: []string{"alpha"}, + namespaceField: []string{"payments"}, + podField: []string{"api-7d9f"}, + containerField: []string{"api"}, + }, + } +} diff --git a/internal/connector/github/action_plan.go b/internal/connector/github/action_plan.go new file mode 100644 index 0000000..878eb21 --- /dev/null +++ b/internal/connector/github/action_plan.go @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "path" + "sort" + "strings" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/connector" + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/intentargs" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + openPRProtocolVersion = "gitops-open-pr/" + APIVersion + openPRTargetKind = "Repository" + maxIntentIDBytes = 253 + maxBaseRefBytes = 128 + maxPRTitleBytes = 256 + maxPRBodyBytes = 8 << 10 + maxCommitMessageBytes = 512 + maxChanges = 32 + maxChangePathBytes = 512 + maxFileContentBytes = 16 << 10 + maxTotalContentBytes = 48 << 10 +) + +const openPRSchema = `{ + "$schema":"https://json-schema.org/draft/2020-12/schema", + "type":"object", + "properties":{ + "base_ref":{"type":"string","minLength":1,"maxLength":128}, + "expected_base_sha":{"type":"string","pattern":"^[0-9a-f]{40}([0-9a-f]{24})?$"}, + "title":{"type":"string","minLength":1,"maxLength":256}, + "body":{"type":"string","maxLength":8192}, + "commit_message":{"type":"string","minLength":1,"maxLength":512}, + "changes":{ + "type":"array", + "minItems":1, + "maxItems":32, + "items":{ + "type":"object", + "properties":{ + "operation":{"type":"string","enum":["create","update","delete"]}, + "path":{"type":"string","minLength":1,"maxLength":512}, + "content":{"type":"string","maxLength":16384}, + "expected_blob_sha":{"type":"string","pattern":"^[0-9a-f]{40}([0-9a-f]{24})?$"} + }, + "required":["operation","path"], + "additionalProperties":false + } + } + }, + "required":["base_ref","expected_base_sha","title","commit_message","changes"], + "additionalProperties":false +}` + +// OpenPRPlannerConfig fixes the only repository and base branch this planner may address. +// It intentionally contains no credential, endpoint URL, or HTTP dependency. +type OpenPRPlannerConfig struct { + Host string + Owner string + Repository string + BaseRef string +} + +// OpenPRPlanner produces deterministic, inspectable GitHub API plans without performing I/O. +type OpenPRPlanner struct { + config OpenPRPlannerConfig + schema *intentargs.Schema +} + +type openPRArgs struct { + BaseRef string `json:"base_ref"` + ExpectedBaseSHA string `json:"expected_base_sha"` + Title string `json:"title"` + Body string `json:"body,omitempty"` + CommitMessage string `json:"commit_message"` + Changes []fileChange `json:"changes"` +} + +type fileChange struct { + Operation string `json:"operation"` + Path string `json:"path"` + Content *string `json:"content,omitempty"` + ExpectedBlobSHA *string `json:"expected_blob_sha,omitempty"` +} + +type plannedChange struct { + Operation string `json:"operation"` + Path string `json:"path"` + ExpectedBlobSHA string `json:"expected_blob_sha,omitempty"` + ContentDigest string `json:"content_digest,omitempty"` +} + +type refCheckParams struct { + BaseRef string `json:"base_ref"` + ExpectedSHA string `json:"expected_sha"` +} + +type commitLookupParams struct { + CommitSHA string `json:"commit_sha"` +} + +type treeVerifyParams struct { + Tree string `json:"tree"` + Changes []plannedChange `json:"changes"` +} + +type treePlanParams struct { + BaseTree string `json:"base_tree"` + Changes []plannedChange `json:"changes"` +} + +type commitPlanParams struct { + ParentSHA string `json:"parent_sha"` + Tree string `json:"tree"` + MessageDigest string `json:"message_digest"` +} + +type refCreateParams struct { + HeadRef string `json:"head_ref"` + Commit string `json:"commit"` +} + +type pullLookupParams struct { + HeadRef string `json:"head_ref"` + BaseRef string `json:"base_ref"` +} + +type pullCreateParams struct { + HeadRef string `json:"head_ref"` + BaseRef string `json:"base_ref"` + TitleDigest string `json:"title_digest"` + BodyDigest string `json:"body_digest"` +} + +// NewOpenPRPlanner constructs the planning-only GitHub typed-action adapter. +func NewOpenPRPlanner(config OpenPRPlannerConfig) (*OpenPRPlanner, error) { + if err := validateOpenPRConfig(config); err != nil { + return nil, err + } + schema, err := intentargs.Compile(json.RawMessage(openPRSchema)) + if err != nil { + return nil, fmt.Errorf("construct GitHub pull request planner: argument schema is invalid") + } + return &OpenPRPlanner{config: config, schema: schema}, nil +} + +// Kind returns the canonical GitHub connector kind. +func (*OpenPRPlanner) Kind() string { return Kind } + +// Capabilities declares planning only. Execution remains blocked on E3 and E5. +func (*OpenPRPlanner) Capabilities() []connector.Capability { + return []connector.Capability{connector.CapPlan} +} + +// Descriptor binds gitops.open-pr to its exact handler-owned argument schema. +func (*OpenPRPlanner) Descriptor() connector.Descriptor { + return connector.Descriptor{ + Kind: Kind, ConnKind: connector.KindTypedAction, + WireVersions: []connector.WireVersion{connector.CurrentWireVersion()}, AdapterVersion: openPRProtocolVersion, Owner: "sith", + Capabilities: []connector.Capability{connector.CapPlan}, + Verbs: []intent.Verb{intent.VerbGitOpsOpenPR}, + ArgSchemas: map[intent.Verb]json.RawMessage{intent.VerbGitOpsOpenPR: json.RawMessage(openPRSchema)}, + } +} + +// Plan returns a deterministic digest-only API plan. It never performs a provider request and +// never retains file content, PR text, credentials, or caller-controlled references in the plan. +func (planner *OpenPRPlanner) Plan(_ context.Context, request connector.Intent) (connector.ActionPlan, error) { + if planner == nil || planner.schema == nil { + return connector.ActionPlan{}, fmt.Errorf("plan GitHub pull request: planner is required") + } + if err := validateOpenPRIntent(planner.config, request); err != nil { + return connector.ActionPlan{}, err + } + args, canonical, err := planner.canonicalizeOpenPRArgs(request.Args) + if err != nil { + return connector.ActionPlan{}, err + } + identityInput := strings.Join([]string{request.Workspace, planner.config.Host, planner.config.Owner, planner.config.Repository}, "\x00") + "\x00" + identity := sha256.Sum256(append([]byte(identityInput), canonical...)) + headRef := "sith/intent-" + hex.EncodeToString(identity[:12]) + + planned := make([]plannedChange, 0, len(args.Changes)) + hunks := make([]fleet.DiffHunk, 0, len(args.Changes)) + for _, change := range args.Changes { + entry := plannedChange{Operation: change.Operation, Path: change.Path} + observed := "absent" + desired := "deleted" + if change.ExpectedBlobSHA != nil { + entry.ExpectedBlobSHA = *change.ExpectedBlobSHA + observed = "blob:" + *change.ExpectedBlobSHA + } + if change.Content != nil { + entry.ContentDigest = digestText(*change.Content) + desired = entry.ContentDigest + } + planned = append(planned, entry) + hunks = append(hunks, fleet.DiffHunk{Path: change.Path, Observed: observed, Desired: desired}) + } + + target := normalizedOpenPRTarget(planner.config) + steps, err := openPRPlanSteps(args, planned, headRef) + if err != nil { + return connector.ActionPlan{}, err + } + return connector.ActionPlan{ + IntentID: request.ID, Verb: intent.VerbGitOpsOpenPR, Target: target, + Diff: fleet.Diff{Ref: target, Drifted: true, Hunks: hunks}, + Steps: steps, Reversible: true, + }, nil +} + +// CanonicalizeOpenPRArgs applies the exact handler-owned schema and semantic policy without +// planning or performing I/O. A provenance resolver may use this seam to prove that its output is +// acceptable to the live handler, but the returned target and arguments are not authorization, +// approval, or execution capability. +func (planner *OpenPRPlanner) CanonicalizeOpenPRArgs(arguments json.RawMessage) (fleet.ResourceRef, json.RawMessage, error) { + _, canonical, err := planner.canonicalizeOpenPRArgs(arguments) + if err != nil { + return fleet.ResourceRef{}, nil, err + } + return normalizedOpenPRTarget(planner.config), append(json.RawMessage(nil), canonical...), nil +} + +func (planner *OpenPRPlanner) canonicalizeOpenPRArgs(arguments json.RawMessage) (openPRArgs, json.RawMessage, error) { + if planner == nil || planner.schema == nil { + return openPRArgs{}, nil, fmt.Errorf("plan GitHub pull request: planner is required") + } + if err := planner.schema.Validate(arguments); err != nil { + return openPRArgs{}, nil, fmt.Errorf("plan GitHub pull request: args are invalid") + } + + var args openPRArgs + if err := json.Unmarshal(arguments, &args); err != nil { + return openPRArgs{}, nil, fmt.Errorf("plan GitHub pull request: decode validated args") + } + if err := validateOpenPRArgs(planner.config, args); err != nil { + return openPRArgs{}, nil, err + } + args.Changes = append([]fileChange(nil), args.Changes...) + sort.Slice(args.Changes, func(left, right int) bool { return args.Changes[left].Path < args.Changes[right].Path }) + + canonical, err := json.Marshal(args) + if err != nil { + return openPRArgs{}, nil, fmt.Errorf("plan GitHub pull request: canonicalize args") + } + return args, json.RawMessage(canonical), nil +} + +func validateOpenPRConfig(config OpenPRPlannerConfig) error { + if validateHost(config.Host) != nil || validatePathComponent("owner", config.Owner, maxOwnerBytes) != nil || + validatePathComponent("repository", config.Repository, maxRepositoryBytes) != nil || + strings.HasSuffix(strings.ToLower(config.Repository), ".git") || !validBaseRef(config.BaseRef) { + return fmt.Errorf("construct GitHub pull request planner: repository policy is invalid") + } + return nil +} + +func validateOpenPRIntent(config OpenPRPlannerConfig, request connector.Intent) error { + if err := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(request.Workspace)); err != nil { + return fmt.Errorf("plan GitHub pull request: workspace is invalid") + } + if validateBoundedText(request.ID, maxIntentIDBytes, false, false) != nil { + return fmt.Errorf("plan GitHub pull request: intent identifier is invalid") + } + if request.Verb != intent.VerbGitOpsOpenPR { + return fmt.Errorf("plan GitHub pull request: verb is invalid") + } + if request.Target.SourceKind != Kind || request.Target.Scope != config.Host || request.Target.Kind != openPRTargetKind || + request.Target.Namespace != config.Owner || request.Target.Name != config.Repository || len(request.Target.Attributes) != 0 { + return fmt.Errorf("plan GitHub pull request: target is outside repository policy") + } + return nil +} + +func validateOpenPRArgs(config OpenPRPlannerConfig, args openPRArgs) error { + if args.BaseRef != config.BaseRef || !validCommitSHA(args.ExpectedBaseSHA) || + validateBoundedText(args.Title, maxPRTitleBytes, false, false) != nil || + validateBoundedText(args.Body, maxPRBodyBytes, true, true) != nil || + validateBoundedText(args.CommitMessage, maxCommitMessageBytes, false, false) != nil || + len(args.Changes) == 0 || len(args.Changes) > maxChanges { + return fmt.Errorf("plan GitHub pull request: args violate repository policy") + } + + seen := make(map[string]struct{}, len(args.Changes)) + totalContent := 0 + for _, change := range args.Changes { + if !validChangePath(change.Path) { + return fmt.Errorf("plan GitHub pull request: change path is invalid") + } + key := strings.ToLower(change.Path) + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("plan GitHub pull request: change paths collide") + } + seen[key] = struct{}{} + content := "" + if change.Content != nil { + content = *change.Content + } + if !validFileContent(content) || len(content) > maxFileContentBytes { + return fmt.Errorf("plan GitHub pull request: change content is invalid") + } + switch change.Operation { + case "create": + if change.Content == nil || change.ExpectedBlobSHA != nil { + return fmt.Errorf("plan GitHub pull request: create precondition is invalid") + } + case "update": + if change.Content == nil || change.ExpectedBlobSHA == nil || !validCommitSHA(*change.ExpectedBlobSHA) { + return fmt.Errorf("plan GitHub pull request: update precondition is invalid") + } + case "delete": + if change.Content != nil || change.ExpectedBlobSHA == nil || !validCommitSHA(*change.ExpectedBlobSHA) { + return fmt.Errorf("plan GitHub pull request: delete precondition is invalid") + } + default: + return fmt.Errorf("plan GitHub pull request: change operation is invalid") + } + totalContent += len(content) + if totalContent > maxTotalContentBytes { + return fmt.Errorf("plan GitHub pull request: aggregate content is too large") + } + } + return nil +} + +func validBaseRef(value string) bool { + if validateBoundedText(value, maxBaseRefBytes, false, false) != nil || strings.HasPrefix(value, ".") || + strings.HasPrefix(value, "-") || strings.HasPrefix(strings.ToLower(value), "refs/") || + strings.EqualFold(value, "HEAD") || validCommitSHA(value) || + strings.HasPrefix(value, "/") || strings.HasSuffix(value, ".") || strings.HasSuffix(value, "/") || + strings.Contains(value, "..") || strings.Contains(value, "//") || strings.Contains(value, "@{") || + strings.HasSuffix(strings.ToLower(value), ".lock") { + return false + } + for _, character := range value { + if unicode.IsSpace(character) || strings.ContainsRune("~^:?*[]\\", character) { + return false + } + } + for _, component := range strings.Split(value, "/") { + if component == "" || strings.HasPrefix(component, ".") || strings.HasSuffix(strings.ToLower(component), ".lock") { + return false + } + } + return true +} + +func validChangePath(value string) bool { + if validateBoundedText(value, maxChangePathBytes, false, false) != nil || strings.HasPrefix(value, "/") || + strings.HasSuffix(value, "/") || strings.Contains(value, "\\") || path.Clean(value) != value { + return false + } + components := strings.Split(value, "/") + for _, character := range value { + letter := (character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z') + digit := character >= '0' && character <= '9' + if !letter && !digit && character != '-' && character != '_' && character != '.' && character != '/' { + return false + } + } + for index, component := range components { + lower := strings.ToLower(component) + if component == "" || component == "." || component == ".." || lower == ".git" || lower == ".gitmodules" { + return false + } + if index == 0 && lower == ".github" && len(components) > 1 && strings.ToLower(components[1]) == "workflows" { + return false + } + } + return true +} + +func validateBoundedText(value string, maximum int, allowEmpty, allowMultiline bool) error { + if (!allowEmpty && value == "") || len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("text is invalid") + } + for _, character := range value { + if unicode.IsControl(character) && (!allowMultiline || (character != '\n' && character != '\t')) { + return fmt.Errorf("text is invalid") + } + } + return nil +} + +func validFileContent(value string) bool { + if !utf8.ValidString(value) || strings.ContainsRune(value, '\x00') { + return false + } + for _, character := range value { + if unicode.IsControl(character) && character != '\n' && character != '\t' { + return false + } + } + return true +} + +func digestText(value string) string { + digest := sha256.Sum256([]byte(value)) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +func normalizedOpenPRTarget(config OpenPRPlannerConfig) fleet.ResourceRef { + return fleet.ResourceRef{SourceKind: Kind, Scope: config.Host, Kind: openPRTargetKind, Namespace: config.Owner, Name: config.Repository} +} + +func openPRPlanSteps(args openPRArgs, changes []plannedChange, headRef string) ([]connector.PlanStep, error) { + values := []struct { + description string + api string + params any + }{ + {"verify configured base ref", "GET /repos/{owner}/{repo}/git/ref/heads/{base}", refCheckParams{args.BaseRef, args.ExpectedBaseSHA}}, + {"resolve the pinned base commit tree", "GET /repos/{owner}/{repo}/git/commits/{commit}", commitLookupParams{args.ExpectedBaseSHA}}, + {"verify exact path preconditions", "GET /repos/{owner}/{repo}/git/trees/{tree}", treeVerifyParams{"verified-base-tree", changes}}, + {"construct one immutable tree", "POST /repos/{owner}/{repo}/git/trees", treePlanParams{"verified-base-tree", changes}}, + {"construct one immutable commit", "POST /repos/{owner}/{repo}/git/commits", commitPlanParams{args.ExpectedBaseSHA, "planned-tree", digestText(args.CommitMessage)}}, + {"create deterministic proposal ref", "POST /repos/{owner}/{repo}/git/refs", refCreateParams{headRef, "planned-commit"}}, + {"reconcile an idempotent existing pull request", "GET /repos/{owner}/{repo}/pulls", pullLookupParams{headRef, args.BaseRef}}, + {"create the human-reviewed pull request", "POST /repos/{owner}/{repo}/pulls", pullCreateParams{headRef, args.BaseRef, digestText(args.Title), digestText(args.Body)}}, + } + steps := make([]connector.PlanStep, 0, len(values)) + for _, value := range values { + encoded, err := json.Marshal(value.params) + if err != nil { + return nil, fmt.Errorf("plan GitHub pull request: encode API plan") + } + steps = append(steps, connector.PlanStep{Description: value.description, API: value.api, Params: encoded}) + } + return steps, nil +} diff --git a/internal/connector/github/action_plan_test.go b/internal/connector/github/action_plan_test.go new file mode 100644 index 0000000..d12a4e5 --- /dev/null +++ b/internal/connector/github/action_plan_test.go @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "sync" + "testing" + + "github.com/ArdurAI/sith/internal/connector" + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/intentargs" +) + +var ( + openPRBaseSHA = strings.Repeat("a", 40) + openPRUpdateSHA = strings.Repeat("b", 40) + openPRDeleteSHA = strings.Repeat("c", 40) +) + +func TestOpenPRPlannerRegistersOnlyBoundedPlanningCapability(t *testing.T) { + t.Parallel() + planner := newTestOpenPRPlanner(t) + registry := connector.NewRegistry() + if err := registry.Register(func() (connector.Connector, error) { return planner, nil }); err != nil { + t.Fatalf("Register() error = %v", err) + } + descriptors := registry.Descriptors() + if len(descriptors) != 1 { + t.Fatalf("descriptors = %#v", descriptors) + } + descriptor := descriptors[0] + if descriptor.Kind != Kind || descriptor.ConnKind != connector.KindTypedAction || descriptor.AdapterVersion != "gitops-open-pr/2026-03-10" || + !slices.Equal(descriptor.WireVersions, []connector.WireVersion{connector.CurrentWireVersion()}) || + !slices.Equal(descriptor.Capabilities, []connector.Capability{connector.CapPlan}) || + !slices.Equal(descriptor.Verbs, []intent.Verb{intent.VerbGitOpsOpenPR}) || len(descriptor.ArgSchemas) != 1 { + t.Fatalf("descriptor = %#v", descriptor) + } + if _, err := registry.PlannerForVerb(intent.VerbGitOpsOpenPR); err != nil { + t.Fatalf("PlannerForVerb() error = %v", err) + } + if _, err := registry.ExecutorForVerb(intent.VerbGitOpsOpenPR); !strings.Contains(err.Error(), "execute") { + t.Fatalf("ExecutorForVerb() error = %v, want unavailable execute", err) + } + if _, err := registry.VerifierForVerb(intent.VerbGitOpsOpenPR); !strings.Contains(err.Error(), "verify") { + t.Fatalf("VerifierForVerb() error = %v, want unavailable verify", err) + } +} + +func TestNewOpenPRPlannerRejectsUnsafeRepositoryPolicies(t *testing.T) { + t.Parallel() + valid := testOpenPRConfig() + tests := []struct { + name string + mutate func(*OpenPRPlannerConfig) + }{ + {"host scheme", func(config *OpenPRPlannerConfig) { config.Host = "https://github.com" }}, + {"uppercase host", func(config *OpenPRPlannerConfig) { config.Host = "GitHub.com" }}, + {"owner path", func(config *OpenPRPlannerConfig) { config.Owner = "ArdurAI/other" }}, + {"repository suffix", func(config *OpenPRPlannerConfig) { config.Repository = "sith.git" }}, + {"base traversal", func(config *OpenPRPlannerConfig) { config.BaseRef = "release/../main" }}, + {"base reflog", func(config *OpenPRPlannerConfig) { config.BaseRef = "main@{1}" }}, + {"base lock", func(config *OpenPRPlannerConfig) { config.BaseRef = "main.lock" }}, + {"base hidden", func(config *OpenPRPlannerConfig) { config.BaseRef = ".hidden" }}, + {"base option", func(config *OpenPRPlannerConfig) { config.BaseRef = "-delete" }}, + {"base symbolic HEAD", func(config *OpenPRPlannerConfig) { config.BaseRef = "HEAD" }}, + {"base full ref", func(config *OpenPRPlannerConfig) { config.BaseRef = "refs/heads/dev" }}, + {"base commit shaped", func(config *OpenPRPlannerConfig) { config.BaseRef = strings.Repeat("a", 40) }}, + {"base control", func(config *OpenPRPlannerConfig) { config.BaseRef = "main\nother" }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + config := valid + test.mutate(&config) + if planner, err := NewOpenPRPlanner(config); err == nil || planner != nil { + t.Fatalf("NewOpenPRPlanner() = %#v, %v, want rejection", planner, err) + } + }) + } +} + +func TestOpenPRPlannerCanonicalizesArgumentsWithoutPlanning(t *testing.T) { + t.Parallel() + planner := newTestOpenPRPlanner(t) + request := validOpenPRIntent(t) + + target, canonical, err := planner.CanonicalizeOpenPRArgs(request.Args) + if err != nil { + t.Fatalf("CanonicalizeOpenPRArgs() error = %v", err) + } + wantTarget := fleet.ResourceRef{SourceKind: Kind, Scope: "github.com", Kind: openPRTargetKind, Namespace: "ArdurAI", Name: "sith"} + if !target.Equal(wantTarget) || len(target.Attributes) != 0 { + t.Fatalf("target = %#v, want %#v", target, wantTarget) + } + var got openPRArgs + if err := json.Unmarshal(canonical, &got); err != nil { + t.Fatalf("decode canonical args: %v", err) + } + if len(got.Changes) != 3 || got.Changes[0].Path != "config/new.yaml" || + got.Changes[1].Path != "deploy/api.yaml" || got.Changes[2].Path != "deploy/old.yaml" { + t.Fatalf("canonical changes = %#v, want stable path ordering", got.Changes) + } + + canonical[0] = '!' + _, second, err := planner.CanonicalizeOpenPRArgs(request.Args) + if err != nil || len(second) == 0 || second[0] != '{' { + t.Fatalf("second canonicalization = %q, %v, want isolated result", second, err) + } + if _, _, err := planner.CanonicalizeOpenPRArgs([]byte(`{"base_ref":"dev","changes":[]}`)); err == nil { + t.Fatal("CanonicalizeOpenPRArgs() accepted incomplete arguments") + } + var nilPlanner *OpenPRPlanner + if _, _, err := nilPlanner.CanonicalizeOpenPRArgs(request.Args); err == nil { + t.Fatal("nil CanonicalizeOpenPRArgs() accepted arguments") + } +} + +func TestOpenPRPlannerProducesDeterministicDigestOnlyPlan(t *testing.T) { + t.Parallel() + planner := newTestOpenPRPlanner(t) + registry := connector.NewRegistry() + if err := registry.Register(func() (connector.Connector, error) { return planner, nil }); err != nil { + t.Fatal(err) + } + + first := validOpenPRIntent(t) + second := validOpenPRIntent(t) + var decoded openPRArgs + if err := json.Unmarshal(second.Args, &decoded); err != nil { + t.Fatal(err) + } + slices.Reverse(decoded.Changes) + second.Args = encodeOpenPRArgs(t, decoded) + + firstPlan, err := registry.Plan(context.Background(), first) + if err != nil { + t.Fatalf("first Plan() error = %v", err) + } + secondPlan, err := registry.Plan(context.Background(), second) + if err != nil { + t.Fatalf("second Plan() error = %v", err) + } + wantTarget := fleet.ResourceRef{SourceKind: Kind, Scope: "github.com", Kind: openPRTargetKind, Namespace: "ArdurAI", Name: "sith"} + if firstPlan.IntentID != "intent-228" || firstPlan.Verb != intent.VerbGitOpsOpenPR || !firstPlan.Reversible || + !firstPlan.Target.Equal(wantTarget) || len(firstPlan.Target.Attributes) != 0 { + t.Fatalf("plan identity = %#v", firstPlan) + } + if !firstPlan.Diff.Ref.Equal(firstPlan.Target) || !firstPlan.Diff.Drifted || len(firstPlan.Diff.Hunks) != 3 || len(firstPlan.Steps) != 8 { + t.Fatalf("plan diff/steps = %#v / %#v", firstPlan.Diff, firstPlan.Steps) + } + for index, wantPath := range []string{"config/new.yaml", "deploy/api.yaml", "deploy/old.yaml"} { + if firstPlan.Diff.Hunks[index].Path != wantPath { + t.Fatalf("hunk paths = %#v", firstPlan.Diff.Hunks) + } + } + if !slices.Equal(firstPlan.Diff.Hunks, secondPlan.Diff.Hunks) { + t.Fatalf("equivalent change order produced different hunks:\n%#v\n%#v", firstPlan.Diff.Hunks, secondPlan.Diff.Hunks) + } + + firstHead := plannedHeadRef(t, firstPlan) + secondHead := plannedHeadRef(t, secondPlan) + if firstHead != secondHead || !strings.HasPrefix(firstHead, "sith/intent-") || len(firstHead) != len("sith/intent-")+24 { + t.Fatalf("head refs = %q and %q", firstHead, secondHead) + } + for _, step := range firstPlan.Steps { + if strings.Contains(step.API, "PATCH") || strings.Contains(step.API, "/contents/") || strings.Contains(step.API, "git/refs/{ref}") { + t.Fatalf("plan contains forbidden mutation API: %#v", step) + } + } + encoded, err := json.Marshal(firstPlan) + if err != nil { + t.Fatal(err) + } + for _, secret := range []string{"replicas: 3", "old manifest", "sensitive PR title", "private rationale", "secret commit message"} { + if strings.Contains(string(encoded), secret) { + t.Fatalf("plan leaked raw input %q: %s", secret, encoded) + } + } + for _, expected := range []string{"sha256:", "config/new.yaml", "verified-base-tree", "planned-commit"} { + if !strings.Contains(string(encoded), expected) { + t.Fatalf("plan is missing inspectable marker %q: %s", expected, encoded) + } + } + + secondBeforeMutation := slices.Clone(second.Args) + first.Args[0] = '!' + if !slices.Equal(secondBeforeMutation, second.Args) { + t.Fatal("test setup unexpectedly shared argument storage") + } + if plannedHeadRef(t, firstPlan) != firstHead { + t.Fatal("plan retained mutable caller argument storage") + } +} + +func TestOpenPRPlannerRejectsSchemaAndSemanticAttacksWithoutEcho(t *testing.T) { + t.Parallel() + planner := newTestOpenPRPlanner(t) + valid := validOpenPRIntent(t) + marker := "do-not-echo-sensitive-marker" + tests := []struct { + name string + mutate func(*connector.Intent) + }{ + {"duplicate JSON member", func(request *connector.Intent) { + request.Args = []byte(strings.Replace(string(request.Args), `"base_ref":"dev"`, `"base_ref":"dev","base_ref":"dev"`, 1)) + }}, + {"unknown schema member", mutateOpenPRArgs(t, func(args *map[string]any) { (*args)["credential"] = marker })}, + {"wrong base", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.BaseRef = marker })}, + {"uppercase base SHA", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.ExpectedBaseSHA = strings.Repeat("A", 40) })}, + {"title newline", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Title = "title\n" + marker })}, + {"body carriage return", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Body = "body\r" + marker })}, + {"commit tab", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.CommitMessage = "message\t" + marker })}, + {"no changes", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes = nil })}, + {"create missing content", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[0].Content = nil })}, + {"create unexpected SHA", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[0].ExpectedBlobSHA = stringPointer(openPRUpdateSHA) })}, + {"update missing SHA", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[1].ExpectedBlobSHA = nil })}, + {"delete has content", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[2].Content = stringPointer(marker) })}, + {"case-colliding paths", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[1].Path = strings.ToUpper(args.Changes[0].Path) })}, + {"content NUL", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[0].Content = stringPointer("data\x00" + marker) })}, + {"content control", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { args.Changes[0].Content = stringPointer("data\x01" + marker) })}, + {"oversized file", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { + args.Changes[0].Content = stringPointer(strings.Repeat("x", maxFileContentBytes+1)) + })}, + {"oversized aggregate", mutateTypedOpenPRArgs(t, func(args *openPRArgs) { + content := strings.Repeat("x", (maxTotalContentBytes/4)+1) + args.Changes = []fileChange{ + {Operation: "create", Path: "a.yaml", Content: stringPointer(content)}, + {Operation: "create", Path: "b.yaml", Content: stringPointer(content)}, + {Operation: "create", Path: "c.yaml", Content: stringPointer(content)}, + {Operation: "create", Path: "d.yaml", Content: stringPointer(content)}, + } + })}, + {"target host", func(request *connector.Intent) { request.Target.Scope = marker + ".example" }}, + {"target attributes", func(request *connector.Intent) { request.Target.Attributes = map[string]string{"token": marker} }}, + {"intent identifier", func(request *connector.Intent) { request.ID = marker + "\nother" }}, + {"wrong verb", func(request *connector.Intent) { request.Verb = intent.VerbArgoCDSync }}, + } + for _, pathAttack := range []string{ + "/etc/passwd", "../secret", "deploy/../secret", "deploy\\secret", ".git/config", "x/.GIT/config", + ".gitmodules", "nested/.gitmodules", ".github/workflows/release.yml", ".GITHUB/WORKFLOWS/release.yml", + "deploy/se cret.yaml", "deploy/π.yaml", "deploy//api.yaml", "deploy/api.yaml/", + } { + pathAttack := pathAttack + tests = append(tests, struct { + name string + mutate func(*connector.Intent) + }{"path " + fmt.Sprintf("%q", pathAttack), mutateTypedOpenPRArgs(t, func(args *openPRArgs) { + args.Title = marker + args.Changes[0].Path = pathAttack + })}) + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + request := cloneConnectorIntent(valid) + test.mutate(&request) + _, err := planner.Plan(context.Background(), request) + if err == nil { + t.Fatal("Plan() accepted malicious request") + } + if strings.Contains(err.Error(), marker) || len(err.Error()) > 160 { + t.Fatalf("error leaked or was unbounded: %q", err) + } + }) + } +} + +func TestOpenPRPlannerIsConcurrentAndImmutable(t *testing.T) { + t.Parallel() + planner := newTestOpenPRPlanner(t) + request := validOpenPRIntent(t) + want, err := planner.Plan(context.Background(), request) + if err != nil { + t.Fatal(err) + } + wantJSON, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + + const workers = 64 + errors := make(chan error, workers) + var wait sync.WaitGroup + for index := 0; index < workers; index++ { + wait.Add(1) + go func() { + defer wait.Done() + got, planErr := planner.Plan(context.Background(), cloneConnectorIntent(request)) + if planErr != nil { + errors <- planErr + return + } + gotJSON, marshalErr := json.Marshal(got) + if marshalErr != nil { + errors <- marshalErr + return + } + if !slices.Equal(gotJSON, wantJSON) { + errors <- fmt.Errorf("nondeterministic plan") + } + }() + } + wait.Wait() + close(errors) + for err := range errors { + t.Error(err) + } +} + +func FuzzOpenPRPlannerNeverPanicsOrEchoesArgs(f *testing.F) { + planner := newTestOpenPRPlanner(f) + valid := validOpenPRIntent(f) + f.Add([]byte(slices.Clone(valid.Args))) + f.Add([]byte(`{"base_ref":"dev","changes":[]}`)) + f.Add([]byte(`{"base_ref":"marker","base_ref":"marker"}`)) + f.Fuzz(func(t *testing.T, document []byte) { + request := cloneConnectorIntent(valid) + request.Args = document + plan, err := planner.Plan(context.Background(), request) + if err != nil { + if len(err.Error()) > 160 { + t.Fatalf("unbounded error length %d", len(err.Error())) + } + return + } + encoded, marshalErr := json.Marshal(plan) + if marshalErr != nil { + t.Fatal(marshalErr) + } + if len(encoded) > 32<<10 || strings.Contains(string(encoded), "replicas: 3") { + t.Fatalf("invalid or leaking plan: %s", encoded) + } + }) +} + +func newTestOpenPRPlanner(t testing.TB) *OpenPRPlanner { + t.Helper() + planner, err := NewOpenPRPlanner(testOpenPRConfig()) + if err != nil { + t.Fatalf("NewOpenPRPlanner() error = %v", err) + } + return planner +} + +func testOpenPRConfig() OpenPRPlannerConfig { + return OpenPRPlannerConfig{Host: "github.com", Owner: "ArdurAI", Repository: "sith", BaseRef: "dev"} +} + +func validOpenPRIntent(t testing.TB) connector.Intent { + t.Helper() + return connector.Intent{ + ID: "intent-228", Workspace: "workspace-a", Actor: "user:operator", Verb: intent.VerbGitOpsOpenPR, + Target: fleet.ResourceRef{SourceKind: Kind, Scope: "github.com", Kind: openPRTargetKind, Namespace: "ArdurAI", Name: "sith"}, + Args: encodeOpenPRArgs(t, openPRArgs{ + BaseRef: "dev", ExpectedBaseSHA: openPRBaseSHA, Title: "sensitive PR title", Body: "private rationale", + CommitMessage: "secret commit message", + Changes: []fileChange{ + {Operation: "create", Path: "config/new.yaml", Content: stringPointer("")}, + {Operation: "update", Path: "deploy/api.yaml", Content: stringPointer("replicas: 3\n"), ExpectedBlobSHA: stringPointer(openPRUpdateSHA)}, + {Operation: "delete", Path: "deploy/old.yaml", ExpectedBlobSHA: stringPointer(openPRDeleteSHA)}, + }, + }), + Justification: "propose desired-state change", Signature: "future-signed-intent", + } +} + +func encodeOpenPRArgs(t testing.TB, args openPRArgs) json.RawMessage { + t.Helper() + document, err := json.Marshal(args) + if err != nil { + t.Fatalf("marshal args: %v", err) + } + return document +} + +func plannedHeadRef(t testing.TB, plan connector.ActionPlan) string { + t.Helper() + for _, step := range plan.Steps { + if step.Description != "create deterministic proposal ref" { + continue + } + var params refCreateParams + if err := json.Unmarshal(step.Params, ¶ms); err != nil { + t.Fatalf("decode ref params: %v", err) + } + return params.HeadRef + } + t.Fatal("plan has no create-ref step") + return "" +} + +func mutateTypedOpenPRArgs(t *testing.T, mutate func(*openPRArgs)) func(*connector.Intent) { + t.Helper() + return func(request *connector.Intent) { + var args openPRArgs + if err := json.Unmarshal(request.Args, &args); err != nil { + t.Fatal(err) + } + mutate(&args) + request.Args = encodeOpenPRArgs(t, args) + } +} + +func mutateOpenPRArgs(t *testing.T, mutate func(*map[string]any)) func(*connector.Intent) { + t.Helper() + return func(request *connector.Intent) { + var args map[string]any + if err := json.Unmarshal(request.Args, &args); err != nil { + t.Fatal(err) + } + mutate(&args) + document, err := json.Marshal(args) + if err != nil { + t.Fatal(err) + } + request.Args = document + } +} + +func cloneConnectorIntent(request connector.Intent) connector.Intent { + request.Args = slices.Clone(request.Args) + request.EvidenceRefs = slices.Clone(request.EvidenceRefs) + if request.Target.Attributes != nil { + original := request.Target.Attributes + request.Target.Attributes = map[string]string{} + for key, value := range original { + request.Target.Attributes[key] = value + } + } + return request +} + +func stringPointer(value string) *string { return &value } + +func TestOpenPRSchemaRemainsBoundedAndExact(t *testing.T) { + t.Parallel() + schema, err := intentargs.Compile(json.RawMessage(openPRSchema)) + if err != nil { + t.Fatalf("Compile() error = %v", err) + } + valid := validOpenPRIntent(t) + if err := schema.Validate(valid.Args); err != nil { + t.Fatalf("Validate(valid) error = %v", err) + } +} diff --git a/internal/connector/github/boundary_test.go b/internal/connector/github/boundary_test.go new file mode 100644 index 0000000..34d5a0e --- /dev/null +++ b/internal/connector/github/boundary_test.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var allowedProductionImports = map[string]bool{ + "bytes": true, + "context": true, + "crypto/sha256": true, + "encoding/hex": true, + "encoding/json": true, + "fmt": true, + "io": true, + "path": true, + "sort": true, + "strconv": true, + "strings": true, + "time": true, + "unicode": true, + "unicode/utf8": true, + "github.com/ArdurAI/sith/internal/connector": true, + "github.com/ArdurAI/sith/internal/fleet": true, + "github.com/ArdurAI/sith/internal/intent": true, + "github.com/ArdurAI/sith/internal/intentargs": true, + "github.com/ArdurAI/sith/internal/tenancy": true, +} + +var allowedProductionDeclarations = map[string]bool{ + "APIVersion": true, + "CanonicalizeOpenPRArgs": true, + "Capabilities": true, + "Descriptor": true, + "Kind": true, + "NewOpenPRPlanner": true, + "OpenPRPlanner": true, + "OpenPRPlannerConfig": true, + "Plan": true, + "ProtocolVersion": true, + "Projection": true, + "ProjectMergedPullRequest": true, + "ProjectFailedWorkflowRun": true, + "UnmarshalJSON": true, + "WorkflowRunProjection": true, + "WorkflowRunProtocolVersion": true, + "changeObservation": true, + "canonicalizeOpenPRArgs": true, + "commitLookupParams": true, + "consumeUniqueJSON": true, + "matchingDelimiter": true, + "maxBaseRefBytes": true, + "maxClockSkew": true, + "maxChangePathBytes": true, + "maxChanges": true, + "maxCommitMessageBytes": true, + "maxFactPayloadBytes": true, + "maxFileContentBytes": true, + "maxHostBytes": true, + "maxIntentIDBytes": true, + "maxJSONDepth": true, + "maxOwnerBytes": true, + "maxPRBodyBytes": true, + "maxPRTitleBytes": true, + "maxRepositoryBytes": true, + "maxResourceName": true, + "maxResponseBytes": true, + "maxTotalContentBytes": true, + "maxWorkspaceBytes": true, + "mergedObservation": true, + "normalizedOpenPRTarget": true, + "openPRArgs": true, + "openPRPlanSteps": true, + "openPRProtocolVersion": true, + "openPRSchema": true, + "openPRTargetKind": true, + "plannedChange": true, + "pullCommit": true, + "pullCreateParams": true, + "pullLookupParams": true, + "pullResponse": true, + "workflowRunFailureKind": true, + "workflowRunObservation": true, + "workflowRunRepository": true, + "workflowRunResponse": true, + "refCheckParams": true, + "refCreateParams": true, + "rejectDuplicateJSON": true, + "requiredCommitSHA": true, + "treePlanParams": true, + "treeVerifyParams": true, + "commitPlanParams": true, + "fileChange": true, + "digestText": true, + "validCommitSHA": true, + "validBaseRef": true, + "validChangePath": true, + "validFileContent": true, + "validWorkflowRunRepositoryIdentity": true, + "validateBoundedText": true, + "validateOpenPRArgs": true, + "validateOpenPRConfig": true, + "validateOpenPRIntent": true, + "validateHost": true, + "validatePathComponent": true, + "validateProjection": true, + "validateResponseIdentity": true, + "validateText": true, + "validateWorkflowRunProjection": true, + "failedWorkflowRunObservation": true, + "workflowRunResourceName": true, +} + +func TestProjectorHasNoIOCredentialPersistenceOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read github package: %v", err) + } + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if !allowedProductionImports[path] { + t.Fatalf("projector imports unreviewed package %q", path) + } + } + for _, node := range file.Decls { + switch declaration := node.(type) { + case *ast.FuncDecl: + if !allowedProductionDeclarations[declaration.Name.Name] { + t.Errorf("projector declares unreviewed function or method %s", declaration.Name.Name) + } + case *ast.GenDecl: + for _, specification := range declaration.Specs { + switch typed := specification.(type) { + case *ast.TypeSpec: + if !allowedProductionDeclarations[typed.Name.Name] { + t.Errorf("projector declares unreviewed type %s", typed.Name.Name) + } + case *ast.ValueSpec: + for _, name := range typed.Names { + if !allowedProductionDeclarations[name.Name] { + t.Errorf("projector declares unreviewed value %s", name.Name) + } + } + } + } + } + } + ast.Inspect(file, func(node ast.Node) bool { + if _, ok := node.(*ast.InterfaceType); ok { + t.Errorf("projector declares an injected interface seam") + } + return true + }) + } +} diff --git a/internal/connector/github/project.go b/internal/connector/github/project.go new file mode 100644 index 0000000..8d4efff --- /dev/null +++ b/internal/connector/github/project.go @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package github normalizes read-only GitHub pull-request and workflow-run evidence for Sith's +// operational graph. +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for GitHub read evidence. + Kind = "github" + // APIVersion is the GitHub REST API version whose response contracts are normalized here. + APIVersion = "2026-03-10" + // ProtocolVersion identifies the normalized Get-a-pull-request fact contract. + ProtocolVersion = "pulls/" + APIVersion + + maxResponseBytes = 512 << 10 + maxWorkspaceBytes = 253 + maxHostBytes = 253 + maxOwnerBytes = 100 + maxRepositoryBytes = 100 + maxResourceName = 253 + maxFactPayloadBytes = 4 << 10 + maxJSONDepth = 64 + maxClockSkew = 5 * time.Minute +) + +// Projection supplies one already-authorized GitHub Get-a-pull-request response. +// ProjectMergedPullRequest does not perform discovery, network access, credential loading, +// persistence, repository-to-workload correlation, or mutation. +type Projection struct { + Workspace string + Host string + Owner string + Repository string + PullNumber int64 + ObservedAt time.Time + Response []byte +} + +type pullResponse struct { + Number *int64 `json:"number"` + State *string `json:"state"` + Draft *bool `json:"draft"` + Merged *bool `json:"merged"` + MergedAt *string `json:"merged_at"` + MergeCommitSHA *string `json:"merge_commit_sha"` + Head *pullCommit `json:"head"` + Base *pullCommit `json:"base"` +} + +type pullCommit struct { + SHA *string `json:"sha"` +} + +func (response *pullResponse) UnmarshalJSON(document []byte) error { + type exactField struct { + name string + target any + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(document, &fields); err != nil { + return fmt.Errorf("pull request response must be a JSON object") + } + for _, field := range []exactField{ + {name: "number", target: &response.Number}, + {name: "state", target: &response.State}, + {name: "draft", target: &response.Draft}, + {name: "merged", target: &response.Merged}, + {name: "merged_at", target: &response.MergedAt}, + {name: "merge_commit_sha", target: &response.MergeCommitSHA}, + {name: "head", target: &response.Head}, + {name: "base", target: &response.Base}, + } { + value, exists := fields[field.name] + if !exists { + continue + } + if err := json.Unmarshal(value, field.target); err != nil { + return fmt.Errorf("%s is invalid", field.name) + } + } + return nil +} + +func (commit *pullCommit) UnmarshalJSON(document []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(document, &fields); err != nil { + return fmt.Errorf("commit must be a JSON object") + } + value, exists := fields["sha"] + if !exists { + return nil + } + if err := json.Unmarshal(value, &commit.SHA); err != nil { + return fmt.Errorf("sha is invalid") + } + return nil +} + +type changeObservation struct { + PullNumber int64 `json:"pull_number"` + ChangeKind string `json:"change_kind"` + HeadSHA string `json:"head_sha"` + BaseSHA string `json:"base_sha"` + MergeCommitSHA string `json:"merge_commit_sha"` + MergedAt time.Time `json:"merged_at"` +} + +// ProjectMergedPullRequest returns one deterministic, bounded TIMELINE fact only for an +// internally consistent merged pull request. A valid response for an unmerged pull request +// abstains with no facts; pre-merge test-merge SHAs are never promoted to merge evidence. +func ProjectMergedPullRequest(input Projection) ([]fleet.GraphFact, error) { + if err := validateProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode GitHub pull request response: %w", err) + } + + var response pullResponse + decoder := json.NewDecoder(bytes.NewReader(input.Response)) + if err := decoder.Decode(&response); err != nil { + return nil, fmt.Errorf("decode GitHub pull request response: %w", err) + } + if err := validateResponseIdentity(response, input.PullNumber); err != nil { + return nil, err + } + if !*response.Merged { + if response.MergedAt != nil { + return nil, fmt.Errorf("unmerged GitHub pull request must not have merged_at") + } + return []fleet.GraphFact{}, nil + } + + observation, err := mergedObservation(response, input.ObservedAt) + if err != nil { + return nil, err + } + encoded, err := json.Marshal(observation) + if err != nil { + return nil, fmt.Errorf("encode GitHub pull request change fact: %w", err) + } + if len(encoded) > maxFactPayloadBytes { + return nil, fmt.Errorf("GitHub pull request change fact exceeds %d encoded bytes", maxFactPayloadBytes) + } + + resourceName := input.Repository + "#" + strconv.FormatInt(input.PullNumber, 10) + nativeID := input.Owner + "/" + resourceName + "@" + observation.MergeCommitSHA + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, + Scope: input.Host, + Kind: "PullRequest", + Namespace: input.Owner, + Name: resourceName, + }, + Kind: fleet.FactChange, + Observed: encoded, + ObservedAt: observation.MergedAt, + Source: input.Host, + Provenance: fleet.Provenance{ + Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID, + }, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensTimeline, + } + if err := fact.Validate(input.Workspace); err != nil { + return nil, fmt.Errorf("validate GitHub pull request change fact: %w", err) + } + return []fleet.GraphFact{fact}, nil +} + +func validateProjection(input Projection) error { + if err := validateText("workspace", input.Workspace, maxWorkspaceBytes); err != nil { + return err + } + if err := validateHost(input.Host); err != nil { + return err + } + if err := validatePathComponent("owner", input.Owner, maxOwnerBytes); err != nil { + return err + } + if err := validatePathComponent("repository", input.Repository, maxRepositoryBytes); err != nil { + return err + } + if strings.HasSuffix(strings.ToLower(input.Repository), ".git") { + return fmt.Errorf("repository must not include the .git suffix") + } + if input.PullNumber <= 0 { + return fmt.Errorf("pull number must be positive") + } + resourceName := input.Repository + "#" + strconv.FormatInt(input.PullNumber, 10) + if len(resourceName) > maxResourceName { + return fmt.Errorf("pull request resource name exceeds %d bytes", maxResourceName) + } + if input.ObservedAt.IsZero() { + return fmt.Errorf("projection observation time is required") + } + if len(input.Response) == 0 { + return fmt.Errorf("GitHub pull request response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("GitHub pull request response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("GitHub pull request response must be valid UTF-8") + } + return nil +} + +func validateResponseIdentity(response pullResponse, pullNumber int64) error { + if response.Number == nil { + return fmt.Errorf("GitHub pull request number is required") + } + if *response.Number != pullNumber { + return fmt.Errorf("GitHub pull request number does not match trusted caller identity") + } + if response.State == nil { + return fmt.Errorf("GitHub pull request state is required") + } + if *response.State != "open" && *response.State != "closed" { + return fmt.Errorf("GitHub pull request state must be open or closed") + } + if response.Draft == nil { + return fmt.Errorf("GitHub pull request draft state is required") + } + if response.Merged == nil { + return fmt.Errorf("GitHub pull request merged state is required") + } + return nil +} + +func mergedObservation(response pullResponse, observedAt time.Time) (changeObservation, error) { + if *response.State != "closed" { + return changeObservation{}, fmt.Errorf("merged GitHub pull request must be closed") + } + if *response.Draft { + return changeObservation{}, fmt.Errorf("merged GitHub pull request must not be a draft") + } + if response.MergedAt == nil || *response.MergedAt == "" { + return changeObservation{}, fmt.Errorf("merged GitHub pull request merged_at is required") + } + mergedAt, err := time.Parse(time.RFC3339Nano, *response.MergedAt) + if err != nil || mergedAt.IsZero() { + return changeObservation{}, fmt.Errorf("merged GitHub pull request merged_at must be a non-zero RFC3339 timestamp") + } + mergedAt = mergedAt.UTC() + if mergedAt.After(observedAt.UTC().Add(maxClockSkew)) { + return changeObservation{}, fmt.Errorf("GitHub pull request merge time exceeds allowed collection clock skew") + } + + headSHA, err := requiredCommitSHA("head.sha", response.Head) + if err != nil { + return changeObservation{}, err + } + baseSHA, err := requiredCommitSHA("base.sha", response.Base) + if err != nil { + return changeObservation{}, err + } + if response.MergeCommitSHA == nil || !validCommitSHA(*response.MergeCommitSHA) { + return changeObservation{}, fmt.Errorf("merged GitHub pull request merge_commit_sha is invalid") + } + return changeObservation{ + PullNumber: *response.Number, + ChangeKind: "pull-request-merged", + HeadSHA: headSHA, + BaseSHA: baseSHA, + MergeCommitSHA: *response.MergeCommitSHA, + MergedAt: mergedAt, + }, nil +} + +func requiredCommitSHA(label string, commit *pullCommit) (string, error) { + if commit == nil || commit.SHA == nil || !validCommitSHA(*commit.SHA) { + return "", fmt.Errorf("merged GitHub pull request %s is invalid", label) + } + return *commit.SHA, nil +} + +func validCommitSHA(value string) bool { + if len(value) != 40 && len(value) != 64 { + return false + } + for _, character := range value { + if (character < '0' || character > '9') && (character < 'a' || character > 'f') { + return false + } + } + return true +} + +func validateHost(value string) error { + if err := validateText("host", value, maxHostBytes); err != nil { + return err + } + if strings.ToLower(value) != value || strings.HasPrefix(value, ".") || strings.HasSuffix(value, ".") { + return fmt.Errorf("host is invalid") + } + for _, label := range strings.Split(value, ".") { + if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return fmt.Errorf("host is invalid") + } + for _, character := range label { + if (character < 'a' || character > 'z') && (character < '0' || character > '9') && character != '-' { + return fmt.Errorf("host is invalid") + } + } + } + return nil +} + +func validatePathComponent(label, value string, maximum int) error { + if err := validateText(label, value, maximum); err != nil { + return err + } + if value == "." || value == ".." { + return fmt.Errorf("%s is invalid", label) + } + for _, character := range value { + if (character < 'a' || character > 'z') && (character < 'A' || character > 'Z') && + (character < '0' || character > '9') && character != '-' && character != '_' && character != '.' { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func validateText(label, value string, maximum int) error { + if value == "" || len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("%s is invalid", label) + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func rejectDuplicateJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || closing != matchingDelimiter(delimiter) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func matchingDelimiter(open json.Delim) json.Delim { + if open == '{' { + return '}' + } + return ']' +} diff --git a/internal/connector/github/project_test.go b/internal/connector/github/project_test.go new file mode 100644 index 0000000..c2c123b --- /dev/null +++ b/internal/connector/github/project_test.go @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +var ( + headSHA = strings.Repeat("a", 40) + baseSHA = strings.Repeat("b", 40) + mergeSHA = strings.Repeat("c", 40) +) + +func TestProjectMergedPullRequestEmitsSanitizedUnattachedTimelineFact(t *testing.T) { + t.Parallel() + input := validProjection(t) + facts, err := ProjectMergedPullRequest(input) + if err != nil { + t.Fatalf("ProjectMergedPullRequest() error = %v", err) + } + if len(facts) != 1 { + t.Fatalf("fact count = %d, want 1", len(facts)) + } + fact := facts[0] + wantMergedAt := time.Date(2026, 7, 16, 20, 30, 0, 123000000, time.UTC) + if fact.Fact.Kind != fleet.FactChange || fact.Lens != fleet.LensTimeline || fact.Entity != nil { + t.Fatalf("unexpected graph contract: %#v", fact) + } + wantRef := fleet.ResourceRef{ + SourceKind: Kind, + Scope: "github.example.com", + Kind: "PullRequest", + Namespace: "ArdurAI", + Name: "sith#42", + } + if fact.Fact.Workspace != "workspace-a" || !fact.Fact.Ref.Equal(wantRef) || fact.Fact.Ref.Attributes != nil { + t.Fatalf("unexpected trusted identity: %#v", fact.Fact) + } + if fact.Fact.Source != "github.example.com" || !fact.Fact.ObservedAt.Equal(wantMergedAt) || + fact.Fact.Provenance != (fleet.Provenance{ + Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: "ArdurAI/sith#42@" + mergeSHA, + }) { + t.Fatalf("unexpected provenance: %#v", fact.Fact) + } + var observation changeObservation + if err := json.Unmarshal(fact.Fact.Observed, &observation); err != nil { + t.Fatalf("decode observation: %v", err) + } + if observation != (changeObservation{ + PullNumber: 42, ChangeKind: "pull-request-merged", HeadSHA: headSHA, + BaseSHA: baseSHA, MergeCommitSHA: mergeSHA, MergedAt: wantMergedAt, + }) { + t.Fatalf("observation = %#v", observation) + } + if len(fact.Fact.Observed) > maxFactPayloadBytes { + t.Fatalf("encoded fact is %d bytes, limit %d", len(fact.Fact.Observed), maxFactPayloadBytes) + } + + encoded, err := json.Marshal(facts) + if err != nil { + t.Fatalf("marshal facts: %v", err) + } + for _, forbidden := range []string{ + "body-secret", "title-secret", "token-secret", "user-secret", "label-secret", + "attacker.example", "attacker-owner", "attacker-repository", "head-secret", "base-secret", + } { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("projected fact retained forbidden response value %q: %s", forbidden, encoded) + } + } + graph, err := fleet.NewGraph(input.Workspace, facts) + if err != nil { + t.Fatalf("NewGraph() error = %v", err) + } + if len(graph.Nodes) != 0 || len(graph.Unattached) != 1 { + t.Fatalf("graph nodes/unattached = %d/%d, want 0/1", len(graph.Nodes), len(graph.Unattached)) + } + if APIVersion != "2026-03-10" || ProtocolVersion != "pulls/2026-03-10" { + t.Fatalf("unexpected API/protocol versions %q/%q", APIVersion, ProtocolVersion) + } +} + +func TestProjectMergedPullRequestIsDeterministicAcrossDiscardedSourceData(t *testing.T) { + t.Parallel() + first := validProjection(t) + second := validProjection(t) + second.Response = mutateResponse(t, second.Response, func(response map[string]any) { + response["title"] = "different title" + response["body"] = "different body" + response["unknown_additive_field"] = map[string]any{"safe_to_drop": true} + }) + firstFacts, err := ProjectMergedPullRequest(first) + if err != nil { + t.Fatalf("first projection error = %v", err) + } + secondFacts, err := ProjectMergedPullRequest(second) + if err != nil { + t.Fatalf("second projection error = %v", err) + } + firstJSON, _ := json.Marshal(firstFacts) + secondJSON, _ := json.Marshal(secondFacts) + if !slices.Equal(firstJSON, secondJSON) { + t.Fatalf("discarded source fields changed projection\nfirst: %s\nsecond: %s", firstJSON, secondJSON) + } +} + +func TestProjectMergedPullRequestAbstainsForValidUnmergedResponses(t *testing.T) { + t.Parallel() + for _, state := range []string{"open", "closed"} { + state := state + t.Run(state, func(t *testing.T) { + t.Parallel() + input := validProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["state"] = state + response["draft"] = state == "open" + response["merged"] = false + response["merged_at"] = nil + // GitHub documents this as a test-merge SHA before merge. It must not create a fact. + response["merge_commit_sha"] = strings.Repeat("d", 40) + }) + facts, err := ProjectMergedPullRequest(input) + if err != nil { + t.Fatalf("ProjectMergedPullRequest() error = %v", err) + } + if len(facts) != 0 { + t.Fatalf("facts = %#v, want honest abstention", facts) + } + }) + } +} + +func TestProjectMergedPullRequestAcceptsSHA256CommitIdentifiers(t *testing.T) { + t.Parallel() + input := validProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["merge_commit_sha"] = strings.Repeat("1", 64) + response["head"] = map[string]any{"sha": strings.Repeat("2", 64)} + response["base"] = map[string]any{"sha": strings.Repeat("3", 64)} + }) + facts, err := ProjectMergedPullRequest(input) + if err != nil { + t.Fatalf("ProjectMergedPullRequest() error = %v", err) + } + if len(facts) != 1 { + t.Fatalf("fact count = %d, want 1", len(facts)) + } +} + +func TestProjectMergedPullRequestAllowsBoundedCollectionClockSkew(t *testing.T) { + t.Parallel() + input := validProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["merged_at"] = input.ObservedAt.UTC().Add(maxClockSkew).Format(time.RFC3339Nano) + }) + facts, err := ProjectMergedPullRequest(input) + if err != nil { + t.Fatalf("ProjectMergedPullRequest() error = %v", err) + } + if len(facts) != 1 { + t.Fatalf("fact count = %d, want 1", len(facts)) + } +} + +func TestProjectMergedPullRequestRejectsMalformedOrInconsistentEvidence(t *testing.T) { + t.Parallel() + valid := validProjection(t) + + tests := []struct { + name string + mutate func(*Projection) + }{ + {"missing workspace", func(input *Projection) { input.Workspace = "" }}, + {"padded workspace", func(input *Projection) { input.Workspace = " workspace-a" }}, + {"control workspace", func(input *Projection) { input.Workspace = "workspace-a\nother" }}, + {"oversized workspace", func(input *Projection) { input.Workspace = strings.Repeat("w", maxWorkspaceBytes+1) }}, + {"missing host", func(input *Projection) { input.Host = "" }}, + {"uppercase host", func(input *Projection) { input.Host = "GitHub.com" }}, + {"scheme in host", func(input *Projection) { input.Host = "https://github.com" }}, + {"port in host", func(input *Projection) { input.Host = "github.com:443" }}, + {"empty host label", func(input *Projection) { input.Host = "github..com" }}, + {"hyphen host edge", func(input *Projection) { input.Host = "-github.com" }}, + {"oversized host label", func(input *Projection) { input.Host = strings.Repeat("a", 64) + ".com" }}, + {"invalid owner", func(input *Projection) { input.Owner = "owner/other" }}, + {"dot owner", func(input *Projection) { input.Owner = "." }}, + {"oversized owner", func(input *Projection) { input.Owner = strings.Repeat("o", maxOwnerBytes+1) }}, + {"invalid repository", func(input *Projection) { input.Repository = "repo/other" }}, + {"dot repository", func(input *Projection) { input.Repository = ".." }}, + {"git suffix", func(input *Projection) { input.Repository = "sith.GIT" }}, + {"oversized repository", func(input *Projection) { input.Repository = strings.Repeat("r", maxRepositoryBytes+1) }}, + {"zero pull number", func(input *Projection) { input.PullNumber = 0 }}, + {"negative pull number", func(input *Projection) { input.PullNumber = -1 }}, + {"zero observed time", func(input *Projection) { input.ObservedAt = time.Time{} }}, + {"empty response", func(input *Projection) { input.Response = nil }}, + {"oversized response", func(input *Projection) { input.Response = bytes.Repeat([]byte(" "), maxResponseBytes+1) }}, + {"invalid UTF-8", func(input *Projection) { input.Response = []byte{'{', 0xff, '}'} }}, + {"malformed JSON", func(input *Projection) { input.Response = []byte(`{"number":`) }}, + {"null document", func(input *Projection) { input.Response = []byte(`null`) }}, + {"duplicate root member", func(input *Projection) { + input.Response = []byte(`{"number":42,"number":42,"state":"closed","draft":false,"merged":false}`) + }}, + {"duplicate discarded nested member", func(input *Projection) { + input.Response = []byte(`{"number":42,"state":"open","draft":false,"merged":false,"unknown":{"secret":"a","secret":"b"}}`) + }}, + {"trailing JSON", func(input *Projection) { input.Response = append(input.Response, []byte(` {}`)...) }}, + {"excessive nesting", func(input *Projection) { + input.Response = []byte(strings.Repeat("[", maxJSONDepth+1) + strings.Repeat("]", maxJSONDepth+1)) + }}, + {"missing number", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "number") })}, + {"null number", mutateProjectionResponse(t, func(response map[string]any) { response["number"] = nil })}, + {"fractional number", mutateProjectionResponse(t, func(response map[string]any) { response["number"] = 42.5 })}, + {"mismatched number", mutateProjectionResponse(t, func(response map[string]any) { response["number"] = 43 })}, + {"wrong-case number alias", mutateProjectionResponse(t, func(response map[string]any) { + delete(response, "number") + response["Number"] = 42 + })}, + {"missing state", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "state") })}, + {"invalid state", mutateProjectionResponse(t, func(response map[string]any) { response["state"] = "merged" })}, + {"non-string state", mutateProjectionResponse(t, func(response map[string]any) { response["state"] = true })}, + {"missing draft", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "draft") })}, + {"non-boolean draft", mutateProjectionResponse(t, func(response map[string]any) { response["draft"] = "false" })}, + {"missing merged", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "merged") })}, + {"non-boolean merged", mutateProjectionResponse(t, func(response map[string]any) { response["merged"] = "true" })}, + {"unmerged with merged_at", mutateProjectionResponse(t, func(response map[string]any) { response["merged"] = false })}, + {"merged open", mutateProjectionResponse(t, func(response map[string]any) { response["state"] = "open" })}, + {"merged draft", mutateProjectionResponse(t, func(response map[string]any) { response["draft"] = true })}, + {"missing merged_at", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "merged_at") })}, + {"null merged_at", mutateProjectionResponse(t, func(response map[string]any) { response["merged_at"] = nil })}, + {"non-string merged_at", mutateProjectionResponse(t, func(response map[string]any) { response["merged_at"] = 1 })}, + {"invalid merged_at", mutateProjectionResponse(t, func(response map[string]any) { response["merged_at"] = "yesterday" })}, + {"zero merged_at", mutateProjectionResponse(t, func(response map[string]any) { response["merged_at"] = "0001-01-01T00:00:00Z" })}, + {"future merged_at", mutateProjectionResponse(t, func(response map[string]any) { + response["merged_at"] = valid.ObservedAt.UTC().Add(maxClockSkew + time.Nanosecond).Format(time.RFC3339Nano) + })}, + {"missing head", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "head") })}, + {"null head", mutateProjectionResponse(t, func(response map[string]any) { response["head"] = nil })}, + {"non-object head", mutateProjectionResponse(t, func(response map[string]any) { response["head"] = "secret" })}, + {"missing head SHA", mutateProjectionResponse(t, func(response map[string]any) { response["head"] = map[string]any{} })}, + {"wrong-case head SHA alias", mutateProjectionResponse(t, func(response map[string]any) { + response["head"] = map[string]any{"SHA": headSHA} + })}, + {"control head SHA", mutateProjectionResponse(t, func(response map[string]any) { response["head"] = map[string]any{"sha": headSHA + "\n"} })}, + {"missing base", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "base") })}, + {"missing base SHA", mutateProjectionResponse(t, func(response map[string]any) { response["base"] = map[string]any{} })}, + {"missing merge SHA", mutateProjectionResponse(t, func(response map[string]any) { delete(response, "merge_commit_sha") })}, + {"null merge SHA", mutateProjectionResponse(t, func(response map[string]any) { response["merge_commit_sha"] = nil })}, + {"short merge SHA", mutateProjectionResponse(t, func(response map[string]any) { response["merge_commit_sha"] = "abc" })}, + {"uppercase merge SHA", mutateProjectionResponse(t, func(response map[string]any) { response["merge_commit_sha"] = strings.Repeat("A", 40) })}, + {"non-hex merge SHA", mutateProjectionResponse(t, func(response map[string]any) { response["merge_commit_sha"] = strings.Repeat("z", 40) })}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := valid + input.Response = slices.Clone(valid.Response) + test.mutate(&input) + if facts, err := ProjectMergedPullRequest(input); err == nil { + t.Fatalf("ProjectMergedPullRequest() facts = %#v, want rejection", facts) + } + }) + } +} + +func TestProjectMergedPullRequestErrorDoesNotEchoUntrustedValues(t *testing.T) { + t.Parallel() + input := validProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["merge_commit_sha"] = "do-not-echo-this-secret" + response["body"] = "also-secret" + }) + _, err := ProjectMergedPullRequest(input) + if err == nil { + t.Fatal("ProjectMergedPullRequest() accepted invalid merge SHA") + } + for _, secret := range []string{"do-not-echo-this-secret", "also-secret"} { + if strings.Contains(err.Error(), secret) { + t.Fatalf("error echoed untrusted value %q: %v", secret, err) + } + } +} + +func TestProjectMergedPullRequestReturnsBoundedFieldErrorForOverlongNumber(t *testing.T) { + t.Parallel() + literal := strings.Repeat("9", 64<<10) + input := validProjection(t) + input.Response = []byte(`{"number":` + literal + `,"state":"closed","draft":false,"merged":true}`) + _, err := ProjectMergedPullRequest(input) + if err == nil { + t.Fatal("ProjectMergedPullRequest() accepted overlong pull number") + } + want := "decode GitHub pull request response: number is invalid" + if err.Error() != want { + t.Fatalf("error = %q, want fixed field error %q", err, want) + } + if len(err.Error()) > 128 || strings.Contains(err.Error(), literal[:64]) { + t.Fatalf("error retained untrusted numeric literal: length=%d error=%q", len(err.Error()), err) + } +} + +func validProjection(t *testing.T) Projection { + t.Helper() + return Projection{ + Workspace: "workspace-a", Host: "github.example.com", Owner: "ArdurAI", Repository: "sith", PullNumber: 42, + ObservedAt: time.Date(2026, 7, 16, 17, 0, 0, 0, time.FixedZone("CDT", -5*60*60)), + Response: validMergedResponse(t), + } +} + +func validMergedResponse(t testing.TB) []byte { + t.Helper() + return encodeResponse(t, map[string]any{ + "number": 42, "state": "closed", "draft": false, "merged": true, + "merged_at": "2026-07-16T15:30:00.123-05:00", "merge_commit_sha": mergeSHA, + "head": map[string]any{ + "sha": headSHA, "label": "attacker-owner:head-secret", + "repo": map[string]any{"name": "attacker-repository", "temp_clone_token": "token-secret"}, + }, + "base": map[string]any{ + "sha": baseSHA, "label": "attacker-owner:base-secret", + "repo": map[string]any{"name": "attacker-repository", "url": "https://attacker.example/base"}, + }, + "url": "https://attacker.example/repos/attacker-owner/attacker-repository/pulls/42", + "title": "title-secret", "body": "body-secret\nwith-control", "user": map[string]any{"login": "user-secret"}, + "labels": []any{map[string]any{"name": "label-secret"}}, + }) +} + +func mutateProjectionResponse(t *testing.T, mutate func(map[string]any)) func(*Projection) { + t.Helper() + return func(input *Projection) { + input.Response = mutateResponse(t, input.Response, mutate) + } +} + +func mutateResponse(t *testing.T, document []byte, mutate func(map[string]any)) []byte { + t.Helper() + var response map[string]any + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := decoder.Decode(&response); err != nil { + t.Fatalf("decode test response: %v", err) + } + mutate(response) + return encodeResponse(t, response) +} + +func encodeResponse(t testing.TB, response map[string]any) []byte { + t.Helper() + encoded, err := json.Marshal(response) + if err != nil { + t.Fatalf("encode test response: %v", err) + } + return encoded +} + +func TestValidCommitSHARejectsAllOtherLengths(t *testing.T) { + t.Parallel() + for _, length := range []int{0, 1, 39, 41, 63, 65, 128} { + if validCommitSHA(strings.Repeat("a", length)) { + t.Fatalf("validCommitSHA() accepted %d characters", length) + } + } + for _, length := range []int{40, 64} { + if !validCommitSHA(strings.Repeat("0", length)) { + t.Fatalf("validCommitSHA() rejected %d lowercase hexadecimal characters", length) + } + } +} + +func FuzzProjectMergedPullRequestNeverEmitsInvalidOrUnboundedFacts(f *testing.F) { + f.Add(validMergedResponse(f)) + f.Add([]byte(`{"number":42,"state":"open","draft":false,"merged":false,"merged_at":null}`)) + f.Add([]byte(`{"number":42,"number":43}`)) + f.Add([]byte(`null`)) + f.Fuzz(func(t *testing.T, response []byte) { + input := Projection{ + Workspace: "workspace-a", Host: "github.com", Owner: "ArdurAI", Repository: "sith", PullNumber: 42, + ObservedAt: time.Date(2026, 7, 16, 21, 0, 0, 0, time.UTC), Response: response, + } + facts, err := ProjectMergedPullRequest(input) + if err != nil { + return + } + if len(facts) > 1 { + t.Fatalf("fact count = %d, want at most 1", len(facts)) + } + for _, fact := range facts { + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("emitted invalid fact: %v", err) + } + if fact.Entity != nil || fact.Lens != fleet.LensTimeline || fact.Fact.Kind != fleet.FactChange { + t.Fatalf("emitted fact outside contract: %#v", fact) + } + if len(fact.Fact.Observed) > maxFactPayloadBytes { + t.Fatalf("emitted %d-byte payload, limit %d", len(fact.Fact.Observed), maxFactPayloadBytes) + } + } + }) +} + +func ExampleProjectMergedPullRequest() { + response := []byte(fmt.Sprintf( + `{"number":42,"state":"closed","draft":false,"merged":true,"merged_at":"2026-07-16T20:30:00Z","merge_commit_sha":%q,"head":{"sha":%q},"base":{"sha":%q}}`, + mergeSHA, headSHA, baseSHA, + )) + facts, err := ProjectMergedPullRequest(Projection{ + Workspace: "workspace-a", Host: "github.com", Owner: "ArdurAI", Repository: "sith", PullNumber: 42, + ObservedAt: time.Date(2026, 7, 16, 21, 0, 0, 0, time.UTC), Response: response, + }) + fmt.Println(len(facts), err) + // Output: 1 +} diff --git a/internal/connector/github/workflow_run.go b/internal/connector/github/workflow_run.go new file mode 100644 index 0000000..2d85365 --- /dev/null +++ b/internal/connector/github/workflow_run.go @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // WorkflowRunProtocolVersion identifies the normalized Get-a-workflow-run fact contract. + WorkflowRunProtocolVersion = "workflow-runs/" + APIVersion + + workflowRunFailureKind = "workflow-run-failed" +) + +// WorkflowRunProjection supplies one already-authorized GitHub Get-a-workflow-run response. +// ProjectFailedWorkflowRun performs no discovery, network access, credential loading, persistence, +// repository-to-workload correlation, or mutation. +type WorkflowRunProjection struct { + Workspace string + Host string + Owner string + Repository string + RunID int64 + ObservedAt time.Time + Response []byte +} + +type workflowRunResponse struct { + ID *int64 `json:"id"` + WorkflowID *int64 `json:"workflow_id"` + RunAttempt *int64 `json:"run_attempt"` + Status *string `json:"status"` + Conclusion *string `json:"conclusion"` + UpdatedAt *string `json:"updated_at"` + Repository *workflowRunRepository `json:"repository"` +} + +type workflowRunRepository struct { + FullName *string `json:"full_name"` +} + +func (response *workflowRunResponse) UnmarshalJSON(document []byte) error { + type exactField struct { + name string + target any + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(document, &fields); err != nil { + return fmt.Errorf("workflow-run response must be a JSON object") + } + for _, field := range []exactField{ + {name: "id", target: &response.ID}, + {name: "workflow_id", target: &response.WorkflowID}, + {name: "run_attempt", target: &response.RunAttempt}, + {name: "status", target: &response.Status}, + {name: "conclusion", target: &response.Conclusion}, + {name: "updated_at", target: &response.UpdatedAt}, + {name: "repository", target: &response.Repository}, + } { + value, exists := fields[field.name] + if !exists { + continue + } + if err := json.Unmarshal(value, field.target); err != nil { + return fmt.Errorf("%s is invalid", field.name) + } + } + return nil +} + +func (repository *workflowRunRepository) UnmarshalJSON(document []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(document, &fields); err != nil { + return fmt.Errorf("workflow-run repository must be a JSON object") + } + value, exists := fields["full_name"] + if !exists { + return nil + } + if err := json.Unmarshal(value, &repository.FullName); err != nil { + return fmt.Errorf("full_name is invalid") + } + return nil +} + +type workflowRunObservation struct { + RunID int64 `json:"run_id"` + WorkflowID int64 `json:"workflow_id"` + RunAttempt int64 `json:"run_attempt"` + ChangeKind string `json:"change_kind"` + Conclusion string `json:"conclusion"` + EventAt time.Time `json:"event_at"` +} + +// ProjectFailedWorkflowRun returns one deterministic, bounded TIMELINE fact only when GitHub +// reports an internally consistent completed workflow run with an explicit failure conclusion. +// Valid incomplete and non-failure completed runs abstain with no facts. +func ProjectFailedWorkflowRun(input WorkflowRunProjection) ([]fleet.GraphFact, error) { + if err := validateWorkflowRunProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode GitHub workflow-run response: %w", err) + } + + var response workflowRunResponse + decoder := json.NewDecoder(bytes.NewReader(input.Response)) + if err := decoder.Decode(&response); err != nil { + return nil, fmt.Errorf("decode GitHub workflow-run response: %w", err) + } + observation, failed, err := failedWorkflowRunObservation(response, input) + if err != nil { + return nil, err + } + if !failed { + return []fleet.GraphFact{}, nil + } + + encoded, err := json.Marshal(observation) + if err != nil { + return nil, fmt.Errorf("encode GitHub workflow-run change fact: %w", err) + } + if len(encoded) > maxFactPayloadBytes { + return nil, fmt.Errorf("GitHub workflow-run change fact exceeds %d encoded bytes", maxFactPayloadBytes) + } + + resourceName := workflowRunResourceName(input.Repository, input.RunID, observation.RunAttempt) + nativeID := input.Owner + "/" + resourceName + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, + Scope: input.Host, + Kind: "WorkflowRun", + Namespace: input.Owner, + Name: resourceName, + }, + Kind: fleet.FactChange, + Observed: encoded, + ObservedAt: observation.EventAt, + Source: input.Host, + Provenance: fleet.Provenance{ + Adapter: Kind, ProtocolV: WorkflowRunProtocolVersion, NativeID: nativeID, + }, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensTimeline, + } + if err := fact.Validate(input.Workspace); err != nil { + return nil, fmt.Errorf("validate GitHub workflow-run change fact: %w", err) + } + return []fleet.GraphFact{fact}, nil +} + +func validateWorkflowRunProjection(input WorkflowRunProjection) error { + if err := validateText("workspace", input.Workspace, maxWorkspaceBytes); err != nil { + return err + } + if err := validateHost(input.Host); err != nil { + return err + } + if err := validatePathComponent("owner", input.Owner, maxOwnerBytes); err != nil { + return err + } + if err := validatePathComponent("repository", input.Repository, maxRepositoryBytes); err != nil { + return err + } + if strings.HasSuffix(strings.ToLower(input.Repository), ".git") { + return fmt.Errorf("repository must not include the .git suffix") + } + if input.RunID <= 0 { + return fmt.Errorf("workflow run ID must be positive") + } + if len(workflowRunResourceName(input.Repository, input.RunID, 1)) > maxResourceName { + return fmt.Errorf("workflow-run resource name exceeds %d bytes", maxResourceName) + } + if input.ObservedAt.IsZero() { + return fmt.Errorf("projection observation time is required") + } + if len(input.Response) == 0 { + return fmt.Errorf("GitHub workflow-run response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("GitHub workflow-run response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("GitHub workflow-run response must be valid UTF-8") + } + return nil +} + +func failedWorkflowRunObservation( + response workflowRunResponse, + input WorkflowRunProjection, +) (workflowRunObservation, bool, error) { + if response.ID == nil || *response.ID != input.RunID { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run ID does not match trusted caller identity") + } + if response.WorkflowID == nil || *response.WorkflowID <= 0 { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow ID must be positive") + } + if response.RunAttempt == nil || *response.RunAttempt <= 0 { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run attempt must be positive") + } + if response.Repository == nil || response.Repository.FullName == nil || + !validWorkflowRunRepositoryIdentity(*response.Repository.FullName, input.Owner, input.Repository) { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run repository does not match trusted caller identity") + } + if response.Status == nil { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run status is required") + } + switch *response.Status { + case "queued", "in_progress", "requested", "waiting", "pending", "completed": + default: + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run status %q is unsupported", *response.Status) + } + if response.UpdatedAt == nil || *response.UpdatedAt == "" { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run updated_at is required") + } + eventAt, err := time.Parse(time.RFC3339Nano, *response.UpdatedAt) + if err != nil || eventAt.IsZero() { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run updated_at must be a non-zero RFC3339 timestamp") + } + eventAt = eventAt.UTC() + if eventAt.After(input.ObservedAt.UTC().Add(maxClockSkew)) { + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run update time exceeds allowed collection clock skew") + } + + if *response.Status != "completed" { + if response.Conclusion != nil { + return workflowRunObservation{}, false, fmt.Errorf("incomplete GitHub workflow run must not have a conclusion") + } + return workflowRunObservation{}, false, nil + } + if response.Conclusion == nil { + return workflowRunObservation{}, false, fmt.Errorf("completed GitHub workflow-run conclusion is required") + } + failed := false + switch *response.Conclusion { + case "failure", "timed_out", "startup_failure": + failed = true + case "action_required", "cancelled", "neutral", "skipped", "stale", "success": //nolint:misspell // GitHub's wire value uses British spelling. + default: + return workflowRunObservation{}, false, fmt.Errorf("GitHub workflow-run conclusion %q is unsupported", *response.Conclusion) + } + if !failed { + return workflowRunObservation{}, false, nil + } + + return workflowRunObservation{ + RunID: *response.ID, + WorkflowID: *response.WorkflowID, + RunAttempt: *response.RunAttempt, + ChangeKind: workflowRunFailureKind, + Conclusion: *response.Conclusion, + EventAt: eventAt, + }, true, nil +} + +func validWorkflowRunRepositoryIdentity(fullName, owner, repository string) bool { + responseOwner, responseRepository, found := strings.Cut(fullName, "/") + if !found || strings.Contains(responseRepository, "/") || + validatePathComponent("response owner", responseOwner, maxOwnerBytes) != nil || + validatePathComponent("response repository", responseRepository, maxRepositoryBytes) != nil || + strings.HasSuffix(strings.ToLower(responseRepository), ".git") { + return false + } + return strings.EqualFold(responseOwner, owner) && strings.EqualFold(responseRepository, repository) +} + +func workflowRunResourceName(repository string, runID, attempt int64) string { + return repository + "#" + strconv.FormatInt(runID, 10) + "-attempt-" + strconv.FormatInt(attempt, 10) +} diff --git a/internal/connector/github/workflow_run_test.go b/internal/connector/github/workflow_run_test.go new file mode 100644 index 0000000..72803f5 --- /dev/null +++ b/internal/connector/github/workflow_run_test.go @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const testWorkflowRunID int64 = 30433642 + +func TestProjectFailedWorkflowRunEmitsSanitizedUnattachedTimelineFact(t *testing.T) { + t.Parallel() + for _, conclusion := range []string{"failure", "timed_out", "startup_failure"} { + conclusion := conclusion + t.Run(conclusion, func(t *testing.T) { + t.Parallel() + input := validWorkflowRunProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["conclusion"] = conclusion + }) + facts, err := ProjectFailedWorkflowRun(input) + if err != nil { + t.Fatalf("ProjectFailedWorkflowRun() error = %v", err) + } + if len(facts) != 1 { + t.Fatalf("fact count = %d, want 1", len(facts)) + } + fact := facts[0] + wantEventAt := time.Date(2026, 7, 18, 19, 55, 0, 123000000, time.UTC) + if fact.Entity != nil || fact.Fact.Kind != fleet.FactChange || fact.Lens != fleet.LensTimeline { + t.Fatalf("unexpected graph contract: %#v", fact) + } + wantRef := fleet.ResourceRef{ + SourceKind: Kind, + Scope: "github.example.com", + Kind: "WorkflowRun", + Namespace: "ArdurAI", + Name: "sith#30433642-attempt-2", + } + if fact.Fact.Workspace != "workspace-a" || !fact.Fact.Ref.Equal(wantRef) || fact.Fact.Ref.Attributes != nil { + t.Fatalf("unexpected trusted identity: %#v", fact.Fact) + } + wantProvenance := fleet.Provenance{ + Adapter: Kind, ProtocolV: WorkflowRunProtocolVersion, + NativeID: "ArdurAI/sith#30433642-attempt-2", + } + if fact.Fact.Source != "github.example.com" || !fact.Fact.ObservedAt.Equal(wantEventAt) || + fact.Fact.Provenance != wantProvenance { + t.Fatalf("unexpected evidence metadata: %#v", fact.Fact) + } + var observation workflowRunObservation + if err := json.Unmarshal(fact.Fact.Observed, &observation); err != nil { + t.Fatalf("decode observation: %v", err) + } + wantObservation := workflowRunObservation{ + RunID: testWorkflowRunID, WorkflowID: 159038, RunAttempt: 2, + ChangeKind: workflowRunFailureKind, Conclusion: conclusion, EventAt: wantEventAt, + } + if observation != wantObservation { + t.Fatalf("observation = %#v, want %#v", observation, wantObservation) + } + if len(fact.Fact.Observed) > maxFactPayloadBytes { + t.Fatalf("encoded fact is %d bytes, limit %d", len(fact.Fact.Observed), maxFactPayloadBytes) + } + + encoded, err := json.Marshal(facts) + if err != nil { + t.Fatalf("marshal facts: %v", err) + } + for _, forbidden := range []string{ + "workflow-title-secret", "branch-secret", "commit-message-secret", "actor-secret", + "token-secret", "logs-secret", "jobs-secret", "artifact-secret", "attacker.example", + } { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("projected fact retained forbidden response value %q: %s", forbidden, encoded) + } + } + graph, err := fleet.NewGraph(input.Workspace, facts) + if err != nil { + t.Fatalf("NewGraph() error = %v", err) + } + if len(graph.Nodes) != 0 || len(graph.Unattached) != 1 { + t.Fatalf("graph nodes/unattached = %d/%d, want 0/1", len(graph.Nodes), len(graph.Unattached)) + } + }) + } + if WorkflowRunProtocolVersion != "workflow-runs/2026-03-10" { + t.Fatalf("workflow-run protocol version = %q", WorkflowRunProtocolVersion) + } +} + +func TestProjectFailedWorkflowRunIsDeterministicAcrossDiscardedSourceData(t *testing.T) { + t.Parallel() + first := validWorkflowRunProjection(t) + second := validWorkflowRunProjection(t) + second.Response = mutateResponse(t, second.Response, func(response map[string]any) { + response["name"] = "different discarded name" + response["display_title"] = "different discarded title" + response["head_branch"] = "different-discarded-branch" + response["ID"] = 7 + response["STATUS"] = "queued" + response["repository"].(map[string]any)["FULL_NAME"] = "attacker/other" + response["unknown_additive_field"] = map[string]any{"safe_to_drop": true} + }) + firstFacts, err := ProjectFailedWorkflowRun(first) + if err != nil { + t.Fatalf("first projection error = %v", err) + } + secondFacts, err := ProjectFailedWorkflowRun(second) + if err != nil { + t.Fatalf("second projection error = %v", err) + } + firstJSON, _ := json.Marshal(firstFacts) + secondJSON, _ := json.Marshal(secondFacts) + if !slices.Equal(firstJSON, secondJSON) { + t.Fatalf("discarded source fields changed projection\nfirst: %s\nsecond: %s", firstJSON, secondJSON) + } +} + +func TestProjectFailedWorkflowRunAbstainsForNonFailureRuns(t *testing.T) { + t.Parallel() + tests := make(map[string]func(map[string]any)) + for _, status := range []string{"queued", "in_progress", "requested", "waiting", "pending"} { + status := status + tests[status] = func(response map[string]any) { + response["status"] = status + response["conclusion"] = nil + } + } + for _, conclusion := range []string{ + "action_required", "cancelled", "neutral", "skipped", "stale", "success", //nolint:misspell // GitHub's wire value uses British spelling. + } { + conclusion := conclusion + tests["completed-"+conclusion] = func(response map[string]any) { + response["status"] = "completed" + response["conclusion"] = conclusion + } + } + for name, mutate := range tests { + name, mutate := name, mutate + t.Run(name, func(t *testing.T) { + t.Parallel() + input := validWorkflowRunProjection(t) + input.Response = mutateResponse(t, input.Response, mutate) + facts, err := ProjectFailedWorkflowRun(input) + if err != nil { + t.Fatalf("ProjectFailedWorkflowRun() error = %v", err) + } + if len(facts) != 0 { + t.Fatalf("facts = %#v, want honest abstention", facts) + } + }) + } +} + +func TestProjectFailedWorkflowRunAcceptsGitHubCaseInsensitiveRepositoryIdentity(t *testing.T) { + t.Parallel() + input := validWorkflowRunProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["repository"].(map[string]any)["full_name"] = "ardurai/SITH" + }) + facts, err := ProjectFailedWorkflowRun(input) + if err != nil || len(facts) != 1 { + t.Fatalf("ProjectFailedWorkflowRun() = %#v, %v, want one fact", facts, err) + } +} + +func TestProjectFailedWorkflowRunRejectsNonCanonicalResponseRepositoryIdentity(t *testing.T) { + t.Parallel() + for _, fullName := range []string{ + "ArdurAI/sith/other", "ArdurAI/sith.git", "ArdurAI/.", "ArdurAI/..", "ArdurAI/sith secret", + "ArdurA\u212a/sith", + } { + fullName := fullName + t.Run(fullName, func(t *testing.T) { + t.Parallel() + input := validWorkflowRunProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["repository"].(map[string]any)["full_name"] = fullName + }) + if _, err := ProjectFailedWorkflowRun(input); err == nil { + t.Fatal("ProjectFailedWorkflowRun() error = nil") + } + }) + } +} + +func TestProjectFailedWorkflowRunRejectsMalformedOrInconsistentEvidence(t *testing.T) { + t.Parallel() + valid := validWorkflowRunProjection(t) + future := valid.ObservedAt.Add(maxClockSkew + time.Nanosecond).Format(time.RFC3339Nano) + tests := []struct { + name string + mutate func(*WorkflowRunProjection) + }{ + {name: "missing workspace", mutate: func(input *WorkflowRunProjection) { input.Workspace = "" }}, + {name: "invalid host", mutate: func(input *WorkflowRunProjection) { input.Host = "https://github.com" }}, + {name: "invalid owner", mutate: func(input *WorkflowRunProjection) { input.Owner = "owner/other" }}, + {name: "invalid repository", mutate: func(input *WorkflowRunProjection) { input.Repository = "sith.git" }}, + {name: "invalid run ID", mutate: func(input *WorkflowRunProjection) { input.RunID = 0 }}, + {name: "missing observation time", mutate: func(input *WorkflowRunProjection) { input.ObservedAt = time.Time{} }}, + {name: "missing response", mutate: func(input *WorkflowRunProjection) { input.Response = nil }}, + {name: "oversized response", mutate: func(input *WorkflowRunProjection) { input.Response = bytes.Repeat([]byte("x"), maxResponseBytes+1) }}, + {name: "invalid UTF-8", mutate: func(input *WorkflowRunProjection) { input.Response = []byte{0xff} }}, + {name: "null response", mutate: func(input *WorkflowRunProjection) { input.Response = []byte("null") }}, + {name: "duplicate member", mutate: func(input *WorkflowRunProjection) { input.Response = []byte(`{"id":30433642,"id":30433643}`) }}, + {name: "mismatched run ID", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["id"] = 7 })}, + {name: "mixed-case run ID alias", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { + delete(response, "id") + response["ID"] = testWorkflowRunID + })}, + {name: "missing workflow ID", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { delete(response, "workflow_id") })}, + {name: "zero workflow ID", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["workflow_id"] = 0 })}, + {name: "missing attempt", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { delete(response, "run_attempt") })}, + {name: "zero attempt", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["run_attempt"] = 0 })}, + {name: "missing repository", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { delete(response, "repository") })}, + {name: "mixed-case repository full-name alias", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { + repository := response["repository"].(map[string]any) + delete(repository, "full_name") + repository["FULL_NAME"] = "ArdurAI/sith" + })}, + {name: "mismatched repository", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { + response["repository"].(map[string]any)["full_name"] = "ArdurAI/other" + })}, + {name: "missing status", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { delete(response, "status") })}, + {name: "mixed-case status alias", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { + delete(response, "status") + response["STATUS"] = "completed" + })}, + {name: "unknown status", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["status"] = "running" })}, + {name: "incomplete with conclusion", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["status"] = "queued" })}, + {name: "completed without conclusion", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["conclusion"] = nil })}, + {name: "unknown conclusion", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["conclusion"] = "soft_failure" })}, + {name: "missing update time", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { delete(response, "updated_at") })}, + {name: "malformed update time", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["updated_at"] = "not-a-time" })}, + {name: "future update time", mutate: mutateWorkflowRunResponse(t, func(response map[string]any) { response["updated_at"] = future })}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := valid + input.Response = append([]byte(nil), valid.Response...) + test.mutate(&input) + if _, err := ProjectFailedWorkflowRun(input); err == nil { + t.Fatal("ProjectFailedWorkflowRun() error = nil") + } + }) + } +} + +func TestProjectFailedWorkflowRunAllowsBoundedCollectionClockSkew(t *testing.T) { + t.Parallel() + input := validWorkflowRunProjection(t) + input.Response = mutateResponse(t, input.Response, func(response map[string]any) { + response["updated_at"] = input.ObservedAt.Add(maxClockSkew).Format(time.RFC3339Nano) + }) + facts, err := ProjectFailedWorkflowRun(input) + if err != nil || len(facts) != 1 { + t.Fatalf("ProjectFailedWorkflowRun() = %#v, %v, want one fact", facts, err) + } +} + +func validWorkflowRunProjection(t testing.TB) WorkflowRunProjection { + t.Helper() + return WorkflowRunProjection{ + Workspace: "workspace-a", Host: "github.example.com", Owner: "ArdurAI", Repository: "sith", + RunID: testWorkflowRunID, ObservedAt: time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC), + Response: validFailedWorkflowRunResponse(t), + } +} + +func validFailedWorkflowRunResponse(t testing.TB) []byte { + t.Helper() + return encodeResponse(t, map[string]any{ + "id": testWorkflowRunID, "workflow_id": 159038, "run_attempt": 2, + "status": "completed", "conclusion": "failure", "updated_at": "2026-07-18T14:55:00.123-05:00", + "repository": map[string]any{ + "full_name": "ArdurAI/sith", "description": "repository-metadata-secret", + "url": "https://attacker.example/repository", + }, + "name": "workflow-title-secret", "display_title": "commit-message-secret", + "head_branch": "branch-secret", "head_sha": strings.Repeat("a", 40), + "actor": map[string]any{"login": "actor-secret", "token": "token-secret"}, + "logs_url": "https://attacker.example/logs-secret", + "jobs_url": "https://attacker.example/jobs-secret", + "artifacts_url": "https://attacker.example/artifact-secret", + }) +} + +func mutateWorkflowRunResponse(t *testing.T, mutate func(map[string]any)) func(*WorkflowRunProjection) { + t.Helper() + return func(input *WorkflowRunProjection) { + input.Response = mutateResponse(t, input.Response, mutate) + } +} + +func FuzzProjectFailedWorkflowRunNeverEmitsInvalidOrUnboundedFacts(f *testing.F) { + f.Add(validFailedWorkflowRunResponse(f)) + f.Add([]byte(`{"id":30433642,"workflow_id":1,"run_attempt":1,"status":"queued","conclusion":null,"updated_at":"2026-07-18T19:55:00Z","repository":{"full_name":"ArdurAI/sith"}}`)) + f.Add([]byte(`{"id":30433642,"id":7}`)) + f.Add([]byte(`null`)) + f.Fuzz(func(t *testing.T, response []byte) { + input := WorkflowRunProjection{ + Workspace: "workspace-a", Host: "github.com", Owner: "ArdurAI", Repository: "sith", + RunID: testWorkflowRunID, ObservedAt: time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC), + Response: response, + } + facts, err := ProjectFailedWorkflowRun(input) + if err != nil { + return + } + if len(facts) > 1 { + t.Fatalf("fact count = %d, want at most 1", len(facts)) + } + for _, fact := range facts { + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("emitted invalid fact: %v", err) + } + if fact.Entity != nil || fact.Lens != fleet.LensTimeline || fact.Fact.Kind != fleet.FactChange { + t.Fatalf("emitted fact outside contract: %#v", fact) + } + if len(fact.Fact.Observed) > maxFactPayloadBytes { + t.Fatalf("emitted %d-byte payload, limit %d", len(fact.Fact.Observed), maxFactPayloadBytes) + } + } + }) +} + +func ExampleProjectFailedWorkflowRun() { + response := []byte(`{"id":30433642,"workflow_id":159038,"run_attempt":1,"status":"completed","conclusion":"failure","updated_at":"2026-07-18T19:55:00Z","repository":{"full_name":"ArdurAI/sith"}}`) + facts, err := ProjectFailedWorkflowRun(WorkflowRunProjection{ + Workspace: "workspace-a", Host: "github.com", Owner: "ArdurAI", Repository: "sith", + RunID: testWorkflowRunID, ObservedAt: time.Date(2026, 7, 18, 20, 0, 0, 0, time.UTC), + Response: response, + }) + fmt.Println(len(facts), err) + // Output: 1 +} + +func TestWorkflowRunObservationShapeIsClosed(t *testing.T) { + t.Parallel() + typeOf := reflect.TypeOf(workflowRunObservation{}) + want := []string{"run_id", "workflow_id", "run_attempt", "change_kind", "conclusion", "event_at"} + got := make([]string, 0, typeOf.NumField()) + for index := 0; index < typeOf.NumField(); index++ { + got = append(got, strings.Split(typeOf.Field(index).Tag.Get("json"), ",")[0]) + } + if !slices.Equal(got, want) { + t.Fatalf("workflow-run observation fields = %q, want %q", got, want) + } +} diff --git a/internal/connector/kubeconfig/adapter.go b/internal/connector/kubeconfig/adapter.go index dd2869d..44c7b57 100644 --- a/internal/connector/kubeconfig/adapter.go +++ b/internal/connector/kubeconfig/adapter.go @@ -48,10 +48,19 @@ var supportedKinds = []string{ type probeFunc func(ctx context.Context, config *rest.Config) error type dynamicFactory func(config *rest.Config) (dynamic.Interface, error) type resourceResolver func(ctx context.Context, config *rest.Config, kind string) (resourceSpec, error) + +type tableRequest struct { + namespace string + name string + labelSelector string + rowBudget int + retainKeys map[string]struct{} +} + type tablePrinter func( ctx context.Context, spec resourceSpec, - namespace, name, labelSelector string, + request tableRequest, ) (map[string][]fleet.DisplayField, error) type tableFactory func(config *rest.Config) (tablePrinter, error) @@ -209,6 +218,7 @@ func withTableFactory(factory tableFactory) Option { // Adapter discovers contexts and performs independent local client-go reads. type Adapter struct { settings options + gate *operationGate mu sync.RWMutex discovered bool @@ -270,6 +280,7 @@ func defaultOptions() options { func newAdapter(settings options) *Adapter { return &Adapter{ settings: settings, + gate: newOperationGate(), scopes: make(map[string]connector.Scope), clients: make(map[string]dynamic.Interface), watchers: make(map[string]dynamic.Interface), @@ -293,11 +304,12 @@ func (*Adapter) Capabilities() []connector.Capability { // Descriptor returns immutable registration metadata for this adapter. func (adapter *Adapter) Descriptor() connector.Descriptor { return connector.Descriptor{ - Kind: adapter.Kind(), - ConnKind: connector.KindReadAdapter, - ProtocolV: protocolVersion, - Owner: "sith-core", - Capabilities: adapter.Capabilities(), + Kind: adapter.Kind(), + ConnKind: connector.KindReadAdapter, + WireVersions: []connector.WireVersion{connector.CurrentWireVersion()}, + AdapterVersion: protocolVersion, + Owner: "sith-core", + Capabilities: adapter.Capabilities(), } } @@ -413,7 +425,7 @@ func (adapter *Adapter) probeContext( probeConfig := rest.CopyConfig(restConfig) probeConfig.Timeout = adapter.settings.probeTimeout - _, err = callWithTimeout(ctx, adapter.settings.probeTimeout, func(probeCtx context.Context) (struct{}, error) { + _, err = callWithTimeout(ctx, adapter.gate, name, adapter.settings.probeTimeout, func(probeCtx context.Context) (struct{}, error) { return struct{}{}, adapter.settings.probe(probeCtx, probeConfig) }) if err != nil { @@ -470,17 +482,76 @@ type operationResult[T any] struct { err error } +// operationGate bounds work that can outlive its caller because client-go transports do not all +// honor context cancellation. Each kubeconfig context is isolated to two retained operations, so +// repeated calls cannot accumulate while unrelated contexts continue independently. +type operationGate struct { + mu sync.Mutex + byScope map[string]int + changed chan struct{} +} + +func newOperationGate() *operationGate { + return &operationGate{byScope: make(map[string]int), changed: make(chan struct{})} +} + +func (gate *operationGate) acquire(ctx context.Context, scope string) error { + for { + gate.mu.Lock() + if err := ctx.Err(); err != nil { + gate.mu.Unlock() + return err + } + if gate.byScope[scope] < 2 { + gate.byScope[scope]++ + gate.mu.Unlock() + return nil + } + changed := gate.changed + gate.mu.Unlock() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-changed: + } + } +} + +func (gate *operationGate) release(scope string) { + gate.mu.Lock() + gate.byScope[scope]-- + if gate.byScope[scope] == 0 { + delete(gate.byScope, scope) + } + close(gate.changed) + gate.changed = make(chan struct{}) + gate.mu.Unlock() +} + func callWithTimeout[T any]( ctx context.Context, + gate *operationGate, + scope string, timeout time.Duration, operation func(context.Context) (T, error), ) (T, error) { operationCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + if err := gate.acquire(operationCtx, scope); err != nil { + var zero T + return zero, err + } + if err := operationCtx.Err(); err != nil { + gate.release(scope) + var zero T + return zero, err + } result := make(chan operationResult[T], 1) // client-go's exec authenticator uses exec.Command rather than CommandContext. Isolating the // call keeps one auth helper that ignores cancellation from stalling the rest of the fleet. go func() { + defer gate.release(scope) value, err := operation(operationCtx) result <- operationResult[T]{value: value, err: err} }() diff --git a/internal/connector/kubeconfig/adapter_test.go b/internal/connector/kubeconfig/adapter_test.go index e6d9b69..6df94b1 100644 --- a/internal/connector/kubeconfig/adapter_test.go +++ b/internal/connector/kubeconfig/adapter_test.go @@ -77,6 +77,7 @@ func TestGenericResourceResolutionIsCached(t *testing.T) { }}, ) var resolveCalls atomic.Int32 + var tableRequests []tableRequest adapter, err := New( WithLoadingRules(testLoadingRules(t, testConfig("alpha"))), withProbe(func(_ context.Context, _ *rest.Config) error { return nil }), @@ -90,8 +91,9 @@ func TestGenericResourceResolutionIsCached(t *testing.T) { }), withTableFactory(func(_ *rest.Config) (tablePrinter, error) { return func( - _ context.Context, _ resourceSpec, _, _, _ string, + _ context.Context, _ resourceSpec, request tableRequest, ) (map[string][]fleet.DisplayField, error) { + tableRequests = append(tableRequests, request) return map[string][]fleet.DisplayField{ tableObjectKey("apps", "sample"): {{Name: "Name", Value: "sample"}, {Name: "Ready", Value: "1/1"}}, }, nil @@ -116,6 +118,17 @@ func TestGenericResourceResolutionIsCached(t *testing.T) { if resolveCalls.Load() != 1 { t.Fatalf("resolver calls = %d, want one cached resolution", resolveCalls.Load()) } + if len(tableRequests) != 2 { + t.Fatalf("table requests = %#v, want query-budget retention keys", tableRequests) + } + for index, request := range tableRequests { + if request.rowBudget != 1 || len(request.retainKeys) != 1 { + t.Fatalf("table request %d = %#v, want query-budget retention keys", index, request) + } + if _, ok := request.retainKeys[tableObjectKey("apps", "sample")]; !ok { + t.Fatalf("table request %d retain keys = %#v, want only the selected fact", index, request.retainKeys) + } + } evidence, err := adapter.Read(context.Background(), result.Facts[0].Ref) if err != nil { @@ -129,10 +142,25 @@ func TestGenericResourceResolutionIsCached(t *testing.T) { func TestWatchStreamsSnapshotUpsertAndDelete(t *testing.T) { t.Parallel() client := fakeClient(pod("api-0", "apps", "registry/api:v1", nil)) + var bootstrapOptionsMu sync.Mutex + var bootstrapOptions []metav1.ListOptions + recordingClient := listOptionsRecordingClient{ + Interface: client, + record: func(options metav1.ListOptions) { + bootstrapOptionsMu.Lock() + bootstrapOptions = append(bootstrapOptions, options) + bootstrapOptionsMu.Unlock() + }, + } + client.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + list := podList("", pod("api-0", "apps", "registry/api:v1", nil)) + list.SetResourceVersion("2") + return true, list, nil + }) adapter, err := New( WithLoadingRules(testLoadingRules(t, testConfig("alpha"))), withProbe(func(_ context.Context, _ *rest.Config) error { return nil }), - withDynamicFactory(func(_ *rest.Config) (dynamic.Interface, error) { return client, nil }), + withDynamicFactory(func(_ *rest.Config) (dynamic.Interface, error) { return recordingClient, nil }), ) if err != nil { t.Fatalf("New() error = %v", err) @@ -145,9 +173,18 @@ func TestWatchStreamsSnapshotUpsertAndDelete(t *testing.T) { } snapshot := receiveWatchEvent(ctx, t, events) if snapshot.Type != connector.WatchSnapshot || snapshot.Scope != "alpha" || len(snapshot.Facts) != 1 { - t.Fatalf("snapshot = %#v", snapshot) + t.Fatalf("snapshot = %#v, error = %v", snapshot, snapshot.Err) } waitForWatchAction(ctx, t, client) + bootstrapOptionsMu.Lock() + observedOptions := append([]metav1.ListOptions(nil), bootstrapOptions...) + bootstrapOptionsMu.Unlock() + if !slices.ContainsFunc(observedOptions, func(options metav1.ListOptions) bool { + return options.Limit == watchBootstrapPageSize && options.Continue == "" + }) { + t.Fatalf("watch list options = %#v, want bounded bootstrap request", observedOptions) + } + assertWatchResourceVersion(t, client, "2") created := pod("api-1", "apps", "registry/api:v2", nil) if _, err := client.Resource(schema.GroupVersionResource{Version: "v1", Resource: "pods"}). @@ -168,6 +205,53 @@ func TestWatchStreamsSnapshotUpsertAndDelete(t *testing.T) { } } +type listOptionsRecordingClient struct { + dynamic.Interface + record func(metav1.ListOptions) +} + +func (client listOptionsRecordingClient) Resource( + resource schema.GroupVersionResource, +) dynamic.NamespaceableResourceInterface { + return &listOptionsRecordingNamespaceable{ + NamespaceableResourceInterface: client.Interface.Resource(resource), + record: client.record, + } +} + +type listOptionsRecordingNamespaceable struct { + dynamic.NamespaceableResourceInterface + record func(metav1.ListOptions) +} + +func (resource *listOptionsRecordingNamespaceable) Namespace(namespace string) dynamic.ResourceInterface { + return &listOptionsRecordingResource{ + ResourceInterface: resource.NamespaceableResourceInterface.Namespace(namespace), + record: resource.record, + } +} + +func (resource *listOptionsRecordingNamespaceable) List( + ctx context.Context, + options metav1.ListOptions, +) (*unstructured.UnstructuredList, error) { + resource.record(options) + return resource.NamespaceableResourceInterface.List(ctx, options) +} + +type listOptionsRecordingResource struct { + dynamic.ResourceInterface + record func(metav1.ListOptions) +} + +func (resource *listOptionsRecordingResource) List( + ctx context.Context, + options metav1.ListOptions, +) (*unstructured.UnstructuredList, error) { + resource.record(options) + return resource.ResourceInterface.List(ctx, options) +} + func TestDiscoverIsIndependentAndPreservesLastSeen(t *testing.T) { t.Parallel() firstObserved := time.Date(2026, time.July, 10, 12, 0, 0, 0, time.UTC) @@ -234,6 +318,7 @@ func TestDiscoverIsIndependentAndPreservesLastSeen(t *testing.T) { func TestDiscoverTimesOutProbeThatIgnoresContext(t *testing.T) { t.Parallel() release := make(chan struct{}) + t.Cleanup(func() { close(release) }) adapter, err := New( WithLoadingRules(testLoadingRules(t, testConfig("blocked"))), WithProbeTimeout(20*time.Millisecond), @@ -247,7 +332,6 @@ func TestDiscoverTimesOutProbeThatIgnoresContext(t *testing.T) { } started := time.Now() discovery, err := adapter.Discover(context.Background()) - close(release) if err != nil { t.Fatalf("Discover() error = %v", err) } @@ -259,6 +343,52 @@ func TestDiscoverTimesOutProbeThatIgnoresContext(t *testing.T) { } } +func TestDiscoverBoundsProbeThatIgnoresContextWithoutStallingPeers(t *testing.T) { + t.Parallel() + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + var blockedCalls atomic.Int32 + var peerCalls atomic.Int32 + adapter, err := New( + WithLoadingRules(testLoadingRules(t, testConfig("blocked-a", "blocked-b", "peer"))), + WithProbeTimeout(20*time.Millisecond), + WithMaxConcurrency(4), + withProbe(func(_ context.Context, config *rest.Config) error { + if config.Host == "https://peer.invalid" { + peerCalls.Add(1) + return nil + } + blockedCalls.Add(1) + <-release + return nil + }), + withDynamicFactory(func(_ *rest.Config) (dynamic.Interface, error) { return fakeClient(), nil }), + ) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + for run := 1; run <= 3; run++ { + discovery, err := adapter.Discover(context.Background()) + if err != nil { + t.Fatalf("Discover() run %d error = %v", run, err) + } + if !slices.Equal(discovery.Unreachable, []string{"blocked-a", "blocked-b"}) { + t.Fatalf("Discover() run %d unreachable = %v, want [blocked-a blocked-b]", run, discovery.Unreachable) + } + if len(discovery.Scopes) != 3 || !discovery.Scopes[2].Reachable { + t.Fatalf("Discover() run %d scopes = %#v, want reachable peer", run, discovery.Scopes) + } + } + + if got := blockedCalls.Load(); got != 4 { + t.Fatalf("blocked probe calls = %d, want each blocked context bounded at 2", got) + } + if got := peerCalls.Load(); got != 3 { + t.Fatalf("peer probe calls = %d, want one healthy probe per discovery", got) + } +} + func TestQueryAndReadReturnSourceStampedEvidenceWithPartialCoverage(t *testing.T) { t.Parallel() observedAt := time.Date(2026, time.July, 10, 13, 0, 0, 0, time.UTC) @@ -338,6 +468,185 @@ func TestQueryAndReadReturnSourceStampedEvidenceWithPartialCoverage(t *testing.T } } +func TestQueryPaginatesWithinDeterministicMultiScopeBudget(t *testing.T) { + t.Parallel() + clients := map[string]*dynamicfake.FakeDynamicClient{ + "https://alpha.invalid": fakeClient(), + "https://beta.invalid": fakeClient(), + } + var alphaOptions, betaOptions []metav1.ListOptions + clients["https://alpha.invalid"].PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + options, ok := action.(interface{ GetListOptions() metav1.ListOptions }) + if !ok { + return true, nil, errors.New("list action did not expose list options") + } + alphaOptions = append(alphaOptions, options.GetListOptions()) + switch options.GetListOptions().Continue { + case "": + return true, podList("alpha-next", pod("zeta", "apps", "registry/zeta:v1", nil)), nil + case "alpha-next": + return true, podList("", pod("alpha", "apps", "registry/alpha:v1", nil)), nil + default: + return true, nil, fmt.Errorf("unexpected alpha continue token %q", options.GetListOptions().Continue) + } + }) + clients["https://beta.invalid"].PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + options, ok := action.(interface{ GetListOptions() metav1.ListOptions }) + if !ok { + return true, nil, errors.New("list action did not expose list options") + } + betaOptions = append(betaOptions, options.GetListOptions()) + switch options.GetListOptions().Continue { + case "": + return true, podList("beta-next", pod("bravo", "apps", "registry/bravo:v1", nil)), nil + case "beta-next": + return true, podList("", pod("charlie", "apps", "registry/charlie:v1", nil)), nil + default: + return true, nil, fmt.Errorf("unexpected beta continue token %q", options.GetListOptions().Continue) + } + }) + + adapter, err := New( + WithLoadingRules(testLoadingRules(t, testConfig("beta", "alpha"))), + WithMaxConcurrency(2), + withProbe(func(_ context.Context, _ *rest.Config) error { return nil }), + withDynamicFactory(func(config *rest.Config) (dynamic.Interface, error) { + return clients[config.Host], nil + }), + ) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + result, err := adapter.Query(context.Background(), fleet.Query{ + Selector: fleet.Selector{ResourceKind: "Pod", Namespace: "apps"}, + Limit: 3, + }) + if err != nil { + t.Fatalf("Query() error = %v", err) + } + if len(alphaOptions) != 2 || alphaOptions[0].Limit != queryListPageSize || alphaOptions[0].Continue != "" || + alphaOptions[1].Limit != queryListPageSize || alphaOptions[1].Continue != "alpha-next" { + t.Fatalf("alpha list options = %#v, want bounded pages with continuation", alphaOptions) + } + if len(betaOptions) != 2 || betaOptions[0].Limit != queryListPageSize || betaOptions[0].Continue != "" || + betaOptions[1].Limit != queryListPageSize || betaOptions[1].Continue != "beta-next" { + t.Fatalf("beta list options = %#v, want bounded pages with continuation", betaOptions) + } + if result.Coverage.Requested != 2 || result.Coverage.Reachable != 2 || + len(result.Coverage.Truncated) != 0 || !result.Coverage.Complete() { + t.Fatalf("Coverage = %#v, want complete paginated coverage", result.Coverage) + } + want := []string{"alpha/alpha", "alpha/zeta", "beta/bravo"} + got := make([]string, 0, len(result.Facts)) + for _, fact := range result.Facts { + got = append(got, fact.Ref.Scope+"/"+fact.Ref.Name) + } + if !slices.Equal(got, want) { + t.Fatalf("facts = %v, want deterministic global order %v", got, want) + } +} + +func TestQueryScopeReportsTruncationAtItemBudget(t *testing.T) { + t.Parallel() + client := fakeClient() + var options []metav1.ListOptions + client.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + listOptions, ok := action.(interface{ GetListOptions() metav1.ListOptions }) + if !ok { + return true, nil, errors.New("list action did not expose list options") + } + options = append(options, listOptions.GetListOptions()) + return true, podList("next", pod("alpha", "apps", "registry/alpha:v1", nil), pod("bravo", "apps", "registry/bravo:v1", nil)), nil + }) + adapter := &Adapter{settings: defaultOptions()} + result := adapter.queryScope( + context.Background(), "alpha", client, nil, resourceSpecs["pod"], false, "", + fleet.Query{Kinds: []fleet.FactKind{fleet.FactInventory}, Selector: fleet.Selector{ResourceKind: "Pod", Namespace: "apps"}}, + 2, + ) + if result.err != nil { + t.Fatalf("queryScope() error = %v", result.err) + } + if len(options) != 1 || options[0].Limit != 2 || options[0].Continue != "" { + t.Fatalf("list options = %#v, want one page capped at the remaining item budget", options) + } + if len(result.facts) != 2 || !result.truncated { + t.Fatalf("queryScope() = %#v, want two facts and explicit truncation", result) + } +} + +func TestQueryContinuationTimeoutDiscardsPartialPages(t *testing.T) { + t.Parallel() + client := fakeClient() + secondPage := make(chan struct{}) + release := make(chan struct{}) + finished := make(chan struct{}) + client.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + options, ok := action.(interface{ GetListOptions() metav1.ListOptions }) + if !ok { + return true, nil, errors.New("list action did not expose list options") + } + if options.GetListOptions().Continue == "" { + return true, podList("next", pod("first", "apps", "registry/first:v1", nil)), nil + } + close(secondPage) + <-release + close(finished) + return true, podList("", pod("second", "apps", "registry/second:v1", nil)), nil + }) + adapter, err := New( + WithLoadingRules(testLoadingRules(t, testConfig("alpha"))), + WithRequestTimeout(20*time.Millisecond), + withProbe(func(_ context.Context, _ *rest.Config) error { return nil }), + withDynamicFactory(func(_ *rest.Config) (dynamic.Interface, error) { return client, nil }), + ) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + result, err := adapter.Query(context.Background(), fleet.Query{ + Selector: fleet.Selector{ResourceKind: "Pod", Namespace: "apps"}, + Limit: 2, + }) + <-secondPage + close(release) + <-finished + if err != nil { + t.Fatalf("Query() error = %v", err) + } + if len(result.Facts) != 0 || result.Coverage.Reachable != 0 || + !slices.Equal(result.Coverage.Unreachable, []string{"alpha"}) || len(result.Coverage.Truncated) != 0 { + t.Fatalf("Query() = %#v, want timed-out continuation to discard all partial facts", result) + } +} + +func TestQueryScopeItemBudgets(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + scopes int + want []int + }{ + {name: "no scopes", want: nil}, + {name: "sorted remainder", scopes: 3, want: []int{3334, 3333, 3333}}, + {name: "even split", scopes: 2, want: []int{queryListItemBudget / 2, queryListItemBudget / 2}}, + {name: "more scopes than budget", scopes: queryListItemBudget + 1, want: append(make([]int, queryListItemBudget), 0)}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + for index := range min(test.scopes, queryListItemBudget) { + if test.name == "more scopes than budget" { + test.want[index] = 1 + } + } + if got := queryScopeItemBudgets(test.scopes); !slices.Equal(got, test.want) { + t.Fatalf("queryScopeItemBudgets(%d) = %v, want %v", test.scopes, got, test.want) + } + }) + } +} + func TestInvalidInputsFailBeforeDiscovery(t *testing.T) { t.Parallel() probeCalls := 0 @@ -545,6 +854,137 @@ func TestDefaultProbeExecCredentialMixedCloudIsolatedAndMemoryOnly(t *testing.T) assertSandboxExcludesSecrets(t, sandbox, secretMaterial) } +func TestDefaultProbeExecCredentialTimeoutIsContained(t *testing.T) { + if os.Getenv("SITH_EXEC_HELPER") == "1" { + runExecCredentialHelper() + } + + sandbox := t.TempDir() + startPath := filepath.Join(sandbox, "helper-started") + releasePath := filepath.Join(sandbox, "release-helper") + t.Cleanup(func() { _ = os.WriteFile(releasePath, []byte("release"), 0o600) }) + const tokenEnv = "SITH_EXEC_TEST_BLOCKED_TOKEN" + const token = "blocked-ephemeral-token-181" + t.Setenv(tokenEnv, token) + + var blockedRequests atomic.Int32 + blockedServer := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + blockedRequests.Add(1) + if request.URL.Path != "/version" { + http.NotFound(writer, request) + return + } + if request.Header.Get("Authorization") != "Bearer "+token { + http.Error(writer, "unauthorized", http.StatusUnauthorized) + return + } + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"gitVersion":"v1.36.1"}`)) + })) + t.Cleanup(blockedServer.Close) + + var peerRequests atomic.Int32 + peerServer := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + peerRequests.Add(1) + if request.URL.Path != "/version" { + http.NotFound(writer, request) + return + } + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(`{"gitVersion":"v1.36.1"}`)) + })) + t.Cleanup(peerServer.Close) + + config := clientcmdapi.NewConfig() + for name, server := range map[string]*httptest.Server{"blocked": blockedServer, "peer": peerServer} { + config.Clusters[name] = &clientcmdapi.Cluster{ + Server: server.URL, + CertificateAuthorityData: pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", Bytes: server.Certificate().Raw, + }), + } + config.AuthInfos[name] = &clientcmdapi.AuthInfo{} + config.Contexts[name] = &clientcmdapi.Context{Cluster: name, AuthInfo: name} + } + config.AuthInfos["blocked"].Exec = &clientcmdapi.ExecConfig{ + Command: os.Args[0], + Args: []string{"-test.run=TestDefaultProbeExecCredentialTimeoutIsContained"}, + APIVersion: "client.authentication.k8s.io/v1", + InteractiveMode: clientcmdapi.NeverExecInteractiveMode, + Env: []clientcmdapi.ExecEnvVar{ + {Name: "SITH_EXEC_HELPER", Value: "1"}, + {Name: "SITH_EXEC_TOKEN_ENV", Value: tokenEnv}, + {Name: "SITH_EXEC_START_PATH", Value: startPath}, + {Name: "SITH_EXEC_RELEASE_PATH", Value: releasePath}, + }, + } + + adapter, err := New( + WithLoadingRules(testLoadingRules(t, *config)), + WithProbeTimeout(100*time.Millisecond), + WithMaxConcurrency(4), + withDynamicFactory(func(_ *rest.Config) (dynamic.Interface, error) { return fakeClient(), nil }), + ) + if err != nil { + t.Fatalf("New() error = %v", err) + } + for run := 1; run <= 3; run++ { + discovery, err := adapter.Discover(context.Background()) + if err != nil { + t.Fatalf("Discover() run %d error = %v", run, err) + } + if !slices.Equal(discovery.Unreachable, []string{"blocked"}) { + t.Fatalf("Discover() run %d unreachable = %v, want [blocked]", run, discovery.Unreachable) + } + } + if got := peerRequests.Load(); got != 3 { + t.Fatalf("peer requests = %d, want one request per discovery", got) + } + if got := blockedRequests.Load(); got != 0 { + t.Fatalf("blocked requests before helper release = %d, want 0", got) + } + started, err := os.ReadFile(startPath) + if err != nil { + t.Fatalf("read helper start marker: %v", err) + } + if got := string(started); got != "started\n" { + t.Fatalf("helper start markers = %q, want one process", got) + } + + if err := os.WriteFile(releasePath, []byte("release"), 0o600); err != nil { + t.Fatalf("release exec helper: %v", err) + } + deadline := time.Now().Add(5 * time.Second) + for { + adapter.gate.mu.Lock() + active := adapter.gate.byScope["blocked"] + adapter.gate.mu.Unlock() + if active == 0 { + break + } + if time.Now().After(deadline) { + t.Fatalf("retained operations = %d after helper release, want 0", active) + } + time.Sleep(10 * time.Millisecond) + } + discovery, err := adapter.Discover(context.Background()) + if err != nil { + t.Fatalf("Discover() after helper release error = %v", err) + } + if len(discovery.Unreachable) != 0 { + t.Fatalf("Discover() after helper release unreachable = %v, want none", discovery.Unreachable) + } + if blockedRequests.Load() == 0 { + t.Fatal("blocked context made no authenticated request after helper release") + } + discoveryJSON, err := json.Marshal(discovery) + if err != nil { + t.Fatalf("marshal discovery: %v", err) + } + assertNoSecretMaterial(t, "fleet discovery", discoveryJSON, []string{token}) + assertSandboxExcludesSecrets(t, sandbox, []string{token}) +} + func runExecCredentialHelper() { var input struct { APIVersion string `json:"apiVersion"` @@ -556,6 +996,25 @@ func runExecCredentialHelper() { if tokenEnv == "" || os.Getenv(tokenEnv) == "" { os.Exit(2) } + if startPath := os.Getenv("SITH_EXEC_START_PATH"); startPath != "" { + marker, err := os.OpenFile(startPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + os.Exit(2) + } + if _, err := marker.WriteString("started\n"); err != nil || marker.Close() != nil { + os.Exit(2) + } + releasePath := os.Getenv("SITH_EXEC_RELEASE_PATH") + deadline := time.Now().Add(30 * time.Second) + for { + if _, err := os.Stat(releasePath); err == nil { + break + } else if !errors.Is(err, fs.ErrNotExist) || time.Now().After(deadline) { + os.Exit(2) + } + time.Sleep(10 * time.Millisecond) + } + } status := map[string]any{"token": os.Getenv(tokenEnv)} certificateEnv, keyEnv := os.Getenv("SITH_EXEC_CERTIFICATE_ENV"), os.Getenv("SITH_EXEC_KEY_ENV") if certificateEnv != "" || keyEnv != "" { @@ -632,6 +1091,33 @@ func fakeClient(objects ...runtime.Object) *dynamicfake.FakeDynamicClient { return fakeClientWithKinds(listKinds, objects...) } +func podList(continueToken string, objects ...*unstructured.Unstructured) *unstructured.UnstructuredList { + list := &unstructured.UnstructuredList{Items: make([]unstructured.Unstructured, 0, len(objects))} + list.SetAPIVersion("v1") + list.SetKind("PodList") + list.SetContinue(continueToken) + list.SetResourceVersion("test-resource-version") + for _, object := range objects { + list.Items = append(list.Items, *object.DeepCopy()) + } + return list +} + +func assertWatchResourceVersion(t *testing.T, client *dynamicfake.FakeDynamicClient, want string) { + t.Helper() + for _, action := range client.Actions() { + watchAction, ok := action.(k8stesting.WatchAction) + if !ok { + continue + } + if got := watchAction.GetWatchRestrictions().ResourceVersion; got != want { + t.Fatalf("watch resourceVersion = %q, want %q", got, want) + } + return + } + t.Fatal("watch action not found") +} + func fakeClientWithKinds( listKinds map[schema.GroupVersionResource]string, objects ...runtime.Object, diff --git a/internal/connector/kubeconfig/directory.go b/internal/connector/kubeconfig/directory.go index 4065eb4..2a79af7 100644 --- a/internal/connector/kubeconfig/directory.go +++ b/internal/connector/kubeconfig/directory.go @@ -28,6 +28,12 @@ const ( var errImportLimit = errors.New("kubeconfig directory import limit reached") +type directoryImportHooks struct { + afterRootInspection func() + afterRootOpen func() + beforeFileOpen func(string) +} + type importedConfig struct { raw *clientcmdapi.Config metadata map[string]contextMetadata @@ -43,6 +49,10 @@ type contextMetadata struct { // returned config is namespaced by an opaque per-file identifier so duplicate context names cannot // silently shadow one another. func loadDirectory(root string) (importedConfig, error) { + return loadDirectoryWithHooks(root, directoryImportHooks{}) +} + +func loadDirectoryWithHooks(root string, hooks directoryImportHooks) (importedConfig, error) { root = strings.TrimSpace(root) if root == "" { return importedConfig{}, fmt.Errorf("kubeconfig directory is required") @@ -58,18 +68,28 @@ func loadDirectory(root string) (importedConfig, error) { if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { return importedConfig{}, fmt.Errorf("kubeconfig directory must be a real directory, not a symlink or file") } + if hooks.afterRootInspection != nil { + hooks.afterRootInspection() + } + rootHandle, err := os.OpenRoot(absolute) + if err != nil { + return importedConfig{}, errors.New("cannot open kubeconfig directory") + } + defer func() { _ = rootHandle.Close() }() + openedRootInfo, err := rootHandle.Stat(".") + if err != nil || !openedRootInfo.IsDir() || !os.SameFile(info, openedRootInfo) { + return importedConfig{}, errors.New("kubeconfig directory changed during import") + } + if hooks.afterRootOpen != nil { + hooks.afterRootOpen() + } result := importedConfig{ raw: clientcmdapi.NewConfig(), metadata: make(map[string]contextMetadata), } entries := 0 - err = filepath.WalkDir(absolute, func(path string, entry fs.DirEntry, walkErr error) error { - relative, relativeErr := filepath.Rel(absolute, path) - if relativeErr != nil { - return errors.New("cannot scan kubeconfig directory") - } - relative = filepath.ToSlash(relative) + err = fs.WalkDir(rootHandle.FS(), ".", func(relative string, entry fs.DirEntry, walkErr error) error { if relative == "." { if walkErr != nil { return errors.New("cannot read kubeconfig directory") @@ -115,7 +135,11 @@ func loadDirectory(root string) (importedConfig, error) { result.diagnostics = append(result.diagnostics, importDiagnostic(relative, message)) return nil } - config, readErr := loadKubeconfigFile(path, fileInfo.Size()) + if hooks.beforeFileOpen != nil { + hooks.beforeFileOpen(relative) + } + origin := filepath.Join(absolute, filepath.FromSlash(relative)) + config, readErr := loadKubeconfigFile(rootHandle, relative, origin, fileInfo) if readErr != nil { result.diagnostics = append(result.diagnostics, importDiagnostic(relative, "invalid kubeconfig")) return nil @@ -138,15 +162,25 @@ func loadDirectory(root string) (importedConfig, error) { return result, nil } -func loadKubeconfigFile(path string, size int64) (*clientcmdapi.Config, error) { - if size < 0 || size > maxImportBytes { - return nil, fmt.Errorf("kubeconfig exceeds the import size limit") - } - file, err := os.Open(path) // #nosec G304 -- path is discovered beneath a user-selected directory without symlink traversal. +func loadKubeconfigFile(root *os.Root, relative, origin string, walkedInfo fs.FileInfo) (*clientcmdapi.Config, error) { + localName := filepath.FromSlash(relative) + file, err := root.Open(localName) if err != nil { return nil, err } defer func() { _ = file.Close() }() + openedInfo, err := file.Stat() + if err != nil || !openedInfo.Mode().IsRegular() || !os.SameFile(walkedInfo, openedInfo) { + return nil, errors.New("kubeconfig entry changed during import") + } + currentInfo, err := root.Lstat(localName) + if err != nil || currentInfo.Mode()&os.ModeSymlink != 0 || !currentInfo.Mode().IsRegular() || + !os.SameFile(openedInfo, currentInfo) { + return nil, errors.New("kubeconfig entry changed during import") + } + if openedInfo.Size() < 0 || openedInfo.Size() > maxImportBytes { + return nil, fmt.Errorf("kubeconfig exceeds the import size limit") + } payload, err := io.ReadAll(io.LimitReader(file, maxImportBytes+1)) if err != nil { return nil, err @@ -158,13 +192,39 @@ func loadKubeconfigFile(path string, size int64) (*clientcmdapi.Config, error) { if err != nil { return nil, err } - setLocationOfOrigin(config, path) + if err := rejectDeferredLocalReferences(config); err != nil { + return nil, err + } + setLocationOfOrigin(config, origin) if err := clientcmd.ResolveLocalPaths(config); err != nil { return nil, err } return config, nil } +func rejectDeferredLocalReferences(config *clientcmdapi.Config) error { + for _, cluster := range config.Clusters { + if cluster != nil && cluster.CertificateAuthority != "" { + return errors.New("directory-imported kubeconfig must embed certificate authority data") + } + } + for _, authInfo := range config.AuthInfos { + if authInfo == nil { + continue + } + if authInfo.ClientCertificate != "" || authInfo.ClientKey != "" { + return errors.New("directory-imported kubeconfig must embed client certificate data") + } + if authInfo.TokenFile != "" { + return errors.New("directory-imported kubeconfig must embed bearer token data") + } + if authInfo.Exec != nil && strings.ContainsAny(authInfo.Exec.Command, `/\`) { + return errors.New("directory-imported kubeconfig exec command must resolve through PATH") + } + } + return nil +} + func setLocationOfOrigin(config *clientcmdapi.Config, path string) { for _, authInfo := range config.AuthInfos { authInfo.LocationOfOrigin = path diff --git a/internal/connector/kubeconfig/directory_test.go b/internal/connector/kubeconfig/directory_test.go index bb169f8..0992200 100644 --- a/internal/connector/kubeconfig/directory_test.go +++ b/internal/connector/kubeconfig/directory_test.go @@ -154,6 +154,207 @@ func TestDirectoryImportBoundsAllTraversalEntries(t *testing.T) { } } +func TestDirectoryImportRejectsRootReplacementBeforeOpen(t *testing.T) { + t.Parallel() + parent := t.TempDir() + directory := filepath.Join(parent, "selected") + writeDirectoryConfig(t, filepath.Join(directory, "original.yaml"), "original", "https://original.invalid") + + imported, err := loadDirectoryWithHooks(directory, directoryImportHooks{ + afterRootInspection: func() { + if err := os.Rename(directory, filepath.Join(parent, "selected-original")); err != nil { + t.Fatal(err) + } + writeDirectoryConfig(t, filepath.Join(directory, "replacement.yaml"), "replacement", "https://replacement.invalid") + }, + }) + if err == nil || !strings.Contains(err.Error(), "changed during import") { + t.Fatalf("loadDirectoryWithHooks(root replacement) = %#v, %v, want identity rejection", imported, err) + } + if strings.Contains(err.Error(), parent) { + t.Fatalf("root replacement error exposed absolute path: %q", err) + } +} + +func TestDirectoryImportStaysOnOpenedRootAfterPathReplacement(t *testing.T) { + t.Parallel() + parent := t.TempDir() + directory := filepath.Join(parent, "selected") + writeDirectoryConfig(t, filepath.Join(directory, "original.yaml"), "original", "https://original.invalid") + + imported, err := loadDirectoryWithHooks(directory, directoryImportHooks{ + afterRootOpen: func() { + if err := os.Rename(directory, filepath.Join(parent, "selected-original")); err != nil { + t.Fatal(err) + } + writeDirectoryConfig(t, filepath.Join(directory, "replacement.yaml"), "replacement", "https://replacement.invalid") + }, + }) + if err != nil { + t.Fatalf("loadDirectoryWithHooks(open-root replacement) error = %v", err) + } + if len(imported.raw.Contexts) != 1 || len(imported.metadata) != 1 || len(imported.diagnostics) != 0 { + t.Fatalf("opened-root import = %#v, want only original descriptor-backed config", imported) + } + for _, metadata := range imported.metadata { + if metadata.displayName != "original" || metadata.origin != "original.yaml" { + t.Fatalf("opened-root metadata = %#v, want original config", metadata) + } + } +} + +func TestDirectoryImportRejectsRegularFileReplacement(t *testing.T) { + t.Parallel() + directory := t.TempDir() + path := filepath.Join(directory, "config.yaml") + parked := filepath.Join(directory, "config-original.yaml") + writeDirectoryConfig(t, path, "original", "https://original.invalid") + + imported, err := loadDirectoryWithHooks(directory, directoryImportHooks{ + beforeFileOpen: func(relative string) { + if relative != "config.yaml" { + return + } + if err := os.Rename(path, parked); err != nil { + t.Fatal(err) + } + writeDirectoryConfig(t, path, "replacement", "https://replacement.invalid") + }, + }) + if err != nil { + t.Fatalf("loadDirectoryWithHooks(file replacement) error = %v", err) + } + assertRejectedRacyImport(t, imported, "config.yaml", directory) +} + +func TestDirectoryImportRejectsExternalSymlinkSwap(t *testing.T) { + t.Parallel() + directory := t.TempDir() + path := filepath.Join(directory, "config.yaml") + parked := filepath.Join(directory, "config-original.yaml") + outside := filepath.Join(t.TempDir(), "outside.yaml") + writeDirectoryConfig(t, path, "original", "https://original.invalid") + writeDirectoryConfig(t, outside, "outside", "https://outside.invalid") + + imported, err := loadDirectoryWithHooks(directory, directoryImportHooks{ + beforeFileOpen: func(relative string) { + if relative != "config.yaml" { + return + } + if err := os.Rename(path, parked); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, path); err != nil { + t.Fatal(err) + } + }, + }) + if err != nil { + t.Fatalf("loadDirectoryWithHooks(symlink swap) error = %v", err) + } + assertRejectedRacyImport(t, imported, "config.yaml", directory, outside) +} + +func TestDirectoryImportRejectsDeferredLocalFileReferences(t *testing.T) { + t.Parallel() + tests := map[string]func(*clientcmdapi.Config){ + "certificate authority": func(config *clientcmdapi.Config) { + config.Clusters["alpha"].CertificateAuthority = "ca.crt" + }, + "client certificate": func(config *clientcmdapi.Config) { + config.AuthInfos["alpha"].ClientCertificate = "client.crt" + }, + "client key": func(config *clientcmdapi.Config) { + config.AuthInfos["alpha"].ClientKey = "client.key" + }, + "token file": func(config *clientcmdapi.Config) { + config.AuthInfos["alpha"].TokenFile = "token" + }, + "path-based exec": func(config *clientcmdapi.Config) { + config.AuthInfos["alpha"].Exec = &clientcmdapi.ExecConfig{Command: "plugins/auth"} + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + directory := t.TempDir() + config := directoryTestConfig("alpha", "https://alpha.invalid") + mutate(config) + if err := clientcmd.WriteToFile(*config, filepath.Join(directory, "config.yaml")); err != nil { + t.Fatal(err) + } + imported, err := loadDirectory(directory) + if err != nil { + t.Fatalf("loadDirectory() error = %v", err) + } + assertRejectedRacyImport(t, imported, "config.yaml", directory) + }) + } +} + +func TestDirectoryImportAllowsPathResolvedExecCommand(t *testing.T) { + t.Parallel() + directory := t.TempDir() + config := directoryTestConfig("alpha", "https://alpha.invalid") + config.AuthInfos["alpha"].Exec = &clientcmdapi.ExecConfig{Command: "cloud-auth"} + if err := clientcmd.WriteToFile(*config, filepath.Join(directory, "config.yaml")); err != nil { + t.Fatal(err) + } + imported, err := loadDirectory(directory) + if err != nil { + t.Fatalf("loadDirectory() error = %v", err) + } + if len(imported.raw.Contexts) != 1 || len(imported.diagnostics) != 0 { + t.Fatalf("PATH exec import = %#v, want accepted config", imported) + } +} + +func TestDirectoryImportRejectsReplacedAncestorEscape(t *testing.T) { + t.Parallel() + directory := t.TempDir() + nested := filepath.Join(directory, "nested") + parked := filepath.Join(directory, "nested-original") + outsideDirectory := t.TempDir() + outside := filepath.Join(outsideDirectory, "config.yaml") + writeDirectoryConfig(t, filepath.Join(nested, "config.yaml"), "original", "https://original.invalid") + writeDirectoryConfig(t, outside, "outside", "https://outside.invalid") + + imported, err := loadDirectoryWithHooks(directory, directoryImportHooks{ + beforeFileOpen: func(relative string) { + if relative != "nested/config.yaml" { + return + } + if err := os.Rename(nested, parked); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outsideDirectory, nested); err != nil { + t.Fatal(err) + } + }, + }) + if err != nil { + t.Fatalf("loadDirectoryWithHooks(ancestor replacement) error = %v", err) + } + assertRejectedRacyImport(t, imported, "nested/config.yaml", directory, outsideDirectory, outside) +} + +func assertRejectedRacyImport(t *testing.T, imported importedConfig, source string, forbidden ...string) { + t.Helper() + if len(imported.raw.Contexts) != 0 || len(imported.diagnostics) != 1 || + imported.diagnostics[0] != importDiagnostic(source, "invalid kubeconfig") { + t.Fatalf("racy import = %#v, want one relative rejection diagnostic", imported) + } + payload, err := json.Marshal(imported.diagnostics) + if err != nil { + t.Fatal(err) + } + for _, value := range forbidden { + if strings.Contains(string(payload), value) { + t.Fatalf("racy import diagnostic leaked %q: %s", value, payload) + } + } +} + func TestDirectoryImportConflictsWithExplicitSource(t *testing.T) { t.Parallel() directory := t.TempDir() @@ -226,10 +427,7 @@ func TestDirectoryImportSkipsBrokenContextReferences(t *testing.T) { func writeDirectoryConfig(t *testing.T, path, contextName, host string) { t.Helper() - config := clientcmdapi.NewConfig() - config.Clusters[contextName] = &clientcmdapi.Cluster{Server: host} - config.AuthInfos[contextName] = &clientcmdapi.AuthInfo{} - config.Contexts[contextName] = &clientcmdapi.Context{Cluster: contextName, AuthInfo: contextName} + config := directoryTestConfig(contextName, host) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { t.Fatal(err) } @@ -237,3 +435,11 @@ func writeDirectoryConfig(t *testing.T, path, contextName, host string) { t.Fatalf("write kubeconfig %s: %v", path, err) } } + +func directoryTestConfig(contextName, host string) *clientcmdapi.Config { + config := clientcmdapi.NewConfig() + config.Clusters[contextName] = &clientcmdapi.Cluster{Server: host} + config.AuthInfos[contextName] = &clientcmdapi.AuthInfo{} + config.Contexts[contextName] = &clientcmdapi.Context{Cluster: contextName, AuthInfo: contextName} + return config +} diff --git a/internal/connector/kubeconfig/local_objects.go b/internal/connector/kubeconfig/local_objects.go index 5c03b68..16ece9c 100644 --- a/internal/connector/kubeconfig/local_objects.go +++ b/internal/connector/kubeconfig/local_objects.go @@ -37,7 +37,7 @@ func (adapter *Adapter) View( return localops.ObjectView{}, err } resource := resourceInterface(client, spec, normalized.Namespace) - object, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { + object, err := callWithTimeout(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { return resource.Get(requestCtx, normalized.Name, metav1.GetOptions{}) }) if err != nil { @@ -74,7 +74,7 @@ func (adapter *Adapter) Describe(ctx context.Context, target localops.Target) (l if selectorValue == "" { selectorKey, selectorValue = "involvedObject.name", object.GetName() } - events, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.UnstructuredList, error) { + events, err := callWithTimeout(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.UnstructuredList, error) { return client.Resource(resourceSpecs["events"].gvr).Namespace(normalized.Namespace).List( requestCtx, metav1.ListOptions{FieldSelector: fields.OneTermEqualSelector(selectorKey, selectorValue).String()}, @@ -109,7 +109,7 @@ func (adapter *Adapter) PreviewApply( return localops.ApplyPreview{}, err } resource := resourceInterface(client, spec, normalized.Namespace) - current, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { + current, err := callWithTimeout(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { return resource.Get(requestCtx, normalized.Name, metav1.GetOptions{}) }) if err != nil { @@ -119,7 +119,7 @@ func (adapter *Adapter) PreviewApply( if err != nil { return localops.ApplyPreview{}, err } - dryRun, err := updateLocalObject(ctx, adapter.settings.requestTimeout, resource, proposed, true) + dryRun, err := updateLocalObject(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, resource, proposed, true) if err != nil { return localops.ApplyPreview{}, err } @@ -145,7 +145,7 @@ func (adapter *Adapter) Apply( return fleet.Evidence{}, err } resource := resourceInterface(client, spec, normalized.Namespace) - current, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { + current, err := callWithTimeout(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { return resource.Get(requestCtx, normalized.Name, metav1.GetOptions{}) }) if err != nil { @@ -155,10 +155,10 @@ func (adapter *Adapter) Apply( if err != nil { return fleet.Evidence{}, err } - if _, err := updateLocalObject(ctx, adapter.settings.requestTimeout, resource, proposed.DeepCopy(), true); err != nil { + if _, err := updateLocalObject(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, resource, proposed.DeepCopy(), true); err != nil { return fleet.Evidence{}, err } - updated, err := updateLocalObject(ctx, adapter.settings.requestTimeout, resource, proposed, false) + updated, err := updateLocalObject(ctx, adapter.gate, normalized.Context, adapter.settings.requestTimeout, resource, proposed, false) if err != nil { return fleet.Evidence{}, err } @@ -300,6 +300,8 @@ func decodeEditedObject( func updateLocalObject( ctx context.Context, + gate *operationGate, + scope string, timeout time.Duration, resource dynamic.ResourceInterface, object *unstructured.Unstructured, @@ -311,7 +313,7 @@ func updateLocalObject( if dryRun { options.DryRun = []string{metav1.DryRunAll} } - updated, err := callWithTimeout(ctx, timeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { + updated, err := callWithTimeout(ctx, gate, scope, timeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { return resource.Update(requestCtx, object, options) }) if err != nil { diff --git a/internal/connector/kubeconfig/local_streams.go b/internal/connector/kubeconfig/local_streams.go index 78a2573..5671c45 100644 --- a/internal/connector/kubeconfig/local_streams.go +++ b/internal/connector/kubeconfig/local_streams.go @@ -3,7 +3,9 @@ package kubeconfig import ( + "bytes" "context" + "encoding/json" "fmt" "io" "net/http" @@ -58,6 +60,55 @@ type portForwardFactory func( var _ localops.Client = (*Adapter)(nil) +// tableResponseGuard keeps untrusted API error bodies inside the adapter's reviewed HTTP boundary. +// Successful Table bodies remain stream-decoded under the tighter page and total budgets in table.go. +type tableResponseGuard struct { + transport http.RoundTripper +} + +func (guard tableResponseGuard) RoundTrip(request *http.Request) (*http.Response, error) { + response, err := guard.transport.RoundTrip(request) + if err != nil || response == nil || + (response.StatusCode >= http.StatusOK && response.StatusCode < http.StatusMultipleChoices) { + return response, err + } + if response.Body != nil { + _ = response.Body.Close() + } + if response.StatusCode < 100 || response.StatusCode > 999 { + return nil, fmt.Errorf("server table returned an invalid HTTP status code") + } + reason := metav1.StatusReasonUnknown + statusCode := int32(response.StatusCode) // #nosec G115 -- net/http status codes are range checked immediately above. + if response.StatusCode == http.StatusGone { + reason = metav1.StatusReasonExpired + } + payload, marshalErr := json.Marshal(metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, + Reason: reason, + Message: "server table request failed", + Code: statusCode, + }) + if marshalErr != nil { + return nil, fmt.Errorf("construct bounded table error response: %w", marshalErr) + } + response.Body = io.NopCloser(bytes.NewReader(payload)) + response.ContentLength = int64(len(payload)) + response.Header.Set("Content-Type", "application/json") + return response, nil +} + +func guardTableErrorResponses(config *rest.Config) { + previous := config.WrapTransport + config.WrapTransport = func(transport http.RoundTripper) http.RoundTripper { + if previous != nil { + transport = previous(transport) + } + return tableResponseGuard{transport: transport} + } +} + // Logs opens one pod log stream in the explicitly selected context. func (adapter *Adapter) Logs( ctx context.Context, diff --git a/internal/connector/kubeconfig/resources.go b/internal/connector/kubeconfig/resources.go index 814b2ae..022b407 100644 --- a/internal/connector/kubeconfig/resources.go +++ b/internal/connector/kubeconfig/resources.go @@ -39,6 +39,14 @@ var ErrUnsupportedSelector = errors.New("query selector is unsupported") // ErrInvalidReference reports a resource address that is incomplete or inconsistent. var ErrInvalidReference = errors.New("resource reference is invalid") +const queryListPageSize int64 = 250 + +const ( + // queryListItemBudget caps objects materialized across every context in one fleet query. + queryListItemBudget = 10_000 + queryListPageBudget = 128 +) + type resourceSpec struct { kind string gvr schema.GroupVersionResource @@ -101,7 +109,7 @@ func (adapter *Adapter) Read(ctx context.Context, ref fleet.ResourceRef) (fleet. } resource := resourceInterface(client, spec, ref.Namespace) - object, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { + object, err := callWithTimeout(ctx, adapter.gate, ref.Scope, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.Unstructured, error) { return resource.Get(requestCtx, ref.Name, metav1.GetOptions{}) }) if err != nil { @@ -143,12 +151,29 @@ func (adapter *Adapter) Query(ctx context.Context, query fleet.Query) (fleet.Que scopes, clients, configs, tables, lastSeen := adapter.stateSnapshot() targets := targetScopeNames(query.Scopes, scopes) + itemBudgets := queryScopeItemBudgets(len(targets)) results := make([]scopeQueryResult, len(targets)) adapter.runBounded(len(targets), func(index int) { name := targets[index] - result, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (scopeQueryResult, error) { + if !scopes[name].Reachable || clients[name] == nil { + results[index] = scopeQueryResult{name: name, err: ErrUnreachableScope} + return + } + scopeSpec := spec + generic := query.Selector.ResourceKind != "" && + (scopeSpec.gvr.Resource == "" || scopeSpec.kind == "ConfigMap" || scopeSpec.kind == "Secret") + if generic { + var err error + scopeSpec, err = adapter.resolveResource(ctx, name, configs[name], query.Selector.ResourceKind) + if err != nil { + results[index] = scopeQueryResult{name: name, err: err} + return + } + } + result, err := callWithTimeout(ctx, adapter.gate, name, adapter.settings.requestTimeout, func(requestCtx context.Context) (scopeQueryResult, error) { return adapter.queryScope( - requestCtx, name, clients[name], configs[name], tables[name], spec, labelSelector.String(), query, + requestCtx, name, clients[name], tables[name], scopeSpec, generic, labelSelector.String(), query, + itemBudgets[index], ), nil }) if err != nil { @@ -173,6 +198,9 @@ func (adapter *Adapter) Query(ctx context.Context, query fleet.Query) (fleet.Que } coverage.Reachable++ facts = append(facts, result.facts...) + if result.truncated { + coverage.Truncated = append(coverage.Truncated, result.name) + } if !result.observedAt.IsZero() { adapter.recordLastSeen(result.name, result.observedAt) } else if isStale(now, lastSeen[result.name], adapter.settings.staleAfter) { @@ -191,6 +219,7 @@ func (adapter *Adapter) Query(ctx context.Context, query fleet.Query) (fleet.Que } sort.Strings(coverage.Unreachable) sort.Strings(coverage.Stale) + sort.Strings(coverage.Truncated) return fleet.QueryResult{Facts: facts, Coverage: coverage}, nil } @@ -198,6 +227,7 @@ type scopeQueryResult struct { name string facts []fleet.Fact observedAt time.Time + truncated bool err error } @@ -205,11 +235,12 @@ func (adapter *Adapter) queryScope( ctx context.Context, name string, client dynamic.Interface, - config *rest.Config, table tablePrinter, spec resourceSpec, + generic bool, labelSelector string, query fleet.Query, + itemBudget int, ) scopeQueryResult { result := scopeQueryResult{name: name} if client == nil { @@ -219,30 +250,59 @@ func (adapter *Adapter) queryScope( if query.Selector.ResourceKind == "" { return result } - // ConfigMap and Secret are statically known for local YAML/edit operations, but remain generic - // fleet lenses so Kubernetes server print columns continue to drive their tabular presentation. - generic := spec.gvr.Resource == "" || spec.kind == "ConfigMap" || spec.kind == "Secret" - if generic { - var err error - spec, err = adapter.resolveResource(ctx, name, config, query.Selector.ResourceKind) - if err != nil { - result.err = err - return result - } - } if !spec.namespaced && query.Selector.Namespace != "" { result.err = fmt.Errorf("%w: namespace cannot select cluster-scoped %s", ErrUnsupportedSelector, spec.kind) return result } + if itemBudget <= 0 { + result.truncated = true + return result + } resource := resourceInterface(client, spec, query.Selector.Namespace) - list, err := resource.List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) - if err != nil { - if spec.kind == "Rollout" && apierrors.IsNotFound(err) { + objects := make([]unstructured.Unstructured, 0, itemBudget) + continueToken := "" + for page := 0; page < queryListPageBudget; page++ { + remaining := itemBudget - len(objects) + if remaining <= 0 { + result.truncated = continueToken != "" + break + } + options := metav1.ListOptions{ + LabelSelector: labelSelector, + Limit: min(queryListPageSize, int64(remaining)), + Continue: continueToken, + } + list, err := resource.List(ctx, options) + if err != nil { + if spec.kind == "Rollout" && apierrors.IsNotFound(err) { + return result + } + result.err = fmt.Errorf("list %s in %s: %w", spec.kind, name, err) return result } - result.err = fmt.Errorf("list %s in %s: %w", spec.kind, name, err) - return result + pageItems := list.Items + if len(pageItems) > remaining { + pageItems = pageItems[:remaining] + result.truncated = true + } + objects = append(objects, pageItems...) + next := list.GetContinue() + if next == "" { + break + } + if next == continueToken { + result.err = fmt.Errorf("list %s in %s: API server repeated a continuation token", spec.kind, name) + return result + } + continueToken = next + if len(objects) >= itemBudget { + result.truncated = true + break + } + if page == queryListPageBudget-1 { + result.truncated = true + } } result.observedAt = adapter.settings.now().UTC() @@ -250,7 +310,22 @@ func (adapter *Adapter) queryScope( result.facts = []fleet.Fact{} return result } - result.facts = make([]fleet.Fact, 0, len(list.Items)) + selectedObjects := make([]unstructured.Unstructured, 0, len(objects)) + retainDisplay := make(map[string]struct{}, len(objects)) + for _, object := range objects { + if query.Selector.Name != "" && object.GetName() != query.Selector.Name { + continue + } + if query.Selector.NamePrefix != "" && !strings.HasPrefix(object.GetName(), query.Selector.NamePrefix) { + continue + } + if query.Selector.Image != "" && !objectUsesImage(object, query.Selector.Image) { + continue + } + selectedObjects = append(selectedObjects, object) + retainDisplay[tableObjectKey(object.GetNamespace(), object.GetName())] = struct{}{} + } + result.facts = make([]fleet.Fact, 0, len(selectedObjects)) display := map[string][]fleet.DisplayField{} if generic { if table == nil { @@ -258,22 +333,18 @@ func (adapter *Adapter) queryScope( return result } var err error - display, err = table(ctx, spec, query.Selector.Namespace, "", labelSelector) + display, err = table(ctx, spec, tableRequest{ + namespace: query.Selector.Namespace, + labelSelector: labelSelector, + rowBudget: len(objects), + retainKeys: retainDisplay, + }) if err != nil { result.err = err return result } } - for _, object := range list.Items { - if query.Selector.Name != "" && object.GetName() != query.Selector.Name { - continue - } - if query.Selector.NamePrefix != "" && !strings.HasPrefix(object.GetName(), query.Selector.NamePrefix) { - continue - } - if query.Selector.Image != "" && !objectUsesImage(object, query.Selector.Image) { - continue - } + for _, object := range selectedObjects { evidence, err := evidenceFromObject(object, spec, name, result.observedAt) if err != nil { result.err = err @@ -285,6 +356,21 @@ func (adapter *Adapter) queryScope( return result } +func queryScopeItemBudgets(scopeCount int) []int { + if scopeCount <= 0 { + return nil + } + budgets := make([]int, scopeCount) + perScope, remainder := queryListItemBudget/scopeCount, queryListItemBudget%scopeCount + for index := range budgets { + budgets[index] = perScope + if index < remainder { + budgets[index]++ + } + } + return budgets +} + func evidenceFromObject( object unstructured.Unstructured, spec resourceSpec, @@ -337,7 +423,7 @@ func (adapter *Adapter) resolveResource( if config == nil { return resourceSpec{}, fmt.Errorf("%w: no client config for %s", ErrUnreachableScope, scope) } - resolved, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(resolveCtx context.Context) (resourceSpec, error) { + resolved, err := callWithTimeout(ctx, adapter.gate, scope, adapter.settings.requestTimeout, func(resolveCtx context.Context) (resourceSpec, error) { return adapter.settings.resolve(resolveCtx, rest.CopyConfig(config), kind) }) if err != nil { diff --git a/internal/connector/kubeconfig/table.go b/internal/connector/kubeconfig/table.go index 45f364d..b6ba41a 100644 --- a/internal/connector/kubeconfig/table.go +++ b/internal/connector/kubeconfig/table.go @@ -6,9 +6,11 @@ import ( "context" "encoding/json" "fmt" + "io" "strconv" "strings" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -16,12 +18,17 @@ import ( "github.com/ArdurAI/sith/internal/fleet" ) -const tableAccept = "application/json;as=Table;g=meta.k8s.io;v=v1" +const ( + tableAccept = "application/json;as=Table;g=meta.k8s.io;v=v1" + tableResponsePageByteLimit = int64(4 << 20) + tableResponseTotalByteBudget = int64(16 << 20) +) func newTablePrinter(config *rest.Config) (tablePrinter, error) { tableConfig := dynamic.ConfigFor(config) tableConfig.GroupVersion = nil tableConfig.APIPath = "/if-you-see-this-search-for-the-break" + guardTableErrorResponses(tableConfig) client, err := rest.UnversionedRESTClientFor(tableConfig) if err != nil { return nil, fmt.Errorf("create table client: %w", err) @@ -29,9 +36,9 @@ func newTablePrinter(config *rest.Config) (tablePrinter, error) { return func( ctx context.Context, spec resourceSpec, - namespace, name, labelSelector string, + request tableRequest, ) (map[string][]fleet.DisplayField, error) { - return requestTable(ctx, client, spec, namespace, name, labelSelector) + return requestTable(ctx, client, spec, request) }, nil } @@ -39,45 +46,159 @@ func requestTable( ctx context.Context, client rest.Interface, spec resourceSpec, - namespace, name, labelSelector string, + request tableRequest, ) (map[string][]fleet.DisplayField, error) { - segments := tableURLSegments(spec, namespace, name) + if request.name == "" && request.rowBudget <= 0 { + return map[string][]fleet.DisplayField{}, nil + } + if request.retainKeys != nil && len(request.retainKeys) == 0 { + return map[string][]fleet.DisplayField{}, nil + } + + remainingKeys := cloneTableKeys(request.retainKeys) + resultCapacity := request.rowBudget + if remainingKeys != nil { + resultCapacity = min(resultCapacity, len(remainingKeys)) + } + if request.name != "" { + resultCapacity = 1 + } + result := make(map[string][]fleet.DisplayField, max(resultCapacity, 0)) + continueToken := "" + seenTokens := make(map[string]struct{}) + rowsRead := 0 + bytesRead := int64(0) + for page := 0; page < queryListPageBudget; page++ { + remainingRows := request.rowBudget - rowsRead + pageLimit := int64(0) + if request.name == "" { + if remainingRows <= 0 { + return result, nil + } + pageLimit = min(queryListPageSize, int64(remainingRows)) + } + remainingBytes := tableResponseTotalByteBudget - bytesRead + if remainingBytes <= 0 { + return nil, fmt.Errorf("server table for %s exceeded the total response byte budget", spec.kind) + } + table, pageBytes, err := requestTablePage( + ctx, client, spec, request, pageLimit, continueToken, min(tableResponsePageByteLimit, remainingBytes), + ) + if err != nil { + return nil, err + } + bytesRead += pageBytes + if len(table.ColumnDefinitions) == 0 { + return nil, fmt.Errorf("server table for %s has no column definitions", spec.kind) + } + if request.name == "" && int64(len(table.Rows)) > pageLimit { + return nil, fmt.Errorf("server table for %s ignored the requested row limit", spec.kind) + } + rowsRead += len(table.Rows) + for _, row := range table.Rows { + rowNamespace, rowName := tableRowIdentity(row, table.ColumnDefinitions, request.namespace) + if rowName == "" { + continue + } + key := tableObjectKey(rowNamespace, rowName) + if remainingKeys != nil { + if _, retain := remainingKeys[key]; !retain { + continue + } + delete(remainingKeys, key) + } + result[key] = tableDisplayFields(row, table.ColumnDefinitions) + } + if request.name != "" || (remainingKeys != nil && len(remainingKeys) == 0) { + return result, nil + } + next := table.GetContinue() + if next == "" { + return result, nil + } + if _, repeated := seenTokens[next]; repeated { + return nil, fmt.Errorf("server table for %s repeated a continuation token", spec.kind) + } + seenTokens[next] = struct{}{} + continueToken = next + if rowsRead >= request.rowBudget { + return result, nil + } + } + return nil, fmt.Errorf("server table for %s exceeded the page budget", spec.kind) +} + +func requestTablePage( + ctx context.Context, + client rest.Interface, + spec resourceSpec, + tableRequest tableRequest, + limit int64, + continueToken string, + byteLimit int64, +) (metav1.Table, int64, error) { + segments := tableURLSegments(spec, tableRequest.namespace, tableRequest.name) request := client.Get().AbsPath(segments...).SetHeader("Accept", tableAccept) - if name == "" { - request = request.VersionedParams(&metav1.ListOptions{LabelSelector: labelSelector}, metav1.ParameterCodec) + if tableRequest.name == "" { + request = request.VersionedParams(&metav1.ListOptions{ + LabelSelector: tableRequest.labelSelector, + Limit: limit, + Continue: continueToken, + }, metav1.ParameterCodec) } else { request = request.VersionedParams(&metav1.GetOptions{}, metav1.ParameterCodec) } - payload, err := request.Do(ctx).Raw() + stream, err := request.Stream(ctx) if err != nil { - return nil, fmt.Errorf("request server table for %s: %w", spec.kind, err) + if ctxErr := ctx.Err(); ctxErr != nil { + return metav1.Table{}, 0, fmt.Errorf("request server table for %s: %w", spec.kind, ctxErr) + } + if apierrors.IsResourceExpired(err) { + return metav1.Table{}, 0, fmt.Errorf("server table continuation for %s expired", spec.kind) + } + return metav1.Table{}, 0, fmt.Errorf("request server table for %s failed", spec.kind) + } + defer func() { _ = stream.Close() }() + payload, err := io.ReadAll(io.LimitReader(stream, byteLimit+1)) + if err != nil { + return metav1.Table{}, 0, fmt.Errorf("read server table for %s: %w", spec.kind, err) + } + if int64(len(payload)) > byteLimit { + return metav1.Table{}, 0, fmt.Errorf("server table for %s exceeded the response page byte limit", spec.kind) } var table metav1.Table if err := json.Unmarshal(payload, &table); err != nil { - return nil, fmt.Errorf("decode server table for %s: %w", spec.kind, err) + return metav1.Table{}, 0, fmt.Errorf("decode server table for %s: %w", spec.kind, err) } - if len(table.ColumnDefinitions) == 0 { - return nil, fmt.Errorf("server table for %s has no column definitions", spec.kind) + return table, int64(len(payload)), nil +} + +func cloneTableKeys(keys map[string]struct{}) map[string]struct{} { + if keys == nil { + return nil } - result := make(map[string][]fleet.DisplayField, len(table.Rows)) - for _, row := range table.Rows { - rowNamespace, rowName := tableRowIdentity(row, table.ColumnDefinitions, namespace) - if rowName == "" { - continue - } - fields := make([]fleet.DisplayField, 0, min(len(row.Cells), len(table.ColumnDefinitions))) - for index, cell := range row.Cells { - if index >= len(table.ColumnDefinitions) { - break - } - column := table.ColumnDefinitions[index] - fields = append(fields, fleet.DisplayField{ - Name: column.Name, Value: tableCellString(cell), Priority: column.Priority, - }) + cloned := make(map[string]struct{}, len(keys)) + for key := range keys { + cloned[key] = struct{}{} + } + return cloned +} + +func tableDisplayFields( + row metav1.TableRow, + columns []metav1.TableColumnDefinition, +) []fleet.DisplayField { + fields := make([]fleet.DisplayField, 0, min(len(row.Cells), len(columns))) + for index, cell := range row.Cells { + if index >= len(columns) { + break } - result[tableObjectKey(rowNamespace, rowName)] = fields + column := columns[index] + fields = append(fields, fleet.DisplayField{ + Name: column.Name, Value: tableCellString(cell), Priority: column.Priority, + }) } - return result, nil + return fields } func tableURLSegments(spec resourceSpec, namespace, name string) []string { diff --git a/internal/connector/kubeconfig/table_test.go b/internal/connector/kubeconfig/table_test.go index 08121bb..5598b4a 100644 --- a/internal/connector/kubeconfig/table_test.go +++ b/internal/connector/kubeconfig/table_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "slices" + "strings" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -53,7 +54,10 @@ func TestTablePrinterRequestsAndDecodesServerColumns(t *testing.T) { } fields, err := printer(context.Background(), resourceSpec{ kind: "ConfigMap", gvr: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, namespaced: true, - }, "apps", "", "app=sample") + }, tableRequest{ + namespace: "apps", labelSelector: "app=sample", rowBudget: 1, + retainKeys: map[string]struct{}{tableObjectKey("apps", "settings"): {}}, + }) if err != nil { t.Fatalf("printer() error = %v", err) } @@ -63,3 +67,192 @@ func TestTablePrinterRequestsAndDecodesServerColumns(t *testing.T) { t.Fatalf("fields = %#v", got) } } + +func TestTablePrinterPaginatesOpaqueTokensAndRetainsSelectedRows(t *testing.T) { + t.Parallel() + const opaqueToken = "opaque+/=token" + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requests++ + writer.Header().Set("Content-Type", "application/json") + switch requests { + case 1: + if request.URL.Query().Get("limit") != "3" || request.URL.Query().Get("continue") != "" { + t.Errorf("first query = %q, want limit=3 without continuation", request.URL.RawQuery) + } + writeTable(t, writer, opaqueToken, + tableRow("apps", "first", "1"), + tableRow("apps", "ignored", "2"), + ) + case 2: + if request.URL.Query().Get("limit") != "1" || request.URL.Query().Get("continue") != opaqueToken { + t.Errorf("second query = %q, want remaining limit and opaque continuation", request.URL.RawQuery) + } + writeTable(t, writer, "", tableRow("apps", "second", "3")) + default: + t.Errorf("unexpected table request %d", requests) + } + })) + defer server.Close() + + printer, err := newTablePrinter(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatalf("newTablePrinter() error = %v", err) + } + fields, err := printer(context.Background(), configMapSpec(), tableRequest{ + namespace: "apps", + rowBudget: 3, + retainKeys: map[string]struct{}{ + tableObjectKey("apps", "first"): {}, + tableObjectKey("apps", "second"): {}, + }, + }) + if err != nil { + t.Fatalf("printer() error = %v", err) + } + if requests != 2 || len(fields) != 2 || fields[tableObjectKey("apps", "first")][1].Value != "1" || + fields[tableObjectKey("apps", "second")][1].Value != "3" { + t.Fatalf("requests/fields = %d/%#v, want two retained rows across two pages", requests, fields) + } + if _, retained := fields[tableObjectKey("apps", "ignored")]; retained { + t.Fatalf("fields = %#v, query-excluded row was retained", fields) + } +} + +func TestTablePrinterRejectsServerThatIgnoresRowLimit(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + writeTable(t, writer, "", tableRow("apps", "first", "1"), tableRow("apps", "second", "2")) + })) + defer server.Close() + + printer, err := newTablePrinter(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatalf("newTablePrinter() error = %v", err) + } + _, err = printer(context.Background(), configMapSpec(), tableRequest{namespace: "apps", rowBudget: 1}) + if err == nil || !strings.Contains(err.Error(), "ignored the requested row limit") { + t.Fatalf("printer() error = %v, want ignored-limit rejection", err) + } +} + +func TestTablePrinterRejectsRepeatedContinuationWithoutRestart(t *testing.T) { + t.Parallel() + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requests++ + if requests == 2 && request.URL.Query().Get("continue") != "cycle" { + t.Errorf("second continuation = %q, want cycle", request.URL.Query().Get("continue")) + } + writer.Header().Set("Content-Type", "application/json") + writeTable(t, writer, "cycle", tableRow("apps", "row", "1")) + })) + defer server.Close() + + printer, err := newTablePrinter(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatalf("newTablePrinter() error = %v", err) + } + _, err = printer(context.Background(), configMapSpec(), tableRequest{namespace: "apps", rowBudget: 3}) + if err == nil || !strings.Contains(err.Error(), "repeated a continuation token") || requests != 2 { + t.Fatalf("printer() requests/error = %d/%v, want fail-closed cycle detection", requests, err) + } +} + +func TestTablePrinterDoesNotRestartRejectedContinuationOrExposeBody(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + statusCode int + reason metav1.StatusReason + wantError string + }{ + {name: "expired", statusCode: http.StatusGone, reason: metav1.StatusReasonExpired, wantError: "expired"}, + {name: "invalid", statusCode: http.StatusBadRequest, reason: metav1.StatusReasonBadRequest, wantError: "failed"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + const secretBody = "credential-like-response-body" + const rejectedToken = "rejected-token" + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requests++ + if requests == 1 { + writer.Header().Set("Content-Type", "application/json") + writeTable(t, writer, rejectedToken, tableRow("apps", "first", "1")) + return + } + if request.URL.Query().Get("continue") != rejectedToken { + t.Errorf("continuation = %q, want opaque rejected token", request.URL.Query().Get("continue")) + } + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(test.statusCode) + _ = json.NewEncoder(writer).Encode(metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, Reason: test.reason, + Message: secretBody, Code: int32(test.statusCode), // #nosec G115 -- test cases use fixed HTTP constants. + }) + })) + defer server.Close() + + printer, err := newTablePrinter(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatalf("newTablePrinter() error = %v", err) + } + _, err = printer(context.Background(), configMapSpec(), tableRequest{namespace: "apps", rowBudget: 2}) + if err == nil || !strings.Contains(err.Error(), test.wantError) || strings.Contains(err.Error(), secretBody) || + strings.Contains(err.Error(), rejectedToken) || requests != 2 { + t.Fatalf("printer() requests/error = %d/%v, want sanitized fail-closed rejection", requests, err) + } + }) + } +} + +func TestTablePrinterBoundsResponsePageBytes(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.Header().Set("Content-Type", "application/json") + _, _ = writer.Write([]byte(strings.Repeat("x", int(tableResponsePageByteLimit+1)))) + })) + defer server.Close() + + printer, err := newTablePrinter(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatalf("newTablePrinter() error = %v", err) + } + _, err = printer(context.Background(), configMapSpec(), tableRequest{namespace: "apps", rowBudget: 1}) + if err == nil || !strings.Contains(err.Error(), "response page byte limit") { + t.Fatalf("printer() error = %v, want bounded response rejection", err) + } +} + +func configMapSpec() resourceSpec { + return resourceSpec{ + kind: "ConfigMap", gvr: schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, namespaced: true, + } +} + +func tableRow(namespace, name, data string) metav1.TableRow { + return metav1.TableRow{ + Cells: []any{name, data}, + Object: runtime.RawExtension{Raw: []byte( + `{"metadata":{"name":"` + name + `","namespace":"` + namespace + `"}}`, + )}, + } +} + +func writeTable(t *testing.T, writer http.ResponseWriter, continueToken string, rows ...metav1.TableRow) { + t.Helper() + if err := json.NewEncoder(writer).Encode(metav1.Table{ + TypeMeta: metav1.TypeMeta{APIVersion: "meta.k8s.io/v1", Kind: "Table"}, + ListMeta: metav1.ListMeta{Continue: continueToken}, + ColumnDefinitions: []metav1.TableColumnDefinition{ + {Name: "Name", Type: "string"}, + {Name: "Data", Type: "integer"}, + }, + Rows: rows, + }); err != nil { + t.Errorf("encode table: %v", err) + } +} diff --git a/internal/connector/kubeconfig/watch.go b/internal/connector/kubeconfig/watch.go index 1720052..768d65e 100644 --- a/internal/connector/kubeconfig/watch.go +++ b/internal/connector/kubeconfig/watch.go @@ -22,12 +22,38 @@ import ( ) const ( - watchBuffer = 256 - initialWatchBackoff = 250 * time.Millisecond - maximumWatchBackoff = 5 * time.Second - watchTimeoutSeconds = int64(300) + watchBuffer = 256 + initialWatchBackoff = 250 * time.Millisecond + maximumWatchBackoff = 5 * time.Second + watchTimeoutSeconds = int64(300) + watchBootstrapPageSize = 250 + watchBootstrapObjectBudget = 10_000 + watchBootstrapPageBudget = 128 ) +type watchBootstrapLimits struct { + pageSize int + objectBudget int + pageBudget int +} + +func defaultWatchBootstrapLimits() watchBootstrapLimits { + return watchBootstrapLimits{ + pageSize: watchBootstrapPageSize, + objectBudget: watchBootstrapObjectBudget, + pageBudget: watchBootstrapPageBudget, + } +} + +type watchBootstrapLister interface { + List(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) +} + +type watchBootstrap struct { + objects []unstructured.Unstructured + resourceVersion string +} + // Watch opens independent list-watch loops for every reachable scope and requested kind. func (adapter *Adapter) Watch(ctx context.Context, kinds ...string) (<-chan connector.WatchEvent, error) { normalized, err := normalizeWatchKinds(kinds) @@ -47,7 +73,8 @@ func (adapter *Adapter) Watch(ctx context.Context, kinds ...string) (<-chan conn go func(scopeName, resourceKind string) { defer waitGroup.Done() sendWatchEvent(ctx, events, connector.WatchEvent{ - Type: connector.WatchError, Kind: resourceKind, Scope: scopeName, Err: ErrUnreachableScope, + Type: connector.WatchError, Workspace: fleet.LocalWorkspace, + Kind: resourceKind, Scope: scopeName, Err: ErrUnreachableScope, }) }(name, kind) continue @@ -94,11 +121,11 @@ func (adapter *Adapter) watchScope( } } resource := resourceInterface(client, spec, "") - list, err := callWithTimeout(ctx, adapter.settings.requestTimeout, func(requestCtx context.Context) (*unstructured.UnstructuredList, error) { - return resource.List(requestCtx, metav1.ListOptions{}) + bootstrap, err := callWithTimeout(ctx, adapter.gate, scope, adapter.settings.requestTimeout, func(requestCtx context.Context) (watchBootstrap, error) { + return loadWatchBootstrap(requestCtx, resource, defaultWatchBootstrapLimits()) }) if err != nil { - if !adapter.reportWatchError(ctx, events, kind, scope, fmt.Errorf("list before watch: %w", err)) || + if !adapter.reportWatchError(ctx, events, kind, scope, err) || !waitForWatchRetry(ctx, backoff) { return } @@ -114,7 +141,7 @@ func (adapter *Adapter) watchScope( } return } - display, err = table(ctx, spec, "", "", "") + display, err = table(ctx, spec, tableRequest{rowBudget: len(bootstrap.objects)}) if err != nil { if !adapter.reportWatchError(ctx, events, kind, scope, err) || !waitForWatchRetry(ctx, backoff) { return @@ -123,7 +150,7 @@ func (adapter *Adapter) watchScope( continue } } - facts, err := factsFromObjects(list.Items, spec, scope, observedAt, display) + facts, err := factsFromObjects(bootstrap.objects, spec, scope, observedAt, display) if err != nil { if !adapter.reportWatchError(ctx, events, kind, scope, err) { return @@ -131,7 +158,8 @@ func (adapter *Adapter) watchScope( return } if !sendWatchEvent(ctx, events, connector.WatchEvent{ - Type: connector.WatchSnapshot, Kind: kind, Scope: scope, Facts: facts, ObservedAt: observedAt, + Type: connector.WatchSnapshot, Workspace: fleet.LocalWorkspace, + Kind: kind, Scope: scope, Facts: facts, ObservedAt: observedAt, }) { return } @@ -139,7 +167,7 @@ func (adapter *Adapter) watchScope( backoff = initialWatchBackoff stream, err := resource.Watch(ctx, metav1.ListOptions{ - ResourceVersion: list.GetResourceVersion(), AllowWatchBookmarks: true, TimeoutSeconds: pointer(watchTimeoutSeconds), + ResourceVersion: bootstrap.resourceVersion, AllowWatchBookmarks: true, TimeoutSeconds: pointer(watchTimeoutSeconds), }) if err != nil { if !adapter.reportWatchError(ctx, events, kind, scope, fmt.Errorf("open watch: %w", err)) || @@ -166,6 +194,71 @@ func (adapter *Adapter) watchScope( } } +func loadWatchBootstrap( + ctx context.Context, + lister watchBootstrapLister, + limits watchBootstrapLimits, +) (watchBootstrap, error) { + if limits.pageSize <= 0 || limits.objectBudget <= 0 || limits.pageBudget <= 0 || + limits.pageSize > limits.objectBudget { + return watchBootstrap{}, errors.New("watch bootstrap limits are invalid") + } + result := watchBootstrap{ + objects: make([]unstructured.Unstructured, 0, min(limits.pageSize, limits.objectBudget)), + } + continueToken := "" + seenTokens := make(map[string]struct{}) + for page := 0; page < limits.pageBudget; page++ { + remaining := limits.objectBudget - len(result.objects) + if remaining <= 0 { + return watchBootstrap{}, fmt.Errorf("watch bootstrap exceeds the %d-object budget", limits.objectBudget) + } + pageLimit := min(limits.pageSize, remaining) + list, err := lister.List(ctx, metav1.ListOptions{ + Limit: int64(pageLimit), // #nosec G115 -- pageLimit is positive and bounded by objectBudget. + Continue: continueToken, + }) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return watchBootstrap{}, fmt.Errorf("watch bootstrap list: %w", ctxErr) + } + if apierrors.IsResourceExpired(err) { + return watchBootstrap{}, errors.New("watch bootstrap continuation expired") + } + return watchBootstrap{}, fmt.Errorf("watch bootstrap list page %d failed", page+1) + } + if list == nil { + return watchBootstrap{}, errors.New("watch bootstrap API server returned an empty list response") + } + if len(list.Items) > pageLimit { + return watchBootstrap{}, errors.New("watch bootstrap API server ignored the requested page limit") + } + resourceVersion := list.GetResourceVersion() + if resourceVersion == "" { + return watchBootstrap{}, errors.New("watch bootstrap returned an empty resourceVersion") + } + if result.resourceVersion == "" { + result.resourceVersion = resourceVersion + } else if resourceVersion != result.resourceVersion { + return watchBootstrap{}, errors.New("watch bootstrap resourceVersion changed between pages") + } + result.objects = append(result.objects, list.Items...) + next := list.GetContinue() + if next == "" { + return result, nil + } + if len(result.objects) >= limits.objectBudget { + return watchBootstrap{}, fmt.Errorf("watch bootstrap exceeds the %d-object budget", limits.objectBudget) + } + if _, repeated := seenTokens[next]; repeated { + return watchBootstrap{}, errors.New("watch bootstrap API server repeated a continuation token") + } + seenTokens[next] = struct{}{} + continueToken = next + } + return watchBootstrap{}, fmt.Errorf("watch bootstrap exceeds the %d-page budget", limits.pageBudget) +} + func (adapter *Adapter) consumeWatch( ctx context.Context, events chan<- connector.WatchEvent, @@ -199,12 +292,15 @@ func (adapter *Adapter) consumeWatch( return err } watchEvent := connector.WatchEvent{ - Kind: kind, Scope: scope, ObservedAt: observedAt, Ref: evidence.Ref, + Workspace: fleet.LocalWorkspace, Kind: kind, Scope: scope, ObservedAt: observedAt, Ref: evidence.Ref, } switch event.Type { case watch.Added, watch.Modified: if generic { - display, tableErr := table(ctx, spec, object.GetNamespace(), object.GetName(), "") + display, tableErr := table(ctx, spec, tableRequest{ + namespace: object.GetNamespace(), + name: object.GetName(), + }) if tableErr != nil { return tableErr } @@ -234,7 +330,7 @@ func (adapter *Adapter) reportWatchError( err error, ) bool { return sendWatchEvent(ctx, events, connector.WatchEvent{ - Type: connector.WatchError, Kind: kind, Scope: scope, Err: err, + Type: connector.WatchError, Workspace: fleet.LocalWorkspace, Kind: kind, Scope: scope, Err: err, }) } diff --git a/internal/connector/kubeconfig/watch_bootstrap_test.go b/internal/connector/kubeconfig/watch_bootstrap_test.go new file mode 100644 index 0000000..3c1897a --- /dev/null +++ b/internal/connector/kubeconfig/watch_bootstrap_test.go @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: Apache-2.0 + +package kubeconfig + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + k8stesting "k8s.io/client-go/testing" + + "github.com/ArdurAI/sith/internal/connector" +) + +func TestLoadWatchBootstrapPaginatesConsistentSnapshot(t *testing.T) { + t.Parallel() + const opaqueToken = "opaque+/=token" + var options []metav1.ListOptions + lister := watchListFunc(func(_ context.Context, current metav1.ListOptions) (*unstructured.UnstructuredList, error) { + options = append(options, current) + switch current.Continue { + case "": + return watchList("rv-7", opaqueToken, "alpha", "bravo"), nil + case opaqueToken: + return watchList("rv-7", "", "charlie"), nil + default: + return nil, fmt.Errorf("unexpected continuation") + } + }) + bootstrap, err := loadWatchBootstrap(context.Background(), lister, watchBootstrapLimits{ + pageSize: 2, objectBudget: 4, pageBudget: 3, + }) + if err != nil { + t.Fatalf("loadWatchBootstrap() error = %v", err) + } + if len(options) != 2 || options[0].Limit != 2 || options[0].Continue != "" || + options[1].Limit != 2 || options[1].Continue != opaqueToken { + t.Fatalf("list options = %#v, want bounded pages with opaque continuation", options) + } + if bootstrap.resourceVersion != "rv-7" || !slices.Equal(watchObjectNames(bootstrap.objects), []string{"alpha", "bravo", "charlie"}) { + t.Fatalf("bootstrap = %#v, want one complete rv-7 snapshot", bootstrap) + } +} + +func TestLoadWatchBootstrapRejectsIncompleteSnapshots(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + limits watchBootstrapLimits + lister watchListFunc + wantErr string + }{ + { + name: "object budget", + limits: watchBootstrapLimits{pageSize: 2, objectBudget: 2, pageBudget: 2}, + lister: func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return watchList("rv-1", "more", "alpha", "bravo"), nil + }, + wantErr: "2-object budget", + }, + { + name: "nil response", + limits: watchBootstrapLimits{pageSize: 1, objectBudget: 2, pageBudget: 2}, + lister: func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return nil, nil + }, + wantErr: "empty list response", + }, + { + name: "ignored page limit", + limits: watchBootstrapLimits{pageSize: 1, objectBudget: 3, pageBudget: 3}, + lister: func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return watchList("rv-1", "", "alpha", "bravo"), nil + }, + wantErr: "ignored the requested page limit", + }, + { + name: "empty resource version", + limits: watchBootstrapLimits{pageSize: 1, objectBudget: 2, pageBudget: 2}, + lister: func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return watchList("", "", "alpha"), nil + }, + wantErr: "empty resourceVersion", + }, + { + name: "changed resource version", + limits: watchBootstrapLimits{pageSize: 1, objectBudget: 3, pageBudget: 3}, + lister: func(_ context.Context, options metav1.ListOptions) (*unstructured.UnstructuredList, error) { + if options.Continue == "" { + return watchList("rv-1", "next", "alpha"), nil + } + return watchList("rv-2", "", "bravo"), nil + }, + wantErr: "resourceVersion changed", + }, + { + name: "continuation cycle", + limits: watchBootstrapLimits{pageSize: 1, objectBudget: 3, pageBudget: 3}, + lister: func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return watchList("rv-1", "cycle", "alpha"), nil + }, + wantErr: "repeated a continuation token", + }, + { + name: "page budget", + limits: watchBootstrapLimits{pageSize: 1, objectBudget: 3, pageBudget: 1}, + lister: func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) { + return watchList("rv-1", "next", "alpha"), nil + }, + wantErr: "1-page budget", + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + bootstrap, err := loadWatchBootstrap(context.Background(), test.lister, test.limits) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("loadWatchBootstrap() = %#v, %v, want %q error", bootstrap, err, test.wantErr) + } + if len(bootstrap.objects) != 0 || bootstrap.resourceVersion != "" { + t.Fatalf("failed bootstrap leaked partial snapshot: %#v", bootstrap) + } + }) + } +} + +func TestLoadWatchBootstrapCancellationDiscardsPartialPages(t *testing.T) { + t.Parallel() + started := make(chan struct{}) + lister := watchListFunc(func(ctx context.Context, options metav1.ListOptions) (*unstructured.UnstructuredList, error) { + if options.Continue == "" { + return watchList("rv-1", "next", "alpha"), nil + } + close(started) + <-ctx.Done() + return nil, ctx.Err() + }) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-started + cancel() + }() + bootstrap, err := loadWatchBootstrap(ctx, lister, watchBootstrapLimits{ + pageSize: 1, objectBudget: 3, pageBudget: 3, + }) + if !errors.Is(err, context.Canceled) || len(bootstrap.objects) != 0 { + t.Fatalf("loadWatchBootstrap() = %#v, %v, want canceled empty result", bootstrap, err) + } +} + +func TestLoadWatchBootstrapSanitizesExpiredContinuation(t *testing.T) { + t.Parallel() + const bodyMarker = "response-body-marker" + lister := watchListFunc(func(_ context.Context, options metav1.ListOptions) (*unstructured.UnstructuredList, error) { + if options.Continue == "" { + return watchList("rv-1", "expired-token", "alpha"), nil + } + return nil, apierrors.NewResourceExpired(bodyMarker) + }) + _, err := loadWatchBootstrap(context.Background(), lister, watchBootstrapLimits{ + pageSize: 1, objectBudget: 3, pageBudget: 3, + }) + if err == nil || !strings.Contains(err.Error(), "continuation expired") || + strings.Contains(err.Error(), bodyMarker) || strings.Contains(err.Error(), "expired-token") { + t.Fatalf("loadWatchBootstrap() error = %v, want sanitized expiration", err) + } +} + +func TestLoadWatchBootstrapSanitizesContinuationFailure(t *testing.T) { + t.Parallel() + const ( + bodyMarker = "generic-response-body-marker" + tokenMarker = "generic-continuation-token" + ) + lister := watchListFunc(func(_ context.Context, options metav1.ListOptions) (*unstructured.UnstructuredList, error) { + if options.Continue == "" { + return watchList("rv-1", tokenMarker, "alpha"), nil + } + return nil, fmt.Errorf("remote failure: %s %s", bodyMarker, tokenMarker) + }) + _, err := loadWatchBootstrap(context.Background(), lister, watchBootstrapLimits{ + pageSize: 1, objectBudget: 3, pageBudget: 3, + }) + if err == nil || !strings.Contains(err.Error(), "list page 2 failed") || + strings.Contains(err.Error(), bodyMarker) || strings.Contains(err.Error(), tokenMarker) { + t.Fatalf("loadWatchBootstrap() error = %v, want sanitized continuation failure", err) + } +} + +func TestWatchBootstrapFailureEmitsErrorWithoutOpeningWatch(t *testing.T) { + t.Parallel() + client := fakeClient() + client.PrependReactor("list", "pods", func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, watchList("rv-1", "cycle", "alpha"), nil + }) + adapter, err := New( + WithLoadingRules(testLoadingRules(t, testConfig("alpha"))), + withProbe(func(context.Context, *rest.Config) error { return nil }), + withDynamicFactory(func(*rest.Config) (dynamic.Interface, error) { return client, nil }), + ) + if err != nil { + t.Fatalf("New() error = %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + events, err := adapter.Watch(ctx, "Pod") + if err != nil { + t.Fatalf("Watch() error = %v", err) + } + event := receiveWatchEvent(ctx, t, events) + if event.Type != connector.WatchError || event.Err == nil || + !strings.Contains(event.Err.Error(), "repeated a continuation token") { + t.Fatalf("watch event = %#v, want fail-closed bootstrap error", event) + } + for _, action := range client.Actions() { + if action.GetVerb() == "watch" { + t.Fatalf("watch opened after failed bootstrap: %#v", action) + } + } +} + +type watchListFunc func(context.Context, metav1.ListOptions) (*unstructured.UnstructuredList, error) + +func (function watchListFunc) List( + ctx context.Context, + options metav1.ListOptions, +) (*unstructured.UnstructuredList, error) { + return function(ctx, options) +} + +func watchList(resourceVersion, continueToken string, names ...string) *unstructured.UnstructuredList { + list := &unstructured.UnstructuredList{Items: make([]unstructured.Unstructured, 0, len(names))} + list.SetAPIVersion("v1") + list.SetKind("PodList") + list.SetResourceVersion(resourceVersion) + list.SetContinue(continueToken) + for _, name := range names { + list.Items = append(list.Items, *pod(name, "apps", "registry.example/test:v1", nil)) + } + return list +} + +func watchObjectNames(objects []unstructured.Unstructured) []string { + names := make([]string, 0, len(objects)) + for _, object := range objects { + names = append(names, object.GetName()) + } + return names +} diff --git a/internal/connector/opencost/boundary_test.go b/internal/connector/opencost/boundary_test.go new file mode 100644 index 0000000..5c2522a --- /dev/null +++ b/internal/connector/opencost/boundary_test.go @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: Apache-2.0 + +package opencost + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +var allowedProductionImports = map[string]bool{ + "bytes": true, + "crypto/sha256": true, + "encoding/hex": true, + "encoding/json": true, + "fmt": true, + "io": true, + "math/big": true, + "sort": true, + "strings": true, + "time": true, + "unicode": true, + "unicode/utf8": true, + "github.com/ArdurAI/sith/internal/fleet": true, + "k8s.io/apimachinery/pkg/util/validation": true, +} + +var allowedProductionFiles = map[string]string{ + "project.go": "bc5a1b2919a50cd0213deb2881c25a3fd43beed790c9d3cbfeb9e147cb72dc96", + "rollup.go": "1dccb66e42dde4be827d2b2ef3a8cfea624dbce374934fd9e8340849cc3ba093", +} + +var allowedProductionDeclarations = map[string]bool{ + "func:ProjectNamespaceCosts": true, + "func:ProjectNamespaceCostSnapshot": true, + "func:RollupWorkspaceCosts": true, + "func:buildFact": true, + "func:consumeUniqueJSON": true, + "func:decodeOptionalField": true, + "func:matchingDelimiter": true, + "func:namespaceCostNativeID": true, + "func:newCostAccumulator": true, + "func:objectFields": true, + "func:observationCostValue": true, + "func:parseCanonicalCost": true, + "func:parseCanonicalTime": true, + "func:parseCostAmount": true, + "func:rejectCaseAliases": true, + "func:rejectDuplicateJSON": true, + "func:sortedScopeKeys": true, + "func:validCostLiteral": true, + "func:validateAllocation": true, + "func:validateAllocationWindow": true, + "func:validateCanonicalTime": true, + "func:validateNamespaceCostFact": true, + "func:validateNamespaceCostSnapshot": true, + "func:validateProjection": true, + "func:validateResponse": true, + "func:validateText": true, + "func:validateWorkspaceRollupRequest": true, + "method:allocationProperties.UnmarshalJSON": true, + "method:allocationRecord.UnmarshalJSON": true, + "method:allocationResponse.UnmarshalJSON": true, + "method:allocationWindow.UnmarshalJSON": true, + "method:costAccumulator.add": true, + "method:costAccumulator.amounts": true, + "type:AllocationQuery": true, + "type:CostAmounts": true, + "type:NamespaceCostSnapshot": true, + "type:Projection": true, + "type:WorkspaceCostCoverage": true, + "type:WorkspaceCostRollup": true, + "type:WorkspaceRollupRequest": true, + "type:allocationProperties": true, + "type:allocationRecord": true, + "type:allocationResponse": true, + "type:allocationWindow": true, + "type:costField": true, + "type:costAccumulator": true, + "type:namespaceCostObservation": true, + "value:Kind": true, + "value:ProtocolVersion": true, + "value:aggregateNamespace": true, + "value:costFields": true, + "value:costScale": true, + "value:currencyUSD": true, + "value:maxAllocations": true, + "value:maxClockSkew": true, + "value:maxCostLiteralBytes": true, + "value:maxCostUnits": true, + "value:maxFactPayloadBytes": true, + "value:maxIdentityText": true, + "value:maxJSONDepth": true, + "value:maxQueryWindow": true, + "value:maxResponseBytes": true, + "value:maxRollupCostUnits": true, + "value:maxRollupFacts": true, + "value:maxRollupInputBytes": true, + "value:maxRollupPayloadBytes": true, + "value:maxRollupScopes": true, +} + +func TestProjectorHasNoIOCredentialPersistenceOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read opencost package: %v", err) + } + seenFiles := make(map[string]bool, len(allowedProductionFiles)) + seenDeclarations := make(map[string]bool, len(allowedProductionDeclarations)) + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + expectedFingerprint, allowed := allowedProductionFiles[entry.Name()] + if !allowed { + t.Errorf("projector contains unreviewed production file %s", entry.Name()) + } + seenFiles[entry.Name()] = true + fileSet := token.NewFileSet() + file, err := parser.ParseFile(fileSet, entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + fingerprint, err := productionStructureFingerprint(fileSet, file) + if err != nil { + t.Fatalf("fingerprint %s: %v", entry.Name(), err) + } + if fingerprint != expectedFingerprint { + t.Errorf("projector production structure changed for %s: got %s", entry.Name(), fingerprint) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if imported.Name != nil { + t.Errorf("projector aliases production import %q as %q", path, imported.Name.Name) + } + if !allowedProductionImports[path] { + t.Errorf("projector imports unreviewed package %q", path) + } + } + for _, declaration := range file.Decls { + for _, key := range productionDeclarationKeys(declaration) { + if !allowedProductionDeclarations[key] { + t.Errorf("projector declares unreviewed symbol %s", key) + } + if seenDeclarations[key] { + t.Errorf("projector declaration %s is duplicated", key) + } + seenDeclarations[key] = true + } + } + ast.Inspect(file, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if ok && isIdentifier(selector.X, "io") && selector.Sel.Name != "EOF" { + t.Errorf("projector uses unreviewed io capability io.%s", selector.Sel.Name) + } + function, ok := node.(*ast.FuncDecl) + if !ok { + return true + } + switch function.Name.Name { + case "Plan", "Execute", "Verify", "Write", "Delete", "Update", "Create", "Reload", "Dial", "Fetch": + t.Errorf("projector declares forbidden capability seam %s", function.Name.Name) + } + return true + }) + } + for name := range allowedProductionFiles { + if !seenFiles[name] { + t.Errorf("reviewed production file %s is missing", name) + } + } + for key := range allowedProductionDeclarations { + if !seenDeclarations[key] { + t.Errorf("reviewed production declaration %s is missing", key) + } + } +} + +func TestPublicProjectorBoundaryIsValueOnlyAndExact(t *testing.T) { + t.Parallel() + file, err := parser.ParseFile(token.NewFileSet(), "project.go", nil, 0) + if err != nil { + t.Fatalf("parse project.go: %v", err) + } + var projector *ast.FuncDecl + var projection, query *ast.TypeSpec + for _, declaration := range file.Decls { + switch value := declaration.(type) { + case *ast.FuncDecl: + if value.Name.Name == "ProjectNamespaceCosts" { + projector = value + } + case *ast.GenDecl: + for _, specification := range value.Specs { + typeSpecification, ok := specification.(*ast.TypeSpec) + if !ok { + continue + } + switch typeSpecification.Name.Name { + case "Projection": + projection = typeSpecification + case "AllocationQuery": + query = typeSpecification + } + } + } + } + if projector == nil || !projectorHasExpectedSignature(projector) { + t.Fatal("ProjectNamespaceCosts must keep the exact value-only Projection to []fleet.GraphFact,error signature") + } + if projection == nil || !structHasExpectedFields(projection, []fieldShape{ + {name: "Workspace", identifier: "string"}, + {name: "Scope", identifier: "string"}, + {name: "CurrencyCode", identifier: "string"}, + {name: "Query", identifier: "AllocationQuery"}, + {name: "CollectedAt", packageName: "time", selected: "Time"}, + {name: "Response", identifier: "byte", slice: true}, + }) { + t.Fatal("Projection boundary changed or gained a capability-bearing field") + } + if query == nil || !structHasExpectedFields(query, []fieldShape{ + {name: "WindowStart", packageName: "time", selected: "Time"}, + {name: "WindowEnd", packageName: "time", selected: "Time"}, + {name: "Step", packageName: "time", selected: "Duration"}, + {name: "Aggregate", identifier: "string"}, + {name: "Filter", identifier: "string"}, + {name: "Accumulate", identifier: "bool"}, + {name: "IncludeIdle", identifier: "bool"}, + {name: "ShareIdle", identifier: "bool"}, + {name: "IdleByNode", identifier: "bool"}, + {name: "ShareLoadBalancer", identifier: "bool"}, + {name: "IncludeAggregatedMetadata", identifier: "bool"}, + {name: "IncludeProportionalAssetCosts", identifier: "bool"}, + }) { + t.Fatal("AllocationQuery boundary changed or gained an unreviewed field") + } +} + +type fieldShape struct { + name string + identifier string + packageName string + selected string + slice bool +} + +func structHasExpectedFields(specification *ast.TypeSpec, expected []fieldShape) bool { + structure, ok := specification.Type.(*ast.StructType) + if !ok || structure.Fields == nil || len(structure.Fields.List) != len(expected) { + return false + } + for index, field := range structure.Fields.List { + want := expected[index] + if len(field.Names) != 1 || field.Names[0].Name != want.name || field.Tag != nil { + return false + } + if want.slice { + slice, ok := field.Type.(*ast.ArrayType) + if !ok || slice.Len != nil || !isIdentifier(slice.Elt, want.identifier) { + return false + } + continue + } + if want.packageName != "" { + if !isSelector(field.Type, want.packageName, want.selected) { + return false + } + continue + } + if !isIdentifier(field.Type, want.identifier) { + return false + } + } + return true +} + +func projectorHasExpectedSignature(declaration *ast.FuncDecl) bool { + if declaration.Recv != nil || declaration.Type.TypeParams != nil || declaration.Type.Params == nil || + len(declaration.Type.Params.List) != 1 || declaration.Type.Results == nil || + len(declaration.Type.Results.List) != 2 { + return false + } + parameter := declaration.Type.Params.List[0] + if len(parameter.Names) != 1 || parameter.Names[0].Name != "input" || !isIdentifier(parameter.Type, "Projection") { + return false + } + firstResult, ok := declaration.Type.Results.List[0].Type.(*ast.ArrayType) + return ok && firstResult.Len == nil && isSelector(firstResult.Elt, "fleet", "GraphFact") && + isIdentifier(declaration.Type.Results.List[1].Type, "error") +} + +func productionDeclarationKeys(declaration ast.Decl) []string { + switch value := declaration.(type) { + case *ast.FuncDecl: + return []string{functionDeclarationKey(value)} + case *ast.GenDecl: + if value.Tok == token.IMPORT { + return nil + } + keys := make([]string, 0, len(value.Specs)) + for _, specification := range value.Specs { + switch typed := specification.(type) { + case *ast.TypeSpec: + keys = append(keys, "type:"+typed.Name.Name) + case *ast.ValueSpec: + for _, name := range typed.Names { + keys = append(keys, "value:"+name.Name) + } + } + } + return keys + default: + return []string{"declaration:"} + } +} + +func functionDeclarationKey(declaration *ast.FuncDecl) string { + if declaration.Recv == nil { + return "func:" + declaration.Name.Name + } + if len(declaration.Recv.List) != 1 { + return "method:." + declaration.Name.Name + } + receiver := declaration.Recv.List[0].Type + if pointer, ok := receiver.(*ast.StarExpr); ok { + receiver = pointer.X + } + identifier, ok := receiver.(*ast.Ident) + if !ok { + return "method:." + declaration.Name.Name + } + return "method:" + identifier.Name + "." + declaration.Name.Name +} + +func productionStructureFingerprint(fileSet *token.FileSet, file *ast.File) (string, error) { + var document bytes.Buffer + if err := format.Node(&document, fileSet, file); err != nil { + return "", err + } + digest := sha256.Sum256(document.Bytes()) + return hex.EncodeToString(digest[:]), nil +} + +func isIdentifier(expression ast.Expr, name string) bool { + identifier, ok := expression.(*ast.Ident) + return ok && identifier.Name == name +} + +func isSelector(expression ast.Expr, packageName, selectedName string) bool { + selector, ok := expression.(*ast.SelectorExpr) + return ok && isIdentifier(selector.X, packageName) && selector.Sel.Name == selectedName +} diff --git a/internal/connector/opencost/project.go b/internal/connector/opencost/project.go new file mode 100644 index 0000000..d7a47c2 --- /dev/null +++ b/internal/connector/opencost/project.go @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package opencost normalizes bounded OpenCost allocation evidence for Sith's +// operational graph. +package opencost + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math/big" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + "k8s.io/apimachinery/pkg/util/validation" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for OpenCost read evidence. + Kind = "opencost" + // ProtocolVersion identifies the normalized namespace-allocation fact contract. + ProtocolVersion = "allocation/namespace-usd-v1" + + maxResponseBytes = 2 << 20 + maxAllocations = 1_024 + maxFactPayloadBytes = 4 << 10 + maxJSONDepth = 64 + maxIdentityText = 253 + maxQueryWindow = 31 * 24 * time.Hour + maxClockSkew = 5 * time.Minute + maxCostUnits = int64(1_000_000_000_000) + maxCostLiteralBytes = 32 + costScale = 5 + + currencyUSD = "USD" + aggregateNamespace = "namespace" +) + +// AllocationQuery records the exact already-authorized OpenCost query contract. The caller must +// use one explicit UTC window, aggregate by namespace, and request one set for the whole window. +type AllocationQuery struct { + WindowStart time.Time + WindowEnd time.Time + Step time.Duration + Aggregate string + Filter string + Accumulate bool + IncludeIdle bool + ShareIdle bool + IdleByNode bool + ShareLoadBalancer bool + IncludeAggregatedMetadata bool + IncludeProportionalAssetCosts bool +} + +// Projection supplies one already-authorized OpenCost allocation response. The trusted caller +// owns endpoint selection, authorization, request execution, and the source-currency assertion. +// ProjectNamespaceCosts performs no discovery, network access, credential loading, persistence, +// process execution, billing, optimization, or mutation. +type Projection struct { + Workspace string + Scope string + CurrencyCode string + Query AllocationQuery + CollectedAt time.Time + Response []byte +} + +type allocationResponse struct { + Code *int + Status *string + Data *[]map[string]json.RawMessage + Message *string + Warning *string +} + +type allocationRecord struct { + Name *string + Properties *allocationProperties + Window *allocationWindow + Start *string + End *string + Amounts map[string]json.RawMessage +} + +type allocationProperties struct { + Cluster *string + Namespace *string +} + +type allocationWindow struct { + Start *string + End *string +} + +type costField struct { + JSONName string + AllowNegative bool + PartOfTotal bool +} + +var costFields = [...]costField{ + {JSONName: "cpuCost", PartOfTotal: true}, + {JSONName: "cpuCostAdjustment", AllowNegative: true, PartOfTotal: true}, + {JSONName: "gpuCost", PartOfTotal: true}, + {JSONName: "gpuCostAdjustment", AllowNegative: true, PartOfTotal: true}, + {JSONName: "ramCost", PartOfTotal: true}, + {JSONName: "ramCostAdjustment", AllowNegative: true, PartOfTotal: true}, + {JSONName: "pvCost", PartOfTotal: true}, + {JSONName: "pvCostAdjustment", AllowNegative: true, PartOfTotal: true}, + {JSONName: "networkCost", PartOfTotal: true}, + {JSONName: "networkCostAdjustment", AllowNegative: true, PartOfTotal: true}, + {JSONName: "loadBalancerCost", PartOfTotal: true}, + {JSONName: "loadBalancerCostAdjustment", AllowNegative: true, PartOfTotal: true}, + {JSONName: "sharedCost", PartOfTotal: true}, + {JSONName: "externalCost", PartOfTotal: true}, + {JSONName: "totalCost"}, +} + +type namespaceCostObservation struct { + Namespace string `json:"namespace"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + Currency string `json:"currency"` + CPUCost string `json:"cpu_cost"` + CPUCostAdjustment string `json:"cpu_cost_adjustment"` + GPUCost string `json:"gpu_cost"` + GPUCostAdjustment string `json:"gpu_cost_adjustment"` + RAMCost string `json:"ram_cost"` + RAMCostAdjustment string `json:"ram_cost_adjustment"` + PVCost string `json:"pv_cost"` + PVCostAdjustment string `json:"pv_cost_adjustment"` + NetworkCost string `json:"network_cost"` + NetworkCostAdjustment string `json:"network_cost_adjustment"` + LoadBalancerCost string `json:"load_balancer_cost"` + LoadBalancerCostAdjustment string `json:"load_balancer_cost_adjustment"` + SharedCost string `json:"shared_cost"` + ExternalCost string `json:"external_cost"` + TotalCost string `json:"total_cost"` +} + +func (response *allocationResponse) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "allocation response") + if err != nil { + return err + } + if err := rejectCaseAliases(fields, []string{"code", "status", "data", "meta", "message", "warning"}); err != nil { + return err + } + for _, field := range []struct { + name string + target any + }{ + {name: "code", target: &response.Code}, + {name: "status", target: &response.Status}, + {name: "data", target: &response.Data}, + {name: "message", target: &response.Message}, + {name: "warning", target: &response.Warning}, + } { + if err := decodeOptionalField(fields, field.name, field.target); err != nil { + return err + } + } + return nil +} + +func (record *allocationRecord) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "allocation") + if err != nil { + return err + } + known := []string{"name", "properties", "window", "start", "end"} + for _, field := range costFields { + known = append(known, field.JSONName) + } + if err := rejectCaseAliases(fields, known); err != nil { + return err + } + for _, field := range []struct { + name string + target any + }{ + {name: "name", target: &record.Name}, + {name: "properties", target: &record.Properties}, + {name: "window", target: &record.Window}, + {name: "start", target: &record.Start}, + {name: "end", target: &record.End}, + } { + if err := decodeOptionalField(fields, field.name, field.target); err != nil { + return err + } + } + record.Amounts = make(map[string]json.RawMessage, len(costFields)) + for _, field := range costFields { + if raw, exists := fields[field.JSONName]; exists { + record.Amounts[field.JSONName] = append(json.RawMessage(nil), raw...) + } + } + return nil +} + +func (properties *allocationProperties) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "allocation properties") + if err != nil { + return err + } + if err := rejectCaseAliases(fields, []string{"cluster", "namespace"}); err != nil { + return err + } + if err := decodeOptionalField(fields, "cluster", &properties.Cluster); err != nil { + return err + } + return decodeOptionalField(fields, "namespace", &properties.Namespace) +} + +func (window *allocationWindow) UnmarshalJSON(document []byte) error { + fields, err := objectFields(document, "allocation window") + if err != nil { + return err + } + if err := rejectCaseAliases(fields, []string{"start", "end"}); err != nil { + return err + } + if err := decodeOptionalField(fields, "start", &window.Start); err != nil { + return err + } + return decodeOptionalField(fields, "end", &window.End) +} + +// ProjectNamespaceCosts returns one deterministic TELEMETRY cost fact per valid Kubernetes +// namespace. A successful empty allocation set abstains with zero facts. Any invalid row returns +// an error and no partial facts. +func ProjectNamespaceCosts(input Projection) ([]fleet.GraphFact, error) { + if err := validateProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode OpenCost allocation response: %w", err) + } + + var response allocationResponse + if err := json.Unmarshal(input.Response, &response); err != nil { + return nil, fmt.Errorf("decode OpenCost allocation response") + } + allocations, err := validateResponse(response) + if err != nil { + return nil, err + } + + namespaces := make([]string, 0, len(allocations)) + for namespace := range allocations { + namespaces = append(namespaces, namespace) + } + sort.Strings(namespaces) + + facts := make([]fleet.GraphFact, 0, len(namespaces)) + for index, namespace := range namespaces { + observation, err := validateAllocation(input, namespace, allocations[namespace]) + if err != nil { + return nil, fmt.Errorf("project OpenCost allocation %d: %w", index, err) + } + fact, err := buildFact(input, observation) + if err != nil { + return nil, err + } + facts = append(facts, fact) + } + return facts, nil +} + +func validateProjection(input Projection) error { + if err := validateText("workspace", input.Workspace, maxIdentityText); err != nil { + return err + } + if err := validateText("scope", input.Scope, maxIdentityText); err != nil || strings.Contains(input.Scope, "/") { + return fmt.Errorf("scope is invalid") + } + if input.CurrencyCode != currencyUSD { + return fmt.Errorf("OpenCost source currency must be USD") + } + if input.Query.Aggregate != aggregateNamespace || input.Query.Filter != "" || input.Query.Accumulate || + input.Query.IncludeIdle || input.Query.ShareIdle || input.Query.IdleByNode || + input.Query.ShareLoadBalancer || input.Query.IncludeAggregatedMetadata || + input.Query.IncludeProportionalAssetCosts { + return fmt.Errorf("OpenCost allocation query contract is invalid") + } + if err := validateCanonicalTime("window start", input.Query.WindowStart); err != nil { + return err + } + if err := validateCanonicalTime("window end", input.Query.WindowEnd); err != nil { + return err + } + if err := validateCanonicalTime("collection time", input.CollectedAt); err != nil { + return err + } + window := input.Query.WindowEnd.Sub(input.Query.WindowStart) + if window <= 0 || window > maxQueryWindow { + return fmt.Errorf("OpenCost allocation window is invalid") + } + if input.Query.Step != window { + return fmt.Errorf("OpenCost allocation step must equal the query window") + } + if input.Query.WindowEnd.After(input.CollectedAt.Add(maxClockSkew)) { + return fmt.Errorf("OpenCost allocation window ends after collection time") + } + if len(input.Response) == 0 { + return fmt.Errorf("OpenCost allocation response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("OpenCost allocation response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("OpenCost allocation response must be valid UTF-8") + } + return nil +} + +func validateResponse(response allocationResponse) (map[string]json.RawMessage, error) { + if response.Code == nil || *response.Code != 200 || response.Data == nil { + return nil, fmt.Errorf("OpenCost allocation response is not a complete success") + } + if response.Status != nil && *response.Status != "success" { + return nil, fmt.Errorf("OpenCost allocation response status is invalid") + } + if response.Message != nil && *response.Message != "" { + return nil, fmt.Errorf("OpenCost allocation response contains a message") + } + if response.Warning != nil && *response.Warning != "" { + return nil, fmt.Errorf("OpenCost allocation response contains a warning") + } + if len(*response.Data) != 1 || (*response.Data)[0] == nil { + return nil, fmt.Errorf("OpenCost allocation response must contain exactly one allocation set") + } + allocations := (*response.Data)[0] + if len(allocations) > maxAllocations { + return nil, fmt.Errorf("OpenCost allocation count exceeds %d", maxAllocations) + } + return allocations, nil +} + +func validateAllocation(input Projection, mapKey string, document json.RawMessage) (namespaceCostObservation, error) { + var record allocationRecord + if err := json.Unmarshal(document, &record); err != nil { + return namespaceCostObservation{}, fmt.Errorf("decode namespace allocation") + } + if record.Name == nil || record.Properties == nil || record.Properties.Cluster == nil || + record.Properties.Namespace == nil || record.Window == nil || record.Window.Start == nil || + record.Window.End == nil || record.Start == nil || record.End == nil { + return namespaceCostObservation{}, fmt.Errorf("namespace allocation identity and window are required") + } + if mapKey != *record.Name || mapKey != *record.Properties.Namespace || + len(validation.IsDNS1123Label(mapKey)) != 0 { + return namespaceCostObservation{}, fmt.Errorf("namespace allocation identity is invalid") + } + if *record.Properties.Cluster != input.Scope { + return namespaceCostObservation{}, fmt.Errorf("namespace allocation cluster does not match trusted scope") + } + if err := validateAllocationWindow(input.Query, *record.Window.Start, *record.Window.End, *record.Start, *record.End); err != nil { + return namespaceCostObservation{}, err + } + + parsed := make(map[string]*big.Rat, len(costFields)) + canonical := make(map[string]string, len(costFields)) + total := new(big.Rat) + for _, field := range costFields { + raw, exists := record.Amounts[field.JSONName] + if !exists { + return namespaceCostObservation{}, fmt.Errorf("namespace allocation cost components are incomplete") + } + amount, normalized, err := parseCostAmount(raw, field.AllowNegative) + if err != nil { + return namespaceCostObservation{}, fmt.Errorf("namespace allocation cost component is invalid") + } + parsed[field.JSONName] = amount + canonical[field.JSONName] = normalized + if field.PartOfTotal { + total.Add(total, amount) + } + } + difference := new(big.Rat).Sub(parsed["totalCost"], total) + difference.Abs(difference) + if difference.Cmp(new(big.Rat).SetFrac64(1, 10_000)) > 0 { + return namespaceCostObservation{}, fmt.Errorf("namespace allocation total does not match cost components") + } + + return namespaceCostObservation{ + Namespace: mapKey, WindowStart: input.Query.WindowStart, WindowEnd: input.Query.WindowEnd, + Currency: currencyUSD, + CPUCost: canonical["cpuCost"], CPUCostAdjustment: canonical["cpuCostAdjustment"], + GPUCost: canonical["gpuCost"], GPUCostAdjustment: canonical["gpuCostAdjustment"], + RAMCost: canonical["ramCost"], RAMCostAdjustment: canonical["ramCostAdjustment"], + PVCost: canonical["pvCost"], PVCostAdjustment: canonical["pvCostAdjustment"], + NetworkCost: canonical["networkCost"], NetworkCostAdjustment: canonical["networkCostAdjustment"], + LoadBalancerCost: canonical["loadBalancerCost"], + LoadBalancerCostAdjustment: canonical["loadBalancerCostAdjustment"], + SharedCost: canonical["sharedCost"], ExternalCost: canonical["externalCost"], + TotalCost: canonical["totalCost"], + }, nil +} + +func validateAllocationWindow(query AllocationQuery, values ...string) error { + if len(values) != 4 { + return fmt.Errorf("namespace allocation window is invalid") + } + want := []time.Time{query.WindowStart, query.WindowEnd, query.WindowStart, query.WindowEnd} + for index, value := range values { + parsed, err := parseCanonicalTime(value) + if err != nil || !parsed.Equal(want[index]) { + return fmt.Errorf("namespace allocation window does not match trusted query") + } + } + return nil +} + +func parseCostAmount(raw json.RawMessage, allowNegative bool) (*big.Rat, string, error) { + value := string(bytes.TrimSpace(raw)) + if len(value) == 0 || len(value) > maxCostLiteralBytes || !validCostLiteral(value) { + return nil, "", fmt.Errorf("invalid cost literal") + } + amount, ok := new(big.Rat).SetString(value) + if !ok || (!allowNegative && amount.Sign() < 0) { + return nil, "", fmt.Errorf("invalid cost amount") + } + absolute := new(big.Rat).Abs(new(big.Rat).Set(amount)) + if absolute.Cmp(new(big.Rat).SetInt64(maxCostUnits)) > 0 { + return nil, "", fmt.Errorf("cost amount exceeds limit") + } + return amount, amount.FloatString(costScale), nil +} + +func validCostLiteral(value string) bool { + if value == "" { + return false + } + index := 0 + if value[index] == '-' { + index++ + if index == len(value) { + return false + } + } + if value[index] == '0' { + index++ + if index < len(value) && value[index] >= '0' && value[index] <= '9' { + return false + } + } else { + if value[index] < '1' || value[index] > '9' { + return false + } + for index < len(value) && value[index] >= '0' && value[index] <= '9' { + index++ + } + } + if index == len(value) { + return true + } + if value[index] != '.' { + return false + } + index++ + fractionStart := index + for index < len(value) && value[index] >= '0' && value[index] <= '9' { + index++ + } + return index == len(value) && index > fractionStart && index-fractionStart <= costScale +} + +func buildFact(input Projection, observation namespaceCostObservation) (fleet.GraphFact, error) { + encoded, err := json.Marshal(observation) + if err != nil { + return fleet.GraphFact{}, fmt.Errorf("encode OpenCost namespace allocation fact: %w", err) + } + if len(encoded) > maxFactPayloadBytes { + return fleet.GraphFact{}, fmt.Errorf("OpenCost namespace allocation fact exceeds %d bytes", maxFactPayloadBytes) + } + identity, err := json.Marshal(struct { + Workspace string `json:"workspace"` + Scope string `json:"scope"` + Observed json.RawMessage `json:"observed"` + }{Workspace: input.Workspace, Scope: input.Scope, Observed: encoded}) + if err != nil { + return fleet.GraphFact{}, fmt.Errorf("encode OpenCost namespace allocation identity: %w", err) + } + digest := sha256.Sum256(identity) + nativeID := "sha256:" + hex.EncodeToString(digest[:]) + entity := fleet.EntityRef{Cluster: input.Scope, Namespace: observation.Namespace} + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, Scope: input.Scope, Kind: "NamespaceCost", + Namespace: observation.Namespace, Name: "allocation", + }, + Kind: fleet.FactCost, Observed: encoded, ObservedAt: input.Query.WindowEnd, + Source: input.Scope, + Provenance: fleet.Provenance{Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID}, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensTelemetry, Entity: &entity, + } + if err := fact.Validate(input.Workspace); err != nil { + return fleet.GraphFact{}, fmt.Errorf("validate OpenCost namespace allocation fact: %w", err) + } + return fact, nil +} + +func validateCanonicalTime(label string, value time.Time) error { + if value.IsZero() || value.Year() < 2000 || value.Year() > 9999 || value.Location() != time.UTC || + value.Nanosecond() != 0 { + return fmt.Errorf("%s is invalid", label) + } + return nil +} + +func parseCanonicalTime(value string) (time.Time, error) { + parsed, err := time.Parse(time.RFC3339, value) + if err != nil || parsed.Year() < 2000 || parsed.Year() > 9999 || + value != parsed.UTC().Format(time.RFC3339) { + return time.Time{}, fmt.Errorf("time is not canonical UTC RFC3339") + } + return parsed, nil +} + +func validateText(label, value string, maximum int) error { + if value == "" || len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("%s is invalid", label) + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func objectFields(document []byte, label string) (map[string]json.RawMessage, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(document, &fields); err != nil || fields == nil { + return nil, fmt.Errorf("%s must be a JSON object", label) + } + return fields, nil +} + +func rejectCaseAliases(fields map[string]json.RawMessage, exact []string) error { + known := make(map[string]string, len(exact)) + for _, name := range exact { + known[strings.ToLower(name)] = name + } + for name := range fields { + canonical, recognized := known[strings.ToLower(name)] + if recognized && name != canonical { + return fmt.Errorf("JSON field %s must use exact case", canonical) + } + } + return nil +} + +func decodeOptionalField(fields map[string]json.RawMessage, name string, target any) error { + value, exists := fields[name] + if !exists { + return nil + } + if bytes.Equal(bytes.TrimSpace(value), []byte("null")) { + return fmt.Errorf("JSON field %s is invalid", name) + } + if err := json.Unmarshal(value, target); err != nil { + return fmt.Errorf("JSON field %s is invalid", name) + } + return nil +} + +func rejectDuplicateJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || closing != matchingDelimiter(delimiter) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func matchingDelimiter(open json.Delim) json.Delim { + if open == '{' { + return '}' + } + return ']' +} diff --git a/internal/connector/opencost/project_test.go b/internal/connector/opencost/project_test.go new file mode 100644 index 0000000..30bc3fc --- /dev/null +++ b/internal/connector/opencost/project_test.go @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: Apache-2.0 + +package opencost + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +var ( + testWindowStart = time.Date(2026, 7, 17, 0, 0, 0, 0, time.UTC) + testWindowEnd = testWindowStart.Add(24 * time.Hour) + testCollectedAt = testWindowEnd.Add(time.Minute) +) + +func TestProjectNamespaceCostsNormalizesSortedPrivateFacts(t *testing.T) { + t.Parallel() + input := testProjection(t, "payments", "infra") + + facts, err := ProjectNamespaceCosts(input) + if err != nil { + t.Fatalf("ProjectNamespaceCosts() error = %v", err) + } + if len(facts) != 2 { + t.Fatalf("fact count = %d, want 2", len(facts)) + } + for index, namespace := range []string{"infra", "payments"} { + fact := facts[index] + if fact.Fact.Kind != fleet.FactCost || fact.Lens != fleet.LensTelemetry { + t.Fatalf("fact %d taxonomy = %s/%s", index, fact.Fact.Kind, fact.Lens) + } + if fact.Fact.Workspace != input.Workspace || fact.Fact.Source != input.Scope || + !fact.Fact.ObservedAt.Equal(testWindowEnd) || fact.Fact.Stale || fact.Fact.StaleFor != "" { + t.Fatalf("fact %d envelope = %#v", index, fact.Fact) + } + if !reflect.DeepEqual(fact.Fact.Ref, fleet.ResourceRef{ + SourceKind: Kind, Scope: input.Scope, Kind: "NamespaceCost", + Namespace: namespace, Name: "allocation", + }) { + t.Fatalf("fact %d ref = %#v", index, fact.Fact.Ref) + } + if fact.Entity == nil || *fact.Entity != (fleet.EntityRef{Cluster: input.Scope, Namespace: namespace}) { + t.Fatalf("fact %d entity = %#v", index, fact.Entity) + } + if fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion || + !strings.HasPrefix(fact.Fact.Provenance.NativeID, "sha256:") || + fact.Fact.Provenance.DeepLink != "" || fact.Fact.Provenance.Collector != "" || + fact.Fact.Display != nil || fact.Fact.Ref.Attributes != nil { + t.Fatalf("fact %d provenance or metadata = %#v", index, fact.Fact) + } + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("fact %d Validate() = %v", index, err) + } + + var observed namespaceCostObservation + if err := json.Unmarshal(fact.Fact.Observed, &observed); err != nil { + t.Fatalf("decode fact %d: %v", index, err) + } + want := namespaceCostObservation{ + Namespace: namespace, WindowStart: testWindowStart, WindowEnd: testWindowEnd, Currency: currencyUSD, + CPUCost: "1.25000", CPUCostAdjustment: "-0.05000", + GPUCost: "2.00000", GPUCostAdjustment: "0.00000", + RAMCost: "0.75000", RAMCostAdjustment: "0.01000", + PVCost: "0.50000", PVCostAdjustment: "0.00000", + NetworkCost: "0.25000", NetworkCostAdjustment: "-0.01000", + LoadBalancerCost: "0.10000", LoadBalancerCostAdjustment: "0.00000", + SharedCost: "0.20000", ExternalCost: "0.30000", TotalCost: "5.30000", + } + if !reflect.DeepEqual(observed, want) { + t.Fatalf("fact %d observed = %#v, want %#v", index, observed, want) + } + encoded, err := json.Marshal(fact) + if err != nil { + t.Fatalf("marshal fact %d: %v", index, err) + } + for _, forbidden := range []string{ + "do-not-retain-provider-id", "do-not-retain-label", "do-not-retain-annotation", + "do-not-retain-controller", "do-not-retain-endpoint", testCollectedAt.Format(time.RFC3339), + } { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("fact %d retained forbidden source data %q", index, forbidden) + } + } + } + + graph, err := fleet.NewGraph(input.Workspace, facts) + if err != nil { + t.Fatalf("NewGraph() error = %v", err) + } + if len(graph.Nodes) != 2 || len(graph.Unattached) != 0 { + t.Fatalf("graph = %#v", graph) + } +} + +func TestProjectNamespaceCostsIsDeterministicAcrossSourceOrderAndUnknownMetadata(t *testing.T) { + t.Parallel() + left := testProjection(t, "zeta", "alpha") + right := testProjection(t, "alpha", "zeta") + left.Response = orderedResponse(t, left.Scope, []string{"zeta", "alpha"}, "first secret") + right.Response = orderedResponse(t, right.Scope, []string{"alpha", "zeta"}, "second secret") + + leftFacts, err := ProjectNamespaceCosts(left) + if err != nil { + t.Fatalf("left projection: %v", err) + } + rightFacts, err := ProjectNamespaceCosts(right) + if err != nil { + t.Fatalf("right projection: %v", err) + } + if !reflect.DeepEqual(leftFacts, rightFacts) { + t.Fatalf("source order or discarded metadata changed facts:\nleft = %#v\nright = %#v", leftFacts, rightFacts) + } +} + +func TestNamespaceCostIdentityBindsOnlyRetainedEvidence(t *testing.T) { + t.Parallel() + base := testProjection(t, "payments") + baseFact := singleFact(t, base) + + collectionChanged := base + collectionChanged.CollectedAt = collectionChanged.CollectedAt.Add(time.Hour) + collectionFact := singleFact(t, collectionChanged) + if collectionFact.Fact.Provenance.NativeID != baseFact.Fact.Provenance.NativeID { + t.Fatal("collection time changed retained fact identity") + } + + unknownChanged := base + unknownChanged.Response = testResponse(t, base.Scope, []string{"payments"}, func(document map[string]any) { + document["meta"] = map[string]any{"discarded": "different secret"} + }) + unknownFact := singleFact(t, unknownChanged) + if unknownFact.Fact.Provenance.NativeID != baseFact.Fact.Provenance.NativeID { + t.Fatal("discarded source metadata changed retained fact identity") + } + + workspaceChanged := base + workspaceChanged.Workspace = "workspace-b" + scopeChanged := projectionFor(t, "workspace-a", "cluster-b", testWindowStart, testWindowEnd, "payments") + namespaceChanged := testProjection(t, "shipping") + windowChanged := projectionFor(t, "workspace-a", "cluster-a", testWindowStart.Add(time.Hour), testWindowEnd.Add(time.Hour), "payments") + costChanged := base + costChanged.Response = testResponse(t, base.Scope, []string{"payments"}, func(document map[string]any) { + allocation := testAllocationFromDocument(document, "payments") + allocation["cpuCost"] = json.Number("1.35") + allocation["totalCost"] = json.Number("5.40") + }) + + for name, input := range map[string]Projection{ + "workspace": workspaceChanged, + "scope": scopeChanged, + "namespace": namespaceChanged, + "window": windowChanged, + "cost": costChanged, + } { + name, input := name, input + t.Run(name, func(t *testing.T) { + t.Parallel() + fact := singleFact(t, input) + if fact.Fact.Provenance.NativeID == baseFact.Fact.Provenance.NativeID { + t.Fatalf("retained %s did not change native identity", name) + } + }) + } +} + +func TestProjectNamespaceCostsAbstainsOnlyForCompleteEmptySet(t *testing.T) { + t.Parallel() + input := testProjection(t) + facts, err := ProjectNamespaceCosts(input) + if err != nil { + t.Fatalf("ProjectNamespaceCosts() error = %v", err) + } + if facts == nil || len(facts) != 0 { + t.Fatalf("facts = %#v, want non-nil empty abstention", facts) + } +} + +func TestProjectNamespaceCostsAcceptsDocumentedRoundingTolerance(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + total string + wantErr bool + }{ + {name: "exact", total: "5.30000"}, + {name: "positive tolerance", total: "5.30010"}, + {name: "negative tolerance", total: "5.29990"}, + {name: "outside tolerance", total: "5.30011", wantErr: true}, + } { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := testProjection(t, "payments") + input.Response = testResponse(t, input.Scope, []string{"payments"}, func(document map[string]any) { + testAllocationFromDocument(document, "payments")["totalCost"] = json.Number(test.total) + }) + facts, err := ProjectNamespaceCosts(input) + if test.wantErr { + if err == nil || facts != nil { + t.Fatalf("ProjectNamespaceCosts() = %#v, %v; want atomic error", facts, err) + } + return + } + if err != nil || len(facts) != 1 { + t.Fatalf("ProjectNamespaceCosts() = %#v, %v", facts, err) + } + }) + } +} + +func TestProjectNamespaceCostsRejectsInvalidProjection(t *testing.T) { + t.Parallel() + valid := testProjection(t, "payments") + tests := []struct { + name string + mutate func(*Projection) + }{ + {name: "workspace", mutate: func(input *Projection) { input.Workspace = "" }}, + {name: "workspace control", mutate: func(input *Projection) { input.Workspace = "workspace\nother" }}, + {name: "scope", mutate: func(input *Projection) { input.Scope = "cluster/other" }}, + {name: "currency", mutate: func(input *Projection) { input.CurrencyCode = "EUR" }}, + {name: "aggregate", mutate: func(input *Projection) { input.Query.Aggregate = "pod" }}, + {name: "filter", mutate: func(input *Projection) { input.Query.Filter = "namespace:payments" }}, + {name: "accumulate", mutate: func(input *Projection) { input.Query.Accumulate = true }}, + {name: "include idle", mutate: func(input *Projection) { input.Query.IncludeIdle = true }}, + {name: "share idle", mutate: func(input *Projection) { input.Query.ShareIdle = true }}, + {name: "idle by node", mutate: func(input *Projection) { input.Query.IdleByNode = true }}, + {name: "share load balancer", mutate: func(input *Projection) { input.Query.ShareLoadBalancer = true }}, + {name: "aggregated metadata", mutate: func(input *Projection) { input.Query.IncludeAggregatedMetadata = true }}, + {name: "asset costs", mutate: func(input *Projection) { input.Query.IncludeProportionalAssetCosts = true }}, + {name: "missing start", mutate: func(input *Projection) { input.Query.WindowStart = time.Time{} }}, + {name: "non UTC start", mutate: func(input *Projection) { + input.Query.WindowStart = input.Query.WindowStart.In(time.FixedZone("offset", 3600)) + }}, + {name: "fractional end", mutate: func(input *Projection) { input.Query.WindowEnd = input.Query.WindowEnd.Add(time.Nanosecond) }}, + {name: "reversed window", mutate: func(input *Projection) { input.Query.WindowStart = input.Query.WindowEnd }}, + {name: "wide window", mutate: func(input *Projection) { + input.Query.WindowStart = input.Query.WindowEnd.Add(-maxQueryWindow - time.Second) + input.Query.Step = input.Query.WindowEnd.Sub(input.Query.WindowStart) + }}, + {name: "step", mutate: func(input *Projection) { input.Query.Step-- }}, + {name: "future window", mutate: func(input *Projection) { + input.CollectedAt = input.Query.WindowEnd.Add(-maxClockSkew - time.Second) + }}, + {name: "missing collection", mutate: func(input *Projection) { input.CollectedAt = time.Time{} }}, + {name: "empty response", mutate: func(input *Projection) { input.Response = nil }}, + {name: "oversized response", mutate: func(input *Projection) { + input.Response = bytes.Repeat([]byte{' '}, maxResponseBytes+1) + }}, + {name: "invalid UTF-8", mutate: func(input *Projection) { input.Response = []byte{0xff} }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := valid + input.Response = slices.Clone(valid.Response) + test.mutate(&input) + if facts, err := ProjectNamespaceCosts(input); err == nil || facts != nil { + t.Fatalf("ProjectNamespaceCosts() = %#v, %v; want atomic error", facts, err) + } + }) + } +} + +func TestProjectNamespaceCostsRejectsInvalidResponsesAtomically(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(map[string]any) + }{ + {name: "missing code", mutate: func(document map[string]any) { delete(document, "code") }}, + {name: "wrong code", mutate: func(document map[string]any) { document["code"] = 206 }}, + {name: "failed status", mutate: func(document map[string]any) { document["status"] = "failed" }}, + {name: "message", mutate: func(document map[string]any) { document["message"] = "do-not-echo" }}, + {name: "warning", mutate: func(document map[string]any) { document["warning"] = "do-not-echo" }}, + {name: "missing data", mutate: func(document map[string]any) { delete(document, "data") }}, + {name: "null data", mutate: func(document map[string]any) { document["data"] = nil }}, + {name: "multiple sets", mutate: func(document map[string]any) { + document["data"] = append(document["data"].([]any), map[string]any{}) + }}, + {name: "null set", mutate: func(document map[string]any) { document["data"] = []any{nil} }}, + {name: "too many allocations", mutate: func(document map[string]any) { + set := make(map[string]any, maxAllocations+1) + for index := 0; index <= maxAllocations; index++ { + namespace := fmt.Sprintf("ns-%04d", index) + set[namespace] = testAllocation("cluster-a", namespace) + } + document["data"] = []any{set} + }}, + {name: "null allocation", mutate: func(document map[string]any) { + testAllocationSet(document)["payments"] = nil + }}, + {name: "missing name", mutate: func(document map[string]any) { + delete(testAllocationFromDocument(document, "payments"), "name") + }}, + {name: "missing properties", mutate: func(document map[string]any) { + delete(testAllocationFromDocument(document, "payments"), "properties") + }}, + {name: "missing window", mutate: func(document map[string]any) { + delete(testAllocationFromDocument(document, "payments"), "window") + }}, + {name: "name mismatch", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["name"] = "shipping" + }}, + {name: "namespace mismatch", mutate: func(document map[string]any) { + testAllocationProperties(document, "payments")["namespace"] = "shipping" + }}, + {name: "cluster mismatch", mutate: func(document map[string]any) { + testAllocationProperties(document, "payments")["cluster"] = "cluster-b" + }}, + {name: "invalid namespace", mutate: func(document map[string]any) { + allocation := testAllocationFromDocument(document, "payments") + delete(testAllocationSet(document), "payments") + allocation["name"] = "Invalid_Namespace" + allocation["properties"].(map[string]any)["namespace"] = "Invalid_Namespace" + testAllocationSet(document)["Invalid_Namespace"] = allocation + }}, + {name: "synthetic namespace", mutate: func(document map[string]any) { + allocation := testAllocationFromDocument(document, "payments") + delete(testAllocationSet(document), "payments") + allocation["name"] = "__idle__" + allocation["properties"].(map[string]any)["namespace"] = "__idle__" + testAllocationSet(document)["__idle__"] = allocation + }}, + {name: "window mismatch", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["window"].(map[string]any)["end"] = testWindowEnd.Add(-time.Second).Format(time.RFC3339) + }}, + {name: "noncanonical window", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["start"] = testWindowStart.Format("2006-01-02T15:04:05+00:00") + }}, + {name: "missing cost", mutate: func(document map[string]any) { + delete(testAllocationFromDocument(document, "payments"), "cpuCost") + }}, + {name: "null cost", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["cpuCost"] = nil + }}, + {name: "string cost", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["cpuCost"] = "1.25" + }}, + {name: "exponent cost", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["cpuCost"] = json.RawMessage("1e0") + }}, + {name: "over-precision cost", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["cpuCost"] = json.RawMessage("1.000001") + }}, + {name: "negative base cost", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["cpuCost"] = json.Number("-1") + }}, + {name: "excessive cost", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["cpuCost"] = json.Number("1000000000000.00001") + }}, + {name: "wrong total", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["totalCost"] = json.Number("6") + }}, + {name: "negative total", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["totalCost"] = json.Number("-1") + }}, + {name: "mixed-case code alias", mutate: func(document map[string]any) { document["Code"] = 200 }}, + {name: "mixed-case cost alias", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["CPUCost"] = 1.25 + }}, + {name: "deep unknown field", mutate: func(document map[string]any) { + testAllocationFromDocument(document, "payments")["unknown"] = json.RawMessage( + strings.Repeat("[", maxJSONDepth+1) + "0" + strings.Repeat("]", maxJSONDepth+1), + ) + }}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + input := testProjection(t, "payments") + input.Response = testResponse(t, input.Scope, []string{"payments"}, test.mutate) + facts, err := ProjectNamespaceCosts(input) + if err == nil || facts != nil { + t.Fatalf("ProjectNamespaceCosts() = %#v, %v; want atomic error", facts, err) + } + if strings.Contains(err.Error(), "do-not-echo") { + t.Fatalf("error echoed untrusted source content: %v", err) + } + }) + } +} + +func TestProjectNamespaceCostsRejectsMalformedDuplicateTrailingAndMixedCaseJSON(t *testing.T) { + t.Parallel() + valid := testProjection(t, "payments") + tests := [][]byte{ + []byte(`{"code":200,"code":200,"data":[{}]}`), + append(slices.Clone(valid.Response), []byte(` {}`)...), + []byte(`{"code":200,"data":[{"payments":{"name":"payments"}}]`), + []byte(`{"Code":200,"data":[{}]}`), + } + for index, response := range tests { + input := valid + input.Response = response + if facts, err := ProjectNamespaceCosts(input); err == nil || facts != nil { + t.Fatalf("case %d = %#v, %v; want atomic error", index, facts, err) + } + } +} + +func TestProjectNamespaceCostsNeverReturnsValidatedPrefix(t *testing.T) { + t.Parallel() + input := testProjection(t, "alpha", "zeta") + input.Response = testResponse(t, input.Scope, []string{"alpha", "zeta"}, func(document map[string]any) { + testAllocationFromDocument(document, "zeta")["totalCost"] = json.Number("999") + }) + if facts, err := ProjectNamespaceCosts(input); err == nil || facts != nil { + t.Fatalf("ProjectNamespaceCosts() = %#v, %v; want no partial facts", facts, err) + } +} + +func TestCostLiteralContract(t *testing.T) { + t.Parallel() + for value, valid := range map[string]bool{ + "0": true, "-0": true, "1": true, "1.2": true, "1.23456": true, + "-1.23456": true, "1000000000000": true, + "": false, "-": false, "+1": false, ".1": false, "1.": false, + "01": false, "-01": false, "1.234567": false, "1e2": false, + "NaN": false, "Inf": false, "1/2": false, " 1": false, + } { + value, valid := value, valid + t.Run(value, func(t *testing.T) { + t.Parallel() + if got := validCostLiteral(value); got != valid { + t.Fatalf("validCostLiteral(%q) = %t, want %t", value, got, valid) + } + }) + } +} + +func FuzzProjectNamespaceCostsNeverPanicsOrReturnsPartial(f *testing.F) { + seed := testResponse(f, "cluster-a", []string{"payments"}, nil) + f.Add(seed) + f.Add([]byte(`{"code":200,"data":[{}]}`)) + f.Add([]byte(`{"code":200,"code":500,"data":[{}]}`)) + f.Fuzz(func(t *testing.T, response []byte) { + input := baseProjection(response) + facts, err := ProjectNamespaceCosts(input) + if err != nil { + if facts != nil { + t.Fatalf("error returned partial facts: %#v, %v", facts, err) + } + return + } + if len(facts) > maxAllocations { + t.Fatalf("fact count = %d", len(facts)) + } + seen := make(map[string]bool, len(facts)) + for index, fact := range facts { + if err := fact.Validate(input.Workspace); err != nil { + t.Fatalf("fact %d failed validation: %v", index, err) + } + if fact.Fact.Kind != fleet.FactCost || fact.Lens != fleet.LensTelemetry || + fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion { + t.Fatalf("fact %d escaped closed taxonomy: %#v", index, fact) + } + if seen[fact.Fact.Provenance.NativeID] { + t.Fatalf("duplicate fact identity %q", fact.Fact.Provenance.NativeID) + } + seen[fact.Fact.Provenance.NativeID] = true + } + }) +} + +func singleFact(t testing.TB, input Projection) fleet.GraphFact { + t.Helper() + facts, err := ProjectNamespaceCosts(input) + if err != nil { + t.Fatalf("ProjectNamespaceCosts() error = %v", err) + } + if len(facts) != 1 { + t.Fatalf("fact count = %d, want 1", len(facts)) + } + return facts[0] +} + +func testProjection(t testing.TB, namespaces ...string) Projection { + t.Helper() + return projectionFor(t, "workspace-a", "cluster-a", testWindowStart, testWindowEnd, namespaces...) +} + +func projectionFor(t testing.TB, workspace, scope string, start, end time.Time, namespaces ...string) Projection { + t.Helper() + input := baseProjection(testResponse(t, scope, namespaces, nil)) + input.Workspace = workspace + input.Scope = scope + input.Query.WindowStart = start + input.Query.WindowEnd = end + input.Query.Step = end.Sub(start) + input.CollectedAt = end.Add(time.Minute) + input.Response = testResponseForWindow(t, scope, namespaces, start, end, nil) + return input +} + +func baseProjection(response []byte) Projection { + return Projection{ + Workspace: "workspace-a", Scope: "cluster-a", CurrencyCode: currencyUSD, + Query: AllocationQuery{ + WindowStart: testWindowStart, WindowEnd: testWindowEnd, + Step: testWindowEnd.Sub(testWindowStart), Aggregate: aggregateNamespace, + }, + CollectedAt: testCollectedAt, + Response: response, + } +} + +func testResponse(t testing.TB, scope string, namespaces []string, mutate func(map[string]any)) []byte { + t.Helper() + return testResponseForWindow(t, scope, namespaces, testWindowStart, testWindowEnd, mutate) +} + +func testResponseForWindow( + t testing.TB, + scope string, + namespaces []string, + start, end time.Time, + mutate func(map[string]any), +) []byte { + t.Helper() + set := make(map[string]any, len(namespaces)) + for _, namespace := range namespaces { + set[namespace] = testAllocationForWindow(scope, namespace, start, end) + } + document := map[string]any{ + "code": 200, "status": "success", "data": []any{set}, + "meta": map[string]any{"endpoint": "do-not-retain-endpoint"}, + } + if mutate != nil { + mutate(document) + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("marshal test response: %v", err) + } + return encoded +} + +func orderedResponse(t testing.TB, scope string, namespaces []string, discarded string) []byte { + t.Helper() + var entries []string + for _, namespace := range namespaces { + allocation, err := json.Marshal(testAllocation(scope, namespace)) + if err != nil { + t.Fatalf("marshal allocation: %v", err) + } + key, err := json.Marshal(namespace) + if err != nil { + t.Fatalf("marshal namespace: %v", err) + } + entries = append(entries, string(key)+":"+string(allocation)) + } + secret, err := json.Marshal(discarded) + if err != nil { + t.Fatalf("marshal discarded metadata: %v", err) + } + return []byte(`{"code":200,"data":[{` + strings.Join(entries, ",") + `}],"meta":{"discarded":` + string(secret) + `}}`) +} + +func testAllocation(scope, namespace string) map[string]any { + return testAllocationForWindow(scope, namespace, testWindowStart, testWindowEnd) +} + +func testAllocationForWindow(scope, namespace string, start, end time.Time) map[string]any { + return map[string]any{ + "name": namespace, + "properties": map[string]any{ + "cluster": scope, "namespace": namespace, + "providerID": "do-not-retain-provider-id", + "labels": map[string]any{"secret": "do-not-retain-label"}, + "annotations": map[string]any{"secret": "do-not-retain-annotation"}, + "controller": "do-not-retain-controller", + }, + "window": map[string]any{"start": start.Format(time.RFC3339), "end": end.Format(time.RFC3339)}, + "start": start.Format(time.RFC3339), "end": end.Format(time.RFC3339), + "cpuCost": json.Number("1.25"), "cpuCostAdjustment": json.Number("-0.05"), + "gpuCost": json.Number("2"), "gpuCostAdjustment": json.Number("0"), + "ramCost": json.Number("0.75"), "ramCostAdjustment": json.Number("0.01"), + "pvCost": json.Number("0.5"), "pvCostAdjustment": json.Number("0"), + "networkCost": json.Number("0.25"), "networkCostAdjustment": json.Number("-0.01"), + "loadBalancerCost": json.Number("0.1"), "loadBalancerCostAdjustment": json.Number("0"), + "sharedCost": json.Number("0.2"), "externalCost": json.Number("0.3"), + "totalCost": json.Number("5.3"), + "rawAllocationOnly": map[string]any{"secret": "do-not-retain-provider-id"}, + } +} + +func testAllocationSet(document map[string]any) map[string]any { + return document["data"].([]any)[0].(map[string]any) +} + +func testAllocationFromDocument(document map[string]any, namespace string) map[string]any { + return testAllocationSet(document)[namespace].(map[string]any) +} + +func testAllocationProperties(document map[string]any, namespace string) map[string]any { + return testAllocationFromDocument(document, namespace)["properties"].(map[string]any) +} diff --git a/internal/connector/opencost/rollup.go b/internal/connector/opencost/rollup.go new file mode 100644 index 0000000..04eaa9c --- /dev/null +++ b/internal/connector/opencost/rollup.go @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: Apache-2.0 + +package opencost + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + "sort" + "time" + + "k8s.io/apimachinery/pkg/util/validation" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + maxRollupScopes = 256 + maxRollupFacts = 4_096 + maxRollupInputBytes = 8 << 20 + maxRollupPayloadBytes = 256 << 10 +) + +const maxRollupCostUnits = maxCostUnits * maxRollupFacts + +// NamespaceCostSnapshot records one successful, already-authorized per-scope projection. +// Presence of a snapshot is the reporting signal, so an empty Facts slice is distinct from a +// scope that has no snapshot. +type NamespaceCostSnapshot struct { + Workspace string `json:"workspace"` + Scope string `json:"scope"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + CurrencyCode string `json:"currency"` + Facts []fleet.GraphFact `json:"facts"` +} + +// WorkspaceRollupRequest supplies the complete expected scope set and each successful scope +// snapshot for one exact allocation window. +type WorkspaceRollupRequest struct { + Workspace string `json:"workspace"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + CurrencyCode string `json:"currency"` + ExpectedScopes []string `json:"expected_scopes"` + Snapshots []NamespaceCostSnapshot `json:"snapshots"` +} + +// WorkspaceCostCoverage makes partial cost reporting impossible to confuse with complete +// coverage. ReportedScopes includes both populated and successful-empty snapshots. +type WorkspaceCostCoverage struct { + ExpectedScopes []string `json:"expected_scopes"` + ReportedScopes []string `json:"reported_scopes"` + EmptyScopes []string `json:"empty_scopes"` + MissingScopes []string `json:"missing_scopes"` + Complete bool `json:"complete"` +} + +// CostAmounts is the exact, canonical component breakdown retained by the workspace rollup. +type CostAmounts struct { + CPUCost string `json:"cpu_cost"` + CPUCostAdjustment string `json:"cpu_cost_adjustment"` + GPUCost string `json:"gpu_cost"` + GPUCostAdjustment string `json:"gpu_cost_adjustment"` + RAMCost string `json:"ram_cost"` + RAMCostAdjustment string `json:"ram_cost_adjustment"` + PVCost string `json:"pv_cost"` + PVCostAdjustment string `json:"pv_cost_adjustment"` + NetworkCost string `json:"network_cost"` + NetworkCostAdjustment string `json:"network_cost_adjustment"` + LoadBalancerCost string `json:"load_balancer_cost"` + LoadBalancerCostAdjustment string `json:"load_balancer_cost_adjustment"` + SharedCost string `json:"shared_cost"` + ExternalCost string `json:"external_cost"` + TotalCost string `json:"total_cost"` +} + +// WorkspaceCostRollup is one deterministic, read-only workspace total. ObservedAt is nil only +// when no expected scope reported a successful snapshot. +type WorkspaceCostRollup struct { + Workspace string `json:"workspace"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + CurrencyCode string `json:"currency"` + ObservedAt *time.Time `json:"observed_at,omitempty"` + NamespaceFacts int `json:"namespace_facts"` + Coverage WorkspaceCostCoverage `json:"coverage"` + Amounts CostAmounts `json:"amounts"` +} + +// ProjectNamespaceCostSnapshot preserves the successful per-scope reporting envelope around the +// existing fact projector. It performs no additional I/O and returns a zero snapshot on error. +func ProjectNamespaceCostSnapshot(input Projection) (NamespaceCostSnapshot, error) { + facts, err := ProjectNamespaceCosts(input) + if err != nil { + return NamespaceCostSnapshot{}, err + } + return NamespaceCostSnapshot{ + Workspace: input.Workspace, Scope: input.Scope, + WindowStart: input.Query.WindowStart, WindowEnd: input.Query.WindowEnd, + CurrencyCode: input.CurrencyCode, Facts: facts, + }, nil +} + +// RollupWorkspaceCosts validates and aggregates successful per-scope snapshots. Missing expected +// scopes remain explicit coverage gaps; they never contribute a synthetic zero-cost fact. +func RollupWorkspaceCosts(input WorkspaceRollupRequest) (WorkspaceCostRollup, error) { + expected, err := validateWorkspaceRollupRequest(input) + if err != nil { + return WorkspaceCostRollup{}, err + } + + byScope := make(map[string]NamespaceCostSnapshot, len(input.Snapshots)) + reported := make([]string, 0, len(input.Snapshots)) + empty := make([]string, 0, len(input.Snapshots)) + accumulator := newCostAccumulator() + factCount := 0 + inputBytes := 0 + for index, snapshot := range input.Snapshots { + if _, exists := byScope[snapshot.Scope]; exists { + return WorkspaceCostRollup{}, fmt.Errorf("OpenCost workspace rollup snapshot %d duplicates scope", index) + } + if _, exists := expected[snapshot.Scope]; !exists { + return WorkspaceCostRollup{}, fmt.Errorf("OpenCost workspace rollup snapshot %d has unexpected scope", index) + } + if err := validateNamespaceCostSnapshot(input, snapshot, accumulator, &factCount, &inputBytes); err != nil { + return WorkspaceCostRollup{}, fmt.Errorf("OpenCost workspace rollup snapshot %d: %w", index, err) + } + byScope[snapshot.Scope] = snapshot + reported = append(reported, snapshot.Scope) + if len(snapshot.Facts) == 0 { + empty = append(empty, snapshot.Scope) + } + } + + sort.Strings(reported) + sort.Strings(empty) + missing := make([]string, 0, len(expected)-len(reported)) + for scope := range expected { + if _, exists := byScope[scope]; !exists { + missing = append(missing, scope) + } + } + sort.Strings(missing) + + amounts, err := accumulator.amounts() + if err != nil { + return WorkspaceCostRollup{}, fmt.Errorf("OpenCost workspace rollup amounts: %w", err) + } + rollup := WorkspaceCostRollup{ + Workspace: input.Workspace, WindowStart: input.WindowStart, WindowEnd: input.WindowEnd, + CurrencyCode: input.CurrencyCode, NamespaceFacts: factCount, + Coverage: WorkspaceCostCoverage{ + ExpectedScopes: sortedScopeKeys(expected), ReportedScopes: reported, + EmptyScopes: empty, MissingScopes: missing, Complete: len(missing) == 0, + }, + Amounts: amounts, + } + if len(reported) != 0 { + observedAt := input.WindowEnd + rollup.ObservedAt = &observedAt + } + encoded, err := json.Marshal(rollup) + if err != nil { + return WorkspaceCostRollup{}, fmt.Errorf("encode OpenCost workspace rollup: %w", err) + } + if len(encoded) > maxRollupPayloadBytes { + return WorkspaceCostRollup{}, fmt.Errorf("OpenCost workspace rollup exceeds %d bytes", maxRollupPayloadBytes) + } + return rollup, nil +} + +func validateWorkspaceRollupRequest(input WorkspaceRollupRequest) (map[string]struct{}, error) { + if err := validateText("workspace", input.Workspace, maxIdentityText); err != nil { + return nil, err + } + if input.CurrencyCode != currencyUSD { + return nil, fmt.Errorf("OpenCost workspace rollup currency must be USD") + } + if err := validateCanonicalTime("window start", input.WindowStart); err != nil { + return nil, err + } + if err := validateCanonicalTime("window end", input.WindowEnd); err != nil { + return nil, err + } + window := input.WindowEnd.Sub(input.WindowStart) + if window <= 0 || window > maxQueryWindow { + return nil, fmt.Errorf("OpenCost workspace rollup window is invalid") + } + if len(input.ExpectedScopes) > maxRollupScopes { + return nil, fmt.Errorf("OpenCost workspace rollup expected scope count exceeds %d", maxRollupScopes) + } + if len(input.Snapshots) > len(input.ExpectedScopes) || len(input.Snapshots) > maxRollupScopes { + return nil, fmt.Errorf("OpenCost workspace rollup snapshot count is invalid") + } + expected := make(map[string]struct{}, len(input.ExpectedScopes)) + for index, scope := range input.ExpectedScopes { + if err := validateText("scope", scope, maxIdentityText); err != nil || bytes.ContainsRune([]byte(scope), '/') { + return nil, fmt.Errorf("OpenCost workspace rollup expected scope %d is invalid", index) + } + if _, exists := expected[scope]; exists { + return nil, fmt.Errorf("OpenCost workspace rollup expected scope %d is duplicated", index) + } + expected[scope] = struct{}{} + } + return expected, nil +} + +func validateNamespaceCostSnapshot( + request WorkspaceRollupRequest, + snapshot NamespaceCostSnapshot, + accumulator *costAccumulator, + factCount *int, + inputBytes *int, +) error { + if snapshot.Workspace != request.Workspace || snapshot.CurrencyCode != request.CurrencyCode || + !snapshot.WindowStart.Equal(request.WindowStart) || !snapshot.WindowEnd.Equal(request.WindowEnd) { + return fmt.Errorf("snapshot envelope does not match requested workspace, window, and currency") + } + if err := validateCanonicalTime("snapshot window start", snapshot.WindowStart); err != nil { + return err + } + if err := validateCanonicalTime("snapshot window end", snapshot.WindowEnd); err != nil { + return err + } + if err := validateText("scope", snapshot.Scope, maxIdentityText); err != nil || + bytes.ContainsRune([]byte(snapshot.Scope), '/') { + return fmt.Errorf("snapshot scope is invalid") + } + if len(snapshot.Facts) > maxAllocations { + return fmt.Errorf("snapshot fact count exceeds %d", maxAllocations) + } + if *factCount+len(snapshot.Facts) > maxRollupFacts { + return fmt.Errorf("workspace fact count exceeds %d", maxRollupFacts) + } + + namespaces := make(map[string]struct{}, len(snapshot.Facts)) + for index, fact := range snapshot.Facts { + observation, observedBytes, err := validateNamespaceCostFact(request, snapshot.Scope, fact) + if err != nil { + return fmt.Errorf("fact %d: %w", index, err) + } + if _, exists := namespaces[observation.Namespace]; exists { + return fmt.Errorf("fact %d duplicates namespace", index) + } + namespaces[observation.Namespace] = struct{}{} + *inputBytes += observedBytes + if *inputBytes > maxRollupInputBytes { + return fmt.Errorf("workspace fact payload exceeds %d bytes", maxRollupInputBytes) + } + if err := accumulator.add(observation); err != nil { + return fmt.Errorf("fact %d amounts: %w", index, err) + } + } + *factCount += len(snapshot.Facts) + return nil +} + +func validateNamespaceCostFact( + request WorkspaceRollupRequest, + scope string, + fact fleet.GraphFact, +) (namespaceCostObservation, int, error) { + if err := fact.Validate(request.Workspace); err != nil { + return namespaceCostObservation{}, 0, fmt.Errorf("graph fact is invalid") + } + if fact.Fact.Kind != fleet.FactCost || fact.Lens != fleet.LensTelemetry || + fact.Fact.Workspace != request.Workspace || fact.Fact.Source != scope || + !fact.Fact.ObservedAt.Equal(request.WindowEnd) || fact.Fact.Stale || fact.Fact.StaleFor != "" { + return namespaceCostObservation{}, 0, fmt.Errorf("fact envelope is invalid") + } + if fact.Fact.Ref.SourceKind != Kind || fact.Fact.Ref.Scope != scope || + fact.Fact.Ref.Kind != "NamespaceCost" || fact.Fact.Ref.Name != "allocation" || + fact.Fact.Ref.Namespace == "" || fact.Fact.Ref.Attributes != nil || fact.Fact.Display != nil { + return namespaceCostObservation{}, 0, fmt.Errorf("fact resource identity or display metadata is invalid") + } + if fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion || + fact.Fact.Provenance.DeepLink != "" || fact.Fact.Provenance.Collector != "" { + return namespaceCostObservation{}, 0, fmt.Errorf("fact provenance is invalid") + } + if fact.Entity == nil || *fact.Entity != (fleet.EntityRef{ + Cluster: scope, Namespace: fact.Fact.Ref.Namespace, + }) { + return namespaceCostObservation{}, 0, fmt.Errorf("fact entity is invalid") + } + if len(fact.Fact.Observed) == 0 || len(fact.Fact.Observed) > maxFactPayloadBytes || + !json.Valid(fact.Fact.Observed) { + return namespaceCostObservation{}, 0, fmt.Errorf("fact payload is invalid") + } + if err := rejectDuplicateJSON(fact.Fact.Observed); err != nil { + return namespaceCostObservation{}, 0, fmt.Errorf("fact payload is not canonical") + } + var observation namespaceCostObservation + if err := json.Unmarshal(fact.Fact.Observed, &observation); err != nil { + return namespaceCostObservation{}, 0, fmt.Errorf("fact payload is invalid") + } + canonical, err := json.Marshal(observation) + if err != nil || !bytes.Equal(canonical, fact.Fact.Observed) { + return namespaceCostObservation{}, 0, fmt.Errorf("fact payload is not canonical") + } + if observation.Namespace != fact.Fact.Ref.Namespace || observation.Currency != request.CurrencyCode || + !observation.WindowStart.Equal(request.WindowStart) || !observation.WindowEnd.Equal(request.WindowEnd) { + return namespaceCostObservation{}, 0, fmt.Errorf("fact payload identity, window, or currency is invalid") + } + if len(validation.IsDNS1123Label(observation.Namespace)) != 0 { + return namespaceCostObservation{}, 0, fmt.Errorf("fact payload namespace is invalid") + } + if err := validateCanonicalTime("fact window start", observation.WindowStart); err != nil { + return namespaceCostObservation{}, 0, err + } + if err := validateCanonicalTime("fact window end", observation.WindowEnd); err != nil { + return namespaceCostObservation{}, 0, err + } + nativeID, err := namespaceCostNativeID(request.Workspace, scope, fact.Fact.Observed) + if err != nil || fact.Fact.Provenance.NativeID != nativeID { + return namespaceCostObservation{}, 0, fmt.Errorf("fact native identity is invalid") + } + return observation, len(fact.Fact.Observed), nil +} + +type costAccumulator struct { + values map[string]*big.Rat +} + +func newCostAccumulator() *costAccumulator { + values := make(map[string]*big.Rat, len(costFields)) + for _, field := range costFields { + values[field.JSONName] = new(big.Rat) + } + return &costAccumulator{values: values} +} + +func (accumulator *costAccumulator) add(observation namespaceCostObservation) error { + if accumulator == nil || len(accumulator.values) != len(costFields) { + return fmt.Errorf("cost accumulator is invalid") + } + total := new(big.Rat) + for _, field := range costFields { + value := observationCostValue(observation, field.JSONName) + amount, err := parseCanonicalCost(value, field.AllowNegative) + if err != nil { + return fmt.Errorf("%s is invalid", field.JSONName) + } + if field.PartOfTotal { + total.Add(total, amount) + } + accumulator.values[field.JSONName].Add(accumulator.values[field.JSONName], amount) + if new(big.Rat).Abs(new(big.Rat).Set(accumulator.values[field.JSONName])).Cmp( + new(big.Rat).SetInt64(maxRollupCostUnits), + ) > 0 { + return fmt.Errorf("%s aggregate exceeds limit", field.JSONName) + } + } + reportedTotal, err := parseCanonicalCost(observation.TotalCost, false) + if err != nil { + return fmt.Errorf("totalCost is invalid") + } + difference := new(big.Rat).Sub(reportedTotal, total) + difference.Abs(difference) + if difference.Cmp(new(big.Rat).SetFrac64(1, 10_000)) > 0 { + return fmt.Errorf("totalCost does not match components") + } + return nil +} + +func (accumulator *costAccumulator) amounts() (CostAmounts, error) { + if accumulator == nil || len(accumulator.values) != len(costFields) { + return CostAmounts{}, fmt.Errorf("cost accumulator is invalid") + } + value := func(name string) string { + return accumulator.values[name].FloatString(costScale) + } + return CostAmounts{ + CPUCost: value("cpuCost"), CPUCostAdjustment: value("cpuCostAdjustment"), + GPUCost: value("gpuCost"), GPUCostAdjustment: value("gpuCostAdjustment"), + RAMCost: value("ramCost"), RAMCostAdjustment: value("ramCostAdjustment"), + PVCost: value("pvCost"), PVCostAdjustment: value("pvCostAdjustment"), + NetworkCost: value("networkCost"), NetworkCostAdjustment: value("networkCostAdjustment"), + LoadBalancerCost: value("loadBalancerCost"), + LoadBalancerCostAdjustment: value("loadBalancerCostAdjustment"), + SharedCost: value("sharedCost"), ExternalCost: value("externalCost"), + TotalCost: value("totalCost"), + }, nil +} + +func parseCanonicalCost(value string, allowNegative bool) (*big.Rat, error) { + if !validCostLiteral(value) || len(value) < costScale+2 || + value[len(value)-costScale-1] != '.' { + return nil, fmt.Errorf("cost value is not canonical") + } + amount, ok := new(big.Rat).SetString(value) + if !ok || (!allowNegative && amount.Sign() < 0) || amount.FloatString(costScale) != value { + return nil, fmt.Errorf("cost value is not canonical") + } + if new(big.Rat).Abs(new(big.Rat).Set(amount)).Cmp(new(big.Rat).SetInt64(maxCostUnits)) > 0 { + return nil, fmt.Errorf("cost value exceeds limit") + } + return amount, nil +} + +func observationCostValue(observation namespaceCostObservation, name string) string { + switch name { + case "cpuCost": + return observation.CPUCost + case "cpuCostAdjustment": + return observation.CPUCostAdjustment + case "gpuCost": + return observation.GPUCost + case "gpuCostAdjustment": + return observation.GPUCostAdjustment + case "ramCost": + return observation.RAMCost + case "ramCostAdjustment": + return observation.RAMCostAdjustment + case "pvCost": + return observation.PVCost + case "pvCostAdjustment": + return observation.PVCostAdjustment + case "networkCost": + return observation.NetworkCost + case "networkCostAdjustment": + return observation.NetworkCostAdjustment + case "loadBalancerCost": + return observation.LoadBalancerCost + case "loadBalancerCostAdjustment": + return observation.LoadBalancerCostAdjustment + case "sharedCost": + return observation.SharedCost + case "externalCost": + return observation.ExternalCost + case "totalCost": + return observation.TotalCost + default: + return "" + } +} + +func namespaceCostNativeID(workspace, scope string, observed json.RawMessage) (string, error) { + identity, err := json.Marshal(struct { + Workspace string `json:"workspace"` + Scope string `json:"scope"` + Observed json.RawMessage `json:"observed"` + }{Workspace: workspace, Scope: scope, Observed: observed}) + if err != nil { + return "", err + } + digest := sha256.Sum256(identity) + return "sha256:" + hex.EncodeToString(digest[:]), nil +} + +func sortedScopeKeys(scopes map[string]struct{}) []string { + result := make([]string, 0, len(scopes)) + for scope := range scopes { + result = append(result, scope) + } + sort.Strings(result) + return result +} diff --git a/internal/connector/opencost/rollup_test.go b/internal/connector/opencost/rollup_test.go new file mode 100644 index 0000000..454e7dd --- /dev/null +++ b/internal/connector/opencost/rollup_test.go @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: Apache-2.0 + +package opencost + +import ( + "bytes" + "encoding/json" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestProjectNamespaceCostSnapshotPreservesSuccessfulEmptyCoverage(t *testing.T) { + t.Parallel() + input := testProjection(t) + + snapshot, err := ProjectNamespaceCostSnapshot(input) + if err != nil { + t.Fatalf("ProjectNamespaceCostSnapshot() error = %v", err) + } + if snapshot.Workspace != input.Workspace || snapshot.Scope != input.Scope || + snapshot.CurrencyCode != input.CurrencyCode || + !snapshot.WindowStart.Equal(input.Query.WindowStart) || + !snapshot.WindowEnd.Equal(input.Query.WindowEnd) || + snapshot.Facts == nil || len(snapshot.Facts) != 0 { + t.Fatalf("successful empty snapshot = %#v", snapshot) + } + + facts, err := ProjectNamespaceCosts(input) + if err != nil { + t.Fatalf("ProjectNamespaceCosts() error = %v", err) + } + if !reflect.DeepEqual(snapshot.Facts, facts) { + t.Fatalf("snapshot facts = %#v, compatibility facts = %#v", snapshot.Facts, facts) + } + + invalid := input + invalid.CurrencyCode = "EUR" + failed, err := ProjectNamespaceCostSnapshot(invalid) + if err == nil || !reflect.DeepEqual(failed, NamespaceCostSnapshot{}) { + t.Fatalf("invalid projection = %#v, %v", failed, err) + } +} + +func TestRollupWorkspaceCostsSurfacesPartialAndEmptyCoverage(t *testing.T) { + t.Parallel() + populated := testCostSnapshot(t, "cluster-a", "payments", "infra") + empty := testCostSnapshot(t, "cluster-b") + request := WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, + ExpectedScopes: []string{"cluster-c", "cluster-b", "cluster-a"}, + Snapshots: []NamespaceCostSnapshot{empty, populated}, + } + + rollup, err := RollupWorkspaceCosts(request) + if err != nil { + t.Fatalf("RollupWorkspaceCosts() error = %v", err) + } + if rollup.Workspace != request.Workspace || !rollup.WindowStart.Equal(testWindowStart) || + !rollup.WindowEnd.Equal(testWindowEnd) || rollup.CurrencyCode != currencyUSD || + rollup.ObservedAt == nil || !rollup.ObservedAt.Equal(testWindowEnd) || + rollup.NamespaceFacts != 2 { + t.Fatalf("rollup envelope = %#v", rollup) + } + wantCoverage := WorkspaceCostCoverage{ + ExpectedScopes: []string{"cluster-a", "cluster-b", "cluster-c"}, + ReportedScopes: []string{"cluster-a", "cluster-b"}, + EmptyScopes: []string{"cluster-b"}, + MissingScopes: []string{"cluster-c"}, + Complete: false, + } + if !reflect.DeepEqual(rollup.Coverage, wantCoverage) { + t.Fatalf("coverage = %#v, want %#v", rollup.Coverage, wantCoverage) + } + wantAmounts := CostAmounts{ + CPUCost: "2.50000", CPUCostAdjustment: "-0.10000", + GPUCost: "4.00000", GPUCostAdjustment: "0.00000", + RAMCost: "1.50000", RAMCostAdjustment: "0.02000", + PVCost: "1.00000", PVCostAdjustment: "0.00000", + NetworkCost: "0.50000", NetworkCostAdjustment: "-0.02000", + LoadBalancerCost: "0.20000", LoadBalancerCostAdjustment: "0.00000", + SharedCost: "0.40000", ExternalCost: "0.60000", TotalCost: "10.60000", + } + if rollup.Amounts != wantAmounts { + t.Fatalf("amounts = %#v, want %#v", rollup.Amounts, wantAmounts) + } + encoded, err := json.Marshal(rollup) + if err != nil { + t.Fatalf("marshal rollup: %v", err) + } + if len(encoded) > maxRollupPayloadBytes { + t.Fatalf("rollup bytes = %d", len(encoded)) + } + for _, forbidden := range []string{ + "payments", "infra", "do-not-retain-provider-id", "do-not-retain-label", + "do-not-retain-annotation", "do-not-retain-controller", "do-not-retain-endpoint", + } { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("rollup retained forbidden detail %q: %s", forbidden, encoded) + } + } +} + +func TestRollupWorkspaceCostsIsDeterministicAcrossSnapshotAndFactOrder(t *testing.T) { + t.Parallel() + alpha := testCostSnapshot(t, "cluster-a", "zeta", "alpha") + beta := testCostSnapshot(t, "cluster-b", "payments") + left := WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, ExpectedScopes: []string{"cluster-b", "cluster-a"}, + Snapshots: []NamespaceCostSnapshot{alpha, beta}, + } + right := cloneRollupRequest(t, left) + slices.Reverse(right.ExpectedScopes) + slices.Reverse(right.Snapshots) + slices.Reverse(right.Snapshots[1].Facts) + + leftRollup, err := RollupWorkspaceCosts(left) + if err != nil { + t.Fatalf("left rollup: %v", err) + } + rightRollup, err := RollupWorkspaceCosts(right) + if err != nil { + t.Fatalf("right rollup: %v", err) + } + if !reflect.DeepEqual(leftRollup, rightRollup) { + t.Fatalf("input order changed rollup:\nleft = %#v\nright = %#v", leftRollup, rightRollup) + } + if !leftRollup.Coverage.Complete || len(leftRollup.Coverage.MissingScopes) != 0 || + leftRollup.NamespaceFacts != 3 || leftRollup.Amounts.TotalCost != "15.90000" { + t.Fatalf("complete rollup = %#v", leftRollup) + } +} + +func TestRollupWorkspaceCostsDistinguishesNoReportsFromSuccessfulEmpty(t *testing.T) { + t.Parallel() + missing, err := RollupWorkspaceCosts(WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, ExpectedScopes: []string{"cluster-b", "cluster-a"}, + }) + if err != nil { + t.Fatalf("missing rollup: %v", err) + } + if missing.ObservedAt != nil || missing.NamespaceFacts != 0 || missing.Coverage.Complete || + !reflect.DeepEqual(missing.Coverage.MissingScopes, []string{"cluster-a", "cluster-b"}) || + missing.Amounts != zeroCostAmounts() { + t.Fatalf("all-missing rollup = %#v", missing) + } + + empty, err := RollupWorkspaceCosts(WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, ExpectedScopes: []string{"cluster-a"}, + Snapshots: []NamespaceCostSnapshot{testCostSnapshot(t, "cluster-a")}, + }) + if err != nil { + t.Fatalf("empty rollup: %v", err) + } + if empty.ObservedAt == nil || !empty.ObservedAt.Equal(testWindowEnd) || + !empty.Coverage.Complete || + !reflect.DeepEqual(empty.Coverage.ReportedScopes, []string{"cluster-a"}) || + !reflect.DeepEqual(empty.Coverage.EmptyScopes, []string{"cluster-a"}) || + len(empty.Coverage.MissingScopes) != 0 || empty.Amounts != zeroCostAmounts() { + t.Fatalf("successful-empty rollup = %#v", empty) + } + + noScopes, err := RollupWorkspaceCosts(WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, + }) + if err != nil { + t.Fatalf("zero-scope rollup: %v", err) + } + if noScopes.ObservedAt != nil || !noScopes.Coverage.Complete || + noScopes.Coverage.ExpectedScopes == nil || noScopes.Coverage.ReportedScopes == nil || + noScopes.Coverage.EmptyScopes == nil || noScopes.Coverage.MissingScopes == nil { + t.Fatalf("zero-scope rollup = %#v", noScopes) + } +} + +func TestRollupWorkspaceCostsRejectsInvalidRequestsAtomically(t *testing.T) { + t.Parallel() + base := WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, ExpectedScopes: []string{"cluster-a"}, + Snapshots: []NamespaceCostSnapshot{testCostSnapshot(t, "cluster-a", "payments")}, + } + fixedOffset := time.FixedZone("offset", -5*60*60) + tests := map[string]func(*WorkspaceRollupRequest){ + "workspace": func(value *WorkspaceRollupRequest) { value.Workspace = " workspace-a" }, + "currency": func(value *WorkspaceRollupRequest) { value.CurrencyCode = "EUR" }, + "window start": func(value *WorkspaceRollupRequest) { value.WindowStart = time.Time{} }, + "window end": func(value *WorkspaceRollupRequest) { value.WindowEnd = value.WindowStart }, + "window duration": func(value *WorkspaceRollupRequest) { + value.WindowEnd = value.WindowStart.Add(maxQueryWindow + time.Second) + }, + "expected duplicate": func(value *WorkspaceRollupRequest) { + value.ExpectedScopes = append(value.ExpectedScopes, value.ExpectedScopes[0]) + }, + "expected invalid": func(value *WorkspaceRollupRequest) { value.ExpectedScopes[0] = "cluster/a" }, + "expected excessive": func(value *WorkspaceRollupRequest) { + value.ExpectedScopes = make([]string, maxRollupScopes+1) + for index := range value.ExpectedScopes { + value.ExpectedScopes[index] = "cluster-" + strings.Repeat("a", index/26) + string(rune('a'+index%26)) + } + value.Snapshots = nil + }, + "snapshot excessive": func(value *WorkspaceRollupRequest) { + value.Snapshots = append(value.Snapshots, cloneCostSnapshot(t, value.Snapshots[0])) + }, + "snapshot unexpected": func(value *WorkspaceRollupRequest) { value.Snapshots[0].Scope = "cluster-b" }, + "snapshot workspace": func(value *WorkspaceRollupRequest) { value.Snapshots[0].Workspace = "workspace-b" }, + "snapshot currency": func(value *WorkspaceRollupRequest) { value.Snapshots[0].CurrencyCode = "EUR" }, + "snapshot window": func(value *WorkspaceRollupRequest) { value.Snapshots[0].WindowEnd = value.WindowEnd.Add(time.Second) }, + "snapshot non-UTC": func(value *WorkspaceRollupRequest) { + value.Snapshots[0].WindowStart = value.WindowStart.In(fixedOffset) + }, + "snapshot fact count": func(value *WorkspaceRollupRequest) { + value.Snapshots[0].Facts = make([]fleet.GraphFact, maxAllocations+1) + }, + } + for name, mutate := range tests { + name, mutate := name, mutate + t.Run(name, func(t *testing.T) { + t.Parallel() + candidate := cloneRollupRequest(t, base) + mutate(&candidate) + assertRollupErrorIsAtomic(t, candidate) + }) + } +} + +func TestRollupWorkspaceCostsRevalidatesFactsAtomically(t *testing.T) { + t.Parallel() + base := WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, ExpectedScopes: []string{"cluster-a"}, + Snapshots: []NamespaceCostSnapshot{testCostSnapshot(t, "cluster-a", "payments")}, + } + tests := map[string]func(*fleet.GraphFact){ + "workspace": func(fact *fleet.GraphFact) { fact.Fact.Workspace = "workspace-b" }, + "kind": func(fact *fleet.GraphFact) { fact.Fact.Kind = fleet.FactAlert }, + "lens": func(fact *fleet.GraphFact) { fact.Lens = fleet.LensLive }, + "source": func(fact *fleet.GraphFact) { fact.Fact.Source = "cluster-b" }, + "observed at": func(fact *fleet.GraphFact) { + fact.Fact.ObservedAt = fact.Fact.ObservedAt.Add(time.Second) + }, + "stale": func(fact *fleet.GraphFact) { fact.Fact.Stale = true }, + "stale detail": func(fact *fleet.GraphFact) { fact.Fact.StaleFor = "1m" }, + "source kind": func(fact *fleet.GraphFact) { fact.Fact.Ref.SourceKind = "other" }, + "scope": func(fact *fleet.GraphFact) { fact.Fact.Ref.Scope = "cluster-b" }, + "resource kind": func(fact *fleet.GraphFact) { + fact.Fact.Ref.Kind = "Namespace" + }, + "resource name": func(fact *fleet.GraphFact) { fact.Fact.Ref.Name = "other" }, + "attributes": func(fact *fleet.GraphFact) { fact.Fact.Ref.Attributes = map[string]string{} }, + "display": func(fact *fleet.GraphFact) { fact.Fact.Display = []fleet.DisplayField{} }, + "adapter": func(fact *fleet.GraphFact) { fact.Fact.Provenance.Adapter = "other" }, + "protocol": func(fact *fleet.GraphFact) { fact.Fact.Provenance.ProtocolV = "other" }, + "native identity": func(fact *fleet.GraphFact) { + fact.Fact.Provenance.NativeID = "sha256:" + strings.Repeat("0", 64) + }, + "deep link": func(fact *fleet.GraphFact) { fact.Fact.Provenance.DeepLink = "https://secret.invalid" }, + "collector": func(fact *fleet.GraphFact) { fact.Fact.Provenance.Collector = "secret" }, + "entity nil": func(fact *fleet.GraphFact) { fact.Entity = nil }, + "entity mismatch": func(fact *fleet.GraphFact) { + fact.Entity.Namespace = "other" + }, + "payload whitespace": func(fact *fleet.GraphFact) { + fact.Fact.Observed = append([]byte(" "), fact.Fact.Observed...) + rebindFactNativeID(t, fact) + }, + "payload unknown": func(fact *fleet.GraphFact) { + var document map[string]any + mustUnmarshal(t, fact.Fact.Observed, &document) + document["provider_id"] = "secret" + fact.Fact.Observed = mustMarshal(t, document) + rebindFactNativeID(t, fact) + }, + "payload duplicate": func(fact *fleet.GraphFact) { + fact.Fact.Observed = bytes.Replace( + fact.Fact.Observed, + []byte(`{"namespace":"payments",`), + []byte(`{"namespace":"payments","namespace":"payments",`), + 1, + ) + rebindFactNativeID(t, fact) + }, + "payload oversized": func(fact *fleet.GraphFact) { + fact.Fact.Observed = append(fact.Fact.Observed, bytes.Repeat([]byte(" "), maxFactPayloadBytes)...) + rebindFactNativeID(t, fact) + }, + "payload namespace": func(fact *fleet.GraphFact) { + mutateFactObservation(t, fact, func(value *namespaceCostObservation) { value.Namespace = "Invalid" }) + fact.Fact.Ref.Namespace = "Invalid" + fact.Entity.Namespace = "Invalid" + rebindFactNativeID(t, fact) + }, + "payload window": func(fact *fleet.GraphFact) { + mutateFactObservation(t, fact, func(value *namespaceCostObservation) { + value.WindowEnd = value.WindowEnd.Add(time.Second) + }) + rebindFactNativeID(t, fact) + }, + "payload currency": func(fact *fleet.GraphFact) { + mutateFactObservation(t, fact, func(value *namespaceCostObservation) { value.Currency = "EUR" }) + rebindFactNativeID(t, fact) + }, + "payload noncanonical amount": func(fact *fleet.GraphFact) { + mutateFactObservation(t, fact, func(value *namespaceCostObservation) { value.CPUCost = "1.25" }) + rebindFactNativeID(t, fact) + }, + "payload component total": func(fact *fleet.GraphFact) { + mutateFactObservation(t, fact, func(value *namespaceCostObservation) { value.TotalCost = "5.40000" }) + rebindFactNativeID(t, fact) + }, + } + for name, mutate := range tests { + name, mutate := name, mutate + t.Run(name, func(t *testing.T) { + t.Parallel() + candidate := cloneRollupRequest(t, base) + mutate(&candidate.Snapshots[0].Facts[0]) + assertRollupErrorIsAtomic(t, candidate) + }) + } + + duplicate := cloneRollupRequest(t, base) + duplicate.Snapshots[0].Facts = append( + duplicate.Snapshots[0].Facts, + duplicate.Snapshots[0].Facts[0], + ) + assertRollupErrorIsAtomic(t, duplicate) +} + +func TestCanonicalRollupCostContract(t *testing.T) { + t.Parallel() + for value, valid := range map[string]bool{ + "0.00000": true, "1.25000": true, "-0.05000": true, + "0": false, "1.25": false, "01.00000": false, "-0.00000": false, + "1.250000": false, "1e2": false, " 1.00000": false, + } { + value, valid := value, valid + t.Run(value, func(t *testing.T) { + t.Parallel() + _, err := parseCanonicalCost(value, true) + if (err == nil) != valid { + t.Fatalf("parseCanonicalCost(%q) error = %v, valid = %t", value, err, valid) + } + }) + } +} + +func FuzzRollupWorkspaceCostsNeverPanicsOrReturnsPartial(f *testing.F) { + seed := testCostSnapshot(f, "cluster-a", "payments") + f.Add([]byte(seed.Facts[0].Fact.Observed), uint8(0), "cluster-a") + f.Add([]byte(`{"namespace":"payments"}`), uint8(1), "cluster-b") + f.Add([]byte(`{"namespace":"payments","namespace":"payments"}`), uint8(2), "cluster/a") + f.Fuzz(func(t *testing.T, payload []byte, mode uint8, scope string) { + request := WorkspaceRollupRequest{ + Workspace: "workspace-a", WindowStart: testWindowStart, WindowEnd: testWindowEnd, + CurrencyCode: currencyUSD, ExpectedScopes: []string{"cluster-a"}, + Snapshots: []NamespaceCostSnapshot{testCostSnapshot(t, "cluster-a", "payments")}, + } + switch mode % 4 { + case 0: + request.Snapshots[0].Facts[0].Fact.Observed = append([]byte(nil), payload...) + nativeID, err := namespaceCostNativeID( + request.Workspace, + request.Snapshots[0].Scope, + request.Snapshots[0].Facts[0].Fact.Observed, + ) + if err == nil { + request.Snapshots[0].Facts[0].Fact.Provenance.NativeID = nativeID + } + case 1: + request.ExpectedScopes = []string{scope} + request.Snapshots[0].Scope = scope + case 2: + request.Snapshots[0].Facts = append( + request.Snapshots[0].Facts, + request.Snapshots[0].Facts[0], + ) + case 3: + request.ExpectedScopes = append(request.ExpectedScopes, scope) + } + rollup, err := RollupWorkspaceCosts(request) + if err != nil { + if !reflect.DeepEqual(rollup, WorkspaceCostRollup{}) { + t.Fatalf("error returned partial rollup: %#v, %v", rollup, err) + } + return + } + if rollup.NamespaceFacts > maxRollupFacts || + rollup.Coverage.Complete != (len(rollup.Coverage.MissingScopes) == 0) || + !slices.IsSorted(rollup.Coverage.ExpectedScopes) || + !slices.IsSorted(rollup.Coverage.ReportedScopes) || + !slices.IsSorted(rollup.Coverage.EmptyScopes) || + !slices.IsSorted(rollup.Coverage.MissingScopes) { + t.Fatalf("invalid successful rollup: %#v", rollup) + } + for _, value := range costAmountValues(rollup.Amounts) { + if _, err := parseCanonicalCost(value, true); err != nil { + t.Fatalf("noncanonical successful amount %q: %v", value, err) + } + } + encoded, err := json.Marshal(rollup) + if err != nil || len(encoded) > maxRollupPayloadBytes { + t.Fatalf("successful rollup encoding = %d bytes, %v", len(encoded), err) + } + }) +} + +func testCostSnapshot(t testing.TB, scope string, namespaces ...string) NamespaceCostSnapshot { + t.Helper() + input := projectionFor(t, "workspace-a", scope, testWindowStart, testWindowEnd, namespaces...) + snapshot, err := ProjectNamespaceCostSnapshot(input) + if err != nil { + t.Fatalf("ProjectNamespaceCostSnapshot() error = %v", err) + } + return snapshot +} + +func cloneCostSnapshot(t testing.TB, value NamespaceCostSnapshot) NamespaceCostSnapshot { + t.Helper() + encoded := mustMarshal(t, value) + var cloned NamespaceCostSnapshot + mustUnmarshal(t, encoded, &cloned) + return cloned +} + +func cloneRollupRequest(t testing.TB, value WorkspaceRollupRequest) WorkspaceRollupRequest { + t.Helper() + encoded := mustMarshal(t, value) + var cloned WorkspaceRollupRequest + mustUnmarshal(t, encoded, &cloned) + return cloned +} + +func mutateFactObservation( + t testing.TB, + fact *fleet.GraphFact, + mutate func(*namespaceCostObservation), +) { + t.Helper() + var observation namespaceCostObservation + mustUnmarshal(t, fact.Fact.Observed, &observation) + mutate(&observation) + fact.Fact.Observed = mustMarshal(t, observation) +} + +func rebindFactNativeID(t testing.TB, fact *fleet.GraphFact) { + t.Helper() + nativeID, err := namespaceCostNativeID(fact.Fact.Workspace, fact.Fact.Ref.Scope, fact.Fact.Observed) + if err != nil { + t.Fatalf("namespaceCostNativeID() error = %v", err) + } + fact.Fact.Provenance.NativeID = nativeID +} + +func assertRollupErrorIsAtomic(t testing.TB, request WorkspaceRollupRequest) { + t.Helper() + rollup, err := RollupWorkspaceCosts(request) + if err == nil || !reflect.DeepEqual(rollup, WorkspaceCostRollup{}) { + t.Fatalf("RollupWorkspaceCosts() = %#v, %v; want zero rollup and error", rollup, err) + } +} + +func zeroCostAmounts() CostAmounts { + return CostAmounts{ + CPUCost: "0.00000", CPUCostAdjustment: "0.00000", + GPUCost: "0.00000", GPUCostAdjustment: "0.00000", + RAMCost: "0.00000", RAMCostAdjustment: "0.00000", + PVCost: "0.00000", PVCostAdjustment: "0.00000", + NetworkCost: "0.00000", NetworkCostAdjustment: "0.00000", + LoadBalancerCost: "0.00000", LoadBalancerCostAdjustment: "0.00000", + SharedCost: "0.00000", ExternalCost: "0.00000", TotalCost: "0.00000", + } +} + +func costAmountValues(amounts CostAmounts) []string { + return []string{ + amounts.CPUCost, amounts.CPUCostAdjustment, + amounts.GPUCost, amounts.GPUCostAdjustment, + amounts.RAMCost, amounts.RAMCostAdjustment, + amounts.PVCost, amounts.PVCostAdjustment, + amounts.NetworkCost, amounts.NetworkCostAdjustment, + amounts.LoadBalancerCost, amounts.LoadBalancerCostAdjustment, + amounts.SharedCost, amounts.ExternalCost, amounts.TotalCost, + } +} + +func mustMarshal(t testing.TB, value any) []byte { + t.Helper() + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal test value: %v", err) + } + return encoded +} + +func mustUnmarshal(t testing.TB, document []byte, target any) { + t.Helper() + if err := json.Unmarshal(document, target); err != nil { + t.Fatalf("unmarshal test value: %v", err) + } +} diff --git a/internal/connector/prometheus/boundary_test.go b/internal/connector/prometheus/boundary_test.go new file mode 100644 index 0000000..d97ada6 --- /dev/null +++ b/internal/connector/prometheus/boundary_test.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package prometheus + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestProjectorHasNoIOOrMutationSeam(t *testing.T) { + t.Parallel() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read prometheus package: %v", err) + } + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), entry.Name(), nil, 0) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if path == "os/exec" || path == "syscall" || path == "plugin" || path == "net" || + strings.HasPrefix(path, "net/") || strings.HasPrefix(path, "google.golang.org/grpc") || + strings.HasPrefix(path, "k8s.io/client-go") { + t.Fatalf("projector imports I/O or execution package %q", path) + } + } + ast.Inspect(file, func(node ast.Node) bool { + declaration, ok := node.(*ast.FuncDecl) + if !ok { + return true + } + switch declaration.Name.Name { + case "Plan", "Execute", "Verify", "Write", "Delete", "Update", "Create", "Reload": + t.Errorf("projector declares forbidden mutation seam %s", declaration.Name.Name) + } + return true + }) + } +} diff --git a/internal/connector/prometheus/project.go b/internal/connector/prometheus/project.go new file mode 100644 index 0000000..7dbbb61 --- /dev/null +++ b/internal/connector/prometheus/project.go @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package prometheus normalizes read-only Prometheus alert evidence for Sith's operational graph. +package prometheus + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // Kind is the stable registry identifier for Prometheus read evidence. + Kind = "prometheus" + // ProtocolVersion identifies the normalized /api/v1/alerts fact contract. + ProtocolVersion = "alerts/v1" + + maxResponseBytes = 512 << 10 + maxAlerts = 256 + maxLabelsPerAlert = 64 + maxLabelNameBytes = 256 + maxLabelValueBytes = 2 << 10 + maxIdentityText = 253 + maxFactPayloadBytes = 8 << 10 + maxJSONDepth = 64 +) + +var projectedLabels = map[string]bool{ + "container": true, + "daemonset": true, + "deployment": true, + "job": true, + "namespace": true, + "node": true, + "pod": true, + "service": true, + "severity": true, + "statefulset": true, +} + +// Projection supplies one already-authorized Prometheus /api/v1/alerts response. ProjectAlerts +// does not perform discovery, network access, credential loading, persistence, or mutation. +type Projection struct { + Workspace string + Scope string + ObservedAt time.Time + Response []byte +} + +type alertsEnvelope struct { + Status string `json:"status"` + Data *alertsData `json:"data"` +} + +type alertsData struct { + Alerts *[]sourceAlert `json:"alerts"` +} + +type sourceAlert struct { + ActiveAt string `json:"activeAt"` + Annotations map[string]string `json:"annotations"` + Labels map[string]string `json:"labels"` + State string `json:"state"` + Value string `json:"value"` +} + +type alertObservation struct { + AlertName string `json:"alert_name"` + State string `json:"state"` + ActiveAt time.Time `json:"active_at"` + Value string `json:"value"` + Labels map[string]string `json:"labels,omitempty"` +} + +type projectedAlert struct { + fact fleet.GraphFact + nativeID string + canonical []byte +} + +// ProjectAlerts returns deterministic, bounded, allowlisted telemetry facts from a Prometheus +// /api/v1/alerts response. It deliberately drops annotations and unknown labels, and leaves alerts +// unattached when their Kubernetes identity cannot be resolved without guessing. +func ProjectAlerts(input Projection) ([]fleet.GraphFact, error) { + if err := validateProjection(input); err != nil { + return nil, err + } + if err := rejectDuplicateJSON(input.Response); err != nil { + return nil, fmt.Errorf("decode Prometheus alerts response: %w", err) + } + var envelope alertsEnvelope + decoder := json.NewDecoder(bytes.NewReader(input.Response)) + if err := decoder.Decode(&envelope); err != nil { + return nil, fmt.Errorf("decode Prometheus alerts response: %w", err) + } + if envelope.Status != "success" { + return nil, fmt.Errorf("prometheus alerts response status must be success") + } + if envelope.Data == nil || envelope.Data.Alerts == nil { + return nil, fmt.Errorf("prometheus alerts response data.alerts is required") + } + alerts := *envelope.Data.Alerts + if len(alerts) > maxAlerts { + return nil, fmt.Errorf("prometheus alert count exceeds %d", maxAlerts) + } + + projected := make([]projectedAlert, 0, len(alerts)) + seen := make(map[string]bool, len(alerts)) + for index, alert := range alerts { + fact, nativeID, canonical, err := projectAlert(input, alert) + if err != nil { + return nil, fmt.Errorf("project Prometheus alert %d: %w", index, err) + } + if seen[nativeID] { + return nil, fmt.Errorf("project Prometheus alert %d: duplicate alert identity", index) + } + seen[nativeID] = true + projected = append(projected, projectedAlert{fact: fact, nativeID: nativeID, canonical: canonical}) + } + sort.Slice(projected, func(left, right int) bool { + if projected[left].nativeID != projected[right].nativeID { + return projected[left].nativeID < projected[right].nativeID + } + return bytes.Compare(projected[left].canonical, projected[right].canonical) < 0 + }) + facts := make([]fleet.GraphFact, len(projected)) + for index := range projected { + facts[index] = projected[index].fact + } + return facts, nil +} + +func validateProjection(input Projection) error { + if err := validateText("workspace", input.Workspace, maxIdentityText, false); err != nil { + return err + } + if err := validateText("scope", input.Scope, maxIdentityText, false); err != nil { + return err + } + if strings.Contains(input.Scope, "/") { + return fmt.Errorf("scope is invalid") + } + if input.ObservedAt.IsZero() { + return fmt.Errorf("projection observation time is required") + } + if len(input.Response) == 0 { + return fmt.Errorf("prometheus alerts response is required") + } + if len(input.Response) > maxResponseBytes { + return fmt.Errorf("prometheus alerts response exceeds %d bytes", maxResponseBytes) + } + if !utf8.Valid(input.Response) { + return fmt.Errorf("prometheus alerts response must be valid UTF-8") + } + return nil +} + +func projectAlert(input Projection, alert sourceAlert) (fleet.GraphFact, string, []byte, error) { + if len(alert.Labels) == 0 { + return fleet.GraphFact{}, "", nil, fmt.Errorf("labels are required") + } + if len(alert.Labels) > maxLabelsPerAlert { + return fleet.GraphFact{}, "", nil, fmt.Errorf("label count exceeds %d", maxLabelsPerAlert) + } + for name, value := range alert.Labels { + if err := validateText("label name", name, maxLabelNameBytes, false); err != nil { + return fleet.GraphFact{}, "", nil, err + } + if err := validateLabelValue(value); err != nil { + return fleet.GraphFact{}, "", nil, err + } + } + alertName, ok := alert.Labels["alertname"] + if !ok { + return fleet.GraphFact{}, "", nil, fmt.Errorf("alertname label is required") + } + if err := validateText("alertname", alertName, maxIdentityText, false); err != nil { + return fleet.GraphFact{}, "", nil, err + } + if alert.State != "pending" && alert.State != "firing" { + return fleet.GraphFact{}, "", nil, fmt.Errorf("alert state must be pending or firing") + } + activeAt, err := time.Parse(time.RFC3339Nano, alert.ActiveAt) + if err != nil { + return fleet.GraphFact{}, "", nil, fmt.Errorf("activeAt must be RFC3339: %w", err) + } + if activeAt.IsZero() { + return fleet.GraphFact{}, "", nil, fmt.Errorf("activeAt must not be zero") + } + value, err := strconv.ParseFloat(alert.Value, 64) + if err != nil || math.IsNaN(value) || math.IsInf(value, 0) { + return fleet.GraphFact{}, "", nil, fmt.Errorf("alert value must be finite") + } + + labels := make(map[string]string) + for name, value := range alert.Labels { + if projectedLabels[name] { + labels[name] = value + } + } + entity, err := resolveEntity(input.Scope, labels) + if err != nil { + return fleet.GraphFact{}, "", nil, err + } + observation := alertObservation{ + AlertName: alertName, + State: alert.State, + ActiveAt: activeAt.UTC(), + Value: strconv.FormatFloat(value, 'g', -1, 64), + Labels: labels, + } + canonical, err := json.Marshal(observation) + if err != nil { + return fleet.GraphFact{}, "", nil, fmt.Errorf("encode alert fact: %w", err) + } + if len(canonical) > maxFactPayloadBytes { + return fleet.GraphFact{}, "", nil, fmt.Errorf("alert fact exceeds %d encoded bytes", maxFactPayloadBytes) + } + fingerprintPayload, err := json.Marshal(struct { + AlertName string `json:"alert_name"` + ActiveAt time.Time `json:"active_at"` + Labels map[string]string `json:"labels,omitempty"` + }{AlertName: alertName, ActiveAt: activeAt.UTC(), Labels: labels}) + if err != nil { + return fleet.GraphFact{}, "", nil, fmt.Errorf("encode alert identity: %w", err) + } + digest := sha256.Sum256(fingerprintPayload) + nativeID := "sha256:" + hex.EncodeToString(digest[:]) + resourceName := "alert-" + hex.EncodeToString(digest[:16]) + + namespace := alert.Labels["namespace"] + fact := fleet.GraphFact{ + Fact: fleet.Fact{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{ + SourceKind: Kind, Scope: input.Scope, Kind: "Alert", Namespace: namespace, Name: resourceName, + }, + Kind: fleet.FactAlert, Observed: canonical, ObservedAt: input.ObservedAt.UTC(), Source: input.Scope, + Provenance: fleet.Provenance{Adapter: Kind, ProtocolV: ProtocolVersion, NativeID: nativeID}, + }, + Workspace: input.Workspace, + }, + Lens: fleet.LensTelemetry, Entity: entity, + } + if err := fact.Validate(input.Workspace); err != nil { + return fleet.GraphFact{}, "", nil, fmt.Errorf("validate Prometheus alert fact: %w", err) + } + return fact, nativeID, canonical, nil +} + +func resolveEntity(scope string, labels map[string]string) (*fleet.EntityRef, error) { + type candidate struct { + kind string + value string + } + candidates := make([]candidate, 0, 5) + for label, kind := range map[string]string{ + "pod": "Pod", "node": "Node", "deployment": "Deployment", "statefulset": "StatefulSet", "daemonset": "DaemonSet", + } { + if value := labels[label]; value != "" { + if err := validateText(label, value, maxIdentityText, false); err != nil { + return nil, err + } + candidates = append(candidates, candidate{kind: kind, value: value}) + } + } + if len(candidates) == 0 { + return nil, nil + } + if len(candidates) != 1 { + return nil, fmt.Errorf("alert has ambiguous Kubernetes identity") + } + namespace := labels["namespace"] + if candidates[0].kind == "Node" { + if namespace != "" { + return nil, fmt.Errorf("node alert must not include a namespace identity") + } + return &fleet.EntityRef{Cluster: scope, Node: candidates[0].value}, nil + } + if err := validateText("namespace", namespace, maxIdentityText, false); err != nil { + return nil, fmt.Errorf("namespaced alert identity: %w", err) + } + if candidates[0].kind == "Pod" { + return &fleet.EntityRef{Cluster: scope, Namespace: namespace, Pod: candidates[0].value}, nil + } + return &fleet.EntityRef{ + Cluster: scope, Namespace: namespace, Kind: candidates[0].kind, Name: candidates[0].value, + }, nil +} + +func validateText(label, value string, maximum int, allowEmpty bool) error { + if (!allowEmpty && value == "") || len(value) > maximum || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("%s is invalid", label) + } + for _, runeValue := range value { + if unicode.IsControl(runeValue) { + return fmt.Errorf("%s is invalid", label) + } + } + return nil +} + +func validateLabelValue(value string) error { + if len(value) > maxLabelValueBytes || !utf8.ValidString(value) { + return fmt.Errorf("label value is invalid") + } + for _, runeValue := range value { + if unicode.IsControl(runeValue) { + return fmt.Errorf("label value is invalid") + } + } + return nil +} + +func rejectDuplicateJSON(document []byte) error { + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + if err := consumeUniqueJSON(decoder, 0); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return fmt.Errorf("JSON contains trailing data") + } + return nil +} + +func consumeUniqueJSON(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, isDelimiter := token.(json.Delim) + if !isDelimiter { + return nil + } + if depth >= maxJSONDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", maxJSONDepth) + } + switch delimiter { + case '{': + seen := make(map[string]bool) + for decoder.More() { + nameToken, err := decoder.Token() + if err != nil { + return err + } + name, ok := nameToken.(string) + if !ok || seen[name] { + return fmt.Errorf("JSON contains a duplicate or invalid object member") + } + seen[name] = true + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + case '[': + for decoder.More() { + if err := consumeUniqueJSON(decoder, depth+1); err != nil { + return err + } + } + default: + return fmt.Errorf("JSON contains an invalid delimiter") + } + closing, err := decoder.Token() + if err != nil || closing != matchingDelimiter(delimiter) { + return fmt.Errorf("JSON contains an invalid closing delimiter") + } + return nil +} + +func matchingDelimiter(open json.Delim) json.Delim { + if open == '{' { + return '}' + } + return ']' +} diff --git a/internal/connector/prometheus/project_test.go b/internal/connector/prometheus/project_test.go new file mode 100644 index 0000000..acb249e --- /dev/null +++ b/internal/connector/prometheus/project_test.go @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: Apache-2.0 + +package prometheus + +import ( + "bytes" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestProjectAlertsEmitsSanitizedTelemetryFacts(t *testing.T) { + t.Parallel() + input := Projection{ + Workspace: "workspace-a", Scope: "cluster-a", + ObservedAt: time.Date(2026, 7, 16, 23, 0, 0, 0, time.FixedZone("CDT", -5*60*60)), + Response: alertsResponse(t, + sourceAlert{ + ActiveAt: "2026-07-16T20:00:00-05:00", State: "firing", Value: "1e+01", + Labels: map[string]string{ + "alertname": "DeploymentUnavailable", "namespace": "payments", "deployment": "api", + "severity": "critical", "cluster": "attacker-cluster", "secret_label": "label-secret", + }, + Annotations: map[string]string{"summary": "annotation-secret"}, + }, + sourceAlert{ + ActiveAt: "2026-07-16T21:00:00Z", State: "pending", Value: "2", + Labels: map[string]string{"alertname": "NodePressure", "node": "worker-a"}, + }, + sourceAlert{ + ActiveAt: "2026-07-16T22:00:00Z", State: "firing", Value: "3", + Labels: map[string]string{"alertname": "ExternalDependencyDown", "service": "payments-db"}, + }, + ), + } + facts, err := ProjectAlerts(input) + if err != nil { + t.Fatalf("ProjectAlerts() error = %v", err) + } + if len(facts) != 3 { + t.Fatalf("fact count = %d, want 3", len(facts)) + } + + byName := make(map[string]fleet.GraphFact, len(facts)) + for _, fact := range facts { + var observation alertObservation + if err := json.Unmarshal(fact.Fact.Observed, &observation); err != nil { + t.Fatalf("decode observation: %v", err) + } + byName[observation.AlertName] = fact + if fact.Fact.Kind != fleet.FactAlert || fact.Lens != fleet.LensTelemetry || + fact.Fact.Ref.SourceKind != Kind || fact.Fact.Ref.Scope != "cluster-a" || + fact.Fact.Provenance.Adapter != Kind || fact.Fact.Provenance.ProtocolV != ProtocolVersion || + !strings.HasPrefix(fact.Fact.Provenance.NativeID, "sha256:") || + fact.Fact.ObservedAt != input.ObservedAt.UTC() { + t.Fatalf("unexpected fact contract: %#v", fact) + } + } + + deployment := byName["DeploymentUnavailable"] + if deployment.Entity == nil || *deployment.Entity != (fleet.EntityRef{ + Cluster: "cluster-a", Namespace: "payments", Kind: "Deployment", Name: "api", + }) || deployment.Fact.Ref.Namespace != "payments" { + t.Fatalf("deployment fact = %#v", deployment) + } + var deploymentObservation alertObservation + if err := json.Unmarshal(deployment.Fact.Observed, &deploymentObservation); err != nil { + t.Fatalf("decode deployment observation: %v", err) + } + if deploymentObservation.Value != "10" || !deploymentObservation.ActiveAt.Equal(time.Date(2026, 7, 17, 1, 0, 0, 0, time.UTC)) || + deploymentObservation.Labels["severity"] != "critical" || deploymentObservation.Labels["namespace"] != "payments" { + t.Fatalf("deployment observation = %#v", deploymentObservation) + } + encoded, err := json.Marshal(facts) + if err != nil { + t.Fatalf("marshal facts: %v", err) + } + for _, forbidden := range []string{"annotation-secret", "label-secret", "attacker-cluster", "annotations", "secret_label", `\"cluster\"`} { + if bytes.Contains(encoded, []byte(forbidden)) { + t.Fatalf("projected facts retained forbidden value %q: %s", forbidden, encoded) + } + } + + node := byName["NodePressure"] + if node.Entity == nil || *node.Entity != (fleet.EntityRef{Cluster: "cluster-a", Node: "worker-a"}) { + t.Fatalf("node fact = %#v", node) + } + if byName["ExternalDependencyDown"].Entity != nil { + t.Fatalf("unresolvable alert was attached: %#v", byName["ExternalDependencyDown"]) + } + graph, err := fleet.NewGraph(input.Workspace, facts) + if err != nil { + t.Fatalf("NewGraph() error = %v", err) + } + if len(graph.Nodes) != 2 || len(graph.Unattached) != 1 { + t.Fatalf("graph nodes/unattached = %d/%d, want 2/1", len(graph.Nodes), len(graph.Unattached)) + } +} + +func TestProjectAlertsIsDeterministicAcrossSourceOrder(t *testing.T) { + t.Parallel() + firstAlert := sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{"alertname": "A", "namespace": "ns-a", "pod": "pod-a", "severity": "warning"}, + } + secondAlert := sourceAlert{ + ActiveAt: "2026-07-16T21:00:00Z", State: "pending", Value: "2.0", + Labels: map[string]string{"alertname": "B", "namespace": "ns-b", "statefulset": "db"}, + } + base := Projection{Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Date(2026, 7, 16, 22, 0, 0, 0, time.UTC)} + base.Response = alertsResponse(t, firstAlert, secondAlert) + first, err := ProjectAlerts(base) + if err != nil { + t.Fatalf("first ProjectAlerts() error = %v", err) + } + base.Response = alertsResponse(t, secondAlert, firstAlert) + second, err := ProjectAlerts(base) + if err != nil { + t.Fatalf("second ProjectAlerts() error = %v", err) + } + firstJSON, _ := json.Marshal(first) + secondJSON, _ := json.Marshal(second) + if !slices.Equal(firstJSON, secondJSON) { + t.Fatalf("projection depends on response order\nfirst: %s\nsecond: %s", firstJSON, secondJSON) + } +} + +func TestProjectAlertsPreservesValidUTF8LabelValues(t *testing.T) { + t.Parallel() + input := Projection{ + Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), + Response: alertsResponse(t, sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{ + "alertname": "UnicodeSeverity", "severity": " 警告 ", "unknown_utf8": "正常 ✅", + }, + }), + } + facts, err := ProjectAlerts(input) + if err != nil { + t.Fatalf("ProjectAlerts() error = %v", err) + } + var observation alertObservation + if err := json.Unmarshal(facts[0].Fact.Observed, &observation); err != nil { + t.Fatalf("decode observation: %v", err) + } + if observation.Labels["severity"] != " 警告 " { + t.Fatalf("severity = %q", observation.Labels["severity"]) + } + if encoded, _ := json.Marshal(facts); bytes.Contains(encoded, []byte("unknown_utf8")) || bytes.Contains(encoded, []byte("正常")) { + t.Fatalf("unknown UTF-8 label leaked: %s", encoded) + } +} + +func TestProjectAlertsAbstainsWhenThereAreNoActiveAlerts(t *testing.T) { + t.Parallel() + facts, err := ProjectAlerts(Projection{ + Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Response: alertsResponse(t), + }) + if err != nil { + t.Fatalf("ProjectAlerts() error = %v", err) + } + if len(facts) != 0 { + t.Fatalf("facts = %#v, want no fabricated alert evidence", facts) + } +} + +func TestProjectAlertsRejectsMalformedOrAmbiguousEvidence(t *testing.T) { + t.Parallel() + validAlert := sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{"alertname": "DeploymentUnavailable", "namespace": "payments", "deployment": "api"}, + } + valid := Projection{Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now(), Response: alertsResponse(t, validAlert)} + + tests := []struct { + name string + mutate func(*Projection) + }{ + {"missing workspace", func(input *Projection) { input.Workspace = "" }}, + {"invalid scope", func(input *Projection) { input.Scope = " cluster-a" }}, + {"unsafe scope", func(input *Projection) { input.Scope = "cluster/a" }}, + {"zero observed time", func(input *Projection) { input.ObservedAt = time.Time{} }}, + {"empty response", func(input *Projection) { input.Response = nil }}, + {"invalid UTF-8", func(input *Projection) { input.Response = []byte{'{', 0xff, '}'} }}, + {"malformed JSON", func(input *Projection) { input.Response = []byte(`{"status":`) }}, + {"duplicate JSON member", func(input *Projection) { + input.Response = []byte(`{"status":"success","status":"success","data":{"alerts":[]}}`) + }}, + {"trailing JSON", func(input *Projection) { + input.Response = []byte(`{"status":"success","data":{"alerts":[]}} {}`) + }}, + {"excessive JSON nesting", func(input *Projection) { + input.Response = []byte(strings.Repeat("[", maxJSONDepth+1) + strings.Repeat("]", maxJSONDepth+1)) + }}, + {"non-success status", func(input *Projection) { + input.Response = []byte(`{"status":"error","data":{"alerts":[]}}`) + }}, + {"missing data", func(input *Projection) { input.Response = []byte(`{"status":"success"}`) }}, + {"missing alerts", func(input *Projection) { input.Response = []byte(`{"status":"success","data":{}}`) }}, + {"null alerts", func(input *Projection) { input.Response = []byte(`{"status":"success","data":{"alerts":null}}`) }}, + {"missing labels", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Labels = nil })}, + {"missing alertname", mutateAlertResponse(t, func(alert *sourceAlert) { delete(alert.Labels, "alertname") })}, + {"invalid alertname", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Labels["alertname"] = " bad-name" })}, + {"invalid state", mutateAlertResponse(t, func(alert *sourceAlert) { alert.State = "inactive" })}, + {"invalid active time", mutateAlertResponse(t, func(alert *sourceAlert) { alert.ActiveAt = "yesterday" })}, + {"zero active time", mutateAlertResponse(t, func(alert *sourceAlert) { alert.ActiveAt = "0001-01-01T00:00:00Z" })}, + {"invalid value", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Value = "NaN" })}, + {"infinite value", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Value = "+Inf" })}, + {"control label", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Labels["severity"] = "critical\nsecret" })}, + {"oversized label name", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Labels[strings.Repeat("x", maxLabelNameBytes+1)] = "x" })}, + {"oversized label value", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Labels["severity"] = strings.Repeat("x", maxLabelValueBytes+1) })}, + {"ambiguous identity", mutateAlertResponse(t, func(alert *sourceAlert) { alert.Labels["pod"] = "api-123" })}, + {"pod without namespace", mutateAlertResponse(t, func(alert *sourceAlert) { + delete(alert.Labels, "deployment") + delete(alert.Labels, "namespace") + alert.Labels["pod"] = "api-123" + })}, + {"node with namespace", mutateAlertResponse(t, func(alert *sourceAlert) { + delete(alert.Labels, "deployment") + alert.Labels["node"] = "worker-a" + })}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := valid + input.Response = slices.Clone(valid.Response) + test.mutate(&input) + if facts, err := ProjectAlerts(input); err == nil { + t.Fatalf("ProjectAlerts() facts = %#v, error = nil", facts) + } + }) + } +} + +func TestProjectAlertsEnforcesBudgetsAndUniqueIdentity(t *testing.T) { + t.Parallel() + base := Projection{Workspace: "workspace-a", Scope: "cluster-a", ObservedAt: time.Now()} + + t.Run("response bytes", func(t *testing.T) { + input := base + input.Response = bytes.Repeat([]byte{' '}, maxResponseBytes+1) + if _, err := ProjectAlerts(input); err == nil { + t.Fatal("ProjectAlerts() accepted oversized response") + } + }) + + t.Run("alert count", func(t *testing.T) { + alerts := make([]sourceAlert, maxAlerts+1) + for index := range alerts { + alerts[index] = sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{"alertname": fmt.Sprintf("Alert-%d", index)}, + } + } + input := base + input.Response = alertsResponse(t, alerts...) + if _, err := ProjectAlerts(input); err == nil { + t.Fatal("ProjectAlerts() accepted too many alerts") + } + }) + + t.Run("label count", func(t *testing.T) { + alert := sourceAlert{ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", Labels: map[string]string{"alertname": "ManyLabels"}} + for index := 0; index < maxLabelsPerAlert; index++ { + alert.Labels[fmt.Sprintf("unknown_%d", index)] = "x" + } + input := base + input.Response = alertsResponse(t, alert) + if _, err := ProjectAlerts(input); err == nil { + t.Fatal("ProjectAlerts() accepted too many labels") + } + }) + + t.Run("encoded payload", func(t *testing.T) { + alert := sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{ + "alertname": "LargeAlert", "container": strings.Repeat("a", maxLabelValueBytes), + "job": strings.Repeat("b", maxLabelValueBytes), "service": strings.Repeat("c", maxLabelValueBytes), + "severity": strings.Repeat("d", maxLabelValueBytes), + }, + } + input := base + input.Response = alertsResponse(t, alert) + if _, err := ProjectAlerts(input); err == nil { + t.Fatal("ProjectAlerts() accepted oversized encoded fact") + } + }) + + t.Run("duplicate identity", func(t *testing.T) { + alert := sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{"alertname": "Duplicate"}, + } + input := base + input.Response = alertsResponse(t, alert, alert) + if _, err := ProjectAlerts(input); err == nil { + t.Fatal("ProjectAlerts() accepted duplicate alert identity") + } + }) +} + +func alertsResponse(t *testing.T, alerts ...sourceAlert) []byte { + t.Helper() + if alerts == nil { + alerts = make([]sourceAlert, 0) + } + encoded, err := json.Marshal(map[string]any{ + "status": "success", + "data": map[string]any{"alerts": alerts}, + }) + if err != nil { + t.Fatalf("encode alerts response: %v", err) + } + return encoded +} + +func mutateAlertResponse(t *testing.T, mutate func(*sourceAlert)) func(*Projection) { + t.Helper() + return func(input *Projection) { + alert := sourceAlert{ + ActiveAt: "2026-07-16T20:00:00Z", State: "firing", Value: "1", + Labels: map[string]string{"alertname": "DeploymentUnavailable", "namespace": "payments", "deployment": "api"}, + } + mutate(&alert) + input.Response = alertsResponse(t, alert) + } +} diff --git a/internal/connector/registry.go b/internal/connector/registry.go index 6966d9e..2f03e92 100644 --- a/internal/connector/registry.go +++ b/internal/connector/registry.go @@ -3,11 +3,17 @@ package connector import ( + "context" + "encoding/json" "errors" "fmt" "reflect" "sort" + "strings" "sync" + + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/intentargs" ) // ErrNotRegistered reports a lookup for an unknown connector kind. @@ -23,17 +29,19 @@ type registryEntry struct { connector Connector descriptor Descriptor declared map[Capability]struct{} + schemas map[intent.Verb]*intentargs.Schema } // Registry stores one canonical, capability-checked connector per kind. type Registry struct { mu sync.RWMutex entries map[string]registryEntry + verbs map[intent.Verb]string } // NewRegistry returns an empty connector registry. func NewRegistry() *Registry { - return &Registry{entries: make(map[string]registryEntry)} + return &Registry{entries: make(map[string]registryEntry), verbs: make(map[intent.Verb]string)} } // Register builds and validates a connector before atomically adding it. @@ -60,7 +68,15 @@ func (registry *Registry) Register(factory Factory) error { if _, exists := registry.entries[entry.descriptor.Kind]; exists { return fmt.Errorf("register connector %q: kind already registered", entry.descriptor.Kind) } + for _, verb := range entry.descriptor.Verbs { + if owner, exists := registry.verbs[verb]; exists { + return fmt.Errorf("register connector %q: action verb %q already belongs to %q", entry.descriptor.Kind, verb, owner) + } + } registry.entries[entry.descriptor.Kind] = entry + for _, verb := range entry.descriptor.Verbs { + registry.verbs[verb] = entry.descriptor.Kind + } return nil } @@ -177,6 +193,96 @@ func (registry *Registry) VerifierFor(kind string) (Verifier, error) { return verifier, nil } +// PlannerForVerb returns the only registered planner classified for verb. +func (registry *Registry) PlannerForVerb(verb intent.Verb) (Planner, error) { + entry, err := registry.entryForVerb(verb, CapPlan) + if err != nil { + return nil, err + } + planner, ok := entry.connector.(Planner) + if !ok { + return nil, fmt.Errorf("%w: %s does not implement planner", ErrCapability, entry.descriptor.Kind) + } + return planner, nil +} + +// ValidateArgsForVerb rejects malformed or schema-invalid arguments for a registered verb. +func (registry *Registry) ValidateArgsForVerb(verb intent.Verb, args json.RawMessage) error { + if !verb.Valid() { + return fmt.Errorf("%w: unknown action verb", ErrNotRegistered) + } + registry.mu.RLock() + kind, exists := registry.verbs[verb] + schema := registry.entries[kind].schemas[verb] + registry.mu.RUnlock() + if !exists || schema == nil { + return fmt.Errorf("%w: action verb %q", ErrNotRegistered, verb) + } + if err := schema.Validate(args); err != nil { + return fmt.Errorf("validate action verb %q: %w", verb, err) + } + return nil +} + +// Plan validates intent arguments before routing the request to its only registered planner. +func (registry *Registry) Plan(ctx context.Context, request Intent) (ActionPlan, error) { + request.Args = append(json.RawMessage(nil), request.Args...) + if err := registry.ValidateArgsForVerb(request.Verb, request.Args); err != nil { + return ActionPlan{}, err + } + planner, err := registry.PlannerForVerb(request.Verb) + if err != nil { + return ActionPlan{}, err + } + return planner.Plan(ctx, request) +} + +// ExecutorForVerb returns the only registered executor classified for verb. +func (registry *Registry) ExecutorForVerb(verb intent.Verb) (Executor, error) { + entry, err := registry.entryForVerb(verb, CapExecute) + if err != nil { + return nil, err + } + executor, ok := entry.connector.(Executor) + if !ok { + return nil, fmt.Errorf("%w: %s does not implement executor", ErrCapability, entry.descriptor.Kind) + } + return executor, nil +} + +// VerifierForVerb returns the only registered post-condition verifier classified for verb. +func (registry *Registry) VerifierForVerb(verb intent.Verb) (Verifier, error) { + entry, err := registry.entryForVerb(verb, CapVerify) + if err != nil { + return nil, err + } + verifier, ok := entry.connector.(Verifier) + if !ok { + return nil, fmt.Errorf("%w: %s does not implement verifier", ErrCapability, entry.descriptor.Kind) + } + return verifier, nil +} + +func (registry *Registry) entryForVerb(verb intent.Verb, capability Capability) (registryEntry, error) { + if !verb.Valid() { + return registryEntry{}, fmt.Errorf("%w: unknown action verb", ErrNotRegistered) + } + registry.mu.RLock() + kind, exists := registry.verbs[verb] + entry := registry.entries[kind] + registry.mu.RUnlock() + if !exists { + return registryEntry{}, fmt.Errorf("%w: action verb %q", ErrNotRegistered, verb) + } + if entry.descriptor.ConnKind != KindTypedAction { + return registryEntry{}, fmt.Errorf("%w: %s is not a typed-action connector", ErrCapability, kind) + } + if _, declared := entry.declared[capability]; !declared { + return registryEntry{}, fmt.Errorf("%w: %s did not declare %s", ErrCapability, kind, capability) + } + return entry, nil +} + func (registry *Registry) entryFor(kind string, capability Capability, typedAction bool) (registryEntry, error) { registry.mu.RLock() entry, exists := registry.entries[kind] @@ -204,8 +310,13 @@ func validateConnector(candidate Connector) (registryEntry, error) { if !descriptor.ConnKind.Valid() { return registryEntry{}, fmt.Errorf("invalid connector kind %q", descriptor.ConnKind) } - if descriptor.ProtocolV == "" { - return registryEntry{}, fmt.Errorf("protocol version must not be empty") + wireVersions, err := canonicalWireVersions(descriptor.WireVersions) + if err != nil { + return registryEntry{}, err + } + descriptor.WireVersions = wireVersions + if strings.TrimSpace(descriptor.AdapterVersion) == "" { + return registryEntry{}, fmt.Errorf("adapter version must not be empty") } if descriptor.Owner == "" { return registryEntry{}, fmt.Errorf("owner must not be empty") @@ -232,9 +343,9 @@ func validateConnector(candidate Connector) (registryEntry, error) { if len(descriptor.Verbs) == 0 { return registryEntry{}, fmt.Errorf("typed-action connector must declare at least one verb") } - seen := make(map[string]struct{}, len(descriptor.Verbs)) + seen := make(map[intent.Verb]struct{}, len(descriptor.Verbs)) for _, verb := range descriptor.Verbs { - if !ValidVerb(verb) { + if !verb.Valid() { return registryEntry{}, fmt.Errorf("invalid action verb %q", verb) } if _, duplicate := seen[verb]; duplicate { @@ -242,8 +353,28 @@ func validateConnector(candidate Connector) (registryEntry, error) { } seen[verb] = struct{}{} } - } else if len(descriptor.Verbs) != 0 { - return registryEntry{}, fmt.Errorf("non-action connector must not declare action verbs") + if len(descriptor.ArgSchemas) != len(seen) { + return registryEntry{}, fmt.Errorf("typed-action connector must declare exactly one argument schema per verb") + } + schemas := make(map[intent.Verb]*intentargs.Schema, len(seen)) + for verb, document := range descriptor.ArgSchemas { + if _, declared := seen[verb]; !declared { + return registryEntry{}, fmt.Errorf("argument schema belongs to undeclared action verb %q", verb) + } + compiled, err := intentargs.Compile(document) + if err != nil { + return registryEntry{}, fmt.Errorf("compile argument schema for action verb %q: %w", verb, err) + } + schemas[verb] = compiled + } + for verb := range seen { + if schemas[verb] == nil { + return registryEntry{}, fmt.Errorf("action verb %q is missing an argument schema", verb) + } + } + return registryEntry{connector: candidate, descriptor: descriptor, declared: declared, schemas: schemas}, nil + } else if len(descriptor.Verbs) != 0 || len(descriptor.ArgSchemas) != 0 { + return registryEntry{}, fmt.Errorf("non-action connector must not declare action verbs or argument schemas") } return registryEntry{connector: candidate, descriptor: descriptor, declared: declared}, nil @@ -311,7 +442,15 @@ func connectorIsNil(candidate Connector) bool { } func cloneDescriptor(descriptor Descriptor) Descriptor { + descriptor.WireVersions = append([]WireVersion(nil), descriptor.WireVersions...) descriptor.Capabilities = append([]Capability(nil), descriptor.Capabilities...) - descriptor.Verbs = append([]string(nil), descriptor.Verbs...) + descriptor.Verbs = append([]intent.Verb(nil), descriptor.Verbs...) + if descriptor.ArgSchemas != nil { + schemas := make(map[intent.Verb]json.RawMessage, len(descriptor.ArgSchemas)) + for verb, schema := range descriptor.ArgSchemas { + schemas[verb] = append(json.RawMessage(nil), schema...) + } + descriptor.ArgSchemas = schemas + } return descriptor } diff --git a/internal/connector/registry_test.go b/internal/connector/registry_test.go index 028c3cf..912d283 100644 --- a/internal/connector/registry_test.go +++ b/internal/connector/registry_test.go @@ -3,11 +3,17 @@ package connector import ( + "bytes" "context" + "encoding/json" "errors" + "slices" + "sync/atomic" "testing" "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/intentargs" ) type testReader struct { @@ -58,7 +64,9 @@ func (connector identityOnlyConnector) Descriptor() Descriptor { } type testExecutor struct { - descriptor Descriptor + descriptor Descriptor + planCalls *atomic.Int64 + plannedArgs *json.RawMessage } func (connector testExecutor) Kind() string { @@ -73,10 +81,24 @@ func (connector testExecutor) Descriptor() Descriptor { return cloneDescriptor(connector.descriptor) } +func (connector testExecutor) Plan(_ context.Context, request Intent) (ActionPlan, error) { + if connector.planCalls != nil { + connector.planCalls.Add(1) + } + if connector.plannedArgs != nil { + *connector.plannedArgs = request.Args + } + return ActionPlan{IntentID: request.ID, Verb: request.Verb}, nil +} + func (testExecutor) Execute(_ context.Context, plan ActionPlan) (ExecutionResult, error) { return ExecutionResult{IntentID: plan.IntentID, Applied: true}, nil } +func (testExecutor) Verify(_ context.Context, request VerifyRequest) (Verification, error) { + return Verification{Satisfied: request.IntentID != ""}, nil +} + func TestRegistryRegisterAndLookupReader(t *testing.T) { t.Parallel() @@ -105,12 +127,25 @@ func TestRegistryRejectsInvalidConnectors(t *testing.T) { name string connector Connector }{ - {name: "unknown taxonomy", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: "other", ProtocolV: "1.0.0", Owner: "test"}}}, - {name: "declared but not implemented", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, ProtocolV: "1.0.0", Owner: "test", Capabilities: []Capability{CapRead}}}}, - {name: "unknown capability", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, ProtocolV: "1.0.0", Owner: "test", Capabilities: []Capability{"shell"}}}}, - {name: "read adapter with verbs", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, ProtocolV: "1.0.0", Owner: "test", Verbs: []string{"gitops.open-pr"}}}}, - {name: "action without verbs", connector: testExecutor{descriptor: Descriptor{Kind: "bad", ConnKind: KindTypedAction, ProtocolV: "1.0.0", Owner: "test", Capabilities: []Capability{CapExecute}}}}, - {name: "action with unknown verb", connector: testExecutor{descriptor: Descriptor{Kind: "bad", ConnKind: KindTypedAction, ProtocolV: "1.0.0", Owner: "test", Capabilities: []Capability{CapExecute}, Verbs: []string{"shell.exec"}}}}, + {name: "unknown taxonomy", connector: identityOnlyConnector{descriptor: testDescriptor("bad", "other")}}, + {name: "missing wire versions", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, AdapterVersion: "1.0.0", Owner: "test"}}}, + {name: "zero wire major", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, WireVersions: []WireVersion{{Minor: 1}}, AdapterVersion: "1.0.0", Owner: "test"}}}, + {name: "duplicate wire version", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, WireVersions: []WireVersion{CurrentWireVersion(), CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test"}}}, + {name: "missing adapter version", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, WireVersions: []WireVersion{CurrentWireVersion()}, Owner: "test"}}}, + {name: "blank adapter version", connector: identityOnlyConnector{descriptor: Descriptor{Kind: "bad", ConnKind: KindReadAdapter, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: " \t", Owner: "test"}}}, + {name: "declared but not implemented", connector: identityOnlyConnector{descriptor: testDescriptorWithCapabilities("bad", KindReadAdapter, CapRead)}}, + {name: "unknown capability", connector: identityOnlyConnector{descriptor: testDescriptorWithCapabilities("bad", KindReadAdapter, "shell")}}, + {name: "read adapter with verbs", connector: identityOnlyConnector{descriptor: func() Descriptor { + descriptor := testDescriptor("bad", KindReadAdapter) + descriptor.Verbs = []intent.Verb{intent.VerbGitOpsOpenPR} + return descriptor + }()}}, + {name: "action without verbs", connector: testExecutor{descriptor: testDescriptorWithCapabilities("bad", KindTypedAction, CapExecute)}}, + {name: "action with unknown verb", connector: testExecutor{descriptor: func() Descriptor { + descriptor := testDescriptorWithCapabilities("bad", KindTypedAction, CapExecute) + descriptor.Verbs = []intent.Verb{"shell.exec"} + return descriptor + }()}}, } for _, test := range tests { @@ -141,6 +176,26 @@ func TestRegistryRejectsDuplicateKind(t *testing.T) { } } +func TestRegistryRejectsDuplicateCanonicalToolAcrossTaxonomies(t *testing.T) { + t.Parallel() + + registry := NewRegistry() + reader := newTestReader("github") + if err := registry.Register(func() (Connector, error) { return reader, nil }); err != nil { + t.Fatalf("first Register() error = %v", err) + } + brokered := identityOnlyConnector{descriptor: testDescriptor("github", KindBrokeredRead)} + brokered.descriptor.AdapterVersion = "deep-link/v1" + if err := registry.Register(func() (Connector, error) { return brokered, nil }); err == nil { + t.Fatal("second canonical connector for github was accepted") + } + got, ok := registry.ByKind("github") + gotReader, isReader := got.(testReader) + if !ok || !isReader || gotReader.descriptor.AdapterVersion != reader.descriptor.AdapterVersion { + t.Fatalf("canonical github connector changed after rejection: %v/%t", got, ok) + } +} + func TestRegistryWithCapabilityIsDeterministic(t *testing.T) { t.Parallel() @@ -161,17 +216,50 @@ func TestRegistryWithCapabilityIsDeterministic(t *testing.T) { } } +func TestRegistryCanonicalizesWireVersionsAndJSONDomains(t *testing.T) { + t.Parallel() + + reader := newTestReader("versions") + reader.descriptor.WireVersions = []WireVersion{ + {Major: 2, Minor: 0}, + {Major: 1, Minor: 2}, + CurrentWireVersion(), + } + reader.descriptor.AdapterVersion = "search/ecs-v1" + registry := NewRegistry() + if err := registry.Register(func() (Connector, error) { return reader, nil }); err != nil { + t.Fatalf("Register() error = %v", err) + } + + descriptor := registry.Descriptors()[0] + want := []WireVersion{CurrentWireVersion(), {Major: 1, Minor: 2}, {Major: 2, Minor: 0}} + if !slices.Equal(descriptor.WireVersions, want) { + t.Fatalf("WireVersions = %#v, want %#v", descriptor.WireVersions, want) + } + document, err := json.Marshal(descriptor) + if err != nil { + t.Fatalf("json.Marshal() error = %v", err) + } + if !bytes.Contains(document, []byte(`"wire_versions":[{"major":1,"minor":0}`)) || + !bytes.Contains(document, []byte(`"adapter_version":"search/ecs-v1"`)) || + bytes.Contains(document, []byte(`"protocol_version"`)) { + t.Fatalf("descriptor JSON = %s", document) + } +} + func TestRegistryExecutorRequiresTypedAction(t *testing.T) { t.Parallel() registry := NewRegistry() executor := testExecutor{descriptor: Descriptor{ - Kind: "argocd", - ConnKind: KindTypedAction, - ProtocolV: "1.0.0", - Owner: "test", - Capabilities: []Capability{CapExecute}, - Verbs: []string{"argocd.sync"}, + Kind: "argocd", + ConnKind: KindTypedAction, + WireVersions: []WireVersion{CurrentWireVersion()}, + AdapterVersion: "1.0.0", + Owner: "test", + Capabilities: []Capability{CapExecute}, + Verbs: []intent.Verb{intent.VerbArgoCDSync}, + ArgSchemas: testArgSchemas(intent.VerbArgoCDSync), }} if err := registry.Register(func() (Connector, error) { return executor, nil }); err != nil { t.Fatalf("Register() error = %v", err) @@ -203,11 +291,206 @@ func TestRegistryFactoryFailuresAreAtomic(t *testing.T) { } } -func TestValidVerb(t *testing.T) { +func TestRegistryRejectsDuplicateVerbOwnershipAtomically(t *testing.T) { t.Parallel() - if !ValidVerb("gitops.open-pr") || ValidVerb("shell.exec") { - t.Fatal("ValidVerb() does not enforce the closed vocabulary") + registry := NewRegistry() + first := testExecutor{descriptor: Descriptor{ + Kind: "first", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapExecute}, Verbs: []intent.Verb{intent.VerbGitOpsOpenPR}, + ArgSchemas: testArgSchemas(intent.VerbGitOpsOpenPR), + }} + second := testExecutor{descriptor: Descriptor{ + Kind: "second", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapExecute}, Verbs: []intent.Verb{intent.VerbGitOpsOpenPR, intent.VerbArgoCDSync}, + ArgSchemas: testArgSchemas(intent.VerbGitOpsOpenPR, intent.VerbArgoCDSync), + }} + if err := registry.Register(func() (Connector, error) { return first, nil }); err != nil { + t.Fatalf("first Register() error = %v", err) + } + if err := registry.Register(func() (Connector, error) { return second, nil }); err == nil { + t.Fatal("duplicate verb owner was accepted") + } + if _, ok := registry.ByKind("second"); ok { + t.Fatal("failed registration modified kind index") + } + if _, err := registry.ExecutorForVerb(intent.VerbArgoCDSync); !errors.Is(err, ErrNotRegistered) { + t.Fatalf("failed registration modified verb index: %v", err) + } + if executor, err := registry.ExecutorForVerb(intent.VerbGitOpsOpenPR); err != nil || executor.Kind() != "first" { + t.Fatalf("original verb owner = %v, %v", executor, err) + } +} + +func TestRegistryVerbLookupFailsClosed(t *testing.T) { + t.Parallel() + + registry := NewRegistry() + executor := testExecutor{descriptor: Descriptor{ + Kind: "git", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapExecute}, Verbs: []intent.Verb{intent.VerbGitOpsOpenPR}, + ArgSchemas: testArgSchemas(intent.VerbGitOpsOpenPR), + }} + if err := registry.Register(func() (Connector, error) { return executor, nil }); err != nil { + t.Fatalf("Register() error = %v", err) + } + if got, err := registry.ExecutorForVerb(intent.VerbGitOpsOpenPR); err != nil || got.Kind() != "git" { + t.Fatalf("ExecutorForVerb() = %v, %v", got, err) + } + if _, err := registry.ExecutorForVerb("shell.exec"); !errors.Is(err, ErrNotRegistered) { + t.Fatalf("unknown ExecutorForVerb() error = %v", err) + } + if _, err := registry.ExecutorForVerb(intent.VerbArgoCDSync); !errors.Is(err, ErrNotRegistered) { + t.Fatalf("unregistered ExecutorForVerb() error = %v", err) + } + if _, err := registry.PlannerForVerb(intent.VerbGitOpsOpenPR); !errors.Is(err, ErrCapability) { + t.Fatalf("wrong-capability PlannerForVerb() error = %v", err) + } + if _, err := registry.VerifierForVerb(intent.VerbGitOpsOpenPR); !errors.Is(err, ErrCapability) { + t.Fatalf("wrong-capability VerifierForVerb() error = %v", err) + } +} + +func TestRegistryVerbLookupRoutesDeclaredCapabilities(t *testing.T) { + t.Parallel() + + registry := NewRegistry() + action := testExecutor{descriptor: Descriptor{ + Kind: "git", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapPlan, CapExecute, CapVerify}, Verbs: []intent.Verb{intent.VerbGitOpsOpenPR}, + ArgSchemas: testArgSchemas(intent.VerbGitOpsOpenPR), + }} + if err := registry.Register(func() (Connector, error) { return action, nil }); err != nil { + t.Fatalf("Register() error = %v", err) + } + planner, err := registry.PlannerForVerb(intent.VerbGitOpsOpenPR) + if err != nil || planner.Kind() != "git" { + t.Fatalf("PlannerForVerb() = %v, %v", planner, err) + } + executor, err := registry.ExecutorForVerb(intent.VerbGitOpsOpenPR) + if err != nil || executor.Kind() != "git" { + t.Fatalf("ExecutorForVerb() = %v, %v", executor, err) + } + verifier, err := registry.VerifierForVerb(intent.VerbGitOpsOpenPR) + if err != nil || verifier.Kind() != "git" { + t.Fatalf("VerifierForVerb() = %v, %v", verifier, err) + } +} + +func TestRegistryRejectsInvalidSchemaOwnershipAtomically(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + schemas map[intent.Verb]json.RawMessage + }{ + {name: "missing schema", schemas: nil}, + {name: "schema for undeclared verb", schemas: testArgSchemas(intent.VerbGitOpsOpenPR, intent.VerbArgoCDSync)}, + {name: "invalid schema", schemas: map[intent.Verb]json.RawMessage{intent.VerbGitOpsOpenPR: json.RawMessage(`{"type":"object"}`)}}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + registry := NewRegistry() + action := testExecutor{descriptor: Descriptor{ + Kind: "git", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapPlan}, Verbs: []intent.Verb{intent.VerbGitOpsOpenPR}, ArgSchemas: test.schemas, + }} + if err := registry.Register(func() (Connector, error) { return action, nil }); err == nil { + t.Fatal("Register() error = nil, want schema rejection") + } + if len(registry.Descriptors()) != 0 { + t.Fatal("failed schema registration modified registry") + } + if _, err := registry.PlannerForVerb(intent.VerbGitOpsOpenPR); !errors.Is(err, ErrNotRegistered) { + t.Fatalf("failed schema registration modified verb index: %v", err) + } + }) + } +} + +func TestRegistryPlanValidatesBeforePlanner(t *testing.T) { + t.Parallel() + + registry := NewRegistry() + planCalls := &atomic.Int64{} + plannedArgs := &json.RawMessage{} + action := testExecutor{descriptor: Descriptor{ + Kind: "scale", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapPlan}, Verbs: []intent.Verb{intent.VerbDeploymentScale}, + ArgSchemas: map[intent.Verb]json.RawMessage{intent.VerbDeploymentScale: json.RawMessage(`{ + "$schema":"https://json-schema.org/draft/2020-12/schema", + "type":"object", + "properties":{"replicas":{"type":"integer","minimum":0,"maximum":10}}, + "required":["replicas"], + "additionalProperties":false + }`)}, + }, planCalls: planCalls, plannedArgs: plannedArgs} + if err := registry.Register(func() (Connector, error) { return action, nil }); err != nil { + t.Fatalf("Register() error = %v", err) + } + + originalArgs := json.RawMessage(`{"replicas":3}`) + plan, err := registry.Plan(context.Background(), Intent{ + ID: "intent-1", Verb: intent.VerbDeploymentScale, Args: originalArgs, + }) + if err != nil || plan.IntentID != "intent-1" || plan.Verb != intent.VerbDeploymentScale { + t.Fatalf("Plan() = %#v, %v", plan, err) + } + if got := planCalls.Load(); got != 1 { + t.Fatalf("planner calls after valid intent = %d, want 1", got) + } + originalArgs[1] = '!' + if got := string(*plannedArgs); got != `{"replicas":3}` { + t.Fatalf("planner args changed with caller storage: %q", got) + } + for _, args := range []json.RawMessage{ + json.RawMessage(`{"replicas":11}`), + json.RawMessage(`{"replicas":3,"force":true}`), + json.RawMessage(`{"replicas":3,"replicas":4}`), + json.RawMessage(`[]`), + } { + if _, err := registry.Plan(context.Background(), Intent{Verb: intent.VerbDeploymentScale, Args: args}); !errors.Is(err, intentargs.ErrInvalidArgs) { + t.Fatalf("Plan(%s) error = %v, want ErrInvalidArgs", args, err) + } + if got := planCalls.Load(); got != 1 { + t.Fatalf("planner calls after invalid intent = %d, want 1", got) + } + } + if _, err := registry.Plan(context.Background(), Intent{Verb: intent.VerbArgoCDSync, Args: json.RawMessage(`{}`)}); !errors.Is(err, ErrNotRegistered) { + t.Fatalf("Plan(unregistered) error = %v, want ErrNotRegistered", err) + } + if got := planCalls.Load(); got != 1 { + t.Fatalf("planner calls after unregistered intent = %d, want 1", got) + } +} + +func TestRegistryDescriptorsCloneArgumentSchemas(t *testing.T) { + t.Parallel() + + registry := NewRegistry() + action := testExecutor{descriptor: Descriptor{ + Kind: "git", ConnKind: KindTypedAction, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + Capabilities: []Capability{CapPlan}, Verbs: []intent.Verb{intent.VerbGitOpsOpenPR}, + ArgSchemas: testArgSchemas(intent.VerbGitOpsOpenPR), + }} + if err := registry.Register(func() (Connector, error) { return action, nil }); err != nil { + t.Fatalf("Register() error = %v", err) + } + descriptors := registry.Descriptors() + document := descriptors[0].ArgSchemas[intent.VerbGitOpsOpenPR] + document[0] = '!' + delete(descriptors[0].ArgSchemas, intent.VerbGitOpsOpenPR) + descriptors[0].WireVersions[0] = WireVersion{Major: 9, Minor: 9} + + again := registry.Descriptors() + got := again[0].ArgSchemas[intent.VerbGitOpsOpenPR] + if len(got) == 0 || got[0] == '!' { + t.Fatal("Descriptors() exposed mutable argument schema storage") + } + if again[0].WireVersions[0] != CurrentWireVersion() { + t.Fatal("Descriptors() exposed mutable wire-version storage") } } @@ -216,11 +499,32 @@ func newTestReader(kind string) testReader { return testReader{ kind: kind, descriptor: Descriptor{ - Kind: kind, - ConnKind: KindReadAdapter, - ProtocolV: "1.0.0", - Owner: "test", - Capabilities: capabilities, + Kind: kind, + ConnKind: KindReadAdapter, + WireVersions: []WireVersion{CurrentWireVersion()}, + AdapterVersion: "1.0.0", + Owner: "test", + Capabilities: capabilities, }, } } + +func testDescriptor(kind string, connKind ConnectorKind) Descriptor { + return Descriptor{ + Kind: kind, ConnKind: connKind, WireVersions: []WireVersion{CurrentWireVersion()}, AdapterVersion: "1.0.0", Owner: "test", + } +} + +func testDescriptorWithCapabilities(kind string, connKind ConnectorKind, capabilities ...Capability) Descriptor { + descriptor := testDescriptor(kind, connKind) + descriptor.Capabilities = capabilities + return descriptor +} + +func testArgSchemas(verbs ...intent.Verb) map[intent.Verb]json.RawMessage { + result := make(map[intent.Verb]json.RawMessage, len(verbs)) + for _, verb := range verbs { + result[verb] = json.RawMessage(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","additionalProperties":false}`) + } + return result +} diff --git a/internal/connector/source.go b/internal/connector/source.go index 46cadb0..25a533a 100644 --- a/internal/connector/source.go +++ b/internal/connector/source.go @@ -81,6 +81,7 @@ func (source readerSource) Fleet(ctx context.Context) (fleet.FleetResult, error) } coverage.Unreachable = sortedUnique(coverage.Unreachable) coverage.Stale = sortedUnique(coverage.Stale) + coverage.Truncated = sortedUnique(coverage.Truncated) return fleet.FleetResult{Clusters: clusters, Coverage: coverage}, nil } diff --git a/internal/connector/source_test.go b/internal/connector/source_test.go index 134db90..27f1c79 100644 --- a/internal/connector/source_test.go +++ b/internal/connector/source_test.go @@ -23,6 +23,7 @@ func TestAsSourcePreservesCoverageAndScopes(t *testing.T) { reader.query.Coverage.Requested = 2 reader.query.Coverage.Reachable = 1 reader.query.Coverage.Unreachable = []string{"lab", "lab"} + reader.query.Coverage.Truncated = []string{"prod", "prod"} source := AsSource(reader) result, err := source.Fleet(context.Background()) @@ -41,6 +42,9 @@ func TestAsSourcePreservesCoverageAndScopes(t *testing.T) { if len(result.Coverage.Unreachable) != 1 || result.Coverage.Unreachable[0] != "lab" { t.Fatalf("Coverage = %#v", result.Coverage) } + if len(result.Coverage.Truncated) != 1 || result.Coverage.Truncated[0] != "prod" { + t.Fatalf("Coverage = %#v", result.Coverage) + } } func TestAsSourceFallsBackToDiscoveryCoverage(t *testing.T) { diff --git a/internal/connector/version.go b/internal/connector/version.go new file mode 100644 index 0000000..e407905 --- /dev/null +++ b/internal/connector/version.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +package connector + +import ( + "errors" + "fmt" + "sort" +) + +const maxWireVersionsPerOffer = 32 + +var ( + // ErrInvalidWireVersions reports an empty, malformed, or ambiguous version offer. + ErrInvalidWireVersions = errors.New("invalid connector wire versions") + // ErrWireMajorMismatch reports that two valid offers have no major version in common. + ErrWireMajorMismatch = errors.New("connector wire major version mismatch") + // ErrWireMinorUnsupported reports a common major without an explicitly shared minor. + ErrWireMinorUnsupported = errors.New("connector wire minor version is not mutually supported") +) + +// WireVersion identifies one framework transport contract. It is deliberately independent from +// an adapter's opaque evidence and behavior contract version. +type WireVersion struct { + Major uint32 `json:"major"` + Minor uint32 `json:"minor"` +} + +// CurrentWireVersion returns the initial E12 framework wire version. +func CurrentWireVersion() WireVersion { + return WireVersion{Major: 1, Minor: 0} +} + +// String returns the deterministic human-readable major.minor form. +func (version WireVersion) String() string { + return fmt.Sprintf("%d.%d", version.Major, version.Minor) +} + +// NegotiateWireVersion returns the highest exact version explicitly offered by both endpoints. +// Same-major versions are not assumed compatible unless the negotiated minor appears in both +// offers. This keeps protobuf wire compatibility separate from application-level compatibility. +func NegotiateWireVersion(local, remote []WireVersion) (WireVersion, error) { + local, err := canonicalWireVersions(local) + if err != nil { + return WireVersion{}, fmt.Errorf("negotiate connector wire version: local offer: %w", err) + } + remote, err = canonicalWireVersions(remote) + if err != nil { + return WireVersion{}, fmt.Errorf("negotiate connector wire version: remote offer: %w", err) + } + + localVersions := make(map[WireVersion]struct{}, len(local)) + localMajors := make(map[uint32]struct{}, len(local)) + for _, version := range local { + localVersions[version] = struct{}{} + localMajors[version.Major] = struct{}{} + } + + var selected WireVersion + matched := false + commonMajor := false + for _, version := range remote { + if _, ok := localMajors[version.Major]; ok { + commonMajor = true + } + if _, ok := localVersions[version]; !ok { + continue + } + if !matched || wireVersionLess(selected, version) { + selected = version + matched = true + } + } + if matched { + return selected, nil + } + if !commonMajor { + return WireVersion{}, ErrWireMajorMismatch + } + return WireVersion{}, ErrWireMinorUnsupported +} + +func canonicalWireVersions(versions []WireVersion) ([]WireVersion, error) { + if len(versions) == 0 { + return nil, fmt.Errorf("%w: offer must not be empty", ErrInvalidWireVersions) + } + if len(versions) > maxWireVersionsPerOffer { + return nil, fmt.Errorf("%w: offer exceeds %d versions", ErrInvalidWireVersions, maxWireVersionsPerOffer) + } + + canonical := append([]WireVersion(nil), versions...) + seen := make(map[WireVersion]struct{}, len(canonical)) + for _, version := range canonical { + if version.Major == 0 { + return nil, fmt.Errorf("%w: major must be greater than zero", ErrInvalidWireVersions) + } + if _, duplicate := seen[version]; duplicate { + return nil, fmt.Errorf("%w: duplicate version %s", ErrInvalidWireVersions, version) + } + seen[version] = struct{}{} + } + sort.Slice(canonical, func(left, right int) bool { + return wireVersionLess(canonical[left], canonical[right]) + }) + return canonical, nil +} + +func wireVersionLess(left, right WireVersion) bool { + if left.Major != right.Major { + return left.Major < right.Major + } + return left.Minor < right.Minor +} diff --git a/internal/connector/version_boundary_test.go b/internal/connector/version_boundary_test.go new file mode 100644 index 0000000..c3e2d0a --- /dev/null +++ b/internal/connector/version_boundary_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 + +package connector + +import ( + "go/ast" + "go/parser" + "go/token" + "strconv" + "testing" +) + +func TestWireVersionCoreHasNoIOProcessCredentialOrNetworkSeam(t *testing.T) { + t.Parallel() + + file, err := parser.ParseFile(token.NewFileSet(), "version.go", nil, 0) + if err != nil { + t.Fatalf("parse version.go: %v", err) + } + allowedImports := map[string]bool{"errors": true, "fmt": true, "sort": true} + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("unquote import: %v", err) + } + if imported.Name != nil || !allowedImports[path] { + t.Errorf("version core imports unreviewed package %q", path) + } + } + + allowedDeclarations := map[string]bool{ + "CurrentWireVersion": true, + "ErrInvalidWireVersions": true, + "ErrWireMajorMismatch": true, + "ErrWireMinorUnsupported": true, + "NegotiateWireVersion": true, + "String": true, + "WireVersion": true, + "canonicalWireVersions": true, + "maxWireVersionsPerOffer": true, + "wireVersionLess": true, + } + for _, declaration := range file.Decls { + for _, name := range declarationNames(declaration) { + if !allowedDeclarations[name] { + t.Errorf("version core declares unreviewed symbol %s", name) + } + } + } + ast.Inspect(file, func(node ast.Node) bool { + switch node.(type) { + case *ast.ChanType, *ast.GoStmt, *ast.InterfaceType: + t.Errorf("version core gained an asynchronous or injected capability seam") + } + return true + }) +} + +func declarationNames(declaration ast.Decl) []string { + switch typed := declaration.(type) { + case *ast.FuncDecl: + return []string{typed.Name.Name} + case *ast.GenDecl: + var names []string + for _, specification := range typed.Specs { + switch value := specification.(type) { + case *ast.TypeSpec: + names = append(names, value.Name.Name) + case *ast.ValueSpec: + for _, name := range value.Names { + names = append(names, name.Name) + } + } + } + return names + default: + return nil + } +} diff --git a/internal/connector/version_test.go b/internal/connector/version_test.go new file mode 100644 index 0000000..06bcc27 --- /dev/null +++ b/internal/connector/version_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package connector + +import ( + "errors" + "testing" +) + +func TestNegotiateWireVersion(t *testing.T) { + t.Parallel() + + v10 := WireVersion{Major: 1, Minor: 0} + v11 := WireVersion{Major: 1, Minor: 1} + v12 := WireVersion{Major: 1, Minor: 2} + v20 := WireVersion{Major: 2, Minor: 0} + oversized := make([]WireVersion, maxWireVersionsPerOffer+1) + for index := range oversized { + oversized[index] = WireVersion{Major: 1, Minor: uint32(index)} + } + tests := []struct { + name string + local []WireVersion + remote []WireVersion + want WireVersion + isErr error + }{ + {name: "initial version", local: []WireVersion{v10}, remote: []WireVersion{v10}, want: v10}, + {name: "highest explicit minor", local: []WireVersion{v10, v11, v12}, remote: []WireVersion{v11, v10}, want: v11}, + {name: "highest explicit major", local: []WireVersion{v20, v10}, remote: []WireVersion{v10, v20}, want: v20}, + {name: "order independent", local: []WireVersion{v12, v10, v11}, remote: []WireVersion{v10, v11}, want: v11}, + {name: "empty local offer", remote: []WireVersion{v10}, isErr: ErrInvalidWireVersions}, + {name: "empty remote offer", local: []WireVersion{v10}, isErr: ErrInvalidWireVersions}, + {name: "zero major", local: []WireVersion{{Minor: 1}}, remote: []WireVersion{v10}, isErr: ErrInvalidWireVersions}, + {name: "duplicate local", local: []WireVersion{v10, v10}, remote: []WireVersion{v10}, isErr: ErrInvalidWireVersions}, + {name: "duplicate remote", local: []WireVersion{v10}, remote: []WireVersion{v10, v10}, isErr: ErrInvalidWireVersions}, + {name: "oversized local offer", local: oversized, remote: []WireVersion{v10}, isErr: ErrInvalidWireVersions}, + {name: "major mismatch", local: []WireVersion{v10, v11}, remote: []WireVersion{v20}, isErr: ErrWireMajorMismatch}, + {name: "minor not explicit", local: []WireVersion{v10}, remote: []WireVersion{v11}, isErr: ErrWireMinorUnsupported}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + got, err := NegotiateWireVersion(test.local, test.remote) + if test.isErr != nil { + if !errors.Is(err, test.isErr) { + t.Fatalf("NegotiateWireVersion() error = %v, want %v", err, test.isErr) + } + if got != (WireVersion{}) { + t.Fatalf("NegotiateWireVersion() = %v on error, want zero", got) + } + return + } + if err != nil || got != test.want { + t.Fatalf("NegotiateWireVersion() = %v, %v, want %v", got, err, test.want) + } + }) + } +} + +func TestCurrentWireVersion(t *testing.T) { + t.Parallel() + + if got := CurrentWireVersion(); got != (WireVersion{Major: 1, Minor: 0}) || got.String() != "1.0" { + t.Fatalf("CurrentWireVersion() = %v (%q)", got, got.String()) + } +} + +func FuzzNegotiateWireVersionNeverReturnsAnUnofferedVersion(f *testing.F) { + f.Add([]byte{1, 0, 1, 1}, []byte{1, 0}) + f.Add([]byte{2, 0}, []byte{1, 0}) + f.Add([]byte{}, []byte{1, 0}) + f.Fuzz(func(t *testing.T, localBytes, remoteBytes []byte) { + local := wireVersionsFromBytes(localBytes) + remote := wireVersionsFromBytes(remoteBytes) + got, err := NegotiateWireVersion(local, remote) + if err != nil { + if got != (WireVersion{}) { + t.Fatalf("error %v returned non-zero version %v", err, got) + } + return + } + if got.Major == 0 || !containsWireVersion(local, got) || !containsWireVersion(remote, got) { + t.Fatalf("negotiated version %v was not explicitly offered by both sides", got) + } + for _, localVersion := range local { + if !wireVersionLess(got, localVersion) || !containsWireVersion(remote, localVersion) { + continue + } + t.Fatalf("negotiated %v instead of higher common version %v", got, localVersion) + } + }) +} + +func wireVersionsFromBytes(input []byte) []WireVersion { + versions := make([]WireVersion, 0, len(input)/2) + for offset := 0; offset+1 < len(input) && len(versions) < 32; offset += 2 { + versions = append(versions, WireVersion{Major: uint32(input[offset]), Minor: uint32(input[offset+1])}) + } + return versions +} + +func containsWireVersion(versions []WireVersion, want WireVersion) bool { + for _, version := range versions { + if version == want { + return true + } + } + return false +} diff --git a/internal/fleet/coverage_test.go b/internal/fleet/coverage_test.go index bb32234..7beed50 100644 --- a/internal/fleet/coverage_test.go +++ b/internal/fleet/coverage_test.go @@ -33,6 +33,14 @@ func TestCoverageAssessment(t *testing.T) { Stale: []string{"spoke-b"}, }, }, + { + name: "truncated reachable scope is incomplete", + in: Coverage{Requested: 2, Reachable: 2, Truncated: []string{"spoke-b"}}, + want: CoverageAssessment{ + Gaps: []CoverageGap{CoverageGapTruncated}, + Truncated: []string{"spoke-b"}, + }, + }, { name: "unreachable scope is incomplete", in: Coverage{Requested: 2, Reachable: 1, Unreachable: []string{"spoke-b"}}, @@ -79,6 +87,26 @@ func TestCoverageAssessment(t *testing.T) { Inconsistent: true, }, }, + { + name: "more truncated names than reachable scopes is inconsistent", + in: Coverage{Requested: 2, Reachable: 1, Unreachable: []string{"spoke-b"}, Truncated: []string{"spoke-a", "spoke-c"}}, + want: CoverageAssessment{ + Gaps: []CoverageGap{CoverageGapInconsistent, CoverageGapUnreachable, CoverageGapTruncated}, + Unreachable: []string{"spoke-b"}, + Truncated: []string{"spoke-a", "spoke-c"}, + Inconsistent: true, + }, + }, + { + name: "unreachable scope cannot also be truncated", + in: Coverage{Requested: 2, Reachable: 1, Unreachable: []string{"spoke-b"}, Truncated: []string{"spoke-b"}}, + want: CoverageAssessment{ + Gaps: []CoverageGap{CoverageGapInconsistent, CoverageGapUnreachable, CoverageGapTruncated}, + Unreachable: []string{"spoke-b"}, + Truncated: []string{"spoke-b"}, + Inconsistent: true, + }, + }, { name: "duplicate and blank scope names are inconsistent", in: Coverage{Requested: 2, Reachable: 0, Unreachable: []string{"spoke-b", "spoke-b"}, Stale: []string{" "}}, @@ -97,7 +125,8 @@ func TestCoverageAssessment(t *testing.T) { got := test.in.Assessment() if got.Complete != test.want.Complete || got.Unaccounted != test.want.Unaccounted || got.Inconsistent != test.want.Inconsistent || - !slices.Equal(got.Gaps, test.want.Gaps) || !slices.Equal(got.Unreachable, test.want.Unreachable) || !slices.Equal(got.Stale, test.want.Stale) { + !slices.Equal(got.Gaps, test.want.Gaps) || !slices.Equal(got.Unreachable, test.want.Unreachable) || + !slices.Equal(got.Stale, test.want.Stale) || !slices.Equal(got.Truncated, test.want.Truncated) { t.Fatalf("Assessment() = %#v, want %#v", got, test.want) } if complete := test.in.Complete(); complete != test.want.Complete { @@ -110,12 +139,16 @@ func TestCoverageAssessment(t *testing.T) { func TestCoverageAssessmentDoesNotAliasInputSlices(t *testing.T) { t.Parallel() - coverage := Coverage{Requested: 2, Reachable: 1, Unreachable: []string{"spoke-b"}, Stale: []string{"spoke-b"}} + coverage := Coverage{ + Requested: 2, Reachable: 1, Unreachable: []string{"spoke-b"}, Stale: []string{"spoke-b"}, Truncated: []string{"spoke-a"}, + } assessment := coverage.Assessment() assessment.Unreachable[0] = "mutated" assessment.Stale[0] = "mutated" + assessment.Truncated[0] = "mutated" - if !slices.Equal(coverage.Unreachable, []string{"spoke-b"}) || !slices.Equal(coverage.Stale, []string{"spoke-b"}) { + if !slices.Equal(coverage.Unreachable, []string{"spoke-b"}) || !slices.Equal(coverage.Stale, []string{"spoke-b"}) || + !slices.Equal(coverage.Truncated, []string{"spoke-a"}) { t.Fatalf("Assessment() aliased Coverage input: %#v", coverage) } } diff --git a/internal/fleet/model.go b/internal/fleet/model.go index 5af1887..5566469 100644 --- a/internal/fleet/model.go +++ b/internal/fleet/model.go @@ -26,12 +26,13 @@ type Cluster struct { ObservedAt time.Time `json:"observed_at,omitempty"` } -// Coverage summarizes which requested scopes answered and which were unreachable. +// Coverage summarizes which requested scopes answered and which were incomplete or unreachable. type Coverage struct { Requested int `json:"requested"` Reachable int `json:"reachable"` Unreachable []string `json:"unreachable,omitempty"` Stale []string `json:"stale,omitempty"` + Truncated []string `json:"truncated,omitempty"` } // CoverageGap explains why coverage is not safe to treat as complete. @@ -42,6 +43,7 @@ const ( CoverageGapInconsistent CoverageGap = "inconsistent" CoverageGapUnreachable CoverageGap = "unreachable" CoverageGapStale CoverageGap = "stale" + CoverageGapTruncated CoverageGap = "truncated" CoverageGapUnaccounted CoverageGap = "unaccounted" ) @@ -52,6 +54,7 @@ type CoverageAssessment struct { Gaps []CoverageGap `json:"gaps,omitempty"` Unreachable []string `json:"unreachable,omitempty"` Stale []string `json:"stale,omitempty"` + Truncated []string `json:"truncated,omitempty"` Unaccounted int `json:"unaccounted,omitempty"` Inconsistent bool `json:"inconsistent"` } @@ -61,13 +64,16 @@ type CoverageAssessment struct { func (c Coverage) Assessment() CoverageAssessment { unreachable, unreachableValid := coverageScopes(c.Unreachable) stale, staleValid := coverageScopes(c.Stale) + truncated, truncatedValid := coverageScopes(c.Truncated) assessment := CoverageAssessment{ Unreachable: unreachable, Stale: stale, + Truncated: truncated, } if c.Requested < 0 || c.Reachable < 0 || c.Reachable > c.Requested || len(stale) > c.Requested || - !unreachableValid || !staleValid { + len(truncated) > c.Reachable || !unreachableValid || !staleValid || !truncatedValid || + coverageScopesOverlap(unreachable, truncated) { assessment.Inconsistent = true } else { remaining := c.Requested - c.Reachable @@ -88,6 +94,9 @@ func (c Coverage) Assessment() CoverageAssessment { if len(assessment.Stale) != 0 { assessment.Gaps = append(assessment.Gaps, CoverageGapStale) } + if len(assessment.Truncated) != 0 { + assessment.Gaps = append(assessment.Gaps, CoverageGapTruncated) + } if assessment.Unaccounted != 0 { assessment.Gaps = append(assessment.Gaps, CoverageGapUnaccounted) } @@ -95,6 +104,21 @@ func (c Coverage) Assessment() CoverageAssessment { return assessment } +func coverageScopesOverlap(left, right []string) bool { + leftIndex, rightIndex := 0, 0 + for leftIndex < len(left) && rightIndex < len(right) { + switch { + case left[leftIndex] == right[rightIndex]: + return true + case left[leftIndex] < right[rightIndex]: + leftIndex++ + default: + rightIndex++ + } + } + return false +} + // Complete reports whether the coverage is internally consistent and every requested scope answered with fresh data. func (c Coverage) Complete() bool { return c.Assessment().Complete diff --git a/internal/fleet/resource.go b/internal/fleet/resource.go index df604ed..4f9f8ce 100644 --- a/internal/fleet/resource.go +++ b/internal/fleet/resource.go @@ -98,7 +98,9 @@ type DisplayField struct { // Provenance identifies how to trace an observation back to its native source. type Provenance struct { - Adapter string `json:"adapter"` + Adapter string `json:"adapter"` + // ProtocolV is the adapter/evidence contract identifier retained for serialized compatibility. + // It is not the E12 framework wire version used for out-of-process negotiation. ProtocolV string `json:"protocol_version"` NativeID string `json:"native_id,omitempty"` DeepLink string `json:"deep_link,omitempty"` diff --git a/internal/fleetcache/scoped_test.go b/internal/fleetcache/scoped_test.go index 7517194..c31bc22 100644 --- a/internal/fleetcache/scoped_test.go +++ b/internal/fleetcache/scoped_test.go @@ -3,6 +3,8 @@ package fleetcache import ( + "errors" + "slices" "testing" "time" @@ -22,10 +24,12 @@ func TestQueryScopedDerivesWorkspaceOnlyFromSignedScope(t *testing.T) { factA.Workspace = "workspace-a" factB := podFact(t, "cluster-b", "api-b", "Running", "image:b", now) factB.Workspace = "workspace-b" - if err := store.Replace("Pod", fleet.QueryResult{ - Facts: []fleet.Fact{factA, factB}, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, - }); err != nil { - t.Fatal(err) + for workspace, fact := range map[string]fleet.Fact{"workspace-a": factA, "workspace-b": factB} { + if err := store.Replace(workspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } } principal, err := tenancy.NewPrincipal("user:alice", map[tenancy.WorkspaceID]tenancy.Role{"workspace-a": tenancy.RoleReader}) if err != nil { @@ -54,6 +58,107 @@ func TestQueryScopedDerivesWorkspaceOnlyFromSignedScope(t *testing.T) { } } +func TestQueryScopedIsolatesSameNamedScopeMetadataByWorkspace(t *testing.T) { + t.Parallel() + + observedA := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + observedB := observedA.Add(time.Minute) + store := newStore(func() time.Time { return observedB }, time.Hour) + store.SetDiscovery("workspace-a", connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", DisplayName: "A shared", Kinds: []string{"Pod"}, Reachable: true, ObservedAt: observedA, + }}}) + store.SetDiscovery("workspace-b", connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", DisplayName: "B shared", Kinds: []string{"Deployment"}, ObservedAt: observedB, + }}, Unreachable: []string{"shared"}}) + principal, err := tenancy.NewPrincipal("user:alice", map[tenancy.WorkspaceID]tenancy.Role{ + "workspace-a": tenancy.RoleReader, + "workspace-b": tenancy.RoleReader, + }) + if err != nil { + t.Fatal(err) + } + scopeA, err := principal.Scope("workspace-a") + if err != nil { + t.Fatal(err) + } + scopeB, err := principal.Scope("workspace-b") + if err != nil { + t.Fatal(err) + } + + snapshotA, err := store.QueryScoped(scopeA, Query{}) + if err != nil { + t.Fatal(err) + } + snapshotB, err := store.QueryScoped(scopeB, Query{}) + if err != nil { + t.Fatal(err) + } + if len(snapshotA.Scopes) != 1 || snapshotA.Scopes[0].DisplayName != "A shared" || + !snapshotA.Scopes[0].Reachable || snapshotA.Scopes[0].ObservedAt != observedA || + !slices.Equal(snapshotA.Scopes[0].Kinds, []string{"Pod"}) { + t.Fatalf("workspace A scope = %#v, want independent reachable metadata", snapshotA.Scopes) + } + if len(snapshotB.Scopes) != 1 || snapshotB.Scopes[0].DisplayName != "B shared" || + snapshotB.Scopes[0].Reachable || snapshotB.Scopes[0].ObservedAt != observedB || + !slices.Equal(snapshotB.Scopes[0].Kinds, []string{"Deployment"}) { + t.Fatalf("workspace B scope = %#v, want independent unreachable metadata", snapshotB.Scopes) + } + if snapshotA.Coverage.Reachable != 1 || len(snapshotA.Coverage.Unreachable) != 0 || + snapshotB.Coverage.Reachable != 0 || !slices.Equal(snapshotB.Coverage.Unreachable, []string{"shared"}) { + t.Fatalf("coverage A = %#v, B = %#v, want workspace-isolated fail-closed coverage", snapshotA.Coverage, snapshotB.Coverage) + } + factA := podFact(t, "shared", "api-a", "Running", "image:a", observedB) + factA.Workspace = "workspace-a" + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchError, Workspace: "workspace-b", + Kind: "Pod", Scope: "shared", Err: errors.New("watch unavailable"), + }); err != nil { + t.Fatalf("ApplyWatchEvent(error) error = %v", err) + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchUpsert, Workspace: "workspace-a", + Kind: "Pod", Scope: "shared", Fact: factA, ObservedAt: observedB.Add(time.Minute), + }); err != nil { + t.Fatalf("ApplyWatchEvent() error = %v", err) + } + snapshotA, err = store.QueryScoped(scopeA, Query{}) + if err != nil { + t.Fatal(err) + } + snapshotB, err = store.QueryScoped(scopeB, Query{}) + if err != nil { + t.Fatal(err) + } + if snapshotA.Scopes[0].ObservedAt != observedB.Add(time.Minute) || !snapshotA.Scopes[0].Reachable || + snapshotB.Scopes[0].ObservedAt != observedB || snapshotB.Scopes[0].Reachable { + t.Fatalf("workspace-qualified watch changed the wrong shared metadata: A = %#v, B = %#v", snapshotA.Scopes, snapshotB.Scopes) + } + if snapshotA.Coverage.Reachable != 1 || len(snapshotA.Coverage.Unreachable) != 0 || + snapshotB.Coverage.Reachable != 0 || !slices.Equal(snapshotB.Coverage.Unreachable, []string{"shared"}) || + snapshotA.LastError != "" || snapshotB.LastError == "" { + t.Fatalf("workspace-qualified success crossed failure state: A = %#v, B = %#v", snapshotA, snapshotB) + } + + store.SetDiscovery("workspace-b", connector.Discovery{Scopes: []connector.Scope{{ + Name: "cluster-b", DisplayName: "B replacement", Reachable: true, ObservedAt: observedB, + }}}) + snapshotA, err = store.QueryScoped(scopeA, Query{}) + if err != nil { + t.Fatal(err) + } + if len(snapshotA.Scopes) != 1 || snapshotA.Scopes[0].DisplayName != "A shared" || !snapshotA.Scopes[0].Reachable { + t.Fatalf("workspace B refresh changed workspace A metadata: %#v", snapshotA.Scopes) + } + guessedB, err := store.QueryScoped(scopeB, Query{Scopes: []string{"shared"}}) + if err != nil { + t.Fatal(err) + } + if len(guessedB.Scopes) != 1 || guessedB.Scopes[0].Reachable || !guessedB.Scopes[0].ObservedAt.IsZero() || len(guessedB.Scopes[0].Kinds) != 0 { + t.Fatalf("removed workspace B scope exposed workspace A metadata: %#v", guessedB.Scopes) + } +} + func TestQueryScopedRejectsMissingBoundary(t *testing.T) { t.Parallel() @@ -81,8 +186,10 @@ func FuzzQueryScopedNeverLeaksForeignWorkspace(f *testing.F) { factA.Workspace = "workspace-a" factB := podFact(t, "cluster-b", "api-b", "Running", "image:b", now) factB.Workspace = "workspace-b" - if err := store.Replace("Pod", fleet.QueryResult{Facts: []fleet.Fact{factA, factB}}); err != nil { - t.Fatal(err) + for workspace, fact := range map[string]fleet.Fact{"workspace-a": factA, "workspace-b": factB} { + if err := store.Replace(workspace, "Pod", fleet.QueryResult{Facts: []fleet.Fact{fact}}); err != nil { + t.Fatal(err) + } } principal, err := tenancy.NewPrincipal("user:alice", map[tenancy.WorkspaceID]tenancy.Role{"workspace-a": tenancy.RoleReader}) if err != nil { diff --git a/internal/fleetcache/store.go b/internal/fleetcache/store.go index 8a3ab74..81988fa 100644 --- a/internal/fleetcache/store.go +++ b/internal/fleetcache/store.go @@ -12,6 +12,7 @@ import ( "github.com/ArdurAI/sith/internal/connector" "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" ) const defaultFreshFor = 15 * time.Second @@ -43,24 +44,35 @@ type Snapshot struct { Diagnostics []connector.Diagnostic `json:"diagnostics,omitempty"` } +type scopeMetadataKey struct { + workspace string + scope string +} + +type workspaceKindKey struct { + workspace string + kind string +} + // Store owns normalized last-known fleet state and never performs network I/O. type Store struct { mu sync.RWMutex records map[string]map[string]Record - coverage map[string]fleet.Coverage - aliases map[string]string - scopes map[string]connector.Scope + coverage map[workspaceKindKey]fleet.Coverage + aliases map[workspaceKindKey]string + scopes map[scopeMetadataKey]connector.Scope diagnostics map[string][]connector.Diagnostic scopeWorkspaces map[string]map[string]bool - warmed map[string]bool - expected map[string]bool - syncing bool - paused bool - lastError string - updatedAt time.Time - version uint64 - changed chan struct{} + warmed map[workspaceKindKey]bool + expected map[workspaceKindKey]bool + syncing map[string]bool + paused map[string]bool + syncErrors map[string]string + watchErrors map[workspaceKindKey]string + updatedAt map[string]time.Time + versions map[string]uint64 + changed map[string]chan struct{} now func() time.Time freshFor time.Duration } @@ -73,50 +85,64 @@ func New() *Store { func newStore(now func() time.Time, freshFor time.Duration) *Store { return &Store{ records: make(map[string]map[string]Record), - coverage: make(map[string]fleet.Coverage), - aliases: make(map[string]string), - scopes: make(map[string]connector.Scope), + coverage: make(map[workspaceKindKey]fleet.Coverage), + aliases: make(map[workspaceKindKey]string), + scopes: make(map[scopeMetadataKey]connector.Scope), diagnostics: make(map[string][]connector.Diagnostic), scopeWorkspaces: make(map[string]map[string]bool), - warmed: make(map[string]bool), - expected: make(map[string]bool), - changed: make(chan struct{}), + warmed: make(map[workspaceKindKey]bool), + expected: make(map[workspaceKindKey]bool), + syncing: make(map[string]bool), + paused: make(map[string]bool), + syncErrors: make(map[string]string), + watchErrors: make(map[workspaceKindKey]string), + updatedAt: make(map[string]time.Time), + versions: make(map[string]uint64), + changed: make(map[string]chan struct{}), now: now, freshFor: freshFor, } } -// BeginSync marks background reconciliation as active without blocking readers. -func (store *Store) BeginSync(kinds ...string) bool { +// BeginSync marks one workspace's background reconciliation as active without blocking readers. +func (store *Store) BeginSync(workspace string, kinds ...string) bool { + if tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)) != nil { + return false + } store.mu.Lock() defer store.mu.Unlock() - if store.paused || store.syncing { + if store.paused[workspace] || store.syncing[workspace] { return false } - store.syncing = true - store.lastError = "" + store.syncing[workspace] = true + delete(store.syncErrors, workspace) for _, kind := range kinds { if canonical := canonicalKind(kind); canonical != "" { - store.expected[canonical] = true - store.aliases[kindAlias(kind)] = canonical + store.expected[workspaceKindKey{workspace: workspace, kind: canonical}] = true + store.aliases[workspaceKindKey{workspace: workspace, kind: kindAlias(kind)}] = canonical } } - store.notifyLocked() + store.notifyLocked(workspace) return true } // SetDiscovery refreshes one workspace's known context set while preserving last-known facts. -func (store *Store) SetDiscovery(workspace string, discovery connector.Discovery) { +func (store *Store) SetDiscovery(workspace string, discovery connector.Discovery) error { + if err := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)); err != nil { + return fmt.Errorf("set cache discovery: %w", err) + } store.mu.Lock() defer store.mu.Unlock() - if store.paused || strings.TrimSpace(workspace) == "" { - return + if store.paused[workspace] { + return nil } for name, memberships := range store.scopeWorkspaces { + if memberships[workspace] { + delete(store.scopes, scopeMetadataKey{workspace: workspace, scope: name}) + } delete(memberships, workspace) if len(memberships) == 0 { delete(store.scopeWorkspaces, name) - delete(store.scopes, name) } } known := make(map[string]connector.Scope, len(discovery.Scopes)+len(discovery.Unreachable)) @@ -129,15 +155,19 @@ func (store *Store) SetDiscovery(workspace string, discovery connector.Discovery } } for name, scope := range known { - store.scopes[name] = scope + store.scopes[scopeMetadataKey{workspace: workspace, scope: name}] = scope store.markScopeWorkspaceLocked(workspace, name) } store.diagnostics[workspace] = cloneDiagnostics(discovery.Diagnostics) - store.notifyLocked() + store.notifyLocked(workspace) + return nil } -// Replace reconciles one resource kind while preserving last-known rows for failed scopes. -func (store *Store) Replace(kind string, result fleet.QueryResult) error { +// Replace reconciles one workspace's resource kind while preserving last-known rows for failed scopes. +func (store *Store) Replace(workspace, kind string, result fleet.QueryResult) error { + if err := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)); err != nil { + return fmt.Errorf("replace cache records: %w", err) + } canonical := canonicalKind(kind) if canonical == "" { return fmt.Errorf("replace cache records: resource kind is required") @@ -148,40 +178,57 @@ func (store *Store) Replace(kind string, result fleet.QueryResult) error { if err != nil { return err } + if record.Workspace != workspace { + return fmt.Errorf("replace cache records: fact workspace %q does not match mutation workspace %q", record.Workspace, workspace) + } normalized = append(normalized, record) } store.mu.Lock() defer store.mu.Unlock() - if store.paused { + if store.paused[workspace] { return nil } if store.records[canonical] == nil { store.records[canonical] = make(map[string]Record) } - store.aliases[kindAlias(kind)] = canonical + store.aliases[workspaceKindKey{workspace: workspace, kind: kindAlias(kind)}] = canonical for _, record := range normalized { - store.aliases[kindAlias(record.Kind)] = canonical + store.aliases[workspaceKindKey{workspace: workspace, kind: kindAlias(record.Kind)}] = canonical store.markScopeWorkspaceLocked(record.Workspace, record.Cluster) } - unreachable := stringSet(result.Coverage.Unreachable) + preserve := stringSet(result.Coverage.Unreachable) + for _, scope := range result.Coverage.Truncated { + preserve[scope] = struct{}{} + } for key, record := range store.records[canonical] { - if _, failed := unreachable[record.Cluster]; !failed { + if record.Workspace != workspace { + continue + } + if _, incomplete := preserve[record.Cluster]; !incomplete { delete(store.records[canonical], key) } } for _, record := range normalized { store.records[canonical][recordKey(record)] = record } - store.coverage[canonical] = cloneCoverage(result.Coverage) - store.warmed[canonical] = true - store.updatedAt = store.now().UTC() - store.notifyLocked() + key := workspaceKindKey{workspace: workspace, kind: canonical} + store.coverage[key] = cloneCoverage(result.Coverage) + if len(result.Coverage.Unreachable) == 0 { + delete(store.watchErrors, key) + } + store.warmed[key] = true + store.updatedAt[workspace] = store.now().UTC() + store.notifyLocked(workspace) return nil } -// ApplyWatchEvent atomically reconciles one live-reader delta without network access. +// ApplyWatchEvent atomically reconciles one workspace-bound live-reader delta without network access. func (store *Store) ApplyWatchEvent(event connector.WatchEvent) error { + workspace := event.Workspace + if err := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)); err != nil { + return fmt.Errorf("apply watch event: %w", err) + } canonical := canonicalKind(event.Kind) if canonical == "" || strings.TrimSpace(event.Scope) == "" { return fmt.Errorf("apply watch event: kind and scope are required") @@ -202,6 +249,9 @@ func (store *Store) ApplyWatchEvent(event connector.WatchEvent) error { if record.Cluster != event.Scope { return fmt.Errorf("apply watch event: fact scope %q does not match stream scope %q", record.Cluster, event.Scope) } + if record.Workspace != workspace { + return fmt.Errorf("apply watch event: fact workspace %q does not match stream workspace %q", record.Workspace, workspace) + } normalized = append(normalized, record) } case connector.WatchUpsert: @@ -212,6 +262,9 @@ func (store *Store) ApplyWatchEvent(event connector.WatchEvent) error { if record.Cluster != event.Scope { return fmt.Errorf("apply watch event: fact scope %q does not match stream scope %q", record.Cluster, event.Scope) } + if record.Workspace != workspace { + return fmt.Errorf("apply watch event: fact workspace %q does not match stream workspace %q", record.Workspace, workspace) + } normalized = []Record{record} case connector.WatchDelete: if event.Ref.Scope != "" && event.Ref.Scope != event.Scope { @@ -224,115 +277,127 @@ func (store *Store) ApplyWatchEvent(event connector.WatchEvent) error { store.mu.Lock() defer store.mu.Unlock() - if store.paused { + if store.paused[workspace] { return nil } - store.aliases[kindAlias(event.Kind)] = canonical + if !store.scopeWorkspaces[event.Scope][workspace] { + return fmt.Errorf("apply watch event: scope %q is not discovered in workspace %q", event.Scope, workspace) + } + store.aliases[workspaceKindKey{workspace: workspace, kind: kindAlias(event.Kind)}] = canonical if store.records[canonical] == nil { store.records[canonical] = make(map[string]Record) } for _, record := range normalized { - store.aliases[kindAlias(record.Kind)] = canonical + store.aliases[workspaceKindKey{workspace: workspace, kind: kindAlias(record.Kind)}] = canonical store.markScopeWorkspaceLocked(record.Workspace, record.Cluster) } switch event.Type { case connector.WatchSnapshot: for key, record := range store.records[canonical] { - if record.Cluster == event.Scope { + if record.Workspace == workspace && record.Cluster == event.Scope { delete(store.records[canonical], key) } } for _, record := range normalized { store.records[canonical][recordKey(record)] = record } - store.markScopeReachableLocked(canonical, event.Scope, event.ObservedAt) + store.markScopeReachableLocked(workspace, canonical, event.Scope, event.ObservedAt) + coverageKey := workspaceKindKey{workspace: workspace, kind: canonical} + coverage := store.coverage[coverageKey] + coverage.Truncated = removeString(coverage.Truncated, event.Scope) + store.coverage[coverageKey] = coverage case connector.WatchUpsert: store.records[canonical][recordKey(normalized[0])] = normalized[0] - store.markScopeReachableLocked(canonical, event.Scope, event.ObservedAt) + store.markScopeReachableLocked(workspace, canonical, event.Scope, event.ObservedAt) case connector.WatchDelete: for key, record := range store.records[canonical] { - if record.Cluster == event.Scope && record.Namespace == event.Ref.Namespace && record.Name == event.Ref.Name { + if record.Workspace == workspace && record.Cluster == event.Scope && record.Namespace == event.Ref.Namespace && record.Name == event.Ref.Name { delete(store.records[canonical], key) } } - store.markScopeReachableLocked(canonical, event.Scope, event.ObservedAt) + store.markScopeReachableLocked(workspace, canonical, event.Scope, event.ObservedAt) case connector.WatchError: - store.markScopeUnreachableLocked(canonical, event.Scope, event.Err) + store.markScopeUnreachableLocked(workspace, canonical, event.Scope, event.Err) } - store.warmed[canonical] = true - store.updatedAt = store.now().UTC() - store.notifyLocked() + store.warmed[workspaceKindKey{workspace: workspace, kind: canonical}] = true + store.updatedAt[workspace] = store.now().UTC() + store.notifyLocked(workspace) return nil } -func (store *Store) markScopeReachableLocked(kind, scope string, observedAt time.Time) { - current := store.scopes[scope] +func (store *Store) markScopeReachableLocked(workspace, kind, scope string, observedAt time.Time) { + scopeKey := scopeMetadataKey{workspace: workspace, scope: scope} + current := store.scopes[scopeKey] current.Name = scope current.Reachable = true if !observedAt.IsZero() { current.ObservedAt = observedAt } - store.scopes[scope] = current - coverage := store.coverage[kind] - coverage.Requested = max(coverage.Requested, len(store.scopes)) + store.scopes[scopeKey] = current + key := workspaceKindKey{workspace: workspace, kind: kind} + coverage := store.coverage[key] + coverage.Requested = max(coverage.Requested, store.workspaceScopeCountLocked(workspace)) coverage.Unreachable = removeString(coverage.Unreachable, scope) coverage.Stale = removeString(coverage.Stale, scope) coverage.Reachable = max(coverage.Requested-len(coverage.Unreachable), 0) - store.coverage[kind] = coverage - if store.allCoverageCompleteLocked() { - store.lastError = "" + store.coverage[key] = coverage + if len(coverage.Unreachable) == 0 { + delete(store.watchErrors, key) } } -func (store *Store) markScopeUnreachableLocked(kind, scope string, watchErr error) { - coverage := store.coverage[kind] - coverage.Requested = max(coverage.Requested, len(store.scopes)) +func (store *Store) markScopeUnreachableLocked(workspace, kind, scope string, watchErr error) { + key := workspaceKindKey{workspace: workspace, kind: kind} + coverage := store.coverage[key] + coverage.Requested = max(coverage.Requested, store.workspaceScopeCountLocked(workspace)) coverage.Unreachable = appendUniqueSorted(coverage.Unreachable, scope) coverage.Stale = appendUniqueSorted(coverage.Stale, scope) coverage.Reachable = max(coverage.Requested-len(coverage.Unreachable), 0) - store.coverage[kind] = coverage - store.lastError = fmt.Sprintf("watch %s in %s: %v", kind, scope, watchErr) + store.coverage[key] = coverage + store.watchErrors[key] = fmt.Sprintf("watch %s in %s: %v", kind, scope, watchErr) } -func (store *Store) allCoverageCompleteLocked() bool { - for _, coverage := range store.coverage { - if len(coverage.Unreachable) > 0 { - return false - } +// EndSync marks one workspace's reconciliation complete and retains prior data on failure. +func (store *Store) EndSync(workspace string, err error) error { + if validationErr := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)); validationErr != nil { + return fmt.Errorf("end cache sync: %w", validationErr) } - return true -} - -// EndSync marks reconciliation complete and retains any prior data on failure. -func (store *Store) EndSync(err error) { store.mu.Lock() defer store.mu.Unlock() - store.syncing = false + store.syncing[workspace] = false if err != nil { - store.lastError = err.Error() + store.syncErrors[workspace] = err.Error() } else { - store.lastError = "" + delete(store.syncErrors, workspace) } - store.notifyLocked() + store.notifyLocked(workspace) + return nil } -// SetPaused freezes or resumes background mutations while keeping snapshots available. -func (store *Store) SetPaused(paused bool) { +// SetPaused freezes or resumes one workspace's background mutations while keeping snapshots available. +func (store *Store) SetPaused(workspace string, paused bool) error { + if err := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)); err != nil { + return fmt.Errorf("set cache pause: %w", err) + } store.mu.Lock() defer store.mu.Unlock() - if store.paused == paused { - return + if store.paused[workspace] == paused { + return nil } - store.paused = paused - store.notifyLocked() + store.paused[workspace] = paused + store.notifyLocked(workspace) + return nil } -// Paused reports whether background reconciliation is frozen. -func (store *Store) Paused() bool { +// Paused reports whether one workspace's background reconciliation is frozen. +func (store *Store) Paused(workspace string) bool { + if tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)) != nil { + return false + } store.mu.RLock() defer store.mu.RUnlock() - return store.paused + return store.paused[workspace] } // Query returns a deterministic workspace-scoped answer without connector or network access. @@ -344,7 +409,7 @@ func (store *Store) Query(workspace string, query Query) Snapshot { } now := store.now().UTC() records := make([]Record, 0) - selectedKind := store.resolveKindLocked(query.Kind) + selectedKind := store.resolveKindLocked(workspace, query.Kind) matchQuery := query if selectedKind != "" { matchQuery.Kind = "" @@ -384,32 +449,39 @@ func (store *Store) Query(workspace string, query Query) Snapshot { } } } - pending := selectedKind != "" && !store.warmed[selectedKind] + pending := selectedKind != "" && !store.warmed[workspaceKindKey{workspace: workspace, kind: selectedKind}] + lastError := store.lastErrorLocked(workspace, selectedKind) return Snapshot{ - Version: store.version, - State: store.stateLocked(coverage, store.recordCountLocked(workspace), pending), - Syncing: store.syncing, - Paused: store.paused, + Version: store.versions[workspace], + State: store.stateLocked(workspace, coverage, store.recordCountLocked(workspace), pending, lastError), + Syncing: store.syncing[workspace], + Paused: store.paused[workspace], Records: records, Coverage: coverage, - UpdatedAt: store.updatedAt, - LastError: store.lastError, + UpdatedAt: store.updatedAt[workspace], + LastError: lastError, Scopes: store.scopesLocked(workspace, query.Scopes), Diagnostics: cloneDiagnostics(store.diagnostics[workspace]), } } -// WaitForChange blocks until the store advances beyond a known version or the context ends. -func (store *Store) WaitForChange(ctx context.Context, after uint64) (uint64, error) { +// WaitForChange blocks until one workspace advances beyond a known version or the context ends. +func (store *Store) WaitForChange(ctx context.Context, workspace string, after uint64) (uint64, error) { + if err := tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)); err != nil { + return 0, fmt.Errorf("wait for cache change: %w", err) + } for { - store.mu.RLock() - if store.version > after { - version := store.version - store.mu.RUnlock() + store.mu.Lock() + if store.versions[workspace] > after { + version := store.versions[workspace] + store.mu.Unlock() return version, nil } - changed := store.changed - store.mu.RUnlock() + if store.changed[workspace] == nil { + store.changed[workspace] = make(chan struct{}) + } + changed := store.changed[workspace] + store.mu.Unlock() select { case <-ctx.Done(): return 0, ctx.Err() @@ -422,29 +494,40 @@ func (store *Store) coverageLocked(workspace string, query Query, records []Reco targets := store.targetScopesLocked(workspace, query.Scopes) unreachable := make(map[string]struct{}) stale := make(map[string]struct{}) - kind := store.resolveKindLocked(query.Kind) + truncated := make(map[string]struct{}) + kind := store.resolveKindLocked(workspace, query.Kind) if kind != "" { - if !store.warmed[kind] { + key := workspaceKindKey{workspace: workspace, kind: kind} + if !store.warmed[key] { return fleet.Coverage{Requested: len(targets)} } - for _, name := range store.coverage[kind].Unreachable { + for _, name := range store.coverage[key].Unreachable { unreachable[name] = struct{}{} } - for _, name := range store.coverage[kind].Stale { + for _, name := range store.coverage[key].Stale { stale[name] = struct{}{} } + for _, name := range store.coverage[key].Truncated { + truncated[name] = struct{}{} + } } else { - for _, coverage := range store.coverage { + for key, coverage := range store.coverage { + if key.workspace != workspace { + continue + } for _, name := range coverage.Unreachable { unreachable[name] = struct{}{} } for _, name := range coverage.Stale { stale[name] = struct{}{} } + for _, name := range coverage.Truncated { + truncated[name] = struct{}{} + } } } for _, name := range targets { - scope, exists := store.scopes[name] + scope, exists := store.scopes[scopeMetadataKey{workspace: workspace, scope: name}] if !exists || !store.scopeWorkspaces[name][workspace] || !scope.Reachable { unreachable[name] = struct{}{} } @@ -467,16 +550,19 @@ func (store *Store) coverageLocked(workspace string, query Query, records []Reco if _, aged := stale[name]; aged { coverage.Stale = append(coverage.Stale, name) } + if _, partial := truncated[name]; partial { + coverage.Truncated = append(coverage.Truncated, name) + } } return coverage } -func (store *Store) resolveKindLocked(kind string) string { +func (store *Store) resolveKindLocked(workspace, kind string) string { canonical := canonicalKind(kind) if canonical == "" { return "" } - if resolved := store.aliases[kindAlias(kind)]; resolved != "" { + if resolved := store.aliases[workspaceKindKey{workspace: workspace, kind: kindAlias(kind)}]; resolved != "" { return resolved } return canonical @@ -520,7 +606,7 @@ func (store *Store) scopesLocked(workspace string, patterns []string) []connecto names := store.targetScopesLocked(workspace, patterns) result := make([]connector.Scope, 0, len(names)) for _, name := range names { - scope, exists := store.scopes[name] + scope, exists := store.scopes[scopeMetadataKey{workspace: workspace, scope: name}] if !exists || !store.scopeWorkspaces[name][workspace] { scope = connector.Scope{Name: name} } @@ -529,29 +615,73 @@ func (store *Store) scopesLocked(workspace string, patterns []string) []connecto return result } -func (store *Store) stateLocked(coverage fleet.Coverage, recordCount int, pending bool) State { +func (store *Store) stateLocked(workspace string, coverage fleet.Coverage, recordCount int, pending bool, lastError string) State { + syncing := store.syncing[workspace] switch { - case store.paused: + case store.paused[workspace]: return StatePaused - case pending && store.syncing: + case pending && syncing: return StateWarming - case pending && store.lastError == "": + case pending && lastError == "": return StateCold - case recordCount == 0 && store.syncing: + case recordCount == 0 && syncing: return StateWarming - case recordCount == 0 && len(store.warmed) == 0: + case recordCount == 0 && !store.workspaceWarmedLocked(workspace): return StateCold - case coverage.Reachable == 0 && (recordCount > 0 || store.lastError != ""): + case coverage.Reachable == 0 && (recordCount > 0 || lastError != ""): return StateOffline - case len(coverage.Unreachable) > 0 || len(coverage.Stale) > 0 || store.lastError != "": + case len(coverage.Unreachable) > 0 || len(coverage.Stale) > 0 || len(coverage.Truncated) > 0 || lastError != "": return StateDegraded - case store.syncing: + case syncing: return StateWarming default: return StateWarm } } +func (store *Store) lastErrorLocked(workspace, kind string) string { + result := make([]string, 0, 1) + if syncError := store.syncErrors[workspace]; syncError != "" { + result = append(result, syncError) + } + if kind != "" { + if watchError := store.watchErrors[workspaceKindKey{workspace: workspace, kind: kind}]; watchError != "" { + result = append(result, watchError) + } + return strings.Join(result, "; ") + } + keys := make([]workspaceKindKey, 0) + for key, watchError := range store.watchErrors { + if key.workspace == workspace && watchError != "" { + keys = append(keys, key) + } + } + sort.Slice(keys, func(left, right int) bool { return keys[left].kind < keys[right].kind }) + for _, key := range keys { + result = append(result, store.watchErrors[key]) + } + return strings.Join(result, "; ") +} + +func (store *Store) workspaceWarmedLocked(workspace string) bool { + for key, warmed := range store.warmed { + if key.workspace == workspace && warmed { + return true + } + } + return false +} + +func (store *Store) workspaceScopeCountLocked(workspace string) int { + count := 0 + for _, memberships := range store.scopeWorkspaces { + if memberships[workspace] { + count++ + } + } + return count +} + func (store *Store) recordCountLocked(workspace string) int { count := 0 for _, records := range store.records { @@ -572,16 +702,22 @@ func (store *Store) markScopeWorkspaceLocked(workspace, scope string) { store.scopeWorkspaces[scope] = make(map[string]bool) } store.scopeWorkspaces[scope][workspace] = true + key := scopeMetadataKey{workspace: workspace, scope: scope} + if _, exists := store.scopes[key]; !exists { + store.scopes[key] = connector.Scope{Name: scope} + } } -func (store *Store) notifyLocked() { - store.version++ - close(store.changed) - store.changed = make(chan struct{}) +func (store *Store) notifyLocked(workspace string) { + store.versions[workspace]++ + if changed := store.changed[workspace]; changed != nil { + close(changed) + } + store.changed[workspace] = make(chan struct{}) } func recordKey(record Record) string { - return strings.Join([]string{record.Kind, record.Cluster, record.Namespace, record.Name}, "\x00") + return strings.Join([]string{record.Workspace, record.Kind, record.Cluster, record.Namespace, record.Name}, "\x00") } func stringSet(values []string) map[string]struct{} { @@ -633,6 +769,7 @@ func cloneRecord(record Record, includeEvidence bool) Record { func cloneCoverage(coverage fleet.Coverage) fleet.Coverage { coverage.Unreachable = append([]string(nil), coverage.Unreachable...) coverage.Stale = append([]string(nil), coverage.Stale...) + coverage.Truncated = append([]string(nil), coverage.Truncated...) return coverage } diff --git a/internal/fleetcache/store_test.go b/internal/fleetcache/store_test.go index 24dbc32..c56e522 100644 --- a/internal/fleetcache/store_test.go +++ b/internal/fleetcache/store_test.go @@ -20,7 +20,7 @@ func TestStorePreservesLastKnownRowsAndSurfacesCoverage(t *testing.T) { t.Parallel() now := time.Date(2026, time.July, 10, 20, 0, 0, 0, time.UTC) store := newStore(func() time.Time { return now }, 15*time.Second) - if !store.BeginSync() { + if !store.BeginSync(fleet.LocalWorkspace) { t.Fatal("BeginSync() = false, want initial sync") } store.SetDiscovery(fleet.LocalWorkspace, connector.Discovery{Scopes: []connector.Scope{ @@ -34,10 +34,10 @@ func TestStorePreservesLastKnownRowsAndSurfacesCoverage(t *testing.T) { }, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, } - if err := store.Replace("pods", initial); err != nil { + if err := store.Replace(fleet.LocalWorkspace, "pods", initial); err != nil { t.Fatalf("Replace(initial) error = %v", err) } - store.EndSync(nil) + store.EndSync(fleet.LocalWorkspace, nil) snapshot := store.Query(fleet.LocalWorkspace, Query{Kind: "Pod"}) if snapshot.State != StateWarm || len(snapshot.Records) != 2 || !snapshot.Coverage.Complete() { @@ -45,7 +45,7 @@ func TestStorePreservesLastKnownRowsAndSurfacesCoverage(t *testing.T) { } now = now.Add(20 * time.Second) - if !store.BeginSync() { + if !store.BeginSync(fleet.LocalWorkspace) { t.Fatal("BeginSync(second) = false") } store.SetDiscovery(fleet.LocalWorkspace, connector.Discovery{ @@ -59,10 +59,10 @@ func TestStorePreservesLastKnownRowsAndSurfacesCoverage(t *testing.T) { Facts: []fleet.Fact{podFact(t, "alpha", "api-1", "Running", "registry/api:v2", now)}, Coverage: fleet.Coverage{Requested: 2, Reachable: 1, Unreachable: []string{"beta"}}, } - if err := store.Replace("Pod", partial); err != nil { + if err := store.Replace(fleet.LocalWorkspace, "Pod", partial); err != nil { t.Fatalf("Replace(partial) error = %v", err) } - store.EndSync(nil) + store.EndSync(fleet.LocalWorkspace, nil) snapshot = store.Query(fleet.LocalWorkspace, Query{Kind: "pods"}) if snapshot.State != StateDegraded || len(snapshot.Records) != 2 { @@ -76,6 +76,50 @@ func TestStorePreservesLastKnownRowsAndSurfacesCoverage(t *testing.T) { } } +func TestStorePreservesRowsOutsideTruncatedPrefix(t *testing.T) { + t.Parallel() + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + store.SetDiscovery(fleet.LocalWorkspace, connector.Discovery{Scopes: []connector.Scope{{Name: "alpha", Reachable: true, ObservedAt: now}}}) + if err := store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{ + podFact(t, "alpha", "api-0", "Running", "registry/api:v1", now), + podFact(t, "alpha", "worker-0", "Running", "registry/worker:v1", now), + }, + Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatalf("Replace(initial) error = %v", err) + } + now = now.Add(time.Second) + if err := store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{podFact(t, "alpha", "api-0", "Running", "registry/api:v2", now)}, + Coverage: fleet.Coverage{Requested: 1, Reachable: 1, Truncated: []string{"alpha"}}, + }); err != nil { + t.Fatalf("Replace(truncated) error = %v", err) + } + + snapshot := store.Query(fleet.LocalWorkspace, Query{Kind: "Pod"}) + if snapshot.State != StateDegraded || len(snapshot.Records) != 2 || snapshot.Records[0].Name != "api-0" || snapshot.Records[1].Name != "worker-0" || + !slices.Equal(snapshot.Coverage.Truncated, []string{"alpha"}) || snapshot.Coverage.Complete() { + t.Fatalf("truncated snapshot = %#v, want prefix update plus preserved last-known row", snapshot) + } + + now = now.Add(time.Second) + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchSnapshot, Workspace: fleet.LocalWorkspace, Scope: "alpha", Kind: "Pod", ObservedAt: now, + Facts: []fleet.Fact{ + podFact(t, "alpha", "api-0", "Running", "registry/api:v3", now), + podFact(t, "alpha", "worker-0", "Running", "registry/worker:v2", now), + }, + }); err != nil { + t.Fatalf("ApplyWatchEvent(snapshot) error = %v", err) + } + snapshot = store.Query(fleet.LocalWorkspace, Query{Kind: "Pod"}) + if snapshot.State != StateWarm || len(snapshot.Records) != 2 || len(snapshot.Coverage.Truncated) != 0 || !snapshot.Coverage.Complete() { + t.Fatalf("complete watch snapshot = %#v, want truncation cleared and warm coverage", snapshot) + } +} + func TestStoreSearchGrammarRunsOnlyOnNormalizedCache(t *testing.T) { t.Parallel() now := time.Date(2026, time.July, 10, 20, 0, 0, 0, time.UTC) @@ -91,7 +135,7 @@ func TestStoreSearchGrammarRunsOnlyOnNormalizedCache(t *testing.T) { }, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, } - if err := store.Replace("Pod", result); err != nil { + if err := store.Replace(fleet.LocalWorkspace, "Pod", result); err != nil { t.Fatalf("Replace() error = %v", err) } @@ -115,10 +159,12 @@ func TestStoreQueryRequiresAndEnforcesWorkspace(t *testing.T) { local := podFact(t, "alpha", "local-api", "Running", "registry/api:v1", now) other := podFact(t, "beta", "other-api", "Running", "registry/api:v1", now) other.Workspace = "other" - if err := store.Replace("Pod", fleet.QueryResult{ - Facts: []fleet.Fact{local, other}, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, - }); err != nil { - t.Fatal(err) + for workspace, fact := range map[string]fleet.Fact{fleet.LocalWorkspace: local, "other": other} { + if err := store.Replace(workspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } } if records := store.Query("", Query{Kind: "Pod"}).Records; len(records) != 0 { t.Fatalf("unscoped records = %#v", records) @@ -148,7 +194,7 @@ func TestStoreSearchesCanonicalCVEFactsByImageAndIdentifier(t *testing.T) { Ref: fleet.ResourceRef{SourceKind: "scanner", Scope: "alpha", Kind: "Image", Name: "payments-v4"}, Kind: fleet.FactCVE, Observed: payload, ObservedAt: now, Source: "scanner", }, Workspace: fleet.LocalWorkspace} - if err := store.Replace("CVE", fleet.QueryResult{ + if err := store.Replace(fleet.LocalWorkspace, "CVE", fleet.QueryResult{ Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, }); err != nil { t.Fatal(err) @@ -173,7 +219,7 @@ func TestStoreMapsGenericResourceAliasToAdvertisedKind(t *testing.T) { "kind": "ConfigMap", "metadata": map[string]any{"name": "settings", "namespace": "apps"}, }, now) - if err := store.Replace("configmaps", fleet.QueryResult{ + if err := store.Replace(fleet.LocalWorkspace, "configmaps", fleet.QueryResult{ Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, }); err != nil { t.Fatalf("Replace() error = %v", err) @@ -196,7 +242,7 @@ func TestStoreAppliesWatchDeltasAndPreservesFailedScopeRows(t *testing.T) { }}) alpha := podFact(t, "alpha", "api-0", "Running", "image:v1", now) beta := podFact(t, "beta", "api-0", "Running", "image:v1", now) - if err := store.Replace("Pod", fleet.QueryResult{ + if err := store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{ Facts: []fleet.Fact{alpha, beta}, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, }); err != nil { t.Fatalf("Replace() error = %v", err) @@ -205,12 +251,14 @@ func TestStoreAppliesWatchDeltasAndPreservesFailedScopeRows(t *testing.T) { now = now.Add(time.Second) updated := podFact(t, "alpha", "api-1", "Running", "image:v2", now) if err := store.ApplyWatchEvent(connector.WatchEvent{ - Type: connector.WatchSnapshot, Kind: "pods", Scope: "alpha", Facts: []fleet.Fact{updated}, ObservedAt: now, + Type: connector.WatchSnapshot, Workspace: fleet.LocalWorkspace, + Kind: "pods", Scope: "alpha", Facts: []fleet.Fact{updated}, ObservedAt: now, }); err != nil { t.Fatalf("ApplyWatchEvent(snapshot) error = %v", err) } if err := store.ApplyWatchEvent(connector.WatchEvent{ - Type: connector.WatchError, Kind: "Pod", Scope: "beta", Err: errors.New("watch disconnected"), + Type: connector.WatchError, Workspace: fleet.LocalWorkspace, + Kind: "Pod", Scope: "beta", Err: errors.New("watch disconnected"), }); err != nil { t.Fatalf("ApplyWatchEvent(error) error = %v", err) } @@ -223,7 +271,7 @@ func TestStoreAppliesWatchDeltasAndPreservesFailedScopeRows(t *testing.T) { } if err := store.ApplyWatchEvent(connector.WatchEvent{ - Type: connector.WatchDelete, Kind: "pods", Scope: "alpha", + Type: connector.WatchDelete, Workspace: fleet.LocalWorkspace, Kind: "pods", Scope: "alpha", Ref: fleet.ResourceRef{Scope: "alpha", Kind: "Pod", Namespace: "apps", Name: "api-1"}, ObservedAt: now, }); err != nil { t.Fatalf("ApplyWatchEvent(delete) error = %v", err) @@ -236,9 +284,10 @@ func TestStoreAppliesWatchDeltasAndPreservesFailedScopeRows(t *testing.T) { func TestStoreRejectsCrossScopeWatchFacts(t *testing.T) { t.Parallel() store := New() + store.SetDiscovery(fleet.LocalWorkspace, connector.Discovery{Scopes: []connector.Scope{{Name: "alpha", Reachable: true}}}) wrongScope := podFact(t, "beta", "api-0", "Running", "image:v1", time.Now().UTC()) err := store.ApplyWatchEvent(connector.WatchEvent{ - Type: connector.WatchUpsert, Kind: "Pod", Scope: "alpha", Fact: wrongScope, + Type: connector.WatchUpsert, Workspace: fleet.LocalWorkspace, Kind: "Pod", Scope: "alpha", Fact: wrongScope, }) if err == nil || !strings.Contains(err.Error(), "does not match stream scope") { t.Fatalf("ApplyWatchEvent() error = %v, want scope rejection", err) @@ -252,12 +301,12 @@ func TestStorePauseAndChangeNotification(t *testing.T) { t.Parallel() store := New() initial := store.Query(fleet.LocalWorkspace, Query{}).Version - store.SetPaused(true) - version, err := store.WaitForChange(context.Background(), initial) + store.SetPaused(fleet.LocalWorkspace, true) + version, err := store.WaitForChange(context.Background(), fleet.LocalWorkspace, initial) if err != nil || version <= initial { t.Fatalf("WaitForChange() = %d, %v", version, err) } - if store.BeginSync() { + if store.BeginSync(fleet.LocalWorkspace) { t.Fatal("BeginSync() while paused = true") } if state := store.Query(fleet.LocalWorkspace, Query{}).State; state != StatePaused { @@ -266,7 +315,7 @@ func TestStorePauseAndChangeNotification(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := store.WaitForChange(ctx, version); err == nil { + if _, err := store.WaitForChange(ctx, fleet.LocalWorkspace, version); err == nil { t.Fatal("WaitForChange(canceled) error = nil") } } @@ -281,7 +330,7 @@ func TestStoreConcurrentSnapshotsAreImmutable(t *testing.T) { waitGroup.Add(2) go func(index int) { defer waitGroup.Done() - _ = store.Replace("Pod", fleet.QueryResult{ + _ = store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{ Facts: []fleet.Fact{podFact(t, "alpha", "pod-"+string(rune('a'+index)), "Running", "image:v1", now)}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, }) @@ -309,7 +358,7 @@ func TestReplaceRejectsInvalidEvidenceAtomically(t *testing.T) { Ref: fleet.ResourceRef{Scope: "alpha", Kind: "Pod", Name: "bad"}, Observed: json.RawMessage(`{"broken"`), }}}} - if err := store.Replace("Pod", result); err == nil { + if err := store.Replace(fleet.LocalWorkspace, "Pod", result); err == nil { t.Fatal("Replace(invalid) error = nil") } if snapshot := store.Query(fleet.LocalWorkspace, Query{}); len(snapshot.Records) != 0 { @@ -398,14 +447,14 @@ func TestStoreReportsOfflineLastKnownAndPendingLens(t *testing.T) { if pending := store.Query(fleet.LocalWorkspace, Query{Kind: "Node"}); pending.Coverage.Reachable != 0 || pending.State != StateCold { t.Fatalf("pending snapshot = %#v, want zero covered cold lens", pending) } - if err := store.Replace("Pod", fleet.QueryResult{ + if err := store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{ Facts: []fleet.Fact{podFact(t, "alpha", "api-0", "Running", "image:v1", now)}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, }); err != nil { t.Fatalf("Replace() error = %v", err) } store.SetDiscovery(fleet.LocalWorkspace, connector.Discovery{Scopes: []connector.Scope{{Name: "alpha", ObservedAt: now}}, Unreachable: []string{"alpha"}}) - store.EndSync(context.DeadlineExceeded) + store.EndSync(fleet.LocalWorkspace, context.DeadlineExceeded) offline := store.Query(fleet.LocalWorkspace, Query{Kind: "Pod", Text: []string{"no-match"}}) if offline.State != StateOffline || len(offline.Records) != 0 { t.Fatalf("offline snapshot = %#v, want zero matches over retained offline data", offline) diff --git a/internal/fleetcache/workspace_mutation_test.go b/internal/fleetcache/workspace_mutation_test.go new file mode 100644 index 0000000..3d4d240 --- /dev/null +++ b/internal/fleetcache/workspace_mutation_test.go @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: Apache-2.0 + +package fleetcache + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/connector" + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestStoreRejectsWorkspaceMismatchedMutationsAtomically(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 18, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + if err := store.SetDiscovery("workspace-a", connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + original := podFact(t, "shared", "api-0", "Running", "registry/api:original", now) + original.Workspace = "workspace-a" + if err := store.Replace("workspace-a", "Pod", fleet.QueryResult{Facts: []fleet.Fact{original}}); err != nil { + t.Fatal(err) + } + + foreign := podFact(t, "shared", "api-0", "Running", "registry/api:foreign", now) + foreign.Workspace = "workspace-b" + validUpdate := podFact(t, "shared", "api-0", "Running", "registry/api:valid-update", now) + validUpdate.Workspace = "workspace-a" + if err := store.Replace("workspace-a", "Pod", fleet.QueryResult{Facts: []fleet.Fact{foreign}}); err == nil || + !strings.Contains(err.Error(), "does not match mutation workspace") { + t.Fatalf("Replace(foreign fact) error = %v, want workspace mismatch", err) + } + if err := store.Replace("workspace-a", "Pod", fleet.QueryResult{Facts: []fleet.Fact{validUpdate, foreign}}); err == nil || + !strings.Contains(err.Error(), "does not match mutation workspace") { + t.Fatalf("Replace(mixed facts) error = %v, want workspace mismatch", err) + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchUpsert, Workspace: "workspace-a", Kind: "Pod", Scope: "shared", Fact: foreign, + }); err == nil || !strings.Contains(err.Error(), "does not match stream workspace") { + t.Fatalf("ApplyWatchEvent(foreign fact) error = %v, want workspace mismatch", err) + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchSnapshot, Workspace: "workspace-a", Kind: "Pod", Scope: "shared", + Facts: []fleet.Fact{validUpdate, foreign}, + }); err == nil || !strings.Contains(err.Error(), "does not match stream workspace") { + t.Fatalf("ApplyWatchEvent(mixed snapshot) error = %v, want workspace mismatch", err) + } + + records := store.Query("workspace-a", Query{Kind: "Pod"}).Records + if len(records) != 1 || len(records[0].Images) != 1 || records[0].Images[0] != "registry/api:original" { + t.Fatalf("records after rejected mutations = %#v, want original row unchanged", records) + } + if records := store.Query("workspace-b", Query{Kind: "Pod"}).Records; len(records) != 0 { + t.Fatalf("workspace B records = %#v, want no leaked mutation", records) + } +} + +func TestStoreWatchSnapshotAndDeleteAreWorkspaceIsolated(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 18, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + for _, workspace := range []string{"workspace-a", "workspace-b"} { + if err := store.SetDiscovery(workspace, connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + fact := podFact(t, "shared", "api-0", "Running", "registry/api:"+workspace, now) + fact.Workspace = workspace + if err := store.Replace(workspace, "Pod", fleet.QueryResult{Facts: []fleet.Fact{fact}}); err != nil { + t.Fatal(err) + } + } + updatedA := podFact(t, "shared", "api-0", "Running", "registry/api:workspace-a-updated", now) + updatedA.Workspace = "workspace-a" + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchUpsert, Workspace: "workspace-a", Kind: "Pod", Scope: "shared", Fact: updatedA, + }); err != nil { + t.Fatal(err) + } + recordsB := store.Query("workspace-b", Query{Kind: "Pod"}).Records + if len(recordsB) != 1 || recordsB[0].Images[0] != "registry/api:workspace-b" { + t.Fatalf("workspace B records after workspace A upsert = %#v, want untouched original", recordsB) + } + + replacement := podFact(t, "shared", "api-1", "Running", "registry/api:replacement", now) + replacement.Workspace = "workspace-a" + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchSnapshot, Workspace: "workspace-a", Kind: "Pod", Scope: "shared", + Facts: []fleet.Fact{replacement}, ObservedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchDelete, Workspace: "workspace-a", Kind: "Pod", Scope: "shared", + Ref: fleet.ResourceRef{Scope: "shared", Namespace: "apps", Name: "api-1"}, ObservedAt: now, + }); err != nil { + t.Fatal(err) + } + + if records := store.Query("workspace-a", Query{Kind: "Pod"}).Records; len(records) != 0 { + t.Fatalf("workspace A records after delete = %#v, want empty", records) + } + records := store.Query("workspace-b", Query{Kind: "Pod"}).Records + if len(records) != 1 || records[0].Name != "api-0" || records[0].Images[0] != "registry/api:workspace-b" { + t.Fatalf("workspace B records = %#v, want untouched original", records) + } +} + +func TestStoreSyncLifecycleIsWorkspaceIsolated(t *testing.T) { + t.Parallel() + + store := New() + if !store.BeginSync("workspace-a", "Pod") { + t.Fatal("BeginSync(workspace-a) = false") + } + if snapshot := store.Query("workspace-a", Query{Kind: "Pod"}); !snapshot.Syncing || snapshot.State != StateWarming { + t.Fatalf("workspace A snapshot = %#v, want warming", snapshot) + } + if snapshot := store.Query("workspace-b", Query{Kind: "Pod"}); snapshot.Syncing || snapshot.State != StateCold || snapshot.Version != 0 { + t.Fatalf("workspace B snapshot = %#v, want independent cold state", snapshot) + } + if err := store.EndSync("workspace-a", errors.New("sync failed")); err != nil { + t.Fatal(err) + } + if snapshot := store.Query("workspace-b", Query{Kind: "Pod"}); snapshot.LastError != "" || snapshot.State != StateCold { + t.Fatalf("workspace B snapshot = %#v, want no foreign sync error", snapshot) + } + if err := store.SetPaused("workspace-a", true); err != nil { + t.Fatal(err) + } + if snapshot := store.Query("workspace-a", Query{}); !snapshot.Paused || snapshot.State != StatePaused { + t.Fatalf("workspace A snapshot = %#v, want paused", snapshot) + } + if snapshot := store.Query("workspace-b", Query{}); snapshot.Paused || snapshot.State == StatePaused { + t.Fatalf("workspace B snapshot = %#v, want pause isolation", snapshot) + } +} + +func TestStoreRejectsMissingWorkspaceMutationBoundaries(t *testing.T) { + t.Parallel() + + store := New() + if store.BeginSync("", "Pod") { + t.Fatal("BeginSync(empty workspace) = true") + } + if err := store.SetDiscovery("", connector.Discovery{}); err == nil { + t.Fatal("SetDiscovery(empty workspace) error = nil") + } + if err := store.Replace("", "Pod", fleet.QueryResult{}); err == nil { + t.Fatal("Replace(empty workspace) error = nil") + } + if err := store.ApplyWatchEvent(connector.WatchEvent{Type: connector.WatchError, Kind: "Pod", Scope: "shared", Err: errors.New("failed")}); err == nil { + t.Fatal("ApplyWatchEvent(empty workspace) error = nil") + } + if err := store.EndSync("", nil); err == nil { + t.Fatal("EndSync(empty workspace) error = nil") + } + if err := store.SetPaused("", true); err == nil { + t.Fatal("SetPaused(empty workspace) error = nil") + } + if _, err := store.WaitForChange(t.Context(), "", 0); err == nil { + t.Fatal("WaitForChange(empty workspace) error = nil") + } +} + +func TestStorePreservesIdenticalRecordsAcrossWorkspaces(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 18, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + for _, workspace := range []string{"workspace-a", "workspace-b"} { + if err := store.SetDiscovery(workspace, connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + } + factA := podFact(t, "shared", "api-0", "Running", "registry/api:a", now) + factA.Workspace = "workspace-a" + factB := podFact(t, "shared", "api-0", "Running", "registry/api:b", now) + factB.Workspace = "workspace-b" + for workspace, fact := range map[string]fleet.Fact{"workspace-a": factA, "workspace-b": factB} { + if err := store.Replace(workspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } + } + + for workspace, wantImage := range map[string]string{ + "workspace-a": "registry/api:a", + "workspace-b": "registry/api:b", + } { + records := store.Query(workspace, Query{Kind: "Pod"}).Records + if len(records) != 1 || records[0].Workspace != workspace || len(records[0].Images) != 1 || records[0].Images[0] != wantImage { + t.Fatalf("%s records = %#v, want independent image %q", workspace, records, wantImage) + } + } +} + +func TestStoreWatchFailureDoesNotDegradeAnotherWorkspace(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 18, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + if err := store.SetDiscovery("workspace-a", connector.Discovery{Scopes: []connector.Scope{{ + Name: "cluster-a", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + if err := store.SetDiscovery("workspace-b", connector.Discovery{Scopes: []connector.Scope{{ + Name: "cluster-b", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + factA := podFact(t, "cluster-a", "api-a", "Running", "registry/api:a", now) + factA.Workspace = "workspace-a" + factB := podFact(t, "cluster-b", "api-b", "Running", "registry/api:b", now) + factB.Workspace = "workspace-b" + for workspace, fact := range map[string]fleet.Fact{"workspace-a": factA, "workspace-b": factB} { + if err := store.Replace(workspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } + } + deployment := objectFact(t, "Deployment", map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": "payments", "namespace": "apps"}, + "spec": map[string]any{"replicas": 1}, + "status": map[string]any{"availableReplicas": 1, "updatedReplicas": 1}, + }, now) + deployment.Workspace = "workspace-a" + deployment.Ref.Scope = "cluster-a" + deployment.Ref.Namespace = "apps" + deployment.Ref.Name = "payments" + if err := store.Replace("workspace-a", "Deployment", fleet.QueryResult{ + Facts: []fleet.Fact{deployment}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchError, Workspace: "workspace-a", + Kind: "Pod", Scope: "cluster-a", Err: errors.New("watch unavailable"), + }); err != nil { + t.Fatal(err) + } + + snapshotA := store.Query("workspace-a", Query{Kind: "Pod"}) + if snapshotA.LastError == "" || snapshotA.State != StateOffline || len(snapshotA.Scopes) != 1 || !snapshotA.Scopes[0].Reachable || + len(snapshotA.Records) != 1 || snapshotA.Records[0].Name != "api-a" || !snapshotA.Records[0].Stale { + t.Fatalf("workspace A snapshot = %#v, want isolated watch failure", snapshotA) + } + snapshotB := store.Query("workspace-b", Query{Kind: "Pod"}) + if snapshotB.LastError != "" || snapshotB.State != StateWarm || !snapshotB.Coverage.Complete() || + len(snapshotB.Scopes) != 1 || !snapshotB.Scopes[0].Reachable { + t.Fatalf("workspace B snapshot = %#v, want unaffected warm state", snapshotB) + } + deploymentSnapshot := store.Query("workspace-a", Query{Kind: "Deployment"}) + if deploymentSnapshot.State != StateWarm || !deploymentSnapshot.Coverage.Complete() || + len(deploymentSnapshot.Records) != 1 || deploymentSnapshot.Records[0].Name != "payments" { + t.Fatalf("workspace A deployment snapshot = %#v, want healthy kind preserved", deploymentSnapshot) + } +} + +func TestStoreKindAliasesAreWorkspaceIsolated(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 18, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + for workspace, kind := range map[string]string{"workspace-a": "Widget", "workspace-b": "WIDGET"} { + if err := store.SetDiscovery(workspace, connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + fact := objectFact(t, kind, map[string]any{ + "apiVersion": "example.sith.dev/v1", + "kind": kind, + "metadata": map[string]any{"name": "object"}, + }, now) + fact.Workspace = workspace + fact.Ref.Scope = "shared" + if err := store.Replace(workspace, kind, fleet.QueryResult{ + Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } + } + + for workspace, wantKind := range map[string]string{"workspace-a": "Widget", "workspace-b": "WIDGET"} { + records := store.Query(workspace, Query{Kind: "widget"}).Records + if len(records) != 1 || records[0].Workspace != workspace || records[0].Kind != wantKind { + t.Fatalf("%s alias records = %#v, want kind %q", workspace, records, wantKind) + } + } +} + +func TestStoreChangeNotificationsAreWorkspaceIsolated(t *testing.T) { + t.Parallel() + + store := New() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + waitDone := make(chan error, 1) + go func() { + _, err := store.WaitForChange(ctx, "workspace-b", 0) + waitDone <- err + }() + + deadline := time.Now().Add(time.Second) + for { + store.mu.RLock() + registered := store.changed["workspace-b"] != nil + store.mu.RUnlock() + if registered { + break + } + if time.Now().After(deadline) { + t.Fatal("workspace B change waiter did not register") + } + time.Sleep(time.Millisecond) + } + + if err := store.SetDiscovery("workspace-a", connector.Discovery{}); err != nil { + t.Fatal(err) + } + if version := store.Query("workspace-a", Query{}).Version; version != 1 { + t.Fatalf("workspace A version = %d, want 1", version) + } + if version := store.Query("workspace-b", Query{}).Version; version != 0 { + t.Fatalf("workspace B version = %d, want 0", version) + } + select { + case err := <-waitDone: + t.Fatalf("workspace B waiter returned after workspace A mutation: %v", err) + default: + } + cancel() + select { + case err := <-waitDone: + if !errors.Is(err, context.Canceled) { + t.Fatalf("workspace B waiter error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + t.Fatal("workspace B waiter did not return after cancellation") + } +} + +func FuzzStoreForeignWorkspaceMutationCannotChangeEitherWorkspace(f *testing.F) { + for _, workspace := range []string{"workspace-b", "", " workspace-b", "workspace-b\x00suffix", strings.Repeat("x", 257)} { + f.Add(workspace) + } + f.Fuzz(func(t *testing.T, factWorkspace string) { + if factWorkspace == "workspace-a" { + factWorkspace = "workspace-b" + } + now := time.Date(2026, time.July, 16, 18, 0, 0, 0, time.UTC) + store := newStore(func() time.Time { return now }, time.Minute) + for _, workspace := range []string{"workspace-a", "workspace-b"} { + if err := store.SetDiscovery(workspace, connector.Discovery{Scopes: []connector.Scope{{ + Name: "shared", Reachable: true, ObservedAt: now, + }}}); err != nil { + t.Fatal(err) + } + fact := podFact(t, "shared", "api-0", "Running", "registry/api:"+workspace, now) + fact.Workspace = workspace + if err := store.Replace(workspace, "Pod", fleet.QueryResult{Facts: []fleet.Fact{fact}}); err != nil { + t.Fatal(err) + } + } + + foreign := podFact(t, "shared", "api-0", "Running", "registry/api:foreign", now) + foreign.Workspace = factWorkspace + if err := store.Replace("workspace-a", "Pod", fleet.QueryResult{Facts: []fleet.Fact{foreign}}); err == nil { + t.Fatal("Replace(foreign workspace) error = nil") + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchUpsert, Workspace: "workspace-a", Kind: "Pod", Scope: "shared", Fact: foreign, + }); err == nil { + t.Fatal("ApplyWatchEvent(foreign workspace) error = nil") + } + + for _, workspace := range []string{"workspace-a", "workspace-b"} { + records := store.Query(workspace, Query{Kind: "Pod"}).Records + wantImage := "registry/api:" + workspace + if len(records) != 1 || len(records[0].Images) != 1 || records[0].Images[0] != wantImage { + t.Fatalf("%s records after rejected foreign mutation = %#v, want %q", workspace, records, wantImage) + } + } + }) +} diff --git a/internal/fleetrender/table.go b/internal/fleetrender/table.go index ed6b931..5112f04 100644 --- a/internal/fleetrender/table.go +++ b/internal/fleetrender/table.go @@ -114,6 +114,11 @@ func CoverageLine(coverage fleet.Coverage) string { } else { parts = append(parts, fmt.Sprintf("%d unreachable (%s)", len(coverage.Unreachable), strings.Join(coverage.Unreachable, ", "))) } + if len(coverage.Truncated) == 0 { + parts = append(parts, "0 truncated") + } else { + parts = append(parts, fmt.Sprintf("%d truncated (%s)", len(coverage.Truncated), strings.Join(coverage.Truncated, ", "))) + } return strings.Join(parts, " · ") } diff --git a/internal/fleetrender/table_test.go b/internal/fleetrender/table_test.go index ef4614c..2fc4439 100644 --- a/internal/fleetrender/table_test.go +++ b/internal/fleetrender/table_test.go @@ -76,14 +76,16 @@ func TestWriteTextAlwaysIncludesCoverage(t *testing.T) { Columns: []string{"CLUSTER", "NAME"}, Rows: [][]string{{"alpha", "api"}}, Coverage: fleet.Coverage{ - Requested: 3, Reachable: 2, Stale: []string{"beta"}, Unreachable: []string{"gamma"}, + Requested: 3, Reachable: 2, Stale: []string{"beta"}, Unreachable: []string{"gamma"}, Truncated: []string{"alpha"}, }, } var output bytes.Buffer if err := WriteText(&output, table); err != nil { t.Fatalf("WriteText() error = %v", err) } - for _, want := range []string{"CLUSTER", "alpha", "covered 2/3 clusters", "1 stale (beta)", "1 unreachable (gamma)"} { + for _, want := range []string{ + "CLUSTER", "alpha", "covered 2/3 clusters", "1 stale (beta)", "1 unreachable (gamma)", "1 truncated (alpha)", + } { if !strings.Contains(output.String(), want) { t.Errorf("output = %q, want %q", output.String(), want) } diff --git a/internal/hubauth/oidc.go b/internal/hubauth/oidc.go index da3fa8c..ab498a3 100644 --- a/internal/hubauth/oidc.go +++ b/internal/hubauth/oidc.go @@ -6,6 +6,7 @@ import ( "context" "crypto/ed25519" "crypto/rsa" + "crypto/subtle" "crypto/tls" "crypto/x509" "encoding/base64" @@ -21,6 +22,7 @@ import ( "strings" "sync" "time" + "unicode" "github.com/golang-jwt/jwt/v5" @@ -30,6 +32,10 @@ import ( const ( maxUpstreamTokenBytes = 16 * 1024 maxOIDCJSONBytes = 256 * 1024 + maxOIDCClientIDBytes = 256 + maxOIDCCodeBytes = 2048 + maxOIDCCodeVerifier = 128 + oidcBrowserValueBytes = 32 defaultOIDCTimeout = 5 * time.Second defaultOIDCCacheTTL = 15 * time.Minute maximumOIDCCacheTTL = time.Hour @@ -45,6 +51,8 @@ var ( ErrInvalidOIDCToken = errors.New("invalid OIDC token") // ErrOIDCBindingNotFound lets stores report a miss without exposing it through Exchange. ErrOIDCBindingNotFound = errors.New("OIDC binding not found") + // ErrInvalidOIDCBrowserLogin deliberately hides provider and token-exchange details. + ErrInvalidOIDCBrowserLogin = errors.New("invalid OIDC browser login") ) // OIDCProviderConfig pins one upstream token profile. @@ -74,6 +82,26 @@ type OIDCServiceConfig struct { Now func() time.Time } +// OIDCBrowserAuthorizationRequest carries one server-generated authorization-code request. +// State, nonce, and code challenge are opaque values; the service accepts only the S256 shape. +type OIDCBrowserAuthorizationRequest struct { + Issuer string + ClientID string + RedirectURI string + State string + Nonce string + CodeChallenge string +} + +// OIDCBrowserCodeExchange carries one server-side authorization-code redemption. +type OIDCBrowserCodeExchange struct { + Issuer string + ClientID string + RedirectURI string + Code string + CodeVerifier string +} + // OIDCService validates upstream identities and exchanges them for Sith sessions. type OIDCService struct { providers map[string]*oidcProvider @@ -99,6 +127,7 @@ type oidcKeyCache struct { type oidcClaims struct { AuthorizedParty string `json:"azp,omitempty"` + Nonce string `json:"nonce,omitempty"` jwt.RegisteredClaims } @@ -109,8 +138,14 @@ type oidcHeader struct { } type oidcDiscovery struct { - Issuer string `json:"issuer"` - JWKSURL string `json:"jwks_uri"` + Issuer string `json:"issuer"` + JWKSURL string `json:"jwks_uri"` + AuthorizationEndpoint string `json:"authorization_endpoint"` + TokenEndpoint string `json:"token_endpoint"` +} + +type oidcTokenResponse struct { + IDToken string `json:"id_token"` } type oidcJWKSet struct { @@ -217,6 +252,30 @@ func (service *OIDCService) BindIdentity(ctx context.Context, admin tenancy.Scop // Exchange validates a pinned upstream token and resolves one workspace binding server-side. func (service *OIDCService) Exchange(ctx context.Context, workspaceID tenancy.WorkspaceID, rawToken string) (IssuedSession, error) { + return service.exchange(ctx, workspaceID, rawToken, "", true, false) +} + +// ExchangeWithNonce validates a pinned ID token and requires the exact transaction nonce. +// Browser callers use it only after a successful, server-side authorization-code exchange. Unlike +// the raw exchange profile, this accepts a standards-compliant ID token without an nbf claim. +func (service *OIDCService) ExchangeWithNonce( + ctx context.Context, + workspaceID tenancy.WorkspaceID, + rawToken, expectedNonce string, +) (IssuedSession, error) { + if !validOIDCBrowserValue(expectedNonce) { + return IssuedSession{}, ErrInvalidOIDCToken + } + return service.exchange(ctx, workspaceID, rawToken, expectedNonce, false, true) +} + +func (service *OIDCService) exchange( + ctx context.Context, + workspaceID tenancy.WorkspaceID, + rawToken, expectedNonce string, + requireNotBefore bool, + requireExactAudience bool, +) (IssuedSession, error) { if service == nil || ctx == nil || ctx.Err() != nil || tenancy.ValidateWorkspaceID(workspaceID) != nil || rawToken == "" || len(rawToken) > maxUpstreamTokenBytes { return IssuedSession{}, ErrInvalidOIDCToken @@ -250,15 +309,21 @@ func (service *OIDCService) Exchange(ctx context.Context, workspaceID tenancy.Wo } return key, nil }) - if err != nil || token == nil || !token.Valid || claims.IssuedAt == nil || claims.NotBefore == nil || + if err != nil || token == nil || !token.Valid || claims.IssuedAt == nil || (requireNotBefore && claims.NotBefore == nil) || claims.ExpiresAt == nil || validateOIDCSubject(claims.Subject) != nil { return IssuedSession{}, ErrInvalidOIDCToken } lifetime := claims.ExpiresAt.Sub(claims.IssuedAt.Time) - if lifetime <= 0 || lifetime > provider.config.MaxTokenLifetime || claims.NotBefore.Before(claims.IssuedAt.Add(-time.Minute)) { + if lifetime <= 0 || lifetime > provider.config.MaxTokenLifetime || + (claims.NotBefore != nil && claims.NotBefore.Before(claims.IssuedAt.Add(-time.Minute))) { + return IssuedSession{}, ErrInvalidOIDCToken + } + if (requireExactAudience && (len(claims.Audience) != 1 || claims.Audience[0] != provider.config.Audience)) || + (!requireExactAudience && len(claims.Audience) != 1 && claims.AuthorizedParty != provider.config.Audience) || + (claims.AuthorizedParty != "" && claims.AuthorizedParty != provider.config.Audience) { return IssuedSession{}, ErrInvalidOIDCToken } - if len(claims.Audience) != 1 && claims.AuthorizedParty != provider.config.Audience { + if expectedNonce != "" && subtle.ConstantTimeCompare([]byte(claims.Nonce), []byte(expectedNonce)) != 1 { return IssuedSession{}, ErrInvalidOIDCToken } membership, err := service.store.LookupOIDCMembership(ctx, workspaceID, provider.config.Issuer, claims.Subject) @@ -276,6 +341,158 @@ func (service *OIDCService) Exchange(ctx context.Context, workspaceID tenancy.Wo return session, nil } +// BrowserAuthorizationURL creates one pinned OIDC authorization-code request with PKCE S256. +func (service *OIDCService) BrowserAuthorizationURL( + ctx context.Context, + request OIDCBrowserAuthorizationRequest, +) (string, error) { + provider, err := service.browserProvider(request.Issuer) + if err != nil || validateOIDCBrowserAuthorizationRequest(request) != nil || ctx == nil || ctx.Err() != nil { + return "", ErrInvalidOIDCBrowserLogin + } + discovery, err := service.fetchProviderDiscovery(ctx, provider) + if err != nil { + return "", ErrInvalidOIDCBrowserLogin + } + endpoint, err := parsePinnedOIDCBrowserEndpoint(discovery.AuthorizationEndpoint, provider, service.allowPrivateTest) + if err != nil { + return "", ErrInvalidOIDCBrowserLogin + } + query := endpoint.Query() + query.Set("response_type", "code") + query.Set("client_id", request.ClientID) + query.Set("redirect_uri", request.RedirectURI) + query.Set("scope", "openid") + query.Set("state", request.State) + query.Set("nonce", request.Nonce) + query.Set("code_challenge", request.CodeChallenge) + query.Set("code_challenge_method", "S256") + endpoint.RawQuery = query.Encode() + return endpoint.String(), nil +} + +// ExchangeAuthorizationCode redeems one PKCE-bound authorization code without exposing its ID token. +func (service *OIDCService) ExchangeAuthorizationCode( + ctx context.Context, + request OIDCBrowserCodeExchange, +) (string, error) { + provider, err := service.browserProvider(request.Issuer) + if err != nil || validateOIDCBrowserCodeExchange(request) != nil || ctx == nil || ctx.Err() != nil { + return "", ErrInvalidOIDCBrowserLogin + } + discovery, err := service.fetchProviderDiscovery(ctx, provider) + if err != nil { + return "", ErrInvalidOIDCBrowserLogin + } + endpoint, err := parsePinnedOIDCBrowserEndpoint(discovery.TokenEndpoint, provider, service.allowPrivateTest) + if err != nil { + return "", ErrInvalidOIDCBrowserLogin + } + form := url.Values{} + form.Set("grant_type", "authorization_code") + form.Set("client_id", request.ClientID) + form.Set("redirect_uri", request.RedirectURI) + form.Set("code", request.Code) + form.Set("code_verifier", request.CodeVerifier) + httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), strings.NewReader(form.Encode())) + if err != nil { + return "", ErrInvalidOIDCBrowserLogin + } + httpRequest.Header.Set("Accept", "application/json") + httpRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded") + client := *service.client + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + response, err := client.Do(httpRequest) + if err != nil { + return "", ErrInvalidOIDCBrowserLogin + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK || !isOIDCJSON(response.Header.Get("Content-Type")) { + return "", ErrInvalidOIDCBrowserLogin + } + body, err := io.ReadAll(io.LimitReader(response.Body, maxOIDCJSONBytes+1)) + if err != nil || len(body) > maxOIDCJSONBytes || rejectDuplicateJSON(body) != nil { + return "", ErrInvalidOIDCBrowserLogin + } + var tokenResponse oidcTokenResponse + if json.Unmarshal(body, &tokenResponse) != nil || tokenResponse.IDToken == "" || len(tokenResponse.IDToken) > maxUpstreamTokenBytes { + return "", ErrInvalidOIDCBrowserLogin + } + return tokenResponse.IDToken, nil +} + +func (service *OIDCService) browserProvider(issuer string) (*oidcProvider, error) { + if service == nil || issuer == "" || strings.TrimSpace(issuer) != issuer { + return nil, ErrInvalidOIDCBrowserLogin + } + provider := service.providers[issuer] + if provider == nil { + return nil, ErrInvalidOIDCBrowserLogin + } + return provider, nil +} + +func validateOIDCBrowserAuthorizationRequest(request OIDCBrowserAuthorizationRequest) error { + if validateOIDCBrowserClient(request.ClientID) != nil || validateOIDCBrowserRedirectURI(request.RedirectURI) != nil || + !validOIDCBrowserValue(request.State) || !validOIDCBrowserValue(request.Nonce) || !validOIDCBrowserValue(request.CodeChallenge) { + return ErrInvalidOIDCBrowserLogin + } + return nil +} + +func validateOIDCBrowserCodeExchange(request OIDCBrowserCodeExchange) error { + if validateOIDCBrowserClient(request.ClientID) != nil || validateOIDCBrowserRedirectURI(request.RedirectURI) != nil || + !validOIDCCode(request.Code) || !validOIDCCodeVerifier(request.CodeVerifier) { + return ErrInvalidOIDCBrowserLogin + } + return nil +} + +func validateOIDCBrowserClient(clientID string) error { + if clientID == "" || len(clientID) > maxOIDCClientIDBytes || strings.TrimSpace(clientID) != clientID || strings.IndexFunc(clientID, unicode.IsControl) >= 0 { + return ErrInvalidOIDCBrowserLogin + } + return nil +} + +func validateOIDCBrowserRedirectURI(rawURI string) error { + parsed, err := url.Parse(rawURI) + if err != nil || len(rawURI) > 2048 || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || + parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path == "" || parsed.EscapedPath() != parsed.Path { + return ErrInvalidOIDCBrowserLogin + } + return nil +} + +func validOIDCBrowserValue(value string) bool { + if len(value) != base64.RawURLEncoding.EncodedLen(oidcBrowserValueBytes) { + return false + } + decoded, err := base64.RawURLEncoding.DecodeString(value) + return err == nil && len(decoded) == oidcBrowserValueBytes && base64.RawURLEncoding.EncodeToString(decoded) == value +} + +func validOIDCCode(code string) bool { + return code != "" && len(code) <= maxOIDCCodeBytes && strings.TrimSpace(code) == code && strings.IndexFunc(code, unicode.IsControl) < 0 +} + +func validOIDCCodeVerifier(verifier string) bool { + if len(verifier) < 43 || len(verifier) > maxOIDCCodeVerifier { + return false + } + for _, value := range verifier { + if !validPKCEVerifierRune(value) { + return false + } + } + return true +} + +func validPKCEVerifierRune(value rune) bool { + return (value >= 'A' && value <= 'Z') || (value >= 'a' && value <= 'z') || (value >= '0' && value <= '9') || + value == '-' || value == '.' || value == '_' || value == '~' +} + func validateOIDCProvider(config *OIDCProviderConfig, allowPrivateForTest bool) (*url.URL, error) { parsed, err := url.Parse(config.Issuer) if err != nil || len(config.Issuer) > 2048 || validateOIDCURL(parsed, allowPrivateForTest) != nil || parsed.RawQuery != "" || parsed.Fragment != "" || parsed.User != nil || @@ -456,14 +673,10 @@ func (service *OIDCService) providerKeyWithFetcher( } func (service *OIDCService) fetchProviderKeys(ctx context.Context, provider *oidcProvider) (map[string]any, error) { - discoveryURL, _ := url.Parse(strings.TrimSuffix(provider.config.Issuer, "/") + "/.well-known/openid-configuration") - var discovery oidcDiscovery - if err := service.getOIDCJSON(ctx, discoveryURL, &discovery); err != nil { + discovery, err := service.fetchProviderDiscovery(ctx, provider) + if err != nil { return nil, err } - if discovery.Issuer != provider.config.Issuer { - return nil, fmt.Errorf("OIDC discovery issuer mismatch") - } jwksURL, err := url.Parse(discovery.JWKSURL) if err != nil || validateOIDCURL(jwksURL, service.allowPrivateTest) != nil || !sameOrigin(provider.url, jwksURL) { return nil, fmt.Errorf("OIDC JWKS URL escaped pinned issuer origin") @@ -471,6 +684,21 @@ func (service *OIDCService) fetchProviderKeys(ctx context.Context, provider *oid return service.fetchJWKSKeys(ctx, provider, jwksURL) } +func (service *OIDCService) fetchProviderDiscovery(ctx context.Context, provider *oidcProvider) (oidcDiscovery, error) { + if service == nil || provider == nil || ctx == nil || ctx.Err() != nil { + return oidcDiscovery{}, fmt.Errorf("OIDC discovery is unavailable") + } + discoveryURL, _ := url.Parse(strings.TrimSuffix(provider.config.Issuer, "/") + "/.well-known/openid-configuration") + var discovery oidcDiscovery + if err := service.getOIDCJSON(ctx, discoveryURL, &discovery); err != nil { + return oidcDiscovery{}, err + } + if discovery.Issuer != provider.config.Issuer { + return oidcDiscovery{}, fmt.Errorf("OIDC discovery issuer mismatch") + } + return discovery, nil +} + func (service *OIDCService) fetchJWKSKeys(ctx context.Context, provider *oidcProvider, jwksURL *url.URL) (map[string]any, error) { var set oidcJWKSet if err := service.getOIDCJSON(ctx, jwksURL, &set); err != nil { @@ -508,8 +736,7 @@ func (service *OIDCService) getOIDCJSON(ctx context.Context, endpoint *url.URL, if response.StatusCode != http.StatusOK { return fmt.Errorf("fetch OIDC metadata: unexpected HTTP status") } - contentType := strings.ToLower(strings.TrimSpace(strings.Split(response.Header.Get("Content-Type"), ";")[0])) - if contentType != "application/json" && !strings.HasSuffix(contentType, "+json") { + if !isOIDCJSON(response.Header.Get("Content-Type")) { return fmt.Errorf("fetch OIDC metadata: response is not JSON") } body, err := io.ReadAll(io.LimitReader(response.Body, maxOIDCJSONBytes+1)) @@ -522,6 +749,11 @@ func (service *OIDCService) getOIDCJSON(ctx context.Context, endpoint *url.URL, return nil } +func isOIDCJSON(contentType string) bool { + contentType = strings.ToLower(strings.TrimSpace(strings.Split(contentType, ";")[0])) + return contentType == "application/json" || strings.HasSuffix(contentType, "+json") +} + func parseOIDCJWK(jwk oidcJWK) (any, error) { switch jwk.Algorithm { case "RS256": @@ -605,6 +837,15 @@ func parsePinnedOIDCJWKSURL(rawURL string, allowPrivateForTest bool) (*url.URL, return parsed, nil } +func parsePinnedOIDCBrowserEndpoint(rawURL string, provider *oidcProvider, allowPrivateForTest bool) (*url.URL, error) { + parsed, err := url.Parse(rawURL) + if err != nil || provider == nil || validateOIDCURL(parsed, allowPrivateForTest) != nil || parsed.RawQuery != "" || + parsed.Fragment != "" || parsed.User != nil || parsed.Path == "" || parsed.EscapedPath() != parsed.Path || !sameOrigin(provider.url, parsed) { + return nil, fmt.Errorf("OIDC browser endpoint is invalid") + } + return parsed, nil +} + func publicOIDCAddress(ip netip.Addr) bool { if !ip.IsValid() || !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() { return false diff --git a/internal/hubauth/oidc_browser_test.go b/internal/hubauth/oidc_browser_test.go new file mode 100644 index 0000000..6928780 --- /dev/null +++ b/internal/hubauth/oidc_browser_test.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubauth + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func TestOIDCBrowserCodeFlowPinsPKCEAndNonce(t *testing.T) { + now := time.Date(2026, 7, 15, 12, 0, 0, 0, time.UTC) + upstreamPublic, upstreamPrivate, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + provider := newOIDCTestProvider(t, []oidcJWK{ed25519JWK("browser-key", upstreamPublic)}) + service, _, verifier, admin := newOIDCTestFixture(t, provider, &now) + if err := service.BindIdentity(context.Background(), admin, provider.server.URL, "upstream:alice", "user:alice"); err != nil { + t.Fatal(err) + } + state := browserTestValue(0x11) + nonce := browserTestValue(0x22) + verifierValue := strings.Repeat("v", 43) + challengeDigest := sha256.Sum256([]byte(verifierValue)) + challenge := base64.RawURLEncoding.EncodeToString(challengeDigest[:]) + provider.mu.Lock() + provider.metadata = func(issuer string) string { + return `{"issuer":` + quoteJSON(issuer) + `,"jwks_uri":` + quoteJSON(issuer+"/jwks") + + `,"authorization_endpoint":` + quoteJSON(issuer+"/authorize") + `,"token_endpoint":` + quoteJSON(issuer+"/token") + `}` + } + provider.token = func(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost || request.Header.Get("Content-Type") != "application/x-www-form-urlencoded" { + t.Errorf("token request method/content type = %s/%q", request.Method, request.Header.Get("Content-Type")) + http.Error(response, "invalid", http.StatusBadRequest) + return + } + if err := request.ParseForm(); err != nil { + t.Fatal(err) + } + if request.Form.Get("grant_type") != "authorization_code" || request.Form.Get("client_id") != "sith-browser" || + request.Form.Get("redirect_uri") != "https://hub.sith.test/v1/console/oidc/callback" || + request.Form.Get("code") != "provider-code" || request.Form.Get("code_verifier") != verifierValue { + t.Errorf("token request form = %#v", request.Form) + http.Error(response, "invalid", http.StatusBadRequest) + return + } + raw := signOIDCTestToken(t, provider.server.URL, "sith-hub", "upstream:alice", "browser-key", upstreamPrivate, now, func(claims jwt.MapClaims) { + claims["nonce"] = nonce + }) + response.Header().Set("Content-Type", "application/json") + _, _ = response.Write([]byte(`{"id_token":` + quoteJSON(raw) + `}`)) + } + provider.mu.Unlock() + + authorizationURL, err := service.BrowserAuthorizationURL(context.Background(), OIDCBrowserAuthorizationRequest{ + Issuer: provider.server.URL, ClientID: "sith-browser", RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", + State: state, Nonce: nonce, CodeChallenge: challenge, + }) + if err != nil { + t.Fatal(err) + } + parsed, err := url.Parse(authorizationURL) + if err != nil { + t.Fatal(err) + } + if parsed.Path != "/authorize" || parsed.Query().Get("response_type") != "code" || parsed.Query().Get("scope") != "openid" || + parsed.Query().Get("state") != state || parsed.Query().Get("nonce") != nonce || parsed.Query().Get("code_challenge") != challenge || + parsed.Query().Get("code_challenge_method") != "S256" || parsed.Query().Get("code_verifier") != "" { + t.Fatalf("authorization query = %#v", parsed.Query()) + } + rawToken, err := service.ExchangeAuthorizationCode(context.Background(), OIDCBrowserCodeExchange{ + Issuer: provider.server.URL, ClientID: "sith-browser", RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", + Code: "provider-code", CodeVerifier: verifierValue, + }) + if err != nil { + t.Fatal(err) + } + session, err := service.ExchangeWithNonce(context.Background(), "workspace-a", rawToken, nonce) + if err != nil { + t.Fatal(err) + } + principal, err := verifier.Verify(context.Background(), session.AccessToken) + if err != nil { + t.Fatal(err) + } + scope, err := principal.Scope("workspace-a") + if err != nil || scope.Subject() != "user:alice" { + t.Fatalf("browser scope = %#v, error = %v", scope, err) + } + if _, err := service.ExchangeWithNonce(context.Background(), "workspace-a", rawToken, browserTestValue(0x33)); !errors.Is(err, ErrInvalidOIDCToken) { + t.Fatalf("wrong nonce error = %v", err) + } + wrongAudience := signOIDCTestToken(t, provider.server.URL, "other-client", "upstream:alice", "browser-key", upstreamPrivate, now, func(claims jwt.MapClaims) { + claims["nonce"] = nonce + }) + if _, err := service.ExchangeWithNonce(context.Background(), "workspace-a", wrongAudience, nonce); !errors.Is(err, ErrInvalidOIDCToken) { + t.Fatalf("wrong audience error = %v", err) + } + multipleAudiences := signOIDCTestToken(t, provider.server.URL, "sith-hub", "upstream:alice", "browser-key", upstreamPrivate, now, func(claims jwt.MapClaims) { + claims["nonce"] = nonce + claims["aud"] = []string{"sith-hub", "other-client"} + claims["azp"] = "sith-hub" + }) + if _, err := service.ExchangeWithNonce(context.Background(), "workspace-a", multipleAudiences, nonce); !errors.Is(err, ErrInvalidOIDCToken) { + t.Fatalf("multiple browser audiences error = %v", err) + } + if _, err := service.ExchangeWithNonce(context.Background(), "workspace-b", rawToken, nonce); !errors.Is(err, ErrInvalidOIDCToken) { + t.Fatalf("cross-workspace browser exchange error = %v", err) + } + withoutNotBefore := signOIDCTestToken(t, provider.server.URL, "sith-hub", "upstream:alice", "browser-key", upstreamPrivate, now, func(claims jwt.MapClaims) { + claims["nonce"] = nonce + delete(claims, "nbf") + }) + if _, err := service.ExchangeWithNonce(context.Background(), "workspace-a", withoutNotBefore, nonce); err != nil { + t.Fatalf("standards-compliant browser ID token without nbf rejected: %v", err) + } +} + +func TestOIDCBrowserCodeExchangeFailsClosed(t *testing.T) { + now := time.Date(2026, 7, 15, 12, 0, 0, 0, time.UTC) + upstreamPublic, _, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + provider := newOIDCTestProvider(t, []oidcJWK{ed25519JWK("browser-key", upstreamPublic)}) + service, _, _, _ := newOIDCTestFixture(t, provider, &now) + provider.mu.Lock() + provider.metadata = func(issuer string) string { + return `{"issuer":` + quoteJSON(issuer) + `,"jwks_uri":` + quoteJSON(issuer+"/jwks") + + `,"authorization_endpoint":` + quoteJSON(issuer+"/authorize") + `,"token_endpoint":` + quoteJSON(issuer+"/token") + `}` + } + provider.token = func(response http.ResponseWriter, request *http.Request) { + if err := request.ParseForm(); err != nil { + t.Fatal(err) + } + if request.Form.Get("code") == "wrong-verifier" { + http.Error(response, "invalid verifier", http.StatusBadRequest) + return + } + response.Header().Set("Content-Type", "application/json") + if request.Form.Get("code") == "malformed" { + _, _ = response.Write([]byte("not JSON")) + return + } + _, _ = response.Write([]byte(`{"id_token":"one","id_token":"two"}`)) + } + provider.mu.Unlock() + + request := OIDCBrowserCodeExchange{ + Issuer: provider.server.URL, ClientID: "sith-browser", RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", + Code: "provider-code", CodeVerifier: strings.Repeat("v", 43), + } + if _, err := service.ExchangeAuthorizationCode(context.Background(), request); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("duplicate token response error = %v", err) + } + request.Code = "wrong-verifier" + request.CodeVerifier = strings.Repeat("w", 43) + if _, err := service.ExchangeAuthorizationCode(context.Background(), request); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("PKCE verifier mismatch error = %v", err) + } + request.Code = "malformed" + request.CodeVerifier = strings.Repeat("v", 43) + if _, err := service.ExchangeAuthorizationCode(context.Background(), request); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("malformed token response error = %v", err) + } + request.CodeVerifier = "plain" + if _, err := service.ExchangeAuthorizationCode(context.Background(), request); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("plain PKCE verifier error = %v", err) + } + if _, err := service.BrowserAuthorizationURL(context.Background(), OIDCBrowserAuthorizationRequest{ + Issuer: provider.server.URL, ClientID: "sith-browser", RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", + State: browserTestValue(0x11), Nonce: browserTestValue(0x22), CodeChallenge: "plain", + }); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("plain PKCE challenge error = %v", err) + } + if _, err := service.BrowserAuthorizationURL(context.Background(), OIDCBrowserAuthorizationRequest{ + Issuer: "https://other.sith.test", ClientID: "sith-browser", RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", + State: browserTestValue(0x11), Nonce: browserTestValue(0x22), CodeChallenge: browserTestValue(0x33), + }); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("wrong issuer error = %v", err) + } + provider.mu.Lock() + provider.outage = true + provider.mu.Unlock() + request.Code = "provider-code" + request.CodeVerifier = strings.Repeat("v", 43) + if _, err := service.ExchangeAuthorizationCode(context.Background(), request); !errors.Is(err, ErrInvalidOIDCBrowserLogin) { + t.Fatalf("provider outage error = %v", err) + } +} + +func browserTestValue(fill byte) string { + return base64.RawURLEncoding.EncodeToString([]byte(strings.Repeat(string([]byte{fill}), oidcBrowserValueBytes))) +} + +func quoteJSON(value string) string { + return `"` + strings.ReplaceAll(strings.ReplaceAll(value, `\`, `\\`), `"`, `\"`) + `"` +} diff --git a/internal/hubauth/oidc_test.go b/internal/hubauth/oidc_test.go index 5059ddd..a8b3588 100644 --- a/internal/hubauth/oidc_test.go +++ b/internal/hubauth/oidc_test.go @@ -53,11 +53,13 @@ func (store *memoryOIDCStore) LookupOIDCMembership(_ context.Context, workspaceI } type oidcTestProvider struct { - server *httptest.Server - mu sync.Mutex - keys []oidcJWK - outage bool - metadata func(string) string + server *httptest.Server + mu sync.Mutex + keys []oidcJWK + outage bool + metadata func(string) string + authorization http.HandlerFunc + token http.HandlerFunc } func newOIDCTestProvider(t *testing.T, keys []oidcJWK) *oidcTestProvider { @@ -80,6 +82,18 @@ func newOIDCTestProvider(t *testing.T, keys []oidcJWK) *oidcTestProvider { _ = json.NewEncoder(response).Encode(oidcDiscovery{Issuer: provider.server.URL, JWKSURL: provider.server.URL + "/jwks"}) case "/jwks": _ = json.NewEncoder(response).Encode(oidcJWKSet{Keys: provider.keys}) + case "/authorize": + if provider.authorization == nil { + http.NotFound(response, request) + return + } + provider.authorization(response, request) + case "/token": + if provider.token == nil { + http.NotFound(response, request) + return + } + provider.token(response, request) default: http.NotFound(response, request) } @@ -166,6 +180,7 @@ func TestOIDCExchangeRejectsHostileTokens(t *testing.T) { "forged key": signOIDCTestToken(t, provider.server.URL, "sith-hub", "upstream:alice", "key-1", attackerKey, now, nil), "wrong algorithm": signOIDCHSTestToken(t, provider.server.URL, now), "wrong type": signOIDCTestTokenWithHeader(t, provider.server.URL, "sith-hub", "upstream:alice", "key-1", privateKey, now, map[string]any{"typ": "at+jwt"}), + "wrong azp": mutateClaims(func(claims jwt.MapClaims) { claims["azp"] = "other-client" }), } // Replace the helper marker with an actual forbidden JOSE header. tests["hostile jku"] = signOIDCTestTokenWithHeader(t, provider.server.URL, "sith-hub", "upstream:alice", "key-1", privateKey, now, map[string]any{"jku": "https://evil.example/jwks"}) diff --git a/internal/hubdb/app.go b/internal/hubdb/app.go index 28f7b29..1b3ff2f 100644 --- a/internal/hubdb/app.go +++ b/internal/hubdb/app.go @@ -68,8 +68,36 @@ func (database *AppDB) Close() { } } +// Ping verifies that the least-privilege application pool can acquire and ping PostgreSQL. It is +// intentionally scope-free because it executes no tenant query and exists only for Hub readiness. +func (database *AppDB) Ping(ctx context.Context) error { + if database == nil || database.pool == nil || ctx == nil { + return fmt.Errorf("ping hub database: database and context are required") + } + return database.pool.Ping(ctx) +} + // InWorkspace runs one callback in an explicit transaction with transaction-local RLS scope. func (database *AppDB) InWorkspace(ctx context.Context, scope tenancy.Scope, run func(pgx.Tx) error) error { + return database.inWorkspace(ctx, scope, pgx.TxOptions{ + IsoLevel: pgx.ReadCommitted, + AccessMode: pgx.ReadWrite, + }, run) +} + +func (database *AppDB) inWorkspaceReadSnapshot(ctx context.Context, scope tenancy.Scope, run func(pgx.Tx) error) error { + return database.inWorkspace(ctx, scope, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadOnly, + }, run) +} + +func (database *AppDB) inWorkspace( + ctx context.Context, + scope tenancy.Scope, + options pgx.TxOptions, + run func(pgx.Tx) error, +) error { if database == nil || database.pool == nil || ctx == nil { return fmt.Errorf("run workspace transaction: database and context are required") } @@ -80,10 +108,7 @@ func (database *AppDB) InWorkspace(ctx context.Context, scope tenancy.Scope, run if scope.Subject() == "" || !scope.Role().Valid() || scope.RequireWorkspace(workspaceID) != nil { return fmt.Errorf("run workspace transaction: validated workspace scope is required") } - return pgx.BeginTxFunc(ctx, database.pool, pgx.TxOptions{ - IsoLevel: pgx.ReadCommitted, - AccessMode: pgx.ReadWrite, - }, func(tx pgx.Tx) error { + return pgx.BeginTxFunc(ctx, database.pool, options, func(tx pgx.Tx) error { if err := setWorkspaceScope(ctx, tx, workspaceID); err != nil { return err } diff --git a/internal/hubdb/app_test.go b/internal/hubdb/app_test.go index 6a8cc74..fbfbcc5 100644 --- a/internal/hubdb/app_test.go +++ b/internal/hubdb/app_test.go @@ -20,6 +20,9 @@ func TestAppDBRejectsMissingBoundary(t *testing.T) { if err := (*AppDB)(nil).InWorkspace(context.Background(), tenancy.Scope{}, nil); err == nil { t.Fatal("nil database unexpectedly accepted") } + if err := (*AppDB)(nil).Ping(context.Background()); err == nil { + t.Fatal("nil database readiness check unexpectedly succeeded") + } } func TestSecureTransportRejectsMissingTLS(t *testing.T) { diff --git a/internal/hubdb/approvals.go b/internal/hubdb/approvals.go new file mode 100644 index 0000000..eb06cf6 --- /dev/null +++ b/internal/hubdb/approvals.go @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubdb + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "io" + "regexp" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +const ( + approvalGrantIDBytes = 16 + approvalGrantEvidenceVersion = 2 +) + +var ( + approvalGrantIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{22}$`) + + // ErrApprovalGrantUnavailable deliberately covers missing, foreign, mismatched, unauthorized, + // legacy, expired, and already-consumed grants so callers cannot use this boundary as an oracle. + ErrApprovalGrantUnavailable = errors.New("approval grant unavailable") +) + +// ApprovalGrantID is one opaque server-minted approval identifier. It is not a credential: use +// still requires the authenticated proposal scope and the exact immutable proposal binding. +type ApprovalGrantID string + +// String returns the canonical opaque identifier. +func (identifier ApprovalGrantID) String() string { return string(identifier) } + +// CreateApprovalGrant persists one same-workspace, separation-of-duty approval for an exact +// validated proposal. PostgreSQL mints its immutable ten-minute lifetime; no caller controls the +// clock. The row stores no raw target, argument, justification, token, or elicitation data. +func (database *AppDB) CreateApprovalGrant( + ctx context.Context, + approver tenancy.Scope, + binding pep.ApprovalBinding, +) (ApprovalGrantID, error) { + return database.createApprovalGrant(ctx, approver, binding, rand.Reader) +} + +func (database *AppDB) createApprovalGrant( + ctx context.Context, + approver tenancy.Scope, + binding pep.ApprovalBinding, + random io.Reader, +) (ApprovalGrantID, error) { + if database == nil || database.pool == nil || ctx == nil { + return "", fmt.Errorf("create approval grant: database and context are required") + } + if binding.Validate() != nil || random == nil || + approver.Authorize(tenancy.ActionApproveIntent) != nil || + approver.RequireWorkspace(binding.WorkspaceID()) != nil || approver.Subject() == binding.Proposer() { + return "", fmt.Errorf("create approval grant: %w", ErrApprovalGrantUnavailable) + } + + identifier, err := newApprovalGrantID(random) + if err != nil { + return "", fmt.Errorf("create approval grant: mint identifier: %w", err) + } + traceContext, traceID, err := tracing.Ensure(ctx) + if err != nil { + return "", fmt.Errorf("create approval grant: establish trace context: %w", err) + } + err = database.InWorkspace(traceContext, approver, func(tx pgx.Tx) error { + var proposerRole, currentApproverRole tenancy.Role + if err := tx.QueryRow(traceContext, ` + SELECT proposer.role, approver.role + FROM sith.memberships proposer + JOIN sith.memberships approver ON approver.workspace_id = proposer.workspace_id + WHERE proposer.workspace_id = $1 AND proposer.subject = $2 AND approver.subject = $3 + FOR SHARE OF proposer, approver + `, binding.WorkspaceID(), binding.Proposer(), approver.Subject()).Scan(&proposerRole, ¤tApproverRole); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrApprovalGrantUnavailable + } + return fmt.Errorf("verify current approval memberships: %w", err) + } + if !proposerRole.Allows(tenancy.ActionProposeIntent) || currentApproverRole != approver.Role() || + !currentApproverRole.Allows(tenancy.ActionApproveIntent) { + return ErrApprovalGrantUnavailable + } + var approvedAt, expiresAt time.Time + err := tx.QueryRow(traceContext, ` + INSERT INTO sith.approval_grants( + workspace_id, id, intent_id, proposer, approver, resolved_digest, + evidence_version, approved_at, expires_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, statement_timestamp(), + statement_timestamp() + interval '10 minutes') + RETURNING approved_at, expires_at + `, binding.WorkspaceID(), identifier, binding.IntentID(), binding.Proposer(), approver.Subject(), + binding.ResolvedDigest(), approvalGrantEvidenceVersion).Scan(&approvedAt, &expiresAt) + if err != nil { + var postgresErr *pgconn.PgError + if errors.As(err, &postgresErr) && postgresErr.Code == "23505" { + return ErrApprovalGrantUnavailable + } + return fmt.Errorf("persist exact approval grant: %w", err) + } + evidence := expiringApprovalGrantEvidenceDigest( + binding.WorkspaceID(), identifier, binding.IntentID(), binding.Proposer(), + approver.Subject(), binding.ResolvedDigest(), approvedAt, expiresAt, + ) + return appendPolicyAuditEntryTx(traceContext, tx, policyAuditEntry{ + format: approvalExpiryAuditFormatVersion, recordedAt: approvedAt.UTC().Truncate(time.Microsecond), + traceID: traceID, workspaceID: binding.WorkspaceID(), actor: approver.Subject(), + role: approver.Role(), action: tenancy.ActionApproveIntent, verb: approvalAuditVerb, + verdict: pep.VerdictAllow, reasonCode: approvalCreatedEventKind, + eventKind: approvalCreatedEventKind, evidence: evidence, + }) + }) + if err != nil { + if errors.Is(err, ErrApprovalGrantUnavailable) { + return "", fmt.Errorf("create approval grant: %w", ErrApprovalGrantUnavailable) + } + return "", fmt.Errorf("create approval grant: %w", err) + } + return identifier, nil +} + +// ConsumeApprovalGrant atomically spends one exact, unexpired grant using PostgreSQL statement +// time. Concurrent, replaying, legacy, expired, missing, and mismatched consumers share one stable +// refusal; exactly one conditional update can win. +func (database *AppDB) ConsumeApprovalGrant( + ctx context.Context, + proposer tenancy.Scope, + binding pep.ApprovalBinding, + identifier ApprovalGrantID, +) error { + if database == nil || database.pool == nil || ctx == nil { + return fmt.Errorf("consume approval grant: database and context are required") + } + if binding.Validate() != nil || !approvalGrantIDPattern.MatchString(identifier.String()) || + proposer.Authorize(tenancy.ActionProposeIntent) != nil || + proposer.RequireWorkspace(binding.WorkspaceID()) != nil || proposer.Subject() != binding.Proposer() { + return fmt.Errorf("consume approval grant: %w", ErrApprovalGrantUnavailable) + } + + traceContext, traceID, err := tracing.Ensure(ctx) + if err != nil { + return fmt.Errorf("consume approval grant: establish trace context: %w", err) + } + err = database.InWorkspace(traceContext, proposer, func(tx pgx.Tx) error { + var currentRole tenancy.Role + if err := tx.QueryRow(traceContext, ` + SELECT role FROM sith.memberships + WHERE workspace_id = $1 AND subject = $2 + FOR SHARE + `, binding.WorkspaceID(), binding.Proposer()).Scan(¤tRole); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return ErrApprovalGrantUnavailable + } + return fmt.Errorf("verify current proposer membership: %w", err) + } + if currentRole != proposer.Role() || !currentRole.Allows(tenancy.ActionProposeIntent) { + return ErrApprovalGrantUnavailable + } + var ( + returned ApprovalGrantID + returnedApprover string + returnedApprovedAt time.Time + returnedExpiresAt time.Time + returnedConsumedAt time.Time + ) + err := tx.QueryRow(traceContext, ` + UPDATE sith.approval_grants + SET consumed_at = statement_timestamp() + WHERE workspace_id = $1 AND id = $2 AND intent_id = $3 AND proposer = $4 + AND resolved_digest = $5 AND evidence_version = $6 AND consumed_at IS NULL + AND approved_at <= statement_timestamp() AND statement_timestamp() < expires_at + RETURNING id, approver, approved_at, expires_at, consumed_at + `, binding.WorkspaceID(), identifier, binding.IntentID(), binding.Proposer(), binding.ResolvedDigest(), + approvalGrantEvidenceVersion).Scan( + &returned, &returnedApprover, &returnedApprovedAt, &returnedExpiresAt, &returnedConsumedAt, + ) + if errors.Is(err, pgx.ErrNoRows) || err == nil && returned != identifier { + return ErrApprovalGrantUnavailable + } + if err != nil { + return fmt.Errorf("atomically consume exact approval grant: %w", err) + } + evidence := expiringApprovalGrantEvidenceDigest( + binding.WorkspaceID(), returned, binding.IntentID(), binding.Proposer(), + returnedApprover, binding.ResolvedDigest(), returnedApprovedAt, returnedExpiresAt, + ) + return appendPolicyAuditEntryTx(traceContext, tx, policyAuditEntry{ + format: approvalExpiryAuditFormatVersion, recordedAt: returnedConsumedAt.UTC().Truncate(time.Microsecond), + traceID: traceID, workspaceID: binding.WorkspaceID(), actor: proposer.Subject(), + role: proposer.Role(), action: tenancy.ActionProposeIntent, verb: approvalAuditVerb, + verdict: pep.VerdictAllow, reasonCode: approvalConsumedEventKind, + eventKind: approvalConsumedEventKind, evidence: evidence, + }) + }) + if err != nil { + if errors.Is(err, ErrApprovalGrantUnavailable) { + return fmt.Errorf("consume approval grant: %w", ErrApprovalGrantUnavailable) + } + return fmt.Errorf("consume approval grant: %w", err) + } + return nil +} + +func newApprovalGrantID(random io.Reader) (ApprovalGrantID, error) { + if random == nil { + return "", fmt.Errorf("random source is required") + } + payload := make([]byte, approvalGrantIDBytes) + if _, err := io.ReadFull(random, payload); err != nil { + return "", fmt.Errorf("read cryptographic entropy: %w", err) + } + identifier := ApprovalGrantID(base64.RawURLEncoding.EncodeToString(payload)) + if !approvalGrantIDPattern.MatchString(identifier.String()) { + return "", fmt.Errorf("minted identifier is not canonical") + } + return identifier, nil +} diff --git a/internal/hubdb/approvals_test.go b/internal/hubdb/approvals_test.go new file mode 100644 index 0000000..e7bcd32 --- /dev/null +++ b/internal/hubdb/approvals_test.go @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubdb + +import ( + "bytes" + "context" + "errors" + "io/fs" + "strings" + "testing" + + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestApprovalGrantIdentifierIsOpaqueAndCanonical(t *testing.T) { + t.Parallel() + + identifier, err := newApprovalGrantID(bytes.NewReader(bytes.Repeat([]byte{0x42}, approvalGrantIDBytes))) + if err != nil { + t.Fatal(err) + } + if !approvalGrantIDPattern.MatchString(identifier.String()) || len(identifier.String()) != 22 { + t.Fatalf("approval grant identifier = %q", identifier) + } + if _, err := newApprovalGrantID(failingApprovalReader{}); err == nil { + t.Fatal("entropy failure minted an approval identifier") + } + if _, err := newApprovalGrantID(nil); err == nil { + t.Fatal("nil entropy source minted an approval identifier") + } +} + +func TestApprovalGrantBoundaryClassifiesMissingDatabaseAsOperationalError(t *testing.T) { + t.Parallel() + + if _, err := (*AppDB)(nil).CreateApprovalGrant(context.Background(), tenancy.Scope{}, pep.ApprovalBinding{}); err == nil { + t.Fatal("nil database created an approval grant") + } else if errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("nil database error was misclassified as safe approval refusal: %v", err) + } + if err := (*AppDB)(nil).ConsumeApprovalGrant(context.Background(), tenancy.Scope{}, pep.ApprovalBinding{}, "AAAAAAAAAAAAAAAAAAAAAA"); err == nil { + t.Fatal("nil database consumed an approval grant") + } else if errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("nil database error was misclassified as safe approval refusal: %v", err) + } +} + +func TestApprovalGrantExpiryMigrationIsFixedVersionedAndFailClosed(t *testing.T) { + t.Parallel() + + migration, err := fs.ReadFile(migrationFiles, "migrations/0013_approval_grant_expiry.sql") + if err != nil { + t.Fatal(err) + } + text := string(migration) + for _, required := range []string{ + "ADD COLUMN expires_at timestamptz", "ADD COLUMN evidence_version smallint", + "expires_at = approved_at + interval '10 minutes'", "evidence_version IN (1, 2)", + "evidence_version = 1 OR consumed_at IS NULL OR consumed_at < expires_at", + "NO FORCE ROW LEVEL SECURITY", "FORCE ROW LEVEL SECURITY", "format_version IN (1, 2, 3)", + } { + if !strings.Contains(text, required) { + t.Fatalf("approval expiry migration is missing %q", required) + } + } + for _, forbidden := range []string{ + "DELETE FROM sith.approval_grants", "SET consumed_at", "DROP TABLE", "DISABLE ROW LEVEL SECURITY", + } { + if strings.Contains(text, forbidden) { + t.Fatalf("approval expiry migration contains destructive legacy handling %q", forbidden) + } + } + noForce := strings.Index(text, "NO FORCE ROW LEVEL SECURITY") + backfill := strings.Index(text, "UPDATE sith.approval_grants") + restoreForce := strings.LastIndex(text, "FORCE ROW LEVEL SECURITY") + if noForce < 0 || backfill <= noForce || restoreForce <= backfill { + t.Fatal("approval expiry backfill does not restore FORCE RLS around its transactional owner window") + } +} + +func TestApprovalGrantMigrationIsPrivacyMinimizedAndForcedRLS(t *testing.T) { + t.Parallel() + + migration, err := fs.ReadFile(migrationFiles, "migrations/0010_approval_grants.sql") + if err != nil { + t.Fatal(err) + } + text := string(migration) + for _, required := range []string{ + "CREATE TABLE sith.approval_grants", "UNIQUE (workspace_id, intent_id, approver, resolved_digest)", + "proposer <> approver", "consumed_at IS NULL OR consumed_at >= approved_at", + "ENABLE ROW LEVEL SECURITY", "FORCE ROW LEVEL SECURITY", "CREATE POLICY workspace_isolation", + } { + if !strings.Contains(text, required) { + t.Fatalf("approval migration is missing %q", required) + } + } + for _, forbidden := range []string{"jsonb", "argument_payload", "target_payload", "justification", "credential", "token", "elicitation"} { + if strings.Contains(strings.ToLower(text), forbidden) { + t.Fatalf("approval migration contains forbidden payload surface %q", forbidden) + } + } +} + +func FuzzApprovalGrantIDCanonicalVocabulary(f *testing.F) { + f.Add("AAAAAAAAAAAAAAAAAAAAAA") + f.Add("not/canonical") + f.Fuzz(func(t *testing.T, value string) { + matches := approvalGrantIDPattern.MatchString(value) + if matches && len(value) != 22 { + t.Fatalf("pattern accepted non-canonical length %d", len(value)) + } + }) +} + +type failingApprovalReader struct{} + +func (failingApprovalReader) Read([]byte) (int, error) { return 0, errors.New("entropy unavailable") } diff --git a/internal/hubdb/audit.go b/internal/hubdb/audit.go index be53dc4..251d087 100644 --- a/internal/hubdb/audit.go +++ b/internal/hubdb/audit.go @@ -61,27 +61,49 @@ func AuditIsolation(ctx context.Context, database catalogQueryer, appRole string AND a.atttypid = 'text'::regtype AND NOT a.attisdropped ) AS has_scope_column, + policy.policy_count, + policy.expected_policy_count, COALESCE(policy.using_expression, '') AS using_expression, COALESCE(policy.check_expression, '') AS check_expression, - has_table_privilege($1, c.oid, 'SELECT') - AND has_table_privilege($1, c.oid, 'INSERT') - AND has_table_privilege($1, c.oid, 'UPDATE') - AND has_table_privilege($1, c.oid, 'DELETE') AS has_dml, + has_table_privilege($1, c.oid, 'SELECT') AS can_select, + has_table_privilege($1, c.oid, 'INSERT') AS can_insert, + has_any_column_privilege($1, c.oid, 'UPDATE') AS can_update, + has_table_privilege($1, c.oid, 'DELETE') AS can_delete, + CASE WHEN c.relname = 'approval_grants' + THEN has_column_privilege($1, c.oid, 'consumed_at', 'UPDATE') + ELSE false END AS can_consume_approval, + EXISTS ( + SELECT 1 FROM pg_attribute mutable + WHERE mutable.attrelid = c.oid AND mutable.attnum > 0 AND NOT mutable.attisdropped + AND mutable.attname <> 'consumed_at' + AND has_column_privilege($1, c.oid, mutable.attname, 'UPDATE') + ) AS can_update_approval_identity, has_table_privilege($1, c.oid, 'TRUNCATE') - OR has_table_privilege($1, c.oid, 'REFERENCES') + OR has_any_column_privilege($1, c.oid, 'REFERENCES') OR has_table_privilege($1, c.oid, 'TRIGGER') OR has_table_privilege($1, c.oid, 'MAINTAIN') AS has_unsafe_privilege FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace LEFT JOIN LATERAL ( - SELECT pg_get_expr(p.polqual, p.polrelid) AS using_expression, - pg_get_expr(p.polwithcheck, p.polrelid) AS check_expression + SELECT count(*) AS policy_count, + count(*) FILTER (WHERE p.polname = 'workspace_isolation' + AND p.polcmd = '*' + AND p.polpermissive + AND p.polroles = ARRAY[0]::oid[]) AS expected_policy_count, + max(pg_get_expr(p.polqual, p.polrelid)) FILTER ( + WHERE p.polname = 'workspace_isolation' + AND p.polcmd = '*' + AND p.polpermissive + AND p.polroles = ARRAY[0]::oid[] + ) AS using_expression, + max(pg_get_expr(p.polwithcheck, p.polrelid)) FILTER ( + WHERE p.polname = 'workspace_isolation' + AND p.polcmd = '*' + AND p.polpermissive + AND p.polroles = ARRAY[0]::oid[] + ) AS check_expression FROM pg_policy p WHERE p.polrelid = c.oid - AND p.polname = 'workspace_isolation' - AND p.polcmd = '*' - AND p.polpermissive - AND p.polroles = ARRAY[0]::oid[] ) policy ON true WHERE n.nspname = 'sith' AND c.relkind IN ('r', 'p') ORDER BY c.relname @@ -94,17 +116,27 @@ func AuditIsolation(ctx context.Context, database catalogQueryer, appRole string for rows.Next() { tableCount++ var ( - table string - rlsEnabled bool - rlsForced bool - appOwns bool - hasScopeColumn bool - usingExpression string - checkExpression string - hasDML bool - hasUnsafePrivilege bool + table string + rlsEnabled bool + rlsForced bool + appOwns bool + hasScopeColumn bool + policyCount int64 + expectedPolicyCount int64 + usingExpression string + checkExpression string + canSelect bool + canInsert bool + canUpdate bool + canDelete bool + canConsumeApproval bool + canUpdateApprovalIdentity bool + hasUnsafePrivilege bool ) - if err := rows.Scan(&table, &rlsEnabled, &rlsForced, &appOwns, &hasScopeColumn, &usingExpression, &checkExpression, &hasDML, &hasUnsafePrivilege); err != nil { + if err := rows.Scan(&table, &rlsEnabled, &rlsForced, &appOwns, &hasScopeColumn, &policyCount, + &expectedPolicyCount, &usingExpression, &checkExpression, + &canSelect, &canInsert, &canUpdate, &canDelete, &canConsumeApproval, &canUpdateApprovalIdentity, + &hasUnsafePrivilege); err != nil { return fmt.Errorf("audit database isolation: scan catalog: %w", err) } if !rlsEnabled { @@ -123,11 +155,27 @@ func AuditIsolation(ctx context.Context, database catalogQueryer, appRole string if table == "workspaces" { scopeColumn = "id" } - if !validPolicyExpression(usingExpression, scopeColumn) || !validPolicyExpression(checkExpression, scopeColumn) { + if policyCount != 1 || expectedPolicyCount != 1 || + !validPolicyExpression(usingExpression, scopeColumn) || !validPolicyExpression(checkExpression, scopeColumn) { violations = append(violations, table+": complete workspace policy is missing") } - if !hasDML { - violations = append(violations, table+": application DML grant is incomplete") + switch table { + case "approval_grants": + if !canSelect || !canInsert || !canUpdate || canDelete || !canConsumeApproval || canUpdateApprovalIdentity { + violations = append(violations, table+": single-use application privilege contract is invalid") + } + case "policy_audit_entries": + if !canSelect || !canInsert || canUpdate || canDelete { + violations = append(violations, table+": immutable application privilege contract is invalid") + } + case "policy_audit_heads": + if !canSelect || !canInsert || !canUpdate || canDelete { + violations = append(violations, table+": chain-head application privilege contract is invalid") + } + default: + if !canSelect || !canInsert || !canUpdate || !canDelete { + violations = append(violations, table+": application DML grant is incomplete") + } } if hasUnsafePrivilege { violations = append(violations, table+": application role has unsafe table privileges") diff --git a/internal/hubdb/doc.go b/internal/hubdb/doc.go index 226a5e0..74484ed 100644 --- a/internal/hubdb/doc.go +++ b/internal/hubdb/doc.go @@ -1,4 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -// Package hubdb owns the PostgreSQL tenancy boundary for the federated hub. +// Package hubdb owns the PostgreSQL tenancy boundary for the federated hub, including the +// FORCE-RLS, exact-once approval-grant store used by the governed write pipeline. package hubdb diff --git a/internal/hubdb/fleet.go b/internal/hubdb/fleet.go index 2bd6928..6331016 100644 --- a/internal/hubdb/fleet.go +++ b/internal/hubdb/fleet.go @@ -211,6 +211,21 @@ func (database *AppDB) QueryFleet( query fleet.Query, freshness time.Duration, now time.Time, +) (fleet.QueryResult, error) { + return database.queryFleet(ctx, scope, query, freshness, now, queryFleetHooks{}) +} + +type queryFleetHooks struct { + afterClusterStates func(pgx.Tx) error +} + +func (database *AppDB) queryFleet( + ctx context.Context, + scope tenancy.Scope, + query fleet.Query, + freshness time.Duration, + now time.Time, + hooks queryFleetHooks, ) (fleet.QueryResult, error) { if err := requireReadScope(scope); err != nil { return fleet.QueryResult{}, fmt.Errorf("query federated fleet: %w", err) @@ -222,16 +237,22 @@ func (database *AppDB) QueryFleet( if err := validateFreshness(freshness, now); err != nil { return fleet.QueryResult{}, fmt.Errorf("query federated fleet: %w", err) } - states, requested, err := database.clusterStates(ctx, scope, requestedScopes) - if err != nil { - return fleet.QueryResult{}, fmt.Errorf("query federated fleet: %w", err) - } - result := fleet.QueryResult{Coverage: coverageFor(states, requested, freshness, now), Facts: []fleet.Fact{}} - err = database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { + var result fleet.QueryResult + err = database.inWorkspaceReadSnapshot(ctx, scope, func(tx pgx.Tx) error { + states, requested, err := readClusterStates(ctx, tx, scope.WorkspaceID(), requestedScopes) + if err != nil { + return err + } + if hooks.afterClusterStates != nil { + if err := hooks.afterClusterStates(tx); err != nil { + return err + } + } facts, err := queryFacts(ctx, tx, scope.WorkspaceID(), query) if err != nil { return err } + result = fleet.QueryResult{Coverage: coverageFor(states, requested, freshness, now), Facts: facts} for index := range facts { state, exists := states[facts[index].Ref.Scope] if !exists { @@ -240,7 +261,6 @@ func (database *AppDB) QueryFleet( facts[index].Workspace = string(scope.WorkspaceID()) facts[index].Stale, facts[index].StaleFor = staleState(state, freshness, now) } - result.Facts = facts return nil }) if err != nil { @@ -260,36 +280,54 @@ func (database *AppDB) clusterStates( ctx context.Context, scope tenancy.Scope, requestedScopes []string, +) (map[string]clusterState, []string, error) { + var ( + states map[string]clusterState + requested []string + ) + err := database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { + var err error + states, requested, err = readClusterStates(ctx, tx, scope.WorkspaceID(), requestedScopes) + return err + }) + if err != nil { + return nil, nil, err + } + return states, requested, nil +} + +func readClusterStates( + ctx context.Context, + tx pgx.Tx, + workspaceID tenancy.WorkspaceID, + requestedScopes []string, ) (map[string]clusterState, []string, error) { states := make(map[string]clusterState) requested := append([]string(nil), requestedScopes...) - err := database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { - query := `SELECT id, managed_cluster_ref, last_seen, COALESCE(last_error_kind, '') - FROM sith.clusters WHERE workspace_id = $1` - arguments := []any{scope.WorkspaceID()} - if len(requestedScopes) > 0 { - query += " AND id = ANY($2)" - arguments = append(arguments, requestedScopes) - } - query += " ORDER BY id" - rows, err := tx.Query(ctx, query, arguments...) - if err != nil { - return err + query := `SELECT id, managed_cluster_ref, last_seen, COALESCE(last_error_kind, '') + FROM sith.clusters WHERE workspace_id = $1` + arguments := []any{workspaceID} + if len(requestedScopes) > 0 { + query += " AND id = ANY($2)" + arguments = append(arguments, requestedScopes) + } + query += " ORDER BY id" + rows, err := tx.Query(ctx, query, arguments...) + if err != nil { + return nil, nil, err + } + defer rows.Close() + for rows.Next() { + var state clusterState + if err := rows.Scan(&state.id, &state.managedClusterRef, &state.lastSeen, &state.lastErrorKind); err != nil { + return nil, nil, err } - defer rows.Close() - for rows.Next() { - var state clusterState - if err := rows.Scan(&state.id, &state.managedClusterRef, &state.lastSeen, &state.lastErrorKind); err != nil { - return err - } - states[state.id] = state - if len(requestedScopes) == 0 { - requested = append(requested, state.id) - } + states[state.id] = state + if len(requestedScopes) == 0 { + requested = append(requested, state.id) } - return rows.Err() - }) - if err != nil { + } + if err := rows.Err(); err != nil { return nil, nil, err } return states, requested, nil diff --git a/internal/hubdb/migrate.go b/internal/hubdb/migrate.go index 1df88ef..3249f23 100644 --- a/internal/hubdb/migrate.go +++ b/internal/hubdb/migrate.go @@ -160,6 +160,11 @@ func grantApplicationPrivileges(ctx context.Context, tx pgx.Tx, appRole string) "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA sith TO " + identifier, "ALTER DEFAULT PRIVILEGES IN SCHEMA sith GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO " + identifier, "ALTER DEFAULT PRIVILEGES IN SCHEMA sith GRANT USAGE, SELECT ON SEQUENCES TO " + identifier, + "REVOKE UPDATE, DELETE ON sith.policy_audit_entries FROM " + identifier, + "REVOKE UPDATE (workspace_id, sequence, format_version, recorded_at, trace_id, actor, role, action, verb, verdict, reason_code, event_kind, evidence_digest, previous_hash, entry_hash) ON sith.policy_audit_entries FROM " + identifier, + "REVOKE DELETE ON sith.policy_audit_heads FROM " + identifier, + "REVOKE UPDATE, DELETE ON sith.approval_grants FROM " + identifier, + "GRANT UPDATE (consumed_at) ON sith.approval_grants TO " + identifier, } for _, statement := range statements { if _, err := tx.Exec(ctx, statement); err != nil { diff --git a/internal/hubdb/migrations/0009_policy_audit_chain.sql b/internal/hubdb/migrations/0009_policy_audit_chain.sql new file mode 100644 index 0000000..a525986 --- /dev/null +++ b/internal/hubdb/migrations/0009_policy_audit_chain.sql @@ -0,0 +1,57 @@ +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE sith.policy_audit_heads ( + workspace_id text PRIMARY KEY REFERENCES sith.workspaces(id) ON DELETE RESTRICT, + last_sequence bigint NOT NULL DEFAULT 0, + last_hash bytea NOT NULL DEFAULT decode(repeat('00', 32), 'hex'), + CONSTRAINT policy_audit_heads_sequence_valid CHECK (last_sequence >= 0), + CONSTRAINT policy_audit_heads_hash_valid CHECK (octet_length(last_hash) = 32) +); + +CREATE TABLE sith.policy_audit_entries ( + workspace_id text NOT NULL REFERENCES sith.policy_audit_heads(workspace_id) ON DELETE RESTRICT, + sequence bigint NOT NULL, + format_version smallint NOT NULL, + recorded_at timestamptz NOT NULL, + trace_id text NOT NULL, + actor text NOT NULL, + role text NOT NULL, + action text NOT NULL, + verb text NOT NULL, + verdict text NOT NULL, + reason_code text NOT NULL, + previous_hash bytea NOT NULL, + entry_hash bytea NOT NULL, + PRIMARY KEY (workspace_id, sequence), + CONSTRAINT policy_audit_entries_sequence_valid CHECK (sequence > 0), + CONSTRAINT policy_audit_entries_format_valid CHECK (format_version = 1), + CONSTRAINT policy_audit_entries_trace_valid CHECK (trace_id ~ '^[0-9a-f]{32}$'), + CONSTRAINT policy_audit_entries_actor_valid CHECK ( + actor = btrim(actor) AND actor <> '' AND octet_length(actor) <= 256 AND actor !~ '[[:cntrl:]]' + ), + CONSTRAINT policy_audit_entries_role_valid CHECK (role IN ('reader', 'operator', 'approver', 'admin')), + CONSTRAINT policy_audit_entries_action_valid CHECK (action IN ('read', 'propose-intent')), + CONSTRAINT policy_audit_entries_verb_valid CHECK ( + verb = btrim(verb) AND verb <> '' AND octet_length(verb) <= 128 AND verb !~ '[[:cntrl:]]' + ), + CONSTRAINT policy_audit_entries_verdict_valid CHECK (verdict IN ('allow', 'deny', 'require-approval')), + CONSTRAINT policy_audit_entries_reason_valid CHECK ( + reason_code ~ '^[a-z0-9_.-]+$' AND octet_length(reason_code) <= 64 + ), + CONSTRAINT policy_audit_entries_previous_hash_valid CHECK (octet_length(previous_hash) = 32), + CONSTRAINT policy_audit_entries_entry_hash_valid CHECK (octet_length(entry_hash) = 32) +); + +ALTER TABLE sith.policy_audit_heads ENABLE ROW LEVEL SECURITY; +ALTER TABLE sith.policy_audit_heads FORCE ROW LEVEL SECURITY; +CREATE POLICY workspace_isolation ON sith.policy_audit_heads + FOR ALL TO PUBLIC + USING (workspace_id = current_setting('sith.workspace_id', true)) + WITH CHECK (workspace_id = current_setting('sith.workspace_id', true)); + +ALTER TABLE sith.policy_audit_entries ENABLE ROW LEVEL SECURITY; +ALTER TABLE sith.policy_audit_entries FORCE ROW LEVEL SECURITY; +CREATE POLICY workspace_isolation ON sith.policy_audit_entries + FOR ALL TO PUBLIC + USING (workspace_id = current_setting('sith.workspace_id', true)) + WITH CHECK (workspace_id = current_setting('sith.workspace_id', true)); diff --git a/internal/hubdb/migrations/0010_approval_grants.sql b/internal/hubdb/migrations/0010_approval_grants.sql new file mode 100644 index 0000000..a3d8db1 --- /dev/null +++ b/internal/hubdb/migrations/0010_approval_grants.sql @@ -0,0 +1,40 @@ +-- SPDX-License-Identifier: Apache-2.0 + +CREATE TABLE sith.approval_grants ( + workspace_id text NOT NULL REFERENCES sith.workspaces(id) ON DELETE CASCADE, + id text NOT NULL, + intent_id text NOT NULL, + proposer text NOT NULL, + approver text NOT NULL, + resolved_digest text NOT NULL, + approved_at timestamptz NOT NULL, + consumed_at timestamptz, + PRIMARY KEY (workspace_id, id), + UNIQUE (workspace_id, intent_id, approver, resolved_digest), + CONSTRAINT approval_grants_id_valid CHECK (id ~ '^[A-Za-z0-9_-]{22}$'), + CONSTRAINT approval_grants_intent_valid CHECK ( + intent_id = btrim(intent_id) AND intent_id <> '' AND octet_length(intent_id) <= 253 + AND intent_id !~ '[[:cntrl:]]' + ), + CONSTRAINT approval_grants_proposer_valid CHECK ( + proposer = btrim(proposer) AND proposer <> '' AND octet_length(proposer) <= 256 + AND proposer !~ '[[:cntrl:]]' + ), + CONSTRAINT approval_grants_approver_valid CHECK ( + approver = btrim(approver) AND approver <> '' AND octet_length(approver) <= 256 + AND approver !~ '[[:cntrl:]]' + ), + CONSTRAINT approval_grants_separation_of_duty CHECK (proposer <> approver), + CONSTRAINT approval_grants_digest_valid CHECK (resolved_digest ~ '^sha256:[0-9a-f]{64}$'), + CONSTRAINT approval_grants_consumption_valid CHECK (consumed_at IS NULL OR consumed_at >= approved_at) +); +CREATE INDEX approval_grants_pending_intent_idx + ON sith.approval_grants (workspace_id, intent_id, resolved_digest) + WHERE consumed_at IS NULL; + +ALTER TABLE sith.approval_grants ENABLE ROW LEVEL SECURITY; +ALTER TABLE sith.approval_grants FORCE ROW LEVEL SECURITY; +CREATE POLICY workspace_isolation ON sith.approval_grants + FOR ALL TO PUBLIC + USING (workspace_id = current_setting('sith.workspace_id', true)) + WITH CHECK (workspace_id = current_setting('sith.workspace_id', true)); diff --git a/internal/hubdb/migrations/0011_approval_lifecycle_audit.sql b/internal/hubdb/migrations/0011_approval_lifecycle_audit.sql new file mode 100644 index 0000000..9bf425e --- /dev/null +++ b/internal/hubdb/migrations/0011_approval_lifecycle_audit.sql @@ -0,0 +1,43 @@ +-- SPDX-License-Identifier: Apache-2.0 + +ALTER TABLE sith.policy_audit_entries + ADD COLUMN event_kind text, + ADD COLUMN evidence_digest text; + +UPDATE sith.policy_audit_entries +SET event_kind = 'policy-decision', evidence_digest = ''; + +ALTER TABLE sith.policy_audit_entries + ALTER COLUMN event_kind SET NOT NULL, + ALTER COLUMN event_kind SET DEFAULT 'policy-decision', + ALTER COLUMN evidence_digest SET NOT NULL, + ALTER COLUMN evidence_digest SET DEFAULT '', + DROP CONSTRAINT policy_audit_entries_format_valid, + DROP CONSTRAINT policy_audit_entries_action_valid, + ADD CONSTRAINT policy_audit_entries_format_valid CHECK (format_version IN (1, 2)), + ADD CONSTRAINT policy_audit_entries_action_valid CHECK ( + action IN ('read', 'propose-intent', 'approve-intent') + ), + ADD CONSTRAINT policy_audit_entries_kind_valid CHECK ( + event_kind IN ('policy-decision', 'approval-created', 'approval-consumed') + ), + ADD CONSTRAINT policy_audit_entries_evidence_valid CHECK ( + (format_version = 1 AND event_kind = 'policy-decision' AND evidence_digest = '') + OR + (format_version = 2 AND event_kind IN ('approval-created', 'approval-consumed') + AND evidence_digest ~ '^sha256:[0-9a-f]{64}$') + ), + ADD CONSTRAINT policy_audit_entries_lifecycle_shape_valid CHECK ( + format_version = 1 + OR + (format_version = 2 AND verb = 'approval.grant' AND verdict = 'allow' + AND reason_code = event_kind + AND ( + (event_kind = 'approval-created' AND role = 'approver' AND action = 'approve-intent') + OR + (event_kind = 'approval-consumed' AND role = 'operator' AND action = 'propose-intent') + )) + ); + +-- Defaults let pre-0011 writers continue appending format-1 decisions while the schema rolls +-- forward. Upgrade every verifier before enabling format-2 approval lifecycle writers. diff --git a/internal/hubdb/migrations/0012_audit_export_action.sql b/internal/hubdb/migrations/0012_audit_export_action.sql new file mode 100644 index 0000000..ff14875 --- /dev/null +++ b/internal/hubdb/migrations/0012_audit_export_action.sql @@ -0,0 +1,7 @@ +-- SPDX-License-Identifier: Apache-2.0 + +ALTER TABLE sith.policy_audit_entries + DROP CONSTRAINT policy_audit_entries_action_valid, + ADD CONSTRAINT policy_audit_entries_action_valid CHECK ( + action IN ('read', 'export-audit', 'propose-intent', 'approve-intent') + ); diff --git a/internal/hubdb/migrations/0013_approval_grant_expiry.sql b/internal/hubdb/migrations/0013_approval_grant_expiry.sql new file mode 100644 index 0000000..f7da293 --- /dev/null +++ b/internal/hubdb/migrations/0013_approval_grant_expiry.sql @@ -0,0 +1,48 @@ +-- SPDX-License-Identifier: Apache-2.0 + +ALTER TABLE sith.approval_grants + ADD COLUMN expires_at timestamptz, + ADD COLUMN evidence_version smallint; + +-- Existing rows retain their historical format-2 evidence. They receive a forensic expiry value, +-- but the format marker keeps every legacy unconsumed grant outside the new consumption predicate. +-- ApplyMigrations holds one serializable transaction; ALTER takes an access-exclusive lock, and a +-- failure rolls this owner-only FORCE-RLS relaxation back before any application can observe it. +ALTER TABLE sith.approval_grants NO FORCE ROW LEVEL SECURITY; +UPDATE sith.approval_grants +SET expires_at = approved_at + interval '10 minutes', evidence_version = 1; +ALTER TABLE sith.approval_grants FORCE ROW LEVEL SECURITY; + +ALTER TABLE sith.approval_grants + ALTER COLUMN expires_at SET NOT NULL, + ALTER COLUMN evidence_version SET NOT NULL, + ADD CONSTRAINT approval_grants_expiry_valid CHECK ( + expires_at = approved_at + interval '10 minutes' + ), + ADD CONSTRAINT approval_grants_evidence_version_valid CHECK (evidence_version IN (1, 2)), + ADD CONSTRAINT approval_grants_v2_consumption_window_valid CHECK ( + evidence_version = 1 OR consumed_at IS NULL OR consumed_at < expires_at + ); + +ALTER TABLE sith.policy_audit_entries + DROP CONSTRAINT policy_audit_entries_format_valid, + DROP CONSTRAINT policy_audit_entries_evidence_valid, + DROP CONSTRAINT policy_audit_entries_lifecycle_shape_valid, + ADD CONSTRAINT policy_audit_entries_format_valid CHECK (format_version IN (1, 2, 3)), + ADD CONSTRAINT policy_audit_entries_evidence_valid CHECK ( + (format_version = 1 AND event_kind = 'policy-decision' AND evidence_digest = '') + OR + (format_version IN (2, 3) AND event_kind IN ('approval-created', 'approval-consumed') + AND evidence_digest ~ '^sha256:[0-9a-f]{64}$') + ), + ADD CONSTRAINT policy_audit_entries_lifecycle_shape_valid CHECK ( + format_version = 1 + OR + (format_version IN (2, 3) AND verb = 'approval.grant' AND verdict = 'allow' + AND reason_code = event_kind + AND ( + (event_kind = 'approval-created' AND role = 'approver' AND action = 'approve-intent') + OR + (event_kind = 'approval-consumed' AND role = 'operator' AND action = 'propose-intent') + )) + ); diff --git a/internal/hubdb/policy_audit.go b/internal/hubdb/policy_audit.go new file mode 100644 index 0000000..47f5e8c --- /dev/null +++ b/internal/hubdb/policy_audit.go @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubdb + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/ArdurAI/sith/internal/auditrecord" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +const ( + policyAuditFormatVersion int16 = 1 + approvalAuditFormatVersion int16 = 2 + approvalExpiryAuditFormatVersion int16 = 3 + approvalEvidenceHashDomain = "sith-approval-grant-evidence/v1" + approvalExpiryEvidenceHashDomain = "sith-approval-grant-evidence/v2" + policyDecisionEventKind = "policy-decision" + approvalCreatedEventKind = "approval-created" + approvalConsumedEventKind = "approval-consumed" + approvalAuditVerb pep.Verb = "approval.grant" +) + +var approvalEvidencePattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +type policyAuditEntry struct { + sequence int64 + format int16 + recordedAt time.Time + traceID tracing.ID + workspaceID tenancy.WorkspaceID + actor string + role tenancy.Role + action tenancy.Action + verb pep.Verb + verdict pep.Verdict + reasonCode string + eventKind string + evidence string + previousHash []byte + entryHash []byte +} + +// Record implements pep.Auditor by atomically appending one privacy-minimized PEP decision to +// the active workspace chain. The mutable head serializes same-workspace writers; history rows are +// append-only at the application-role privilege boundary. +func (database *AppDB) Record(ctx context.Context, event pep.AuditEvent) error { + if database == nil || database.pool == nil || ctx == nil { + return fmt.Errorf("append policy audit: database and context are required") + } + if err := event.Validate(); err != nil { + return fmt.Errorf("append policy audit: invalid event: %w", err) + } + scope, err := auditEventScope(event) + if err != nil { + return fmt.Errorf("append policy audit: derive workspace scope: %w", err) + } + + return database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { + entry := policyAuditEntry{ + format: policyAuditFormatVersion, + recordedAt: event.At.UTC().Truncate(time.Microsecond), traceID: event.TraceID, + workspaceID: event.WorkspaceID, actor: event.Actor, role: event.Role, action: event.Action, + verb: event.Verb, verdict: event.Verdict, reasonCode: event.ReasonCode, + eventKind: policyDecisionEventKind, + } + return appendPolicyAuditEntryTx(ctx, tx, entry) + }) +} + +func appendPolicyAuditEntryTx(ctx context.Context, tx pgx.Tx, entry policyAuditEntry) error { + if ctx == nil || tx == nil { + return fmt.Errorf("append policy audit entry: context and transaction are required") + } + if err := validatePolicyAuditEntry(entry); err != nil { + return fmt.Errorf("append policy audit entry: %w", err) + } + workspaceID := entry.workspaceID + if _, err := tx.Exec(ctx, ` + INSERT INTO sith.policy_audit_heads(workspace_id) + VALUES ($1) + ON CONFLICT (workspace_id) DO NOTHING + `, workspaceID); err != nil { + return fmt.Errorf("initialize policy audit chain: %w", err) + } + + var lastSequence int64 + var lastHash []byte + if err := tx.QueryRow(ctx, ` + SELECT last_sequence, last_hash + FROM sith.policy_audit_heads + WHERE workspace_id = $1 + FOR UPDATE + `, workspaceID).Scan(&lastSequence, &lastHash); err != nil { + return fmt.Errorf("lock policy audit chain: %w", err) + } + if lastSequence < 0 || lastSequence == math.MaxInt64 || len(lastHash) != sha256.Size { + return fmt.Errorf("policy audit chain head is invalid") + } + + entry.sequence = lastSequence + 1 + entry.previousHash = bytes.Clone(lastHash) + entry.entryHash = policyAuditEntryHash(entry) + if len(entry.entryHash) != sha256.Size { + return fmt.Errorf("hash policy audit entry: portable audit hash is invalid") + } + + if _, err := tx.Exec(ctx, ` + INSERT INTO sith.policy_audit_entries( + workspace_id, sequence, format_version, recorded_at, trace_id, actor, role, action, + verb, verdict, reason_code, event_kind, evidence_digest, previous_hash, entry_hash + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) + `, entry.workspaceID, entry.sequence, entry.format, entry.recordedAt, entry.traceID, + entry.actor, entry.role, entry.action, entry.verb, entry.verdict, entry.reasonCode, + entry.eventKind, entry.evidence, entry.previousHash, entry.entryHash); err != nil { + return fmt.Errorf("insert policy audit entry: %w", err) + } + tag, err := tx.Exec(ctx, ` + UPDATE sith.policy_audit_heads + SET last_sequence = $2, last_hash = $3 + WHERE workspace_id = $1 AND last_sequence = $4 AND last_hash = $5 + `, workspaceID, entry.sequence, entry.entryHash, lastSequence, lastHash) + if err != nil { + return fmt.Errorf("advance policy audit chain: %w", err) + } + if tag.RowsAffected() != 1 { + return fmt.Errorf("advance policy audit chain: locked head changed unexpectedly") + } + return nil +} + +// VerifyPolicyAuditChain checks the complete retained chain and head in one repeatable-read +// workspace snapshot. It detects retained-history mutation, deletion, reordering, and forked links; +// external anchoring is required to detect wholesale replacement by a privileged database owner. +func (database *AppDB) VerifyPolicyAuditChain(ctx context.Context, scope tenancy.Scope) error { + if database == nil || database.pool == nil || ctx == nil { + return fmt.Errorf("verify policy audit chain: database and context are required") + } + return database.inWorkspaceReadSnapshot(ctx, scope, func(tx pgx.Tx) error { + _, _, _, err := readVerifiedPolicyAuditChainTx(ctx, tx, scope, 0) + return err + }) +} + +// ExportPolicyAuditChain returns one complete, verified, privacy-minimized workspace chain. The +// fixed entry ceiling bounds memory and scan cost, and inWorkspaceReadSnapshot commits before this +// method returns, so an HTTP caller cannot hold the database transaction open while encoding. +func (database *AppDB) ExportPolicyAuditChain(ctx context.Context, scope tenancy.Scope) (auditrecord.Export, error) { + if database == nil || database.pool == nil || ctx == nil { + return auditrecord.Export{}, fmt.Errorf("export policy audit chain: database and context are required") + } + if err := scope.Authorize(tenancy.ActionExportAudit); err != nil { + return auditrecord.Export{}, fmt.Errorf("export policy audit chain: admin scope is required") + } + + var result auditrecord.Export + err := database.inWorkspaceReadSnapshot(ctx, scope, func(tx pgx.Tx) error { + entries, headSequence, headHash, err := readVerifiedPolicyAuditChainTx(ctx, tx, scope, auditrecord.MaxEntries) + if err != nil { + return err + } + result = newPolicyAuditExport(scope.WorkspaceID(), entries, headSequence, headHash) + return nil + }) + if err != nil { + return auditrecord.Export{}, fmt.Errorf("export policy audit chain: %w", err) + } + return result, nil +} + +// ExportPolicyAuditPage returns at most MaxEntries consecutive records from one immutable retained +// workspace snapshot. A continuation is validated against retained head and boundary rows inside +// the same forced-RLS repeatable-read transaction; later appends cannot move the declared snapshot. +func (database *AppDB) ExportPolicyAuditPage( + ctx context.Context, + scope tenancy.Scope, + request auditrecord.PageRequest, +) (auditrecord.Page, error) { + if database == nil || database.pool == nil || ctx == nil { + return auditrecord.Page{}, fmt.Errorf("export policy audit page: database and context are required") + } + if err := scope.Authorize(tenancy.ActionExportAudit); err != nil { + return auditrecord.Page{}, fmt.Errorf("export policy audit page: admin scope is required") + } + if err := request.ValidateForWorkspace(scope.WorkspaceID()); err != nil { + return auditrecord.Page{}, fmt.Errorf("export policy audit page: continuation is invalid") + } + + var result auditrecord.Page + err := database.inWorkspaceReadSnapshot(ctx, scope, func(tx pgx.Tx) error { + page, err := readVerifiedPolicyAuditPageTx(ctx, tx, scope, request) + if err != nil { + return err + } + result = page + return nil + }) + if err != nil { + return auditrecord.Page{}, fmt.Errorf("export policy audit page: %w", err) + } + return result, nil +} + +func readVerifiedPolicyAuditPageTx( + ctx context.Context, + tx pgx.Tx, + scope tenancy.Scope, + request auditrecord.PageRequest, +) (auditrecord.Page, error) { + var currentHeadSequence int64 + var currentHeadHash []byte + if err := tx.QueryRow(ctx, ` + SELECT last_sequence, last_hash + FROM sith.policy_audit_heads + WHERE workspace_id = $1 + `, scope.WorkspaceID()).Scan(¤tHeadSequence, ¤tHeadHash); err != nil { + if err == pgx.ErrNoRows { + return auditrecord.Page{}, fmt.Errorf("policy audit chain is not initialized") + } + return auditrecord.Page{}, fmt.Errorf("read policy audit chain head: %w", err) + } + if currentHeadSequence <= 0 || len(currentHeadHash) != sha256.Size { + return auditrecord.Page{}, fmt.Errorf("policy audit chain head is invalid") + } + var retainedCurrentHeadHash []byte + if err := tx.QueryRow(ctx, ` + SELECT entry_hash + FROM sith.policy_audit_entries + WHERE workspace_id = $1 AND sequence = $2 + `, scope.WorkspaceID(), currentHeadSequence).Scan(&retainedCurrentHeadHash); err != nil { + return auditrecord.Page{}, fmt.Errorf("read current policy audit head anchor: %w", err) + } + if len(retainedCurrentHeadHash) != sha256.Size || !bytes.Equal(currentHeadHash, retainedCurrentHeadHash) { + return auditrecord.Page{}, fmt.Errorf("current policy audit head anchor is invalid") + } + + snapshotHeadSequence := request.HeadSequence() + snapshotHeadHash := request.HeadHash() + nextSequence := request.NextSequence() + previousHash := request.PreviousHash() + if request.Initial() { + snapshotHeadSequence = currentHeadSequence + snapshotHeadHash = auditHashString(currentHeadHash) + nextSequence = 1 + previousHash = auditHashString(make([]byte, sha256.Size)) + } else { + if snapshotHeadSequence > currentHeadSequence || nextSequence > snapshotHeadSequence { + return auditrecord.Page{}, fmt.Errorf("policy audit page snapshot is unavailable") + } + var retainedPreviousHash []byte + if err := tx.QueryRow(ctx, ` + SELECT entry_hash + FROM sith.policy_audit_entries + WHERE workspace_id = $1 AND sequence = $2 + `, scope.WorkspaceID(), nextSequence-1).Scan(&retainedPreviousHash); err != nil { + return auditrecord.Page{}, fmt.Errorf("read policy audit page previous anchor: %w", err) + } + if len(retainedPreviousHash) != sha256.Size || auditHashString(retainedPreviousHash) != previousHash { + return auditrecord.Page{}, fmt.Errorf("policy audit page previous anchor is invalid") + } + } + retainedSnapshotHash := retainedCurrentHeadHash + if snapshotHeadSequence != currentHeadSequence { + retainedSnapshotHash = nil + if err := tx.QueryRow(ctx, ` + SELECT entry_hash + FROM sith.policy_audit_entries + WHERE workspace_id = $1 AND sequence = $2 + `, scope.WorkspaceID(), snapshotHeadSequence).Scan(&retainedSnapshotHash); err != nil { + return auditrecord.Page{}, fmt.Errorf("read policy audit page snapshot anchor: %w", err) + } + } + if len(retainedSnapshotHash) != sha256.Size || auditHashString(retainedSnapshotHash) != snapshotHeadHash { + return auditrecord.Page{}, fmt.Errorf("policy audit page snapshot anchor is invalid") + } + + endSequence := snapshotHeadSequence + if snapshotHeadSequence-nextSequence+1 > int64(auditrecord.MaxEntries) { + endSequence = nextSequence + int64(auditrecord.MaxEntries) - 1 + } + rows, err := tx.Query(ctx, ` + SELECT sequence, format_version, recorded_at, trace_id, workspace_id, actor, role, + action, verb, verdict, reason_code, event_kind, evidence_digest, + previous_hash, entry_hash + FROM sith.policy_audit_entries + WHERE workspace_id = $1 AND sequence >= $2 AND sequence <= $3 + ORDER BY sequence + `, scope.WorkspaceID(), nextSequence, endSequence) + if err != nil { + return auditrecord.Page{}, fmt.Errorf("read policy audit page: %w", err) + } + defer rows.Close() + + entries := make([]policyAuditEntry, 0, int(endSequence-nextSequence+1)) + expectedSequence := nextSequence + expectedPrevious := previousHash + for rows.Next() { + var entry policyAuditEntry + if err := rows.Scan(&entry.sequence, &entry.format, &entry.recordedAt, &entry.traceID, + &entry.workspaceID, &entry.actor, &entry.role, &entry.action, &entry.verb, + &entry.verdict, &entry.reasonCode, &entry.eventKind, &entry.evidence, + &entry.previousHash, &entry.entryHash); err != nil { + return auditrecord.Page{}, fmt.Errorf("scan policy audit page: %w", err) + } + if entry.sequence != expectedSequence || entry.workspaceID != scope.WorkspaceID() || + len(entry.previousHash) != sha256.Size || len(entry.entryHash) != sha256.Size || + auditHashString(entry.previousHash) != expectedPrevious { + return auditrecord.Page{}, fmt.Errorf("policy audit page continuity is invalid at sequence %d", expectedSequence) + } + if err := validatePolicyAuditEntry(entry); err != nil { + return auditrecord.Page{}, fmt.Errorf("policy audit page event is invalid at sequence %d", expectedSequence) + } + calculated := policyAuditEntryHash(entry) + if len(calculated) != sha256.Size || !bytes.Equal(entry.entryHash, calculated) { + return auditrecord.Page{}, fmt.Errorf("policy audit page hash is invalid at sequence %d", expectedSequence) + } + entries = append(entries, entry) + expectedPrevious = auditHashString(entry.entryHash) + expectedSequence++ + } + if err := rows.Err(); err != nil { + return auditrecord.Page{}, fmt.Errorf("iterate policy audit page: %w", err) + } + if expectedSequence != endSequence+1 || len(entries) != int(endSequence-nextSequence+1) { + return auditrecord.Page{}, fmt.Errorf("policy audit page retained range is incomplete") + } + if endSequence == snapshotHeadSequence && expectedPrevious != snapshotHeadHash { + return auditrecord.Page{}, fmt.Errorf("policy audit page head does not match snapshot") + } + return newPolicyAuditPage( + scope.WorkspaceID(), entries, snapshotHeadSequence, snapshotHeadHash, nextSequence, previousHash, + ) +} + +func readVerifiedPolicyAuditChainTx( + ctx context.Context, + tx pgx.Tx, + scope tenancy.Scope, + retainLimit int, +) ([]policyAuditEntry, int64, []byte, error) { + var headSequence int64 + var headHash []byte + if err := tx.QueryRow(ctx, ` + SELECT last_sequence, last_hash + FROM sith.policy_audit_heads + WHERE workspace_id = $1 + `, scope.WorkspaceID()).Scan(&headSequence, &headHash); err != nil { + if err == pgx.ErrNoRows { + return nil, 0, nil, fmt.Errorf("policy audit chain is not initialized") + } + return nil, 0, nil, fmt.Errorf("read policy audit chain head: %w", err) + } + if headSequence <= 0 || len(headHash) != sha256.Size { + return nil, 0, nil, fmt.Errorf("policy audit chain head is invalid") + } + if retainLimit < 0 || (retainLimit > 0 && headSequence > int64(retainLimit)) { + return nil, 0, nil, fmt.Errorf("policy audit chain exceeds the online export limit") + } + + rows, err := tx.Query(ctx, ` + SELECT sequence, format_version, recorded_at, trace_id, workspace_id, actor, role, + action, verb, verdict, reason_code, event_kind, evidence_digest, + previous_hash, entry_hash + FROM sith.policy_audit_entries + WHERE workspace_id = $1 + ORDER BY sequence + `, scope.WorkspaceID()) + if err != nil { + return nil, 0, nil, fmt.Errorf("read policy audit chain: %w", err) + } + defer rows.Close() + + var retained []policyAuditEntry + if retainLimit > 0 { + retained = make([]policyAuditEntry, 0, int(headSequence)) + } + expectedSequence := int64(1) + expectedPrevious := make([]byte, sha256.Size) + for rows.Next() { + var entry policyAuditEntry + if err := rows.Scan(&entry.sequence, &entry.format, &entry.recordedAt, &entry.traceID, + &entry.workspaceID, &entry.actor, &entry.role, &entry.action, &entry.verb, + &entry.verdict, &entry.reasonCode, &entry.eventKind, &entry.evidence, + &entry.previousHash, &entry.entryHash); err != nil { + return nil, 0, nil, fmt.Errorf("scan policy audit chain: %w", err) + } + if entry.sequence != expectedSequence || + entry.workspaceID != scope.WorkspaceID() || len(entry.previousHash) != sha256.Size || + len(entry.entryHash) != sha256.Size || !bytes.Equal(entry.previousHash, expectedPrevious) { + return nil, 0, nil, fmt.Errorf("policy audit chain continuity is invalid at sequence %d", expectedSequence) + } + if err := validatePolicyAuditEntry(entry); err != nil { + return nil, 0, nil, fmt.Errorf("policy audit chain event is invalid at sequence %d", expectedSequence) + } + calculated := policyAuditEntryHash(entry) + if len(calculated) != sha256.Size || !bytes.Equal(entry.entryHash, calculated) { + return nil, 0, nil, fmt.Errorf("policy audit chain hash is invalid at sequence %d", expectedSequence) + } + if retainLimit > 0 { + if len(retained) == retainLimit { + return nil, 0, nil, fmt.Errorf("policy audit chain exceeds the online export limit") + } + retained = append(retained, entry) + } + expectedPrevious = bytes.Clone(entry.entryHash) + expectedSequence++ + } + if err := rows.Err(); err != nil { + return nil, 0, nil, fmt.Errorf("iterate policy audit chain: %w", err) + } + retainedSequence := expectedSequence - 1 + if retainedSequence != headSequence || !bytes.Equal(expectedPrevious, headHash) { + return nil, 0, nil, fmt.Errorf("policy audit chain head does not match retained history") + } + return retained, headSequence, bytes.Clone(headHash), nil +} + +func newPolicyAuditExport( + workspaceID tenancy.WorkspaceID, + entries []policyAuditEntry, + headSequence int64, + headHash []byte, +) auditrecord.Export { + records := make([]auditrecord.Entry, len(entries)) + for index, entry := range entries { + records[index] = portablePolicyAuditEntry(entry) + } + return auditrecord.Export{ + Schema: auditrecord.SchemaV1, WorkspaceID: string(workspaceID), + Chain: auditrecord.Chain{ + HashAlgorithm: auditrecord.HashAlgorithm, HeadSequence: headSequence, + HeadHash: auditHashString(headHash), + }, + Entries: records, + } +} + +func newPolicyAuditPage( + workspaceID tenancy.WorkspaceID, + entries []policyAuditEntry, + headSequence int64, + headHash string, + startSequence int64, + previousHash string, +) (auditrecord.Page, error) { + records := make([]auditrecord.Entry, len(entries)) + for index, entry := range entries { + records[index] = portablePolicyAuditEntry(entry) + } + page := auditrecord.Page{ + Schema: auditrecord.PageSchemaV1, WorkspaceID: string(workspaceID), + Snapshot: auditrecord.Chain{ + HashAlgorithm: auditrecord.HashAlgorithm, HeadSequence: headSequence, HeadHash: headHash, + }, + StartSequence: startSequence, PreviousHash: previousHash, Entries: records, + } + endSequence := startSequence + int64(len(records)) - 1 + if endSequence < headSequence { + cursor, err := auditrecord.EncodePageCursor( + workspaceID, headSequence, headHash, endSequence+1, records[len(records)-1].EntryHash, + ) + if err != nil { + return auditrecord.Page{}, fmt.Errorf("construct policy audit page continuation: %w", err) + } + page.NextCursor = cursor + } + if err := page.VerifyForWorkspace(workspaceID); err != nil { + return auditrecord.Page{}, fmt.Errorf("construct policy audit page: %w", err) + } + return page, nil +} + +func auditHashString(value []byte) string { + return "sha256:" + hex.EncodeToString(value) +} + +func auditEventScope(event pep.AuditEvent) (tenancy.Scope, error) { + principal, err := tenancy.NewPrincipal(event.Actor, map[tenancy.WorkspaceID]tenancy.Role{ + event.WorkspaceID: event.Role, + }) + if err != nil { + return tenancy.Scope{}, err + } + return principal.Scope(event.WorkspaceID) +} + +func policyAuditEntryHash(entry policyAuditEntry) []byte { + recomputed, err := auditrecord.RecomputeEntryHash(entry.workspaceID, portablePolicyAuditEntry(entry)) + if err != nil { + return nil + } + decoded, err := hex.DecodeString(strings.TrimPrefix(recomputed, "sha256:")) + if err != nil || len(decoded) != sha256.Size { + return nil + } + return decoded +} + +func portablePolicyAuditEntry(entry policyAuditEntry) auditrecord.Entry { + return auditrecord.Entry{ + Sequence: entry.sequence, FormatVersion: entry.format, + RecordedAt: entry.recordedAt.UTC().Truncate(time.Microsecond), TraceID: string(entry.traceID), + Actor: entry.actor, Role: string(entry.role), Action: string(entry.action), Verb: string(entry.verb), + Verdict: string(entry.verdict), ReasonCode: entry.reasonCode, EventKind: entry.eventKind, + EvidenceDigest: entry.evidence, PreviousHash: auditHashString(entry.previousHash), + EntryHash: auditHashString(entry.entryHash), + } +} + +func validatePolicyAuditEntry(entry policyAuditEntry) error { + switch entry.format { + case policyAuditFormatVersion: + if entry.eventKind != policyDecisionEventKind || entry.evidence != "" { + return fmt.Errorf("format 1 policy audit metadata is invalid") + } + event := pep.AuditEvent{ + At: entry.recordedAt, TraceID: entry.traceID, WorkspaceID: entry.workspaceID, + Actor: entry.actor, Role: entry.role, Action: entry.action, Verb: entry.verb, + Verdict: entry.verdict, ReasonCode: entry.reasonCode, + } + return event.Validate() + case approvalAuditFormatVersion, approvalExpiryAuditFormatVersion: + if entry.recordedAt.IsZero() || entry.recordedAt.After(time.Now().Add(time.Minute)) || + !entry.traceID.Valid() || !approvalEvidencePattern.MatchString(entry.evidence) || + entry.verb != approvalAuditVerb || entry.verdict != pep.VerdictAllow || entry.reasonCode != entry.eventKind { + return fmt.Errorf("approval lifecycle audit metadata is invalid") + } + principal, err := tenancy.NewPrincipal(entry.actor, map[tenancy.WorkspaceID]tenancy.Role{ + entry.workspaceID: entry.role, + }) + if err != nil { + return fmt.Errorf("approval lifecycle audit scope: %w", err) + } + if _, err := principal.Scope(entry.workspaceID); err != nil { + return fmt.Errorf("approval lifecycle audit scope: %w", err) + } + switch entry.eventKind { + case approvalCreatedEventKind: + if entry.role != tenancy.RoleApprover || entry.action != tenancy.ActionApproveIntent { + return fmt.Errorf("approval-created lifecycle actor is invalid") + } + case approvalConsumedEventKind: + if entry.role != tenancy.RoleOperator || entry.action != tenancy.ActionProposeIntent { + return fmt.Errorf("approval-consumed lifecycle actor is invalid") + } + default: + return fmt.Errorf("approval lifecycle event kind is invalid") + } + return nil + default: + return fmt.Errorf("policy audit format is unsupported") + } +} + +func approvalGrantEvidenceDigest( + workspaceID tenancy.WorkspaceID, + identifier ApprovalGrantID, + intentID, proposer, approver, resolvedDigest string, + approvedAt time.Time, +) string { + canonical := make([]byte, 0, 512) + for _, value := range []string{ + approvalEvidenceHashDomain, string(workspaceID), identifier.String(), intentID, + proposer, approver, resolvedDigest, approvedAt.UTC().Truncate(time.Microsecond).Format(time.RFC3339Nano), + } { + canonical = appendCanonicalString(canonical, value) + } + digest := sha256.Sum256(canonical) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +func expiringApprovalGrantEvidenceDigest( + workspaceID tenancy.WorkspaceID, + identifier ApprovalGrantID, + intentID, proposer, approver, resolvedDigest string, + approvedAt, expiresAt time.Time, +) string { + canonical := make([]byte, 0, 512) + for _, value := range []string{ + approvalExpiryEvidenceHashDomain, string(workspaceID), identifier.String(), intentID, + proposer, approver, resolvedDigest, + approvedAt.UTC().Truncate(time.Microsecond).Format(time.RFC3339Nano), + expiresAt.UTC().Truncate(time.Microsecond).Format(time.RFC3339Nano), + } { + canonical = appendCanonicalString(canonical, value) + } + digest := sha256.Sum256(canonical) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +func appendCanonicalString(target []byte, value string) []byte { + target = strconv.AppendInt(target, int64(len(value)), 10) + target = append(target, ':') + return append(target, value...) +} diff --git a/internal/hubdb/policy_audit_test.go b/internal/hubdb/policy_audit_test.go new file mode 100644 index 0000000..24124e6 --- /dev/null +++ b/internal/hubdb/policy_audit_test.go @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubdb + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "io/fs" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/auditrecord" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +func TestPolicyAuditEntryHashBindsEveryField(t *testing.T) { + t.Parallel() + + base := policyAuditTestEntry() + baseline := policyAuditEntryHash(base) + if len(baseline) != 32 || !bytes.Equal(baseline, policyAuditEntryHash(base)) { + t.Fatal("policy audit entry hash is not a stable SHA-256 digest") + } + want, err := hex.DecodeString("061c65c9a8af1fa2a8334e86c10f82845c4d8bbeb3f2cf840b2be47c7e1edc8d") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(baseline, want) { + t.Fatalf("format 1 policy audit hash changed: got %x, want %x", baseline, want) + } + + mutations := map[string]func(*policyAuditEntry){ + "sequence": func(entry *policyAuditEntry) { entry.sequence++ }, + "format": func(entry *policyAuditEntry) { entry.format++ }, + "recorded time": func(entry *policyAuditEntry) { entry.recordedAt = entry.recordedAt.Add(time.Microsecond) }, + "trace": func(entry *policyAuditEntry) { entry.traceID = "ffffffffffffffffffffffffffffffff" }, + "workspace": func(entry *policyAuditEntry) { entry.workspaceID = "workspace-b" }, + "actor": func(entry *policyAuditEntry) { entry.actor = "user:bob" }, + "role": func(entry *policyAuditEntry) { entry.role = tenancy.RoleAdmin }, + "action": func(entry *policyAuditEntry) { entry.action = tenancy.ActionProposeIntent }, + "verb": func(entry *policyAuditEntry) { entry.verb = "deployment.restart" }, + "verdict": func(entry *policyAuditEntry) { entry.verdict = pep.VerdictDeny }, + "reason": func(entry *policyAuditEntry) { entry.reasonCode = "policy-deny" }, + "previous hash": func(entry *policyAuditEntry) { entry.previousHash[0] = 1 }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + changed := base + changed.previousHash = bytes.Clone(base.previousHash) + mutate(&changed) + if bytes.Equal(baseline, policyAuditEntryHash(changed)) { + t.Fatalf("hash did not bind %s", name) + } + }) + } +} + +func TestApprovalAuditEntryHashBindsLifecycleMetadata(t *testing.T) { + t.Parallel() + + base := policyAuditTestEntry() + base.format = approvalAuditFormatVersion + base.role = tenancy.RoleApprover + base.action = tenancy.ActionApproveIntent + base.verb = approvalAuditVerb + base.reasonCode = approvalCreatedEventKind + base.eventKind = approvalCreatedEventKind + base.evidence = "sha256:" + strings.Repeat("a", 64) + baseline := policyAuditEntryHash(base) + + for name, mutate := range map[string]func(*policyAuditEntry){ + "event kind": func(entry *policyAuditEntry) { + entry.eventKind = approvalConsumedEventKind + entry.reasonCode = approvalConsumedEventKind + }, + "evidence": func(entry *policyAuditEntry) { + entry.evidence = "sha256:" + strings.Repeat("b", 64) + }, + } { + t.Run(name, func(t *testing.T) { + changed := base + mutate(&changed) + if bytes.Equal(baseline, policyAuditEntryHash(changed)) { + t.Fatalf("format 2 hash did not bind %s", name) + } + }) + } +} + +func TestExpiringApprovalAuditEntryUsesDistinctFormatDomain(t *testing.T) { + t.Parallel() + + legacy := policyAuditTestEntry() + legacy.format = approvalAuditFormatVersion + legacy.role = tenancy.RoleApprover + legacy.action = tenancy.ActionApproveIntent + legacy.verb = approvalAuditVerb + legacy.reasonCode = approvalCreatedEventKind + legacy.eventKind = approvalCreatedEventKind + legacy.evidence = "sha256:" + strings.Repeat("a", 64) + + expiring := legacy + expiring.format = approvalExpiryAuditFormatVersion + legacyHash, expiringHash := policyAuditEntryHash(legacy), policyAuditEntryHash(expiring) + if len(legacyHash) != 32 || len(expiringHash) != 32 || bytes.Equal(legacyHash, expiringHash) { + t.Fatal("format 2 and format 3 lifecycle records did not use distinct valid hash domains") + } +} + +func TestApprovalGrantEvidenceDigestBindsImmutableGrant(t *testing.T) { + t.Parallel() + + approvedAt := time.Date(2026, time.July, 17, 12, 34, 56, 123456000, time.UTC) + base := approvalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", + "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt, + ) + if !approvalEvidencePattern.MatchString(base) { + t.Fatalf("evidence digest = %q, want canonical SHA-256", base) + } + mutations := []string{ + approvalGrantEvidenceDigest("workspace-b", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt), + approvalGrantEvidenceDigest("workspace-a", "BBBBBBBBBBBBBBBBBBBBBB", "intent-a", "user:operator", "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt), + approvalGrantEvidenceDigest("workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-b", "user:operator", "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt), + approvalGrantEvidenceDigest("workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:other", "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt), + approvalGrantEvidenceDigest("workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", "user:other", "sha256:"+strings.Repeat("a", 64), approvedAt), + approvalGrantEvidenceDigest("workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", "user:approver", "sha256:"+strings.Repeat("b", 64), approvedAt), + approvalGrantEvidenceDigest("workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt.Add(time.Microsecond)), + } + for index, changed := range mutations { + if changed == base { + t.Fatalf("evidence digest did not bind immutable field %d", index) + } + } +} + +func TestExpiringApprovalGrantEvidenceDigestBindsImmutableExpiry(t *testing.T) { + t.Parallel() + + approvedAt := time.Date(2026, time.July, 21, 12, 34, 56, 123456000, time.UTC) + expiresAt := approvedAt.Add(10 * time.Minute) + base := expiringApprovalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", + "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt, expiresAt, + ) + if !approvalEvidencePattern.MatchString(base) { + t.Fatalf("expiring evidence digest = %q, want canonical SHA-256", base) + } + if want := "sha256:25edbb61ecc55494ed155e14b30733b08ab090469a789b79eed3bf871ddbd1b4"; base != want { + t.Fatalf("expiring evidence digest = %q, want golden %q", base, want) + } + if base == approvalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", + "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt, + ) { + t.Fatal("expiring evidence reused the legacy evidence domain") + } + if changed := expiringApprovalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", "intent-a", "user:operator", + "user:approver", "sha256:"+strings.Repeat("a", 64), approvedAt, expiresAt.Add(time.Microsecond), + ); changed == base { + t.Fatal("expiring evidence digest did not bind expires_at") + } +} + +func TestPolicyAuditBoundaryRejectsMissingDatabaseAndInvalidScope(t *testing.T) { + t.Parallel() + + if err := (*AppDB)(nil).Record(context.Background(), pep.AuditEvent{}); err == nil { + t.Fatal("nil database accepted a policy audit event") + } + if err := (*AppDB)(nil).VerifyPolicyAuditChain(context.Background(), tenancy.Scope{}); err == nil { + t.Fatal("nil database verified a policy audit chain") + } + if _, err := (*AppDB)(nil).ExportPolicyAuditChain(context.Background(), tenancy.Scope{}); err == nil { + t.Fatal("nil database exported a policy audit chain") + } + if _, err := (*AppDB)(nil).ExportPolicyAuditPage(context.Background(), tenancy.Scope{}, auditrecord.PageRequest{}); err == nil { + t.Fatal("nil database exported a policy audit page") + } + if _, err := auditEventScope(pep.AuditEvent{}); err == nil { + t.Fatal("invalid audit event produced a workspace scope") + } +} + +func TestPolicyAuditPageProjectionIsPortableAndPrivacyMinimized(t *testing.T) { + t.Parallel() + + entry := policyAuditTestEntry() + entry.entryHash = policyAuditEntryHash(entry) + page, err := newPolicyAuditPage( + "workspace-a", []policyAuditEntry{entry}, 1, auditHashString(entry.entryHash), 1, + auditHashString(entry.previousHash), + ) + if err != nil { + t.Fatal(err) + } + if page.Schema != auditrecord.PageSchemaV1 || page.WorkspaceID != "workspace-a" || + page.Snapshot.HeadSequence != 1 || page.StartSequence != 1 || page.NextCursor != "" || + len(page.Entries) != 1 || page.Entries[0].EntryHash != page.Snapshot.HeadHash { + t.Fatalf("page = %#v", page) + } + if err := page.Verify(); err != nil { + t.Fatalf("portable page Verify() error = %v", err) + } + encoded, err := json.Marshal(page) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"arguments", "selector", "target", "credential", "token", "payload", "justification"} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("portable page leaked forbidden field %q: %s", forbidden, encoded) + } + } +} + +func TestPolicyAuditExportProjectionIsPortableAndPrivacyMinimized(t *testing.T) { + t.Parallel() + + entry := policyAuditTestEntry() + entry.entryHash = policyAuditEntryHash(entry) + exported := newPolicyAuditExport("workspace-a", []policyAuditEntry{entry}, 1, entry.entryHash) + if exported.Schema != auditrecord.SchemaV1 || exported.WorkspaceID != "workspace-a" || + exported.Chain.HashAlgorithm != auditrecord.HashAlgorithm || exported.Chain.HeadSequence != 1 || + exported.Chain.HeadHash != "sha256:"+hex.EncodeToString(entry.entryHash) || len(exported.Entries) != 1 { + t.Fatalf("export = %#v", exported) + } + if err := exported.Verify(); err != nil { + t.Fatalf("portable export Verify() error = %v", err) + } + record := exported.Entries[0] + if record.Sequence != 1 || record.FormatVersion != policyAuditFormatVersion || + record.RecordedAt != entry.recordedAt || record.TraceID != string(entry.traceID) || + record.Action != string(tenancy.ActionRead) || record.Verb != string(pep.VerbFleetRead) || + record.EventKind != policyDecisionEventKind || record.EvidenceDigest != "" || + record.PreviousHash != "sha256:"+strings.Repeat("0", 64) || record.EntryHash != exported.Chain.HeadHash { + t.Fatalf("portable record = %#v", record) + } + encoded, err := json.Marshal(exported) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"arguments", "selector", "target", "credential", "token", "payload", "justification"} { + if strings.Contains(string(encoded), forbidden) { + t.Fatalf("portable export leaked forbidden field %q: %s", forbidden, encoded) + } + } +} + +func TestAuditExportMigrationAddsOnlyClosedAction(t *testing.T) { + t.Parallel() + + migration, err := fs.ReadFile(migrationFiles, "migrations/0012_audit_export_action.sql") + if err != nil { + t.Fatal(err) + } + text := string(migration) + for _, required := range []string{ + "DROP CONSTRAINT policy_audit_entries_action_valid", + "action IN ('read', 'export-audit', 'propose-intent', 'approve-intent')", + } { + if !strings.Contains(text, required) { + t.Fatalf("audit export migration is missing %q", required) + } + } + upperText := strings.ToUpper(text) + for _, forbidden := range []string{"DISABLE ROW LEVEL SECURITY", "NO FORCE ROW LEVEL SECURITY", "GRANT", "DROP POLICY"} { + if strings.Contains(upperText, forbidden) { + t.Fatalf("audit export migration contains unsafe boundary change %q", forbidden) + } + } +} + +func FuzzPolicyAuditEntryHashUsesLengthFraming(f *testing.F) { + f.Add("user:alice", "phase-1-read") + f.Add("a", "bc") + f.Fuzz(func(t *testing.T, left, right string) { + if len(right) < 2 || len(left)+len(right) > 256 { + t.Skip() + } + first := policyAuditTestEntry() + first.actor, first.reasonCode = left, right + second := policyAuditTestEntry() + second.actor, second.reasonCode = left+right[:1], right[1:] + if validatePolicyAuditEntry(first) != nil || validatePolicyAuditEntry(second) != nil { + t.Skip() + } + firstHash, secondHash := policyAuditEntryHash(first), policyAuditEntryHash(second) + if len(firstHash) != 32 || len(secondHash) != 32 { + t.Fatal("valid audit entries did not produce SHA-256 digests") + } + if bytes.Equal(firstHash, secondHash) { + t.Fatal("length-delimited audit fields produced an ambiguous digest") + } + }) +} + +func FuzzApprovalGrantEvidenceDigestUsesLengthFraming(f *testing.F) { + f.Add("a", "bc") + f.Add("user:operator", "user:approver") + f.Fuzz(func(t *testing.T, left, right string) { + if left == "" || right == "" || len(left)+len(right) > 512 { + t.Skip() + } + approvedAt := time.Date(2026, time.July, 17, 12, 34, 56, 123456000, time.UTC) + first := approvalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", left, right, "user:approver", + "sha256:"+strings.Repeat("a", 64), approvedAt, + ) + second := approvalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", left+right, "user:operator", "user:approver", + "sha256:"+strings.Repeat("a", 64), approvedAt, + ) + if first == second { + t.Fatal("length-delimited approval evidence fields produced an ambiguous digest") + } + }) +} + +func FuzzExpiringApprovalGrantEvidenceDigestUsesLengthFraming(f *testing.F) { + f.Add("a", "bc") + f.Add("user:operator", "user:approver") + f.Fuzz(func(t *testing.T, left, right string) { + if left == "" || right == "" || len(left)+len(right) > 512 { + t.Skip() + } + approvedAt := time.Date(2026, time.July, 21, 12, 34, 56, 123456000, time.UTC) + expiresAt := approvedAt.Add(10 * time.Minute) + first := expiringApprovalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", left, right, "user:approver", + "sha256:"+strings.Repeat("a", 64), approvedAt, expiresAt, + ) + second := expiringApprovalGrantEvidenceDigest( + "workspace-a", "AAAAAAAAAAAAAAAAAAAAAA", left+right, "user:operator", "user:approver", + "sha256:"+strings.Repeat("a", 64), approvedAt, expiresAt, + ) + if first == second { + t.Fatal("length-delimited expiring approval evidence fields produced an ambiguous digest") + } + }) +} + +func policyAuditTestEntry() policyAuditEntry { + return policyAuditEntry{ + sequence: 1, format: policyAuditFormatVersion, + recordedAt: time.Date(2026, time.July, 17, 12, 34, 56, 123456000, time.UTC), + traceID: tracing.ID("0123456789abcdef0123456789abcdef"), + workspaceID: "workspace-a", actor: "user:alice", role: tenancy.RoleReader, + action: tenancy.ActionRead, verb: pep.VerbFleetRead, verdict: pep.VerdictAllow, + reasonCode: "phase-1-read", eventKind: policyDecisionEventKind, + previousHash: make([]byte, 32), + } +} diff --git a/internal/hubdb/postgres_integration_test.go b/internal/hubdb/postgres_integration_test.go index 4c25adb..55170e3 100644 --- a/internal/hubdb/postgres_integration_test.go +++ b/internal/hubdb/postgres_integration_test.go @@ -8,24 +8,32 @@ import ( "context" "crypto/ed25519" "crypto/rand" + "crypto/sha256" + "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" + "io/fs" "net/url" "os" "os/exec" "strings" + "sync" "testing" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + "github.com/ArdurAI/sith/internal/auditrecord" "github.com/ArdurAI/sith/internal/fleet" "github.com/ArdurAI/sith/internal/hubauth" "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/intent" "github.com/ArdurAI/sith/internal/pep" "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" ) const ( @@ -58,12 +66,73 @@ func TestPostgresRLSBackstop(t *testing.T) { ownerURL := databaseURL(adminURL, ownerRole, ownerPassword) owner := connectPostgres(t, ctx, ownerURL) defer owner.Close(context.Background()) + applyLegacyMigrations(t, ctx, owner, "0013_approval_grant_expiry.sql") + if err := pgx.BeginTxFunc(ctx, owner, pgx.TxOptions{}, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, `SELECT set_config('sith.workspace_id', 'workspace-legacy', true)`); err != nil { + return err + } + _, err := tx.Exec(ctx, ` + INSERT INTO sith.workspaces(id, name, tenant_key) + VALUES ('workspace-legacy', 'Legacy Workspace', 'legacy-display-key'); + INSERT INTO sith.approval_grants( + workspace_id, id, intent_id, proposer, approver, resolved_digest, approved_at + ) VALUES ( + 'workspace-legacy', 'LLLLLLLLLLLLLLLLLLLLLL', 'intent-legacy-upgrade', + 'user:legacy-operator', 'user:legacy-approver', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + statement_timestamp() - interval '1 hour' + ) + `) + return err + }); err != nil { + t.Fatalf("seed pre-0013 approval grant: %v", err) + } if err := Migrate(ctx, MigrationConfig{OwnerURL: ownerURL, ApplicationRole: appRole, AllowInsecureLocal: true}); err != nil { t.Fatalf("Migrate() error = %v", err) } + var legacyVersioned, legacyLifetimeFixed, legacyUnconsumed bool + if err := pgx.BeginTxFunc(ctx, owner, pgx.TxOptions{}, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, `SELECT set_config('sith.workspace_id', 'workspace-legacy', true)`); err != nil { + return err + } + return tx.QueryRow(ctx, ` + SELECT evidence_version = 1, + expires_at = approved_at + interval '10 minutes', + consumed_at IS NULL + FROM sith.approval_grants + WHERE workspace_id = 'workspace-legacy' AND id = 'LLLLLLLLLLLLLLLLLLLLLL' + `).Scan(&legacyVersioned, &legacyLifetimeFixed, &legacyUnconsumed) + }); err != nil { + t.Fatalf("inspect upgraded legacy approval grant: %v", err) + } + if !legacyVersioned || !legacyLifetimeFixed || !legacyUnconsumed { + t.Fatalf("legacy approval upgrade = versioned:%t lifetime:%t unconsumed:%t", + legacyVersioned, legacyLifetimeFixed, legacyUnconsumed) + } if err := Migrate(ctx, MigrationConfig{OwnerURL: ownerURL, ApplicationRole: appRole, AllowInsecureLocal: true}); err != nil { t.Fatalf("idempotent Migrate() error = %v", err) } + if _, err := owner.Exec(ctx, `GRANT UPDATE (actor) ON sith.policy_audit_entries TO `+pgx.Identifier{appRole}.Sanitize()); err != nil { + t.Fatalf("grant column-level audit mutation fixture: %v", err) + } + if err := AuditIsolation(ctx, owner, appRole); err == nil || !strings.Contains(err.Error(), "immutable application privilege contract is invalid") { + t.Fatalf("column-level audit mutation escaped catalog audit: %v", err) + } + if err := Migrate(ctx, MigrationConfig{OwnerURL: ownerURL, ApplicationRole: appRole, AllowInsecureLocal: true}); err != nil { + t.Fatalf("Migrate() did not repair column-level audit privilege drift: %v", err) + } + var canUpdateAuditColumn bool + if err := owner.QueryRow(ctx, ` + SELECT has_any_column_privilege($1, 'sith.policy_audit_entries', 'UPDATE') + `, appRole).Scan(&canUpdateAuditColumn); err != nil { + t.Fatalf("inspect repaired column-level audit privileges: %v", err) + } + if canUpdateAuditColumn { + t.Fatal("migration retained column-level UPDATE on immutable audit entries") + } + if err := AuditIsolation(ctx, owner, appRole); err != nil { + t.Fatalf("repaired AuditIsolation() error = %v", err) + } seedTenantRows(t, ctx, admin) if _, err := OpenAppDB(ctx, AppConfig{URL: databaseURL(adminURL, appRole, appPassword)}); err == nil { @@ -79,7 +148,7 @@ func TestPostgresRLSBackstop(t *testing.T) { } database, err := OpenAppDB(ctx, AppConfig{ - URL: databaseURL(adminURL, appRole, appPassword), MaxConns: 1, AllowInsecure: true, + URL: databaseURL(adminURL, appRole, appPassword), MaxConns: 2, AllowInsecure: true, }) if err != nil { t.Fatalf("OpenAppDB() error = %v", err) @@ -101,7 +170,10 @@ func TestPostgresRLSBackstop(t *testing.T) { t.Fatalf("workspace A query: %v", err) } - tables := []string{"workspaces", "memberships", "clusters", "fleet_facts", "api_keys", "oidc_bindings", "cloud_identity_bindings"} + tables := []string{ + "workspaces", "memberships", "clusters", "fleet_facts", "api_keys", "oidc_bindings", + "cloud_identity_bindings", "policy_audit_heads", "policy_audit_entries", "approval_grants", + } for _, table := range tables { var count int if err := database.pool.QueryRow(ctx, `SELECT count(*) FROM sith.`+pgx.Identifier{table}.Sanitize()).Scan(&count); err != nil { @@ -129,6 +201,12 @@ func TestPostgresRLSBackstop(t *testing.T) { {name: "OIDC binding", statement: `INSERT INTO sith.oidc_bindings(workspace_id, issuer, upstream_subject, member_subject) VALUES ('workspace-b', 'https://idp.example', 'upstream:mallory', 'user:bob')`}, {name: "cloud identity binding", statement: "INSERT INTO sith.cloud_identity_bindings(workspace_id, provider, realm, upstream_subject, member_subject)\n\t\t\tVALUES ('workspace-b', 'aws', '222222222222', 'AROAX:mallory', 'user:bob')"}, + {name: "approval grant", statement: `INSERT INTO sith.approval_grants( + workspace_id, id, intent_id, proposer, approver, resolved_digest, + evidence_version, approved_at, expires_at + ) VALUES ('workspace-b', 'ZZZZZZZZZZZZZZZZZZZZZZ', 'intent-foreign', 'user:operator', + 'user:approver', 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 2, statement_timestamp(), statement_timestamp() + interval '10 minutes')`}, } for _, test := range foreignWrites { t.Run("foreign "+test.name+" write denied", func(t *testing.T) { @@ -230,15 +308,37 @@ func TestPostgresRLSBackstop(t *testing.T) { t.Fatalf("restored AuditIsolation() error = %v", err) } + if _, err := owner.Exec(ctx, `CREATE POLICY workspace_bypass ON sith.memberships + AS PERMISSIVE FOR ALL TO PUBLIC USING (true) WITH CHECK (true)`); err != nil { + t.Fatal(err) + } + if err := assertWorkspaceAIsolation(ctx, database, scope); err == nil || !strings.Contains(err.Error(), "memberships count = 2") { + t.Fatalf("additional-policy negative control did not expose the isolation invariant: %v", err) + } + if err := AuditIsolation(ctx, owner, appRole); err == nil || !strings.Contains(err.Error(), "complete workspace policy is missing") { + t.Fatalf("additional-policy negative control escaped catalog audit: %v", err) + } + if _, err := owner.Exec(ctx, `DROP POLICY workspace_bypass ON sith.memberships`); err != nil { + t.Fatal(err) + } + if err := assertWorkspaceAIsolation(ctx, database, scope); err != nil { + t.Fatalf("additional-policy removal did not recover invariant: %v", err) + } + if err := AuditIsolation(ctx, owner, appRole); err != nil { + t.Fatalf("additional-policy removal AuditIsolation() error = %v", err) + } + assertAPIKeyStoreIntegration(t, ctx, database, admin) assertOIDCStoreIntegration(t, ctx, database) assertCloudIdentityStoreIntegration(t, ctx, database) assertFleetStoreIntegration(t, ctx, database) + assertPolicyAuditChainIntegration(t, ctx, database, admin, appRole) + assertApprovalGrantIntegration(t, ctx, database, admin, appRole) } func assertWorkspaceAIsolation(ctx context.Context, database *AppDB, scope tenancy.Scope) error { return database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { - for _, table := range []string{"workspaces", "memberships", "clusters", "fleet_facts", "api_keys", "oidc_bindings", "cloud_identity_bindings"} { + for _, table := range []string{"workspaces", "memberships", "clusters", "fleet_facts", "api_keys", "oidc_bindings", "cloud_identity_bindings", "approval_grants"} { var count int if err := tx.QueryRow(ctx, `SELECT count(*) FROM sith.`+pgx.Identifier{table}.Sanitize()).Scan(&count); err != nil { return err @@ -258,6 +358,835 @@ func assertWorkspaceAIsolation(ctx context.Context, database *AppDB, scope tenan }) } +func assertPolicyAuditChainIntegration( + t *testing.T, + ctx context.Context, + database *AppDB, + admin *pgx.Conn, + appRole string, +) { + t.Helper() + + workspaceA := testScope(t, "user:alice", "workspace-a", tenancy.RoleAdmin) + workspaceB := testScope(t, "user:bob", "workspace-b", tenancy.RoleAdmin) + if err := database.Record(ctx, newPolicyAuditEvent(t, workspaceA)); err != nil { + t.Fatalf("Record(workspace A) error = %v", err) + } + if err := database.Record(ctx, newPolicyAuditEvent(t, workspaceB)); err != nil { + t.Fatalf("Record(workspace B) error = %v", err) + } + + const concurrentAppends = 20 + errorsByAppend := make(chan error, concurrentAppends) + var writers sync.WaitGroup + for range concurrentAppends { + event := newPolicyAuditEvent(t, workspaceA) + writers.Add(1) + go func(event pep.AuditEvent) { + defer writers.Done() + errorsByAppend <- database.Record(ctx, event) + }(event) + } + writers.Wait() + close(errorsByAppend) + for err := range errorsByAppend { + if err != nil { + t.Fatalf("concurrent Record() error = %v", err) + } + } + + for name, scope := range map[string]tenancy.Scope{"workspace A": workspaceA, "workspace B": workspaceB} { + if err := database.VerifyPolicyAuditChain(ctx, scope); err != nil { + t.Fatalf("VerifyPolicyAuditChain(%s) error = %v", name, err) + } + } + enforcer, err := pep.NewEnforcer(pep.Config{Hook: pep.AllowReadHook{}, Auditor: database}) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeAuditExport(ctx, workspaceA); err != nil { + t.Fatalf("AuthorizeAuditExport(workspace A) error = %v", err) + } + exportedA, err := database.ExportPolicyAuditChain(ctx, workspaceA) + if err != nil { + t.Fatalf("ExportPolicyAuditChain(workspace A) error = %v", err) + } + if exportedA.Schema != "sith.policy-audit-export/v1" || exportedA.WorkspaceID != "workspace-a" || + exportedA.Chain.HeadSequence != concurrentAppends+2 || len(exportedA.Entries) != concurrentAppends+2 { + t.Fatalf("workspace A export shape = %#v", exportedA) + } + last := exportedA.Entries[len(exportedA.Entries)-1] + if last.Action != string(tenancy.ActionExportAudit) || last.Verb != string(pep.VerbAuditExport) || + last.Verdict != string(pep.VerdictAllow) || last.ReasonCode != "phase-1-audit-export" || + last.EntryHash != exportedA.Chain.HeadHash { + t.Fatalf("workspace A export did not include its authorizing decision: %#v", last) + } + exportedB, err := database.ExportPolicyAuditChain(ctx, workspaceB) + if err != nil || len(exportedB.Entries) != 1 || exportedB.WorkspaceID != "workspace-b" || + exportedB.Entries[0].Actor != "user:bob" || strings.Contains(fmt.Sprintf("%#v", exportedB), "user:alice") { + t.Fatalf("workspace B export/error = %#v/%v", exportedB, err) + } + if err := database.InWorkspace(ctx, workspaceA, func(tx pgx.Tx) error { + var headCount, entryCount, foreignCount int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM sith.policy_audit_heads`).Scan(&headCount); err != nil { + return err + } + if err := tx.QueryRow(ctx, `SELECT count(*) FROM sith.policy_audit_entries`).Scan(&entryCount); err != nil { + return err + } + if err := tx.QueryRow(ctx, ` + SELECT count(*) FROM sith.policy_audit_entries WHERE workspace_id = 'workspace-b' + `).Scan(&foreignCount); err != nil { + return err + } + if headCount != 1 || entryCount != concurrentAppends+2 || foreignCount != 0 { + return fmt.Errorf("workspace A audit view = heads %d entries %d foreign %d", headCount, entryCount, foreignCount) + } + return nil + }); err != nil { + t.Fatalf("policy audit RLS isolation: %v", err) + } + + var canSelect, canInsert, canUpdate, canDelete bool + if err := admin.QueryRow(ctx, ` + SELECT has_table_privilege($1, 'sith.policy_audit_entries', 'SELECT'), + has_table_privilege($1, 'sith.policy_audit_entries', 'INSERT'), + has_any_column_privilege($1, 'sith.policy_audit_entries', 'UPDATE'), + has_table_privilege($1, 'sith.policy_audit_entries', 'DELETE') + `, appRole).Scan(&canSelect, &canInsert, &canUpdate, &canDelete); err != nil { + t.Fatalf("inspect policy audit privileges: %v", err) + } + if !canSelect || !canInsert || canUpdate || canDelete { + t.Fatalf("immutable audit privileges = select:%t insert:%t update:%t delete:%t", canSelect, canInsert, canUpdate, canDelete) + } + for name, statement := range map[string]string{ + "update entry": `UPDATE sith.policy_audit_entries SET actor = 'user:mallory' WHERE sequence = 1`, + "delete entry": `DELETE FROM sith.policy_audit_entries WHERE sequence = 1`, + "delete head": `DELETE FROM sith.policy_audit_heads`, + } { + err := database.InWorkspace(ctx, workspaceA, func(tx pgx.Tx) error { + _, execErr := tx.Exec(ctx, statement) + return execErr + }) + var postgresErr *pgconn.PgError + if !errors.As(err, &postgresErr) || postgresErr.Code != "42501" { + t.Fatalf("%s error = %v, want privilege denial 42501", name, err) + } + } + + var originalPrevious, originalEntryHash, originalHeadHash []byte + if err := admin.QueryRow(ctx, ` + SELECT previous_hash, entry_hash + FROM sith.policy_audit_entries + WHERE workspace_id = 'workspace-a' AND sequence = 2 + `).Scan(&originalPrevious, &originalEntryHash); err != nil { + t.Fatalf("read policy audit tamper fixture: %v", err) + } + if err := admin.QueryRow(ctx, ` + SELECT last_hash FROM sith.policy_audit_heads WHERE workspace_id = 'workspace-a' + `).Scan(&originalHeadHash); err != nil { + t.Fatalf("read policy audit head fixture: %v", err) + } + + assertTamperDetected := func(name, mutate, restore string, mutateArgs, restoreArgs []any) { + t.Helper() + if _, err := admin.Exec(ctx, mutate, mutateArgs...); err != nil { + t.Fatalf("%s mutation: %v", name, err) + } + if err := database.VerifyPolicyAuditChain(ctx, workspaceA); err == nil { + t.Fatalf("%s was not detected", name) + } + if _, err := database.ExportPolicyAuditChain(ctx, workspaceA); err == nil { + t.Fatalf("%s was exported", name) + } + if _, err := admin.Exec(ctx, restore, restoreArgs...); err != nil { + t.Fatalf("%s restore: %v", name, err) + } + if err := database.VerifyPolicyAuditChain(ctx, workspaceA); err != nil { + t.Fatalf("%s restore did not recover chain: %v", name, err) + } + } + assertTamperDetected( + "field mutation", + `UPDATE sith.policy_audit_entries SET actor = 'user:mallory' WHERE workspace_id = 'workspace-a' AND sequence = 2`, + `UPDATE sith.policy_audit_entries SET actor = 'user:alice' WHERE workspace_id = 'workspace-a' AND sequence = 2`, + nil, nil, + ) + assertTamperDetected( + "sequence reordering", + `UPDATE sith.policy_audit_entries SET sequence = 100 WHERE workspace_id = 'workspace-a' AND sequence = 2`, + `UPDATE sith.policy_audit_entries SET sequence = 2 WHERE workspace_id = 'workspace-a' AND sequence = 100`, + nil, nil, + ) + assertTamperDetected( + "previous hash mutation", + `UPDATE sith.policy_audit_entries SET previous_hash = decode(repeat('00', 32), 'hex') WHERE workspace_id = 'workspace-a' AND sequence = 2`, + `UPDATE sith.policy_audit_entries SET previous_hash = $1 WHERE workspace_id = 'workspace-a' AND sequence = 2`, + nil, []any{originalPrevious}, + ) + assertTamperDetected( + "entry hash mutation", + `UPDATE sith.policy_audit_entries SET entry_hash = decode(repeat('00', 32), 'hex') WHERE workspace_id = 'workspace-a' AND sequence = 2`, + `UPDATE sith.policy_audit_entries SET entry_hash = $1 WHERE workspace_id = 'workspace-a' AND sequence = 2`, + nil, []any{originalEntryHash}, + ) + assertTamperDetected( + "head mismatch", + `UPDATE sith.policy_audit_heads SET last_hash = decode(repeat('00', 32), 'hex') WHERE workspace_id = 'workspace-a'`, + `UPDATE sith.policy_audit_heads SET last_hash = $1 WHERE workspace_id = 'workspace-a'`, + nil, []any{originalHeadHash}, + ) + if _, err := admin.Exec(ctx, ` + CREATE TEMP TABLE policy_audit_deleted_fixture AS + SELECT * FROM sith.policy_audit_entries + WHERE workspace_id = 'workspace-a' AND sequence = 2 + `); err != nil { + t.Fatalf("save retained policy audit entry: %v", err) + } + if _, err := admin.Exec(ctx, ` + DELETE FROM sith.policy_audit_entries WHERE workspace_id = 'workspace-a' AND sequence = 2 + `); err != nil { + t.Fatalf("delete retained policy audit entry: %v", err) + } + if err := database.VerifyPolicyAuditChain(ctx, workspaceA); err == nil { + t.Fatal("deleted retained policy audit entry was not detected") + } + if _, err := admin.Exec(ctx, ` + INSERT INTO sith.policy_audit_entries + SELECT * FROM policy_audit_deleted_fixture; + DROP TABLE policy_audit_deleted_fixture + `); err != nil { + t.Fatalf("restore retained policy audit entry: %v", err) + } + if err := database.VerifyPolicyAuditChain(ctx, workspaceA); err != nil { + t.Fatalf("deleted entry restore did not recover chain: %v", err) + } + if _, err := database.ExportPolicyAuditChain(ctx, testScope(t, "user:alice", "workspace-a", tenancy.RoleReader)); err == nil { + t.Fatal("reader scope exported the policy audit chain") + } + + for sequence := exportedA.Chain.HeadSequence + 1; sequence <= auditrecord.MaxEntries; sequence++ { + if err := database.Record(ctx, newPolicyAuditEvent(t, workspaceA)); err != nil { + t.Fatalf("fill bounded export at sequence %d: %v", sequence, err) + } + } + bounded, err := database.ExportPolicyAuditChain(ctx, workspaceA) + if err != nil || len(bounded.Entries) != auditrecord.MaxEntries || bounded.Chain.HeadSequence != auditrecord.MaxEntries { + t.Fatalf("512-entry export/error = %d/%d/%v", len(bounded.Entries), bounded.Chain.HeadSequence, err) + } + if err := database.Record(ctx, newPolicyAuditEvent(t, workspaceA)); err != nil { + t.Fatalf("append export overflow sentinel: %v", err) + } + if oversized, err := database.ExportPolicyAuditChain(ctx, workspaceA); err == nil || oversized.Schema != "" || len(oversized.Entries) != 0 { + t.Fatalf("513-entry export/error = %#v/%v", oversized, err) + } + + firstRequest, err := auditrecord.FirstPage(workspaceA.WorkspaceID()) + if err != nil { + t.Fatal(err) + } + firstPage, err := database.ExportPolicyAuditPage(ctx, workspaceA, firstRequest) + if err != nil || len(firstPage.Entries) != auditrecord.MaxEntries || firstPage.StartSequence != 1 || + firstPage.Snapshot.HeadSequence != auditrecord.MaxEntries+1 || firstPage.NextCursor == "" { + t.Fatalf("first audit page/error = %#v/%v", firstPage, err) + } + if err := database.Record(ctx, newPolicyAuditEvent(t, workspaceA)); err != nil { + t.Fatalf("append after fixed audit snapshot: %v", err) + } + continuation, err := auditrecord.ContinuePage(workspaceA.WorkspaceID(), firstPage.NextCursor) + if err != nil { + t.Fatal(err) + } + finalPage, err := database.ExportPolicyAuditPage(ctx, workspaceA, continuation) + if err != nil || len(finalPage.Entries) != 1 || finalPage.StartSequence != auditrecord.MaxEntries+1 || + finalPage.Snapshot != firstPage.Snapshot || finalPage.NextCursor != "" { + t.Fatalf("final audit page/error = %#v/%v", finalPage, err) + } + var pageVerifier auditrecord.PageSequenceVerifier + if err := pageVerifier.Add(firstPage); err != nil { + t.Fatal(err) + } + if err := pageVerifier.Add(finalPage); err != nil { + t.Fatal(err) + } + if result, err := pageVerifier.Finish(); err != nil || result.Entries != auditrecord.MaxEntries+1 { + t.Fatalf("paged PostgreSQL verification = %#v/%v", result, err) + } + foreignRequest, err := auditrecord.FirstPage("workspace-b") + if err != nil { + t.Fatal(err) + } + if _, err := database.ExportPolicyAuditPage(ctx, workspaceA, foreignRequest); err == nil { + t.Fatal("foreign page request crossed workspace storage boundary") + } + if _, err := database.ExportPolicyAuditPage( + ctx, testScope(t, "user:alice", "workspace-a", tenancy.RoleReader), firstRequest, + ); err == nil { + t.Fatal("reader scope exported an audit page") + } + + cursorPayload, err := base64.RawURLEncoding.DecodeString(firstPage.NextCursor) + if err != nil { + t.Fatal(err) + } + for name, offset := range map[string]int{"snapshot head": 41, "previous anchor": len(cursorPayload) - 1} { + t.Run("paged export rejects altered "+name, func(t *testing.T) { + altered := append([]byte(nil), cursorPayload...) + altered[offset] ^= 1 + request, err := auditrecord.ContinuePage( + workspaceA.WorkspaceID(), base64.RawURLEncoding.EncodeToString(altered), + ) + if err != nil { + t.Fatal(err) + } + if _, err := database.ExportPolicyAuditPage(ctx, workspaceA, request); err == nil { + t.Fatal("altered continuation reached a successful page") + } + }) + } +} + +func assertApprovalGrantIntegration( + t *testing.T, + ctx context.Context, + database *AppDB, + admin *pgx.Conn, + appRole string, +) { + t.Helper() + + if _, err := admin.Exec(ctx, `INSERT INTO sith.memberships(workspace_id, subject, role) VALUES + ('workspace-a', 'user:operator-a', 'operator'), + ('workspace-a', 'user:approver-a', 'approver'), + ('workspace-a', 'user:stale-approver', 'reader'), + ('workspace-b', 'user:operator-b', 'operator'), + ('workspace-b', 'user:approver-b', 'approver')`); err != nil { + t.Fatalf("seed approval memberships: %v", err) + } + + proposerA := testScope(t, "user:operator-a", "workspace-a", tenancy.RoleOperator) + approverA := testScope(t, "user:approver-a", "workspace-a", tenancy.RoleApprover) + proposerB := testScope(t, "user:operator-b", "workspace-b", tenancy.RoleOperator) + approverB := testScope(t, "user:approver-b", "workspace-b", tenancy.RoleApprover) + now := time.Now().UTC().Truncate(time.Microsecond) + + create := func(intentID, arguments string) (pep.ApprovalBinding, ApprovalGrantID) { + t.Helper() + binding := postgresApprovalBinding(t, intentID, "workspace-a", proposerA.Subject(), arguments) + identifier, err := database.CreateApprovalGrant(ctx, approverA, binding) + if err != nil { + t.Fatalf("CreateApprovalGrant(%s) error = %v", intentID, err) + } + if !approvalGrantIDPattern.MatchString(identifier.String()) { + t.Fatalf("CreateApprovalGrant(%s) identifier = %q", intentID, identifier) + } + return binding, identifier + } + + baseBinding, baseID := create("intent-250-base", "replicas=3") + var baseApprovedAt, baseExpiresAt time.Time + var baseEvidenceVersion int16 + if err := admin.QueryRow(ctx, ` + SELECT approved_at, expires_at, evidence_version + FROM sith.approval_grants WHERE workspace_id = 'workspace-a' AND id = $1 + `, baseID).Scan(&baseApprovedAt, &baseExpiresAt, &baseEvidenceVersion); err != nil { + t.Fatalf("inspect expiring approval grant: %v", err) + } + if baseEvidenceVersion != approvalGrantEvidenceVersion || + baseExpiresAt.Sub(baseApprovedAt) != 10*time.Minute || + baseApprovedAt.Before(now.Add(-time.Minute)) || baseApprovedAt.After(time.Now().Add(time.Minute)) { + t.Fatalf("approval lifetime = approved %s expires %s evidence v%d", baseApprovedAt, baseExpiresAt, baseEvidenceVersion) + } + if _, err := database.CreateApprovalGrant(ctx, approverA, baseBinding); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("duplicate approval error = %v", err) + } + + for name, scope := range map[string]tenancy.Scope{ + "reader": testScope(t, "user:reader", "workspace-a", tenancy.RoleReader), + "operator": proposerA, + "admin": testScope(t, "user:admin", "workspace-a", tenancy.RoleAdmin), + } { + if _, err := database.CreateApprovalGrant(ctx, scope, postgresApprovalBinding(t, + "intent-role-"+name, "workspace-a", proposerA.Subject(), name)); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("%s approval error = %v", name, err) + } + } + selfScope := testScope(t, "user:self", "workspace-a", tenancy.RoleApprover) + if _, err := database.CreateApprovalGrant(ctx, selfScope, postgresApprovalBinding( + t, "intent-self", "workspace-a", selfScope.Subject(), "self")); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("self approval error = %v", err) + } + staleScope := testScope(t, "user:stale-approver", "workspace-a", tenancy.RoleApprover) + if _, err := database.CreateApprovalGrant(ctx, staleScope, postgresApprovalBinding( + t, "intent-stale-role", "workspace-a", proposerA.Subject(), "stale")); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("stale approver role error = %v", err) + } + if _, err := database.CreateApprovalGrant(ctx, approverA, postgresApprovalBinding( + t, "intent-missing-proposer", "workspace-a", "user:missing", "missing")); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("missing proposer membership error = %v", err) + } + + wrongDigestBinding := postgresApprovalBinding(t, baseBinding.IntentID(), "workspace-a", proposerA.Subject(), "replicas=4") + if err := database.ConsumeApprovalGrant(ctx, proposerA, wrongDigestBinding, baseID); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("wrong digest consume error = %v", err) + } + wrongIntentBinding := postgresApprovalBinding(t, "intent-250-other", "workspace-a", proposerA.Subject(), "replicas=3") + if err := database.ConsumeApprovalGrant(ctx, proposerA, wrongIntentBinding, baseID); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("wrong intent consume error = %v", err) + } + if err := database.ConsumeApprovalGrant(ctx, proposerB, baseBinding, baseID); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("foreign workspace consume error = %v", err) + } + if err := database.ConsumeApprovalGrant(ctx, proposerA, baseBinding, "XXXXXXXXXXXXXXXXXXXXXX"); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("unknown approval consume error = %v", err) + } + seedGrant := func( + identifier ApprovalGrantID, + binding pep.ApprovalBinding, + evidenceVersion int16, + approvedOffsetMinutes int32, + ) { + t.Helper() + if _, err := admin.Exec(ctx, ` + WITH fixture_time AS ( + SELECT statement_timestamp() + make_interval(mins => $8::integer) AS approved_at + ) + INSERT INTO sith.approval_grants( + workspace_id, id, intent_id, proposer, approver, resolved_digest, + evidence_version, approved_at, expires_at + ) + SELECT $1, $2, $3, $4, $5, $6, $7, + approved_at, approved_at + interval '10 minutes' + FROM fixture_time + `, binding.WorkspaceID(), identifier, binding.IntentID(), binding.Proposer(), approverA.Subject(), + binding.ResolvedDigest(), evidenceVersion, approvedOffsetMinutes); err != nil { + t.Fatalf("seed approval grant %s: %v", identifier, err) + } + } + preApprovalBinding := postgresApprovalBinding( + t, "intent-299-pre-approval", "workspace-a", proposerA.Subject(), "time=before", + ) + expiredBinding := postgresApprovalBinding( + t, "intent-299-expired", "workspace-a", proposerA.Subject(), "time=expired", + ) + legacyBinding := postgresApprovalBinding( + t, "intent-299-legacy", "workspace-a", proposerA.Subject(), "evidence=legacy", + ) + preApprovalID := ApprovalGrantID("IIIIIIIIIIIIIIIIIIIIII") + expiredID := ApprovalGrantID("JJJJJJJJJJJJJJJJJJJJJJ") + legacyID := ApprovalGrantID("KKKKKKKKKKKKKKKKKKKKKK") + seedGrant(preApprovalID, preApprovalBinding, approvalGrantEvidenceVersion, 5) + seedGrant(expiredID, expiredBinding, approvalGrantEvidenceVersion, -15) + seedGrant(legacyID, legacyBinding, 1, 0) + + var preApprovalIsFuture, expiredIsPast, exactTemporalLifetimes bool + if err := admin.QueryRow(ctx, ` + SELECT pre_approval.approved_at > statement_timestamp(), + expired_grant.expires_at < statement_timestamp(), + pre_approval.expires_at = pre_approval.approved_at + interval '10 minutes' + AND expired_grant.expires_at = expired_grant.approved_at + interval '10 minutes' + FROM sith.approval_grants AS pre_approval + CROSS JOIN sith.approval_grants AS expired_grant + WHERE pre_approval.workspace_id = 'workspace-a' AND pre_approval.id = $1 + AND expired_grant.workspace_id = 'workspace-a' AND expired_grant.id = $2 + `, preApprovalID, expiredID).Scan(&preApprovalIsFuture, &expiredIsPast, &exactTemporalLifetimes); err != nil { + t.Fatalf("inspect temporal approval fixtures: %v", err) + } + if !preApprovalIsFuture || !expiredIsPast || !exactTemporalLifetimes { + t.Fatalf("temporal approval fixtures = future:%t expired:%t exact-lifetimes:%t", + preApprovalIsFuture, expiredIsPast, exactTemporalLifetimes) + } + + var refusalHeadBefore int64 + if err := admin.QueryRow(ctx, ` + SELECT last_sequence FROM sith.policy_audit_heads WHERE workspace_id = 'workspace-a' + `).Scan(&refusalHeadBefore); err != nil { + t.Fatalf("read approval audit head before temporal refusals: %v", err) + } + for name, attempt := range map[string]func() error{ + "before approval": func() error { + return database.ConsumeApprovalGrant(ctx, proposerA, preApprovalBinding, preApprovalID) + }, + "expired": func() error { + return database.ConsumeApprovalGrant(ctx, proposerA, expiredBinding, expiredID) + }, + "legacy evidence": func() error { + return database.ConsumeApprovalGrant(ctx, proposerA, legacyBinding, legacyID) + }, + } { + if err := attempt(); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("%s approval consume error = %v", name, err) + } + } + var refusalHeadAfter int64 + var refusedConsumptions int + if err := admin.QueryRow(ctx, ` + SELECT last_sequence FROM sith.policy_audit_heads WHERE workspace_id = 'workspace-a' + `).Scan(&refusalHeadAfter); err != nil { + t.Fatalf("read approval audit head after temporal refusals: %v", err) + } + if err := admin.QueryRow(ctx, ` + SELECT count(*) FROM sith.approval_grants + WHERE workspace_id = 'workspace-a' AND id IN ($1, $2, $3) AND consumed_at IS NOT NULL + `, preApprovalID, expiredID, legacyID).Scan(&refusedConsumptions); err != nil { + t.Fatalf("inspect refused temporal approval rows: %v", err) + } + if refusalHeadAfter != refusalHeadBefore || refusedConsumptions != 0 { + t.Fatalf("temporal refusals changed audit head %d -> %d or consumed %d rows", + refusalHeadBefore, refusalHeadAfter, refusedConsumptions) + } + if err := database.ConsumeApprovalGrant(ctx, proposerA, baseBinding, baseID); err != nil { + t.Fatalf("exact approval consume error = %v", err) + } + if err := database.ConsumeApprovalGrant(ctx, proposerA, baseBinding, baseID); !errors.Is(err, ErrApprovalGrantUnavailable) { + t.Fatalf("replayed approval consume error = %v", err) + } + + concurrentBinding, concurrentID := create("intent-250-concurrent", "image=v2") + results := make(chan error, 2) + var consumers sync.WaitGroup + for range 2 { + consumers.Add(1) + go func() { + defer consumers.Done() + results <- database.ConsumeApprovalGrant(ctx, proposerA, concurrentBinding, concurrentID) + }() + } + consumers.Wait() + close(results) + var successes, refusals int + for err := range results { + switch { + case err == nil: + successes++ + case errors.Is(err, ErrApprovalGrantUnavailable): + refusals++ + default: + t.Fatalf("concurrent approval consume error = %v", err) + } + } + if successes != 1 || refusals != 1 { + t.Fatalf("concurrent consumes = successes %d refusals %d", successes, refusals) + } + + var originalHeadSequence int64 + var originalHeadHash []byte + readAndCorruptHead := func() { + t.Helper() + if err := admin.QueryRow(ctx, ` + SELECT last_sequence, last_hash + FROM sith.policy_audit_heads WHERE workspace_id = 'workspace-a' + `).Scan(&originalHeadSequence, &originalHeadHash); err != nil { + t.Fatalf("read approval audit head fixture: %v", err) + } + if _, err := admin.Exec(ctx, ` + UPDATE sith.policy_audit_heads SET last_sequence = 9223372036854775807 + WHERE workspace_id = 'workspace-a' + `); err != nil { + t.Fatalf("corrupt approval audit head fixture: %v", err) + } + } + restoreHead := func() { + t.Helper() + if _, err := admin.Exec(ctx, ` + UPDATE sith.policy_audit_heads SET last_sequence = $1, last_hash = $2 + WHERE workspace_id = 'workspace-a' + `, originalHeadSequence, originalHeadHash); err != nil { + t.Fatalf("restore approval audit head fixture: %v", err) + } + } + + rollbackCreateBinding := postgresApprovalBinding( + t, "intent-252-create-rollback", "workspace-a", proposerA.Subject(), "rollback=create", + ) + readAndCorruptHead() + _, rollbackCreateErr := database.CreateApprovalGrant(ctx, approverA, rollbackCreateBinding) + restoreHead() + if rollbackCreateErr == nil || errors.Is(rollbackCreateErr, ErrApprovalGrantUnavailable) { + t.Fatalf("audit-failed create error = %v, want operational audit failure", rollbackCreateErr) + } + var rollbackCreateCount int + if err := admin.QueryRow(ctx, ` + SELECT count(*) FROM sith.approval_grants + WHERE workspace_id = 'workspace-a' AND intent_id = 'intent-252-create-rollback' + `).Scan(&rollbackCreateCount); err != nil { + t.Fatalf("inspect audit-failed approval create: %v", err) + } + if rollbackCreateCount != 0 { + t.Fatalf("audit-failed approval create retained %d grants", rollbackCreateCount) + } + + rollbackConsumeBinding, rollbackConsumeID := create("intent-252-consume-rollback", "rollback=consume") + readAndCorruptHead() + rollbackConsumeErr := database.ConsumeApprovalGrant( + ctx, proposerA, rollbackConsumeBinding, rollbackConsumeID, + ) + restoreHead() + if rollbackConsumeErr == nil || errors.Is(rollbackConsumeErr, ErrApprovalGrantUnavailable) { + t.Fatalf("audit-failed consume error = %v, want operational audit failure", rollbackConsumeErr) + } + var rollbackConsumedAt *time.Time + if err := admin.QueryRow(ctx, ` + SELECT consumed_at FROM sith.approval_grants + WHERE workspace_id = 'workspace-a' AND id = $1 + `, rollbackConsumeID).Scan(&rollbackConsumedAt); err != nil { + t.Fatalf("inspect audit-failed approval consume: %v", err) + } + if rollbackConsumedAt != nil { + t.Fatalf("audit-failed approval consume retained timestamp %s", rollbackConsumedAt) + } + if err := database.ConsumeApprovalGrant( + ctx, proposerA, rollbackConsumeBinding, rollbackConsumeID, + ); err != nil { + t.Fatalf("consume after audit rollback error = %v", err) + } + + legacyTraceID, err := tracing.NewID() + if err != nil { + t.Fatalf("mint legacy approval audit trace: %v", err) + } + if err := database.InWorkspace(ctx, approverB, func(tx pgx.Tx) error { + return appendPolicyAuditEntryTx(ctx, tx, policyAuditEntry{ + format: approvalAuditFormatVersion, recordedAt: time.Now().UTC().Truncate(time.Microsecond), + traceID: legacyTraceID, workspaceID: "workspace-b", actor: approverB.Subject(), + role: approverB.Role(), action: tenancy.ActionApproveIntent, verb: approvalAuditVerb, + verdict: pep.VerdictAllow, reasonCode: approvalCreatedEventKind, + eventKind: approvalCreatedEventKind, evidence: "sha256:" + strings.Repeat("c", 64), + }) + }); err != nil { + t.Fatalf("append legacy format-2 approval fixture: %v", err) + } + + bindingB := postgresApprovalBinding(t, "intent-250-b", "workspace-b", proposerB.Subject(), "region=west") + identifierB, err := database.CreateApprovalGrant(ctx, approverB, bindingB) + if err != nil { + t.Fatalf("CreateApprovalGrant(workspace B) error = %v", err) + } + if err := database.InWorkspace(ctx, proposerA, func(tx pgx.Tx) error { + var foreignCount int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM sith.approval_grants WHERE workspace_id = 'workspace-b'`).Scan(&foreignCount); err != nil { + return err + } + if foreignCount != 0 { + return fmt.Errorf("approval RLS exposed %d foreign grants", foreignCount) + } + tag, err := tx.Exec(ctx, `UPDATE sith.approval_grants SET consumed_at = $1 + WHERE workspace_id = 'workspace-b' AND id = $2`, now.Add(time.Second), identifierB) + if err != nil { + return err + } + if tag.RowsAffected() != 0 { + return fmt.Errorf("approval RLS mutated %d foreign grants", tag.RowsAffected()) + } + return nil + }); err != nil { + t.Fatalf("approval RLS isolation: %v", err) + } + + var canConsume, canMutateProposer, canMutateExpiry, canMutateEvidenceVersion, canDelete bool + if err := admin.QueryRow(ctx, ` + SELECT has_column_privilege($1, 'sith.approval_grants', 'consumed_at', 'UPDATE'), + has_column_privilege($1, 'sith.approval_grants', 'proposer', 'UPDATE'), + has_column_privilege($1, 'sith.approval_grants', 'expires_at', 'UPDATE'), + has_column_privilege($1, 'sith.approval_grants', 'evidence_version', 'UPDATE'), + has_table_privilege($1, 'sith.approval_grants', 'DELETE') + `, appRole).Scan( + &canConsume, &canMutateProposer, &canMutateExpiry, &canMutateEvidenceVersion, &canDelete, + ); err != nil { + t.Fatalf("inspect approval privileges: %v", err) + } + if !canConsume || canMutateProposer || canMutateExpiry || canMutateEvidenceVersion || canDelete { + t.Fatalf("approval privileges = consume:%t proposer:%t expiry:%t evidence-version:%t delete:%t", + canConsume, canMutateProposer, canMutateExpiry, canMutateEvidenceVersion, canDelete) + } + for name, statement := range map[string]string{ + "mutate proposer": `UPDATE sith.approval_grants SET proposer = 'user:mallory' WHERE id = 'GGGGGGGGGGGGGGGGGGGGGG'`, + "mutate expiry": `UPDATE sith.approval_grants SET expires_at = expires_at + interval '1 minute' WHERE id = 'GGGGGGGGGGGGGGGGGGGGGG'`, + "mutate evidence version": `UPDATE sith.approval_grants SET evidence_version = 1 WHERE id = 'GGGGGGGGGGGGGGGGGGGGGG'`, + "delete grant": `DELETE FROM sith.approval_grants WHERE id = 'GGGGGGGGGGGGGGGGGGGGGG'`, + } { + err := database.InWorkspace(ctx, proposerA, func(tx pgx.Tx) error { + _, execErr := tx.Exec(ctx, statement) + return execErr + }) + var postgresErr *pgconn.PgError + if !errors.As(err, &postgresErr) || postgresErr.Code != "42501" { + t.Fatalf("%s error = %v, want privilege denial 42501", name, err) + } + } + if err := AuditIsolation(ctx, admin, appRole); err != nil { + t.Fatalf("approval AuditIsolation() error = %v", err) + } + + rows, err := admin.Query(ctx, ` + SELECT event_kind, evidence_digest, actor, role, action, verb, verdict, reason_code + FROM sith.policy_audit_entries + WHERE workspace_id = 'workspace-a' AND format_version = 3 + ORDER BY sequence + `) + if err != nil { + t.Fatalf("read approval lifecycle audit entries: %v", err) + } + defer rows.Close() + kindCounts := map[string]int{} + kindsByEvidence := map[string]map[string]bool{} + for rows.Next() { + var eventKind, evidence, actor, role, action, verb, verdict, reason string + if err := rows.Scan(&eventKind, &evidence, &actor, &role, &action, &verb, &verdict, &reason); err != nil { + t.Fatalf("scan approval lifecycle audit entry: %v", err) + } + if !approvalEvidencePattern.MatchString(evidence) || verb != string(approvalAuditVerb) || + verdict != string(pep.VerdictAllow) || reason != eventKind { + t.Fatalf("invalid approval lifecycle audit metadata for %s", eventKind) + } + switch eventKind { + case approvalCreatedEventKind: + if actor != approverA.Subject() || role != string(tenancy.RoleApprover) || action != string(tenancy.ActionApproveIntent) { + t.Fatalf("invalid approval-created actor metadata: %s/%s/%s", actor, role, action) + } + case approvalConsumedEventKind: + if actor != proposerA.Subject() || role != string(tenancy.RoleOperator) || action != string(tenancy.ActionProposeIntent) { + t.Fatalf("invalid approval-consumed actor metadata: %s/%s/%s", actor, role, action) + } + default: + t.Fatalf("unexpected approval lifecycle event kind %q", eventKind) + } + kindCounts[eventKind]++ + if kindsByEvidence[evidence] == nil { + kindsByEvidence[evidence] = map[string]bool{} + } + kindsByEvidence[evidence][eventKind] = true + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate approval lifecycle audit entries: %v", err) + } + if kindCounts[approvalCreatedEventKind] != 3 || kindCounts[approvalConsumedEventKind] != 3 || len(kindsByEvidence) != 3 { + t.Fatalf("approval lifecycle audit counts = %#v across %d grants, want 3 complete pairs", kindCounts, len(kindsByEvidence)) + } + for evidence, kinds := range kindsByEvidence { + if !kinds[approvalCreatedEventKind] || !kinds[approvalConsumedEventKind] || len(kinds) != 2 { + t.Fatalf("approval lifecycle evidence %s has events %#v, want one create and one consume", evidence, kinds) + } + } + if err := database.VerifyPolicyAuditChain(ctx, proposerA); err != nil { + t.Fatalf("mixed-version workspace A audit chain error = %v", err) + } + if err := database.VerifyPolicyAuditChain(ctx, proposerB); err != nil { + t.Fatalf("mixed-version workspace B audit chain error = %v", err) + } + mixedExport, err := database.ExportPolicyAuditChain( + ctx, testScope(t, "user:bob", "workspace-b", tenancy.RoleAdmin), + ) + if err != nil { + t.Fatalf("mixed-version workspace B export error = %v", err) + } + if len(mixedExport.Entries) != 3 || mixedExport.Entries[0].FormatVersion != policyAuditFormatVersion || + mixedExport.Entries[1].FormatVersion != approvalAuditFormatVersion || + mixedExport.Entries[2].FormatVersion != approvalExpiryAuditFormatVersion { + t.Fatalf("mixed-version workspace B export shape = %#v", mixedExport) + } + if err := mixedExport.Verify(); err != nil { + t.Fatalf("mixed-version PostgreSQL export offline verification error = %v", err) + } +} + +func postgresApprovalBinding( + t *testing.T, + intentID string, + workspaceID tenancy.WorkspaceID, + proposer string, + arguments string, +) pep.ApprovalBinding { + t.Helper() + digest := sha256.Sum256([]byte(arguments)) + input, err := pep.NewProposalInput( + intentID, workspaceID, proposer, intent.VerbDeploymentRestart, + fleet.ResourceRef{SourceKind: "argocd", Scope: "cluster-a", Kind: "deployment", Namespace: "payments", Name: "payments"}, + "sha256:"+hex.EncodeToString(digest[:]), + ) + if err != nil { + t.Fatalf("NewProposalInput() error = %v", err) + } + binding, err := input.ApprovalBinding() + if err != nil { + t.Fatalf("ApprovalBinding() error = %v", err) + } + return binding +} + +func testScope( + t *testing.T, + subject string, + workspaceID tenancy.WorkspaceID, + role tenancy.Role, +) tenancy.Scope { + t.Helper() + principal, err := tenancy.NewPrincipal(subject, map[tenancy.WorkspaceID]tenancy.Role{workspaceID: role}) + if err != nil { + t.Fatal(err) + } + scope, err := principal.Scope(workspaceID) + if err != nil { + t.Fatal(err) + } + return scope +} + +func newPolicyAuditEvent(t *testing.T, scope tenancy.Scope) pep.AuditEvent { + t.Helper() + traceID, err := tracing.NewID() + if err != nil { + t.Fatal(err) + } + return pep.AuditEvent{ + At: time.Now().UTC(), TraceID: traceID, WorkspaceID: scope.WorkspaceID(), Actor: scope.Subject(), + Role: scope.Role(), Action: tenancy.ActionRead, Verb: pep.VerbFleetRead, + Verdict: pep.VerdictAllow, ReasonCode: "phase-1-read", + } +} + +func applyLegacyMigrations( + t *testing.T, + ctx context.Context, + owner *pgx.Conn, + stopBefore string, +) { + t.Helper() + if _, err := owner.Exec(ctx, ` + CREATE SCHEMA sith_meta; + REVOKE ALL ON SCHEMA sith_meta FROM PUBLIC; + CREATE TABLE sith_meta.schema_migrations ( + version text PRIMARY KEY, + checksum bytea NOT NULL, + applied_at timestamptz NOT NULL DEFAULT transaction_timestamp() + ) + `); err != nil { + t.Fatalf("initialize legacy migration ledger: %v", err) + } + entries, err := fs.ReadDir(migrationFiles, "migrations") + if err != nil { + t.Fatalf("read legacy migration fixtures: %v", err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") || entry.Name() >= stopBefore { + continue + } + migration, err := fs.ReadFile(migrationFiles, "migrations/"+entry.Name()) + if err != nil { + t.Fatalf("read legacy migration %s: %v", entry.Name(), err) + } + if _, err := owner.Exec(ctx, string(migration)); err != nil { + t.Fatalf("apply legacy migration %s: %v", entry.Name(), err) + } + checksum := sha256.Sum256(migration) + if _, err := owner.Exec(ctx, ` + INSERT INTO sith_meta.schema_migrations(version, checksum) VALUES ($1, $2) + `, entry.Name(), checksum[:]); err != nil { + t.Fatalf("record legacy migration %s: %v", entry.Name(), err) + } + } +} + func startPostgres(t *testing.T) string { t.Helper() docker := os.Getenv("DOCKER_BIN") @@ -361,6 +1290,16 @@ func seedTenantRows(t *testing.T, ctx context.Context, admin *pgx.Conn) { "INSERT INTO sith.cloud_identity_bindings(workspace_id, provider, realm, upstream_subject, member_subject) VALUES\n" + "('workspace-a', 'aws', '111111111111', 'AROAX:alice', 'user:alice'),\n" + "('workspace-b', 'aws', '222222222222', 'AROAX:bob', 'user:bob')", + `INSERT INTO sith.approval_grants( + workspace_id, id, intent_id, proposer, approver, resolved_digest, + evidence_version, approved_at, expires_at + ) VALUES + ('workspace-a', 'GGGGGGGGGGGGGGGGGGGGGG', 'intent-seed-a', 'user:seed-operator', + 'user:seed-approver', 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 1, statement_timestamp(), statement_timestamp() + interval '10 minutes'), + ('workspace-b', 'HHHHHHHHHHHHHHHHHHHHHH', 'intent-seed-b', 'user:seed-operator', + 'user:seed-approver', 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 1, statement_timestamp(), statement_timestamp() + interval '10 minutes')`, } for _, statement := range statements { if _, err := admin.Exec(ctx, statement); err != nil { @@ -587,6 +1526,7 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap if err != nil || len(queryResult.Facts) != 1 || queryResult.Facts[0].Workspace != "workspace-a" || queryResult.Facts[0].Stale { t.Fatalf("workspace-a health query = %#v, error = %v", queryResult, err) } + assertFleetQuerySnapshotIsolation(t, ctx, database, scope) if err := database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { _, err := tx.Exec(ctx, `INSERT INTO sith.clusters(workspace_id, id, managed_cluster_ref) VALUES ($1, $2, $3)`, scope.WorkspaceID(), "cluster-a2", "ocm/cluster-a2") @@ -645,6 +1585,17 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap if err != nil { t.Fatal(err) } + inventorySearcher, err := hubfleet.NewInventorySearcher(hubfleet.InventorySearcherConfig{ + Querier: database, PEP: postgresReadPEP(t), Freshness: time.Minute, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + inventory, err := inventorySearcher.Search(ctx, scope, hubfleet.InventorySearchRequest{ResourceKind: "Pod", Namespace: "payments"}) + if err != nil || len(inventory.Facts) != 2 || inventory.Facts[0].Ref.Scope != "cluster-a" || inventory.Facts[1].Ref.Scope != "cluster-a2" || + inventory.Coverage.Requested != 2 || inventory.Coverage.Reachable != 2 || len(inventory.Coverage.Stale) != 0 { + t.Fatalf("two-spoke exact inventory search = %#v, error = %v", inventory, err) + } images, err := imageSearcher.Search(ctx, scope, hubfleet.ImageSearchRequest{Digest: digest}) if err != nil || len(images.Facts) != 2 || images.Facts[0].Ref.Scope != "cluster-a" || images.Facts[1].Ref.Scope != "cluster-a2" || images.Coverage.Requested != 2 || images.Coverage.Reachable != 2 || len(images.Coverage.Stale) != 0 { @@ -681,6 +1632,10 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap if err != nil || len(foreignCorrelation.Facts) != 0 { t.Fatalf("cross-workspace correlation = %#v, error = %v", foreignCorrelation, err) } + foreignInventory, err := inventorySearcher.Search(ctx, foreignScope, hubfleet.InventorySearchRequest{ResourceKind: "Pod", Namespace: "payments"}) + if err != nil || len(foreignInventory.Facts) != 0 || foreignInventory.Coverage.Requested != 1 || len(foreignInventory.Coverage.Unreachable) != 1 { + t.Fatalf("cross-workspace inventory search = %#v, error = %v", foreignInventory, err) + } foreignImages, err := imageSearcher.Search(ctx, foreignScope, hubfleet.ImageSearchRequest{Digest: digest}) if err != nil || len(foreignImages.Facts) != 0 || foreignImages.Coverage.Requested != 1 || len(foreignImages.Coverage.Unreachable) != 1 { t.Fatalf("cross-workspace image search = %#v, error = %v", foreignImages, err) @@ -731,6 +1686,107 @@ func assertFleetStoreIntegration(t *testing.T, ctx context.Context, database *Ap } } +func assertFleetQuerySnapshotIsolation(t *testing.T, ctx context.Context, database *AppDB, scope tenancy.Scope) { + t.Helper() + spoke := hubfleet.Spoke{ID: "cluster-snapshot", ManagedClusterRef: "ocm/cluster-snapshot"} + if err := database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { + _, err := tx.Exec(ctx, `INSERT INTO sith.clusters(workspace_id, id, managed_cluster_ref) + VALUES ($1, $2, $3)`, scope.WorkspaceID(), spoke.ID, spoke.ManagedClusterRef) + return err + }); err != nil { + t.Fatalf("register snapshot-isolation spoke: %v", err) + } + + newObserved := time.Date(2026, time.July, 12, 17, 0, 0, 0, time.UTC) + oldObserved := newObserved.Add(-2 * time.Hour) + snapshot := func(observed time.Time, generation int) hubfleet.Snapshot { + return hubfleet.Snapshot{ObservedAt: observed, Facts: []fleet.Evidence{{ + Ref: fleet.ResourceRef{ + SourceKind: hubfleet.SourceKind, + Scope: spoke.ID, + Kind: "Deployment", + Namespace: "payments", + Name: "snapshot-proof", + }, + Kind: fleet.FactInventory, + Observed: []byte(fmt.Sprintf(`{"generation":%d}`, generation)), + ObservedAt: observed, + Source: spoke.ID, + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }}} + } + if err := database.ReplaceSnapshot(ctx, scope, spoke, snapshot(oldObserved, 1), oldObserved); err != nil { + t.Fatalf("store old snapshot: %v", err) + } + + replaced := false + result, err := database.queryFleet(ctx, scope, fleet.Query{ + Kinds: []fleet.FactKind{fleet.FactInventory}, + Scopes: []string{spoke.ID}, + }, 30*time.Minute, newObserved, queryFleetHooks{ + afterClusterStates: func(tx pgx.Tx) error { + var isolation, readOnly, configuredWorkspace string + if err := tx.QueryRow(ctx, `SHOW transaction_isolation`).Scan(&isolation); err != nil { + return err + } + if err := tx.QueryRow(ctx, `SHOW transaction_read_only`).Scan(&readOnly); err != nil { + return err + } + if err := tx.QueryRow(ctx, `SELECT current_setting('sith.workspace_id', true)`).Scan(&configuredWorkspace); err != nil { + return err + } + if isolation != "repeatable read" || readOnly != "on" || configuredWorkspace != string(scope.WorkspaceID()) { + return fmt.Errorf("query transaction = isolation %q, read only %q, workspace %q", isolation, readOnly, configuredWorkspace) + } + var foreignClusters int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM sith.clusters WHERE workspace_id = 'workspace-b'`).Scan(&foreignClusters); err != nil { + return err + } + if foreignClusters != 0 { + return fmt.Errorf("query snapshot exposed %d foreign clusters", foreignClusters) + } + if err := database.ReplaceSnapshot(ctx, scope, spoke, snapshot(newObserved, 2), newObserved); err != nil { + return err + } + replaced = true + return nil + }, + }) + if err != nil { + t.Fatalf("query during committed replacement: %v", err) + } + if !replaced || len(result.Facts) != 1 || result.Coverage.Requested != 1 || result.Coverage.Reachable != 1 || + len(result.Coverage.Unreachable) != 0 { + t.Fatalf("snapshot query = %#v, replacement committed = %t", result, replaced) + } + var payload struct { + Generation int `json:"generation"` + } + if err := json.Unmarshal(result.Facts[0].Observed, &payload); err != nil { + t.Fatalf("decode snapshot generation: %v", err) + } + oldPair := payload.Generation == 1 && result.Facts[0].ObservedAt.Equal(oldObserved) && result.Facts[0].Stale && + len(result.Coverage.Stale) == 1 && result.Coverage.Stale[0] == spoke.ID + newPair := payload.Generation == 2 && result.Facts[0].ObservedAt.Equal(newObserved) && !result.Facts[0].Stale && + len(result.Coverage.Stale) == 0 + if !oldPair && !newPair { + t.Fatalf("mixed cluster state and fact snapshot: generation = %d, result = %#v", payload.Generation, result) + } + + if err := database.InWorkspace(ctx, scope, func(tx pgx.Tx) error { + tag, err := tx.Exec(ctx, `DELETE FROM sith.clusters WHERE workspace_id = $1 AND id = $2`, scope.WorkspaceID(), spoke.ID) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return fmt.Errorf("deleted %d snapshot-isolation spokes", tag.RowsAffected()) + } + return nil + }); err != nil { + t.Fatalf("remove snapshot-isolation spoke: %v", err) + } +} + func postgresReadPEP(t *testing.T) *pep.Enforcer { t.Helper() enforcer, err := pep.NewEnforcer(pep.Config{ diff --git a/internal/hubfleet/collector.go b/internal/hubfleet/collector.go index 40e3ee5..4bab20f 100644 --- a/internal/hubfleet/collector.go +++ b/internal/hubfleet/collector.go @@ -12,6 +12,7 @@ import ( "io" "sort" "strings" + "sync" "time" "unicode" @@ -25,14 +26,16 @@ const ( // SourceKind is the fixed fleet-model source stamp for an OCM-brokered spoke. SourceKind = "ocm-spoke" - protocolVersion = "1.0.0" - defaultSpokeTimeout = 5 * time.Second - defaultSnapshotAge = 5 * time.Minute - maxSpokeTimeout = 30 * time.Second - maxSnapshotAge = time.Hour - maxSnapshotFacts = 1_100 - maxObservedBytes = 256 * 1024 - maxFutureSkew = 30 * time.Second + protocolVersion = "1.0.0" + defaultSpokeTimeout = 5 * time.Second + defaultSnapshotAge = 5 * time.Minute + defaultSpokeConcurrency = 4 + maxSpokeTimeout = 30 * time.Second + maxSnapshotAge = time.Hour + maximumSpokeConcurrency = 64 + maxSnapshotFacts = 1_100 + maxObservedBytes = 256 * 1024 + maxFutureSkew = 30 * time.Second ) var observedKeys = map[fleet.FactKind]map[string]struct{}{ @@ -118,32 +121,37 @@ type Store interface { // CollectorConfig defines bounded collection behavior. type CollectorConfig struct { - Store Store - Transport Transport - PEP *pep.Enforcer - Observer SnapshotObserver - TraceObserver tracing.Observer - SpokeTimeout time.Duration - MaxSnapshotAge time.Duration - Now func() time.Time + LifecycleContext context.Context + Store Store + Transport Transport + PEP *pep.Enforcer + Observer SnapshotObserver + TraceObserver tracing.Observer + SpokeTimeout time.Duration + MaxSnapshotAge time.Duration + MaxConcurrentSpokes int + Now func() time.Time } // Collector collects independent, per-spoke snapshots without allowing one failure to suppress peers. type Collector struct { - store Store - transport Transport - pep *pep.Enforcer - observer SnapshotObserver - tracer tracing.Observer - spokeTimeout time.Duration - maxSnapshotAge time.Duration - now func() time.Time + store Store + transport Transport + pep *pep.Enforcer + refreshes *refreshCoordinator + observer SnapshotObserver + tracer tracing.Observer + spokeTimeout time.Duration + maxSnapshotAge time.Duration + maxConcurrentSpokes int + nowMu sync.Mutex + now func() time.Time } // NewCollector constructs a fail-closed collector with bounded per-spoke work. func NewCollector(config CollectorConfig) (*Collector, error) { - if config.Store == nil || config.Transport == nil || config.PEP == nil { - return nil, fmt.Errorf("new spoke collector: store, transport, and policy enforcer are required") + if config.LifecycleContext == nil || config.Store == nil || config.Transport == nil || config.PEP == nil { + return nil, fmt.Errorf("new spoke collector: lifecycle context, store, transport, and policy enforcer are required") } if config.SpokeTimeout == 0 { config.SpokeTimeout = defaultSpokeTimeout @@ -157,6 +165,12 @@ func NewCollector(config CollectorConfig) (*Collector, error) { if config.MaxSnapshotAge < time.Second || config.MaxSnapshotAge > maxSnapshotAge { return nil, fmt.Errorf("new spoke collector: maximum snapshot age must be between 1s and %s", maxSnapshotAge) } + if config.MaxConcurrentSpokes == 0 { + config.MaxConcurrentSpokes = defaultSpokeConcurrency + } + if config.MaxConcurrentSpokes < 1 || config.MaxConcurrentSpokes > maximumSpokeConcurrency { + return nil, fmt.Errorf("new spoke collector: maximum concurrent spokes must be between 1 and %d", maximumSpokeConcurrency) + } if config.Now == nil { config.Now = time.Now } @@ -167,21 +181,23 @@ func NewCollector(config CollectorConfig) (*Collector, error) { config.TraceObserver = tracing.NoopObserver() } return &Collector{ - store: config.Store, - transport: config.Transport, - pep: config.PEP, - observer: config.Observer, - tracer: config.TraceObserver, - spokeTimeout: config.SpokeTimeout, - maxSnapshotAge: config.MaxSnapshotAge, - now: config.Now, + store: config.Store, + transport: config.Transport, + pep: config.PEP, + refreshes: newRefreshCoordinator(config.LifecycleContext), + observer: config.Observer, + tracer: config.TraceObserver, + spokeTimeout: config.SpokeTimeout, + maxSnapshotAge: config.MaxSnapshotAge, + maxConcurrentSpokes: config.MaxConcurrentSpokes, + now: config.Now, }, nil } -// Collect refreshes every registered spoke independently and returns honest coverage. -// A transport or validation failure is recorded as a closed status and does not fail the peer loop. +// Collect authorizes every caller before joining at most one active refresh for its workspace. +// Different workspaces remain independent, and callers receive only the closed coverage/error result. func (collector *Collector) Collect(ctx context.Context, scope tenancy.Scope) (fleet.Coverage, error) { - if collector == nil || collector.store == nil || collector.transport == nil || collector.pep == nil || ctx == nil { + if collector == nil || collector.store == nil || collector.transport == nil || collector.pep == nil || collector.refreshes == nil || ctx == nil { return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: collector, policy enforcer, and context are required") } if err := scope.Authorize(tenancy.ActionRead); err != nil { @@ -198,6 +214,12 @@ func (collector *Collector) Collect(ctx context.Context, scope tenancy.Scope) (f if err := collector.pep.AuthorizeRead(ctx, scope, pep.NewReadInput(pep.VerbSpokeSnapshotRefresh, nil)); err != nil { return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: %w", err) } + return collector.refreshes.collect(ctx, scope, collector.collectWorkspace) +} + +// collectWorkspace executes one authorized workspace refresh on an internal context that carries +// no caller cancellation, credentials, request values, or request trace identity. +func (collector *Collector) collectWorkspace(ctx context.Context, scope tenancy.Scope) (fleet.Coverage, error) { spokes, err := collector.store.RegisteredSpokes(ctx, scope) if err != nil { return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: list registered spokes: %w", err) @@ -208,58 +230,185 @@ func (collector *Collector) Collect(ctx context.Context, scope tenancy.Scope) (f } coverage := fleet.Coverage{Requested: len(spokes)} - for _, spoke := range spokes { - if err := ctx.Err(); err != nil { - return coverage, fmt.Errorf("collect spoke snapshots: %w", err) - } - attemptedAt := collector.now().UTC() - startedAt := time.Now() - spokeContext, cancel := context.WithTimeout(ctx, collector.spokeTimeout) - snapshot, collectionErr := collector.transport.Snapshot(spokeContext, scope.WorkspaceID(), cloneSpoke(spoke)) - deadlineErr := spokeContext.Err() - cancel() - if err := ctx.Err(); err != nil { - collector.observeSnapshot(SnapshotOutcomeCanceled, time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeCanceled, time.Since(startedAt)) - return coverage, fmt.Errorf("collect spoke snapshots: %w", err) - } - if collectionErr == nil && deadlineErr != nil { - collectionErr = deadlineErr - } - if collectionErr != nil { - if err := collector.recordFailure(ctx, scope, spoke, failureFor(collectionErr), attemptedAt, &coverage); err != nil { - collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) - return coverage, err + if len(spokes) == 0 { + return coverage, nil + } + + collectionContext, cancel := context.WithCancel(ctx) + jobs := make(chan Spoke) + results := make(chan spokeCollectionResult) + workers := min(collector.maxConcurrentSpokes, len(spokes)) + var waitGroup sync.WaitGroup + waitGroup.Add(workers) + for range workers { + go func() { + defer waitGroup.Done() + collector.collectSpokeWorker(collectionContext, scope.WorkspaceID(), jobs, results) + }() + } + active := make(map[string]time.Time, workers) + var shutdownOnce sync.Once + shutdown := func(observeCanceled bool) { + shutdownOnce.Do(func() { + cancel() + close(jobs) + waitGroup.Wait() + if observeCanceled { + for _, startedAt := range active { + collector.observeSnapshot(SnapshotOutcomeCanceled, time.Since(startedAt)) + collector.observeTrace(ctx, tracing.OutcomeCanceled, time.Since(startedAt)) + } } - collector.observeSnapshot(snapshotOutcomeForFailure(failureFor(collectionErr)), time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) - continue + }) + } + defer shutdown(true) + + next := 0 + inFlight := 0 + for next < len(spokes) || inFlight > 0 { + var admission chan<- Spoke + var spoke Spoke + if next < len(spokes) { + admission = jobs + spoke = spokes[next] } - if err := validateSnapshot(spoke, snapshot, attemptedAt, collector.maxSnapshotAge); err != nil { - if failureErr := collector.recordFailure(ctx, scope, spoke, FailureInvalidSnapshot, attemptedAt, &coverage); failureErr != nil { - collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) - return coverage, failureErr + select { + case <-ctx.Done(): + shutdown(true) + return coverage, fmt.Errorf("collect spoke snapshots: %w", ctx.Err()) + case admission <- spoke: + active[spoke.ID] = time.Now() + next++ + inFlight++ + case result := <-results: + if err := ctx.Err(); err != nil { + shutdown(true) + return coverage, fmt.Errorf("collect spoke snapshots: %w", err) + } + delete(active, result.spoke.ID) + inFlight-- + if result.err != nil { + shutdown(true) + return coverage, result.err + } + if err := collector.persistSpokeResult(ctx, scope, result, &coverage); err != nil { + shutdown(true) + return coverage, err } - collector.observeSnapshot(SnapshotOutcomeInvalidSnapshot, time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) - continue - } - if err := collector.store.ReplaceSnapshot(ctx, scope, spoke, cloneSnapshot(snapshot), attemptedAt); err != nil { - collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(startedAt)) - return coverage, fmt.Errorf("collect spoke snapshots: persist %q: %w", spoke.ID, err) } - coverage.Reachable++ - collector.observeSnapshot(SnapshotOutcomeSuccess, time.Since(startedAt)) - collector.observeTrace(ctx, tracing.OutcomeSuccess, time.Since(startedAt)) } + shutdown(false) sort.Strings(coverage.Unreachable) sort.Strings(coverage.Stale) return coverage, nil } +type spokeCollectionResult struct { + spoke Spoke + snapshot Snapshot + failure FailureKind + err error + attemptedAt time.Time + startedAt time.Time +} + +func (collector *Collector) collectSpokeWorker( + ctx context.Context, + workspaceID tenancy.WorkspaceID, + jobs <-chan Spoke, + results chan<- spokeCollectionResult, +) { + for { + select { + case <-ctx.Done(): + return + case spoke, open := <-jobs: + if !open { + return + } + result, ok := collector.collectSpoke(ctx, workspaceID, spoke) + if !ok { + return + } + select { + case results <- result: + case <-ctx.Done(): + return + } + } + } +} + +func (collector *Collector) collectSpoke( + ctx context.Context, + workspaceID tenancy.WorkspaceID, + spoke Spoke, +) (result spokeCollectionResult, ok bool) { + result = spokeCollectionResult{spoke: spoke, startedAt: time.Now()} + defer func() { + if recover() != nil { + result.snapshot = Snapshot{} + result.failure = "" + result.err = errRefreshFlightPanicked + ok = true + } + }() + result.attemptedAt = collector.nowUTC() + spokeContext, cancel := context.WithTimeout(ctx, collector.spokeTimeout) + snapshot, collectionErr := collector.transport.Snapshot(spokeContext, workspaceID, cloneSpoke(spoke)) + deadlineErr := spokeContext.Err() + cancel() + if ctx.Err() != nil { + return spokeCollectionResult{}, false + } + if collectionErr == nil && deadlineErr != nil { + collectionErr = deadlineErr + } + if collectionErr != nil { + result.failure = failureFor(collectionErr) + return result, true + } + if err := validateSnapshot(spoke, snapshot, result.attemptedAt, collector.maxSnapshotAge); err != nil { + result.failure = FailureInvalidSnapshot + return result, true + } + result.snapshot = cloneSnapshot(snapshot) + return result, true +} + +func (collector *Collector) persistSpokeResult( + ctx context.Context, + scope tenancy.Scope, + result spokeCollectionResult, + coverage *fleet.Coverage, +) error { + if result.failure != "" { + if err := collector.recordFailure(ctx, scope, result.spoke, result.failure, result.attemptedAt, coverage); err != nil { + collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(result.startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(result.startedAt)) + return err + } + collector.observeSnapshot(snapshotOutcomeForFailure(result.failure), time.Since(result.startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(result.startedAt)) + return nil + } + if err := collector.store.ReplaceSnapshot(ctx, scope, result.spoke, cloneSnapshot(result.snapshot), result.attemptedAt); err != nil { + collector.observeSnapshot(SnapshotOutcomeStoreError, time.Since(result.startedAt)) + collector.observeTrace(ctx, tracing.OutcomeFailure, time.Since(result.startedAt)) + return fmt.Errorf("collect spoke snapshots: persist %q: %w", result.spoke.ID, err) + } + coverage.Reachable++ + collector.observeSnapshot(SnapshotOutcomeSuccess, time.Since(result.startedAt)) + collector.observeTrace(ctx, tracing.OutcomeSuccess, time.Since(result.startedAt)) + return nil +} + +func (collector *Collector) nowUTC() time.Time { + collector.nowMu.Lock() + defer collector.nowMu.Unlock() + return collector.now().UTC() +} + func (collector *Collector) recordFailure( ctx context.Context, scope tenancy.Scope, diff --git a/internal/hubfleet/collector_concurrency_test.go b/internal/hubfleet/collector_concurrency_test.go new file mode 100644 index 0000000..974cc15 --- /dev/null +++ b/internal/hubfleet/collector_concurrency_test.go @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestCollectorBoundsParallelSpokesAndSortsCoverage(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + store := newMemoryStore(testSpokes("spoke-f", "spoke-a", "spoke-e", "spoke-b", "spoke-d", "spoke-c")) + release := make(chan struct{}) + started := make(chan string, len(store.spokes)) + var active atomic.Int64 + var maximum atomic.Int64 + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Transport: transportFunc(func(_ context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + current := active.Add(1) + defer active.Add(-1) + for previous := maximum.Load(); current > previous && !maximum.CompareAndSwap(previous, current); previous = maximum.Load() { + } + started <- spoke.ID + <-release + if spoke.ID == "spoke-b" || spoke.ID == "spoke-d" || spoke.ID == "spoke-f" { + return Snapshot{}, errors.New("unavailable") + } + return validSnapshot(spoke.ID, now), nil + }), + MaxConcurrentSpokes: 2, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + result := make(chan collectorResult, 1) + scope := readerScope(t, "workspace-a") + go func() { + coverage, collectErr := collector.Collect(context.Background(), scope) + result <- collectorResult{coverage: coverage, err: collectErr} + }() + for range 2 { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("worker did not start") + } + } + select { + case spokeID := <-started: + t.Fatalf("worker bound exceeded before release by %q", spokeID) + case <-time.After(100 * time.Millisecond): + } + close(release) + + completed := <-result + if completed.err != nil { + t.Fatalf("Collect() error = %v", completed.err) + } + if maximum.Load() != 2 { + t.Fatalf("maximum concurrency = %d, want 2", maximum.Load()) + } + if completed.coverage.Requested != 6 || completed.coverage.Reachable != 3 || + !slices.Equal(completed.coverage.Unreachable, []string{"spoke-b", "spoke-d", "spoke-f"}) { + t.Fatalf("coverage = %#v", completed.coverage) + } +} + +func TestCollectorCollapsesSerialTimeoutsIntoParallelWave(t *testing.T) { + t.Parallel() + + store := newMemoryStore(testSpokes("spoke-a", "spoke-b", "spoke-c", "spoke-d")) + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, _ Spoke) (Snapshot, error) { + <-ctx.Done() + return Snapshot{}, ctx.Err() + }), + SpokeTimeout: time.Second, + MaxConcurrentSpokes: 4, + Now: func() time.Time { return time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) }, + }) + if err != nil { + t.Fatal(err) + } + + startedAt := time.Now() + coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")) + elapsed := time.Since(startedAt) + if err != nil { + t.Fatalf("Collect() error = %v", err) + } + if elapsed >= 2500*time.Millisecond { + t.Fatalf("four one-second timeouts completed in %s, want one bounded parallel wave", elapsed) + } + if coverage.Reachable != 0 || len(coverage.Unreachable) != 4 { + t.Fatalf("coverage = %#v", coverage) + } +} + +func TestCollectorPersistsHealthyPeerWhileAnotherWorkerIsBlocked(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + store := newMemoryStore(testSpokes("spoke-blocked", "spoke-healthy")) + releaseBlocked := make(chan struct{}) + blockedStarted := make(chan struct{}) + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Transport: transportFunc(func(_ context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + if spoke.ID == "spoke-blocked" { + close(blockedStarted) + <-releaseBlocked + } + return validSnapshot(spoke.ID, now), nil + }), + MaxConcurrentSpokes: 2, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + result := make(chan error, 1) + scope := readerScope(t, "workspace-a") + go func() { + _, collectErr := collector.Collect(context.Background(), scope) + result <- collectErr + }() + select { + case <-blockedStarted: + case <-time.After(time.Second): + t.Fatal("blocked spoke did not start") + } + if !waitForStoredSnapshot(store, "spoke-healthy", time.Second) { + t.Fatal("healthy peer was not persisted while another worker was blocked") + } + close(releaseBlocked) + if err := <-result; err != nil { + t.Fatalf("Collect() error = %v", err) + } +} + +func TestCollectWorkspaceCancellationStopsAdmissionAndWorkers(t *testing.T) { + t.Parallel() + + store := newMemoryStore(testSpokes("spoke-a", "spoke-b", "spoke-c", "spoke-d", "spoke-e")) + started := make(chan string, len(store.spokes)) + var active atomic.Int64 + observer := &recordingSnapshotObserver{} + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Observer: observer, + Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + active.Add(1) + defer active.Add(-1) + started <- spoke.ID + <-ctx.Done() + return Snapshot{}, ctx.Err() + }), + MaxConcurrentSpokes: 2, + }) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan collectorResult, 1) + scope := readerScope(t, "workspace-a") + go func() { + coverage, collectErr := collector.collectWorkspace(ctx, scope) + result <- collectorResult{coverage: coverage, err: collectErr} + }() + for range 2 { + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("worker did not start") + } + } + cancel() + completed := <-result + if !errors.Is(completed.err, context.Canceled) { + t.Fatalf("collectWorkspace() error = %v, want cancellation", completed.err) + } + if completed.coverage.Requested != len(store.spokes) || completed.coverage.Reachable != 0 { + t.Fatalf("coverage = %#v", completed.coverage) + } + if active.Load() != 0 { + t.Fatalf("active workers after return = %d", active.Load()) + } + if len(observer.events) != 2 { + t.Fatalf("cancellation observations = %#v, want two admitted spokes", observer.events) + } + for _, event := range observer.events { + if event.outcome != SnapshotOutcomeCanceled || event.duration < 0 { + t.Fatalf("cancellation observation = %#v", event) + } + } + select { + case spokeID := <-started: + t.Fatalf("spoke %q was admitted after cancellation", spokeID) + default: + } + store.mu.Lock() + defer store.mu.Unlock() + if len(store.snapshots) != 0 || len(store.failures) != 0 { + t.Fatalf("store mutated after cancellation: snapshots=%v failures=%v", store.snapshots, store.failures) + } +} + +func TestCollectorStoreFailureCancelsWorkersAndLaterAdmission(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + baseStore := newMemoryStore(testSpokes("spoke-a", "spoke-b", "spoke-c", "spoke-d")) + store := &failingReplaceStore{memoryStore: baseStore, failSpoke: "spoke-a"} + started := make(chan string, len(baseStore.spokes)) + secondWorkerStarted := make(chan struct{}) + blockedCanceled := make(chan struct{}) + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + started <- spoke.ID + if spoke.ID == "spoke-a" { + <-secondWorkerStarted + } + if spoke.ID == "spoke-b" { + close(secondWorkerStarted) + <-ctx.Done() + close(blockedCanceled) + return Snapshot{}, ctx.Err() + } + return validSnapshot(spoke.ID, now), nil + }), + MaxConcurrentSpokes: 2, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + _, err = collector.Collect(context.Background(), readerScope(t, "workspace-a")) + if err == nil || !strings.Contains(err.Error(), `persist "spoke-a"`) { + t.Fatalf("Collect() error = %v, want fail-closed store error", err) + } + select { + case <-blockedCanceled: + case <-time.After(time.Second): + t.Fatal("blocked worker was not canceled before return") + } + var admitted []string + for { + select { + case spokeID := <-started: + admitted = append(admitted, spokeID) + default: + if slices.Contains(admitted, "spoke-c") || slices.Contains(admitted, "spoke-d") { + t.Fatalf("later spokes admitted after store failure: %v", admitted) + } + return + } + } +} + +func TestCollectorRecoversPanickingTransportWorkerAndLaterRefresh(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + store := newMemoryStore(testSpokes("spoke-a", "spoke-b", "spoke-c")) + secondWorkerStarted := make(chan struct{}) + blockedCanceled := make(chan struct{}) + var panicFlight atomic.Bool + panicFlight.Store(true) + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + if panicFlight.Load() { + switch spoke.ID { + case "spoke-a": + <-secondWorkerStarted + panicFlight.Store(false) + panic("untrusted transport panic") + case "spoke-b": + close(secondWorkerStarted) + <-ctx.Done() + close(blockedCanceled) + return Snapshot{}, ctx.Err() + } + } + return validSnapshot(spoke.ID, now), nil + }), + MaxConcurrentSpokes: 2, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + if _, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")); !errors.Is(err, errRefreshFlightPanicked) { + t.Fatalf("panicking Collect() error = %v", err) + } + select { + case <-blockedCanceled: + case <-time.After(time.Second): + t.Fatal("peer worker was not canceled after transport panic") + } + coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")) + if err != nil || coverage.Reachable != len(store.spokes) { + t.Fatalf("later Collect() coverage/error = %#v/%v", coverage, err) + } +} + +func TestCollectorJoinsWorkersWhenStorePanicsAndLaterRefresh(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + store := &panickingReplaceStore{memoryStore: newMemoryStore(testSpokes("spoke-a", "spoke-b", "spoke-c"))} + store.panicNext.Store(true) + secondWorkerStarted := make(chan struct{}) + blockedCanceled := make(chan struct{}) + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), + Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + if store.panicNext.Load() { + switch spoke.ID { + case "spoke-a": + <-secondWorkerStarted + case "spoke-b": + close(secondWorkerStarted) + <-ctx.Done() + close(blockedCanceled) + return Snapshot{}, ctx.Err() + } + } + return validSnapshot(spoke.ID, now), nil + }), + MaxConcurrentSpokes: 2, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + if _, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")); !errors.Is(err, errRefreshFlightPanicked) { + t.Fatalf("panicking store Collect() error = %v", err) + } + select { + case <-blockedCanceled: + case <-time.After(time.Second): + t.Fatal("peer worker was not joined after store panic") + } + coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")) + if err != nil || coverage.Reachable != len(store.spokes) { + t.Fatalf("later Collect() coverage/error = %#v/%v", coverage, err) + } +} + +type collectorResult struct { + coverage fleet.Coverage + err error +} + +type failingReplaceStore struct { + *memoryStore + failSpoke string +} + +type panickingReplaceStore struct { + *memoryStore + panicNext atomic.Bool +} + +func (store *panickingReplaceStore) ReplaceSnapshot( + ctx context.Context, + scope tenancy.Scope, + spoke Spoke, + snapshot Snapshot, + attemptedAt time.Time, +) error { + if store.panicNext.CompareAndSwap(true, false) { + panic("store panic") + } + return store.memoryStore.ReplaceSnapshot(ctx, scope, spoke, snapshot, attemptedAt) +} + +func (store *failingReplaceStore) ReplaceSnapshot( + ctx context.Context, + scope tenancy.Scope, + spoke Spoke, + snapshot Snapshot, + attemptedAt time.Time, +) error { + if spoke.ID == store.failSpoke { + return errors.New("forced store failure") + } + return store.memoryStore.ReplaceSnapshot(ctx, scope, spoke, snapshot, attemptedAt) +} + +func newMemoryStore(spokes []Spoke) *memoryStore { + return &memoryStore{spokes: spokes, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)} +} + +func testSpokes(ids ...string) []Spoke { + spokes := make([]Spoke, 0, len(ids)) + for _, id := range ids { + spokes = append(spokes, Spoke{ID: id, ManagedClusterRef: "ocm/" + id}) + } + return spokes +} + +func waitForStoredSnapshot(store *memoryStore, spokeID string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + store.mu.Lock() + _, found := store.snapshots[spokeID] + store.mu.Unlock() + if found { + return true + } + time.Sleep(time.Millisecond) + } + return false +} + +var _ Store = (*failingReplaceStore)(nil) +var _ Store = (*panickingReplaceStore)(nil) diff --git a/internal/hubfleet/collector_test.go b/internal/hubfleet/collector_test.go index 3c7ddb4..bae7c14 100644 --- a/internal/hubfleet/collector_test.go +++ b/internal/hubfleet/collector_test.go @@ -29,8 +29,9 @@ func TestCollectorIsolatesSpokeFailuresAndRetainsStaleSnapshots(t *testing.T) { failures: make(map[string]FailureKind), } collector, err := NewCollector(CollectorConfig{ - Store: store, - PEP: testReadPEP(t), + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), Transport: transportFunc(func(_ context.Context, workspaceID tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { if workspaceID != "workspace-a" { return Snapshot{}, errors.New("wrong workspace") @@ -77,8 +78,9 @@ func TestCollectorBoundsEveryTransportCall(t *testing.T) { failures: make(map[string]FailureKind), } collector, err := NewCollector(CollectorConfig{ - Store: store, - PEP: testReadPEP(t), + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, _ Spoke) (Snapshot, error) { deadline, exists := ctx.Deadline() if !exists || time.Until(deadline) > 1100*time.Millisecond { @@ -170,8 +172,9 @@ func TestCollectorCopiesTransportOwnedSnapshot(t *testing.T) { failures: make(map[string]FailureKind), } collector, err := NewCollector(CollectorConfig{ - Store: store, - PEP: testReadPEP(t), + LifecycleContext: t.Context(), + Store: store, + PEP: testReadPEP(t), Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { return transportSnapshot, nil }), @@ -195,11 +198,30 @@ func TestCollectorAndSourceRejectUnsafeConfiguration(t *testing.T) { t.Parallel() store := &memoryStore{snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)} - if _, err := NewCollector(CollectorConfig{Store: store, Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + transport := transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { return Snapshot{}, nil - }), PEP: testReadPEP(t), SpokeTimeout: 999 * time.Millisecond}); err == nil { + }) + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), Store: store, Transport: transport, PEP: testReadPEP(t), + }) + if err != nil || collector.maxConcurrentSpokes != defaultSpokeConcurrency { + t.Fatalf("default worker configuration = %d/%v", collector.maxConcurrentSpokes, err) + } + if _, err := NewCollector(CollectorConfig{Store: store, Transport: transport, PEP: testReadPEP(t)}); err == nil { + t.Fatal("collector without lifecycle context unexpectedly accepted") + } + if _, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), Store: store, Transport: transport, + PEP: testReadPEP(t), SpokeTimeout: 999 * time.Millisecond, + }); err == nil { t.Fatal("sub-second collector timeout unexpectedly accepted") } + if _, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), Store: store, Transport: transport, + PEP: testReadPEP(t), MaxConcurrentSpokes: maximumSpokeConcurrency + 1, + }); err == nil { + t.Fatal("oversized collector worker pool unexpectedly accepted") + } if _, err := NewSource(SourceConfig{Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { return fleet.FleetResult{}, nil }), Scope: tenancy.Scope{}}); err == nil { diff --git a/internal/hubfleet/correlation.go b/internal/hubfleet/correlation.go index 32b4b49..e925ed5 100644 --- a/internal/hubfleet/correlation.go +++ b/internal/hubfleet/correlation.go @@ -78,7 +78,7 @@ func (correlator *Correlator) Correlate( if err := scope.Authorize(tenancy.ActionRead); err != nil { return fleet.QueryResult{}, fmt.Errorf("correlate fleet: %w", err) } - if err := request.validate(); err != nil { + if err := request.Validate(); err != nil { return fleet.QueryResult{}, fmt.Errorf("correlate fleet: %w", err) } canonicalArguments := strings.Join([]string{request.ResourceKind, request.Name, request.Namespace, request.HealthNot, fmt.Sprintf("%d", request.Limit)}, "\x00") @@ -101,7 +101,8 @@ func (correlator *Correlator) Correlate( return result, nil } -func (request CorrelationRequest) validate() error { +// Validate rejects malformed or unsafe correlation inputs before any policy or storage call. +func (request CorrelationRequest) Validate() error { for _, field := range []struct { name string value string diff --git a/internal/hubfleet/inventory_search.go b/internal/hubfleet/inventory_search.go new file mode 100644 index 0000000..37bc77f --- /dev/null +++ b/internal/hubfleet/inventory_search.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +// InventorySearchRequest selects one closed normalized OCM resource kind, optionally narrowed by +// an exact namespace and exact name. It deliberately excludes prefixes, labels, and raw selectors. +type InventorySearchRequest struct { + ResourceKind string `json:"resource_kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// InventorySearcherConfig defines a read-only, tenant-scoped normalized inventory service. +type InventorySearcherConfig struct { + Querier FleetQuerier + PEP *pep.Enforcer + Freshness time.Duration + Now func() time.Time +} + +// InventorySearcher resolves bounded normalized OCM inventory from persisted Hub state. +type InventorySearcher struct { + querier FleetQuerier + pep *pep.Enforcer + freshness time.Duration + now func() time.Time +} + +// NewInventorySearcher constructs a bounded read-only inventory service. +func NewInventorySearcher(config InventorySearcherConfig) (*InventorySearcher, error) { + if config.Querier == nil || config.PEP == nil { + return nil, fmt.Errorf("new fleet inventory searcher: querier and policy enforcer are required") + } + if config.Freshness == 0 { + config.Freshness = defaultSnapshotAge + } + if config.Freshness < time.Second || config.Freshness > maxSnapshotAge { + return nil, fmt.Errorf("new fleet inventory searcher: freshness must be between 1s and %s", maxSnapshotAge) + } + if config.Now == nil { + config.Now = time.Now + } + return &InventorySearcher{querier: config.Querier, pep: config.PEP, freshness: config.Freshness, now: config.Now}, nil +} + +// Search returns one coverage-honest persisted inventory selection in the signed workspace. +func (searcher *InventorySearcher) Search(ctx context.Context, scope tenancy.Scope, request InventorySearchRequest) (fleet.QueryResult, error) { + if searcher == nil || searcher.querier == nil || searcher.pep == nil || ctx == nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet inventory: searcher, policy enforcer, and context are required") + } + traceContext, _, err := tracing.Ensure(ctx) + if err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet inventory: establish trace context: %w", err) + } + ctx = traceContext + if err := scope.Authorize(tenancy.ActionRead); err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet inventory: %w", err) + } + if err := request.Validate(); err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet inventory: %w", err) + } + canonicalArguments := strings.Join([]string{request.ResourceKind, request.Namespace, request.Name, strconv.Itoa(request.Limit)}, "\x00") + if err := searcher.pep.AuthorizeRead(ctx, scope, pep.NewReadInput(pep.VerbFleetInventorySearch, []byte(canonicalArguments))); err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet inventory: %w", err) + } + result, err := searcher.querier.QueryFleet(ctx, scope, fleet.Query{ + Kinds: []fleet.FactKind{fleet.FactInventory}, + Selector: fleet.Selector{ResourceKind: request.ResourceKind, Namespace: request.Namespace, Name: request.Name}, + Limit: request.Limit, + }, searcher.freshness, searcher.now().UTC()) + if err != nil { + return fleet.QueryResult{}, fmt.Errorf("search fleet inventory: %w", err) + } + return result, nil +} + +// Validate rejects malformed or unsupported inventory inputs before policy or storage access. +func (request InventorySearchRequest) Validate() error { + for _, field := range []struct { + name string + value string + maximum int + empty bool + }{ + {name: "resource kind", value: request.ResourceKind, maximum: 128}, + {name: "resource namespace", value: request.Namespace, maximum: 256, empty: true}, + {name: "resource name", value: request.Name, maximum: 256, empty: true}, + } { + if err := validateText(field.name, field.value, field.maximum, field.empty); err != nil { + return err + } + } + switch request.ResourceKind { + case "Deployment", "Pod", "Rollout": + default: + return fmt.Errorf("resource kind is not supported for Hub inventory") + } + if request.Limit < 0 || request.Limit > 1_000 { + return fmt.Errorf("limit must be between 0 and 1000") + } + return nil +} diff --git a/internal/hubfleet/inventory_search_test.go b/internal/hubfleet/inventory_search_test.go new file mode 100644 index 0000000..61ab298 --- /dev/null +++ b/internal/hubfleet/inventory_search_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestInventorySearcherUsesExactTenantScopedInventoryQuery(t *testing.T) { + t.Parallel() + now := time.Date(2026, time.July, 17, 18, 0, 0, 0, time.UTC) + querier := &recordingFleetQuerier{result: fleet.QueryResult{Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{Ref: fleet.ResourceRef{Scope: "spoke-a", Kind: "Deployment", Name: "payments"}}, Workspace: "workspace-a", + }}}} + searcher, err := NewInventorySearcher(InventorySearcherConfig{Querier: querier, PEP: testReadPEP(t), Freshness: time.Minute, Now: func() time.Time { return now }}) + if err != nil { + t.Fatal(err) + } + result, err := searcher.Search(context.Background(), readerScope(t, "workspace-a"), InventorySearchRequest{ + ResourceKind: "Deployment", Namespace: "payments", Name: "payments", Limit: 257, + }) + if err != nil || len(result.Facts) != 1 { + t.Fatalf("Search() result = %#v, error = %v", result, err) + } + if querier.scope.WorkspaceID() != "workspace-a" || len(querier.query.Kinds) != 1 || querier.query.Kinds[0] != fleet.FactInventory || + querier.query.Selector.ResourceKind != "Deployment" || querier.query.Selector.Namespace != "payments" || + querier.query.Selector.Name != "payments" || querier.query.Selector.NamePrefix != "" || len(querier.query.Selector.Labels) != 0 || + querier.query.Limit != 257 || querier.freshness != time.Minute || !querier.now.Equal(now) { + t.Fatalf("querier call = %#v, freshness = %s, now = %s", querier.query, querier.freshness, querier.now) + } +} + +func TestInventorySearcherRejectsUnsafeRequestsBeforeQuery(t *testing.T) { + t.Parallel() + calls := 0 + searcher, err := NewInventorySearcher(InventorySearcherConfig{ + Querier: fleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + calls++ + return fleet.QueryResult{}, errors.New("unexpected query") + }), + PEP: testReadPEP(t), + }) + if err != nil { + t.Fatal(err) + } + for _, request := range []InventorySearchRequest{ + {}, {ResourceKind: "Secret"}, {ResourceKind: "Service"}, + {ResourceKind: "Deployment", Namespace: " payments"}, {ResourceKind: "Pod", Name: "api", Limit: 1_001}, + } { + if _, err := searcher.Search(context.Background(), readerScope(t, "workspace-a"), request); err == nil { + t.Fatalf("Search(%#v) unexpectedly succeeded", request) + } + } + if calls != 0 { + t.Fatalf("invalid requests reached fleet query %d times", calls) + } +} + +func TestNewInventorySearcherRejectsUnsafeConfiguration(t *testing.T) { + t.Parallel() + if _, err := NewInventorySearcher(InventorySearcherConfig{}); err == nil { + t.Fatal("NewInventorySearcher accepted missing dependencies") + } + if _, err := NewInventorySearcher(InventorySearcherConfig{Querier: &recordingFleetQuerier{}, PEP: testReadPEP(t), Freshness: time.Hour + time.Second}); err == nil { + t.Fatal("NewInventorySearcher accepted excessive freshness") + } +} diff --git a/internal/hubfleet/metrics_test.go b/internal/hubfleet/metrics_test.go index 8a6e5d2..d47d2c6 100644 --- a/internal/hubfleet/metrics_test.go +++ b/internal/hubfleet/metrics_test.go @@ -47,7 +47,8 @@ func TestCollectorObservesClosedSnapshotOutcomes(t *testing.T) { spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind), } collector, err := NewCollector(CollectorConfig{ - Store: store, Transport: test.transport, PEP: testReadPEP(t), Observer: observer, Now: func() time.Time { return now }, + LifecycleContext: t.Context(), Store: store, Transport: test.transport, + PEP: testReadPEP(t), Observer: observer, Now: func() time.Time { return now }, }) if err != nil { t.Fatal(err) @@ -65,6 +66,7 @@ func TestCollectorObservesClosedSnapshotOutcomes(t *testing.T) { func TestCollectorRecoversFromPanickingSnapshotObserver(t *testing.T) { now := time.Date(2026, time.July, 12, 20, 0, 0, 0, time.UTC) collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), Store: &memoryStore{ spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind), }, diff --git a/internal/hubfleet/policy_test.go b/internal/hubfleet/policy_test.go index 1653581..822121e 100644 --- a/internal/hubfleet/policy_test.go +++ b/internal/hubfleet/policy_test.go @@ -20,7 +20,7 @@ func TestHubReadEntrypointsStopBeforeDependenciesWhenPolicyRefuses(t *testing.T) store := &memoryStore{snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)} collector, err := NewCollector(CollectorConfig{ - Store: store, PEP: refusal.enforcer(t), + LifecycleContext: t.Context(), Store: store, PEP: refusal.enforcer(t), Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { return Snapshot{}, errors.New("transport must not run") }), @@ -36,7 +36,11 @@ func TestHubReadEntrypointsStopBeforeDependenciesWhenPolicyRefuses(t *testing.T) } reader := &recordingFleetReader{} - source, err := NewSource(SourceConfig{Reader: reader, Scope: scope, PEP: refusal.enforcer(t)}) + var readObservations int + source, err := NewSource(SourceConfig{ + Reader: reader, Scope: scope, PEP: refusal.enforcer(t), + Observer: fleetReadObserverFunc(func(FleetReadObservation) { readObservations++ }), + }) if err != nil { t.Fatal(err) } @@ -46,6 +50,9 @@ func TestHubReadEntrypointsStopBeforeDependenciesWhenPolicyRefuses(t *testing.T) if reader.calls != 0 { t.Fatalf("source reached fleet reader %d times after refusal", reader.calls) } + if readObservations != 0 { + t.Fatalf("source emitted %d fleet-read observations before authorization", readObservations) + } querier := &recordingFleetQuerier{} correlator, err := NewCorrelator(CorrelatorConfig{Querier: querier, PEP: refusal.enforcer(t), Freshness: time.Minute}) @@ -71,7 +78,18 @@ func TestHubReadEntrypointsStopBeforeDependenciesWhenPolicyRefuses(t *testing.T) if querier.calls != 0 { t.Fatalf("image search reached fleet query %d times after refusal", querier.calls) } - if got, want := refusal.verbs, []pep.Verb{pep.VerbSpokeSnapshotRefresh, pep.VerbFleetRead, pep.VerbFleetCorrelate, pep.VerbFleetImageSearch}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] || got[3] != want[3] { + + inventorySearcher, err := NewInventorySearcher(InventorySearcherConfig{Querier: querier, PEP: refusal.enforcer(t)}) + if err != nil { + t.Fatal(err) + } + if _, err := inventorySearcher.Search(context.Background(), scope, InventorySearchRequest{ResourceKind: "Pod"}); err == nil { + t.Fatal("Inventory Search() unexpectedly bypassed policy refusal") + } + if querier.calls != 0 { + t.Fatalf("inventory search reached fleet query %d times after refusal", querier.calls) + } + if got, want := refusal.verbs, []pep.Verb{pep.VerbSpokeSnapshotRefresh, pep.VerbFleetRead, pep.VerbFleetCorrelate, pep.VerbFleetImageSearch, pep.VerbFleetInventorySearch}; len(got) != len(want) || got[0] != want[0] || got[1] != want[1] || got[2] != want[2] || got[3] != want[3] || got[4] != want[4] { t.Fatalf("policy verbs = %q, want %q", got, want) } } @@ -100,7 +118,7 @@ func (refusal *policyRefusal) enforcer(t *testing.T) *pep.Enforcer { func TestHubReadConstructorsRequirePolicyEnforcer(t *testing.T) { store := &memoryStore{snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)} - if _, err := NewCollector(CollectorConfig{Store: store, Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + if _, err := NewCollector(CollectorConfig{LifecycleContext: t.Context(), Store: store, Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { return Snapshot{}, nil })}); err == nil { t.Fatal("NewCollector() accepted no policy enforcer") @@ -110,6 +128,11 @@ func TestHubReadConstructorsRequirePolicyEnforcer(t *testing.T) { })}); err == nil { t.Fatal("NewCorrelator() accepted no policy enforcer") } + if _, err := NewInventorySearcher(InventorySearcherConfig{Querier: fleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, nil + })}); err == nil { + t.Fatal("NewInventorySearcher() accepted no policy enforcer") + } if _, err := NewSource(SourceConfig{Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { return fleet.FleetResult{}, nil }), Scope: readerScope(t, "workspace-a")}); err == nil { diff --git a/internal/hubfleet/refresh_coordinator.go b/internal/hubfleet/refresh_coordinator.go new file mode 100644 index 0000000..f3937ab --- /dev/null +++ b/internal/hubfleet/refresh_coordinator.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +var errRefreshFlightPanicked = errors.New("collect spoke snapshots: refresh flight panicked") + +type refreshFlight struct { + done chan struct{} + coverage fleet.Coverage + err error +} + +type refreshCoordinator struct { + lifecycle context.Context + mu sync.Mutex + flights map[tenancy.WorkspaceID]*refreshFlight +} + +func newRefreshCoordinator(lifecycle context.Context) *refreshCoordinator { + return &refreshCoordinator{ + lifecycle: valueFreeContext{Context: lifecycle}, + flights: make(map[tenancy.WorkspaceID]*refreshFlight), + } +} + +// valueFreeContext preserves only lifecycle cancellation and deadlines. Refresh work must not +// inherit request-scoped credentials, authorization state, or trace identity from its owner. +type valueFreeContext struct { + context.Context +} + +func (ctx valueFreeContext) Value(any) any { return nil } + +func (ctx valueFreeContext) Deadline() (time.Time, bool) { return ctx.Context.Deadline() } + +func (coordinator *refreshCoordinator) collect( + ctx context.Context, + scope tenancy.Scope, + collect func(context.Context, tenancy.Scope) (fleet.Coverage, error), +) (fleet.Coverage, error) { + if coordinator == nil || ctx == nil || collect == nil { + return fleet.Coverage{}, errors.New("collect spoke snapshots: refresh coordinator, context, and collector are required") + } + if err := ctx.Err(); err != nil { + return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: %w", err) + } + + workspaceID := scope.WorkspaceID() + coordinator.mu.Lock() + flight, exists := coordinator.flights[workspaceID] + if !exists { + flight = &refreshFlight{done: make(chan struct{})} + coordinator.flights[workspaceID] = flight + //nolint:gosec // A flight outlives one caller but remains bounded by the collector lifecycle. + go coordinator.run(workspaceID, flight, scope, collect) + } + coordinator.mu.Unlock() + + return waitForRefresh(ctx, flight) +} + +func (coordinator *refreshCoordinator) run( + workspaceID tenancy.WorkspaceID, + flight *refreshFlight, + scope tenancy.Scope, + collect func(context.Context, tenancy.Scope) (fleet.Coverage, error), +) { + coverage := fleet.Coverage{} + var err error + defer func() { + if recover() != nil { + coverage = fleet.Coverage{} + err = errRefreshFlightPanicked + } + coordinator.finish(workspaceID, flight, coverage, err) + }() + + flightContext, _, traceErr := tracing.Ensure(coordinator.lifecycle) + if traceErr != nil { + err = fmt.Errorf("collect spoke snapshots: establish refresh flight trace: %w", traceErr) + return + } + coverage, err = collect(flightContext, scope) +} + +func (coordinator *refreshCoordinator) finish( + workspaceID tenancy.WorkspaceID, + flight *refreshFlight, + coverage fleet.Coverage, + err error, +) { + coordinator.mu.Lock() + flight.coverage = cloneCoverage(coverage) + flight.err = err + if coordinator.flights[workspaceID] == flight { + delete(coordinator.flights, workspaceID) + } + close(flight.done) + coordinator.mu.Unlock() +} + +func (coordinator *refreshCoordinator) activeFlights() int { + coordinator.mu.Lock() + defer coordinator.mu.Unlock() + return len(coordinator.flights) +} + +func waitForRefresh(ctx context.Context, flight *refreshFlight) (fleet.Coverage, error) { + select { + case <-flight.done: + return cloneCoverage(flight.coverage), flight.err + default: + } + + select { + case <-flight.done: + return cloneCoverage(flight.coverage), flight.err + case <-ctx.Done(): + select { + case <-flight.done: + return cloneCoverage(flight.coverage), flight.err + default: + return fleet.Coverage{}, fmt.Errorf("collect spoke snapshots: %w", ctx.Err()) + } + } +} + +func cloneCoverage(coverage fleet.Coverage) fleet.Coverage { + coverage.Unreachable = append([]string(nil), coverage.Unreachable...) + coverage.Stale = append([]string(nil), coverage.Stale...) + coverage.Truncated = append([]string(nil), coverage.Truncated...) + return coverage +} diff --git a/internal/hubfleet/refresh_coordinator_test.go b/internal/hubfleet/refresh_coordinator_test.go new file mode 100644 index 0000000..d18a1ba --- /dev/null +++ b/internal/hubfleet/refresh_coordinator_test.go @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestCollectorCoalescesAuthorizedRequestsForOneWorkspace(t *testing.T) { + t.Parallel() + + const callers = 24 + now := time.Date(2026, time.July, 16, 20, 0, 0, 0, time.UTC) + started := make(chan struct{}) + release := make(chan struct{}) + store := &memoryStore{ + spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, + snapshots: make(map[string]Snapshot), + failures: make(map[string]FailureKind), + } + var authorizations atomic.Int32 + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.HookFunc(func(context.Context, pep.Request) (pep.Decision, error) { + authorizations.Add(1) + return pep.Decision{Verdict: pep.VerdictAllow, ReasonCode: "test-allow"}, nil + }), + Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { return nil }), + }) + if err != nil { + t.Fatal(err) + } + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: enforcer, + Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + select { + case <-started: + default: + close(started) + } + <-release + return validSnapshot("spoke-a", now), nil + }), + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + results := make(chan fleet.Coverage, callers) + errors := make(chan error, callers) + contexts := make([]*observedDoneContext, callers) + var waiters sync.WaitGroup + for index := range callers { + contexts[index] = newObservedDoneContext(context.Background()) + waiters.Add(1) + go func(ctx context.Context) { + defer waiters.Done() + coverage, collectErr := collector.Collect(ctx, readerScope(t, "workspace-a")) + results <- coverage + errors <- collectErr + }(contexts[index]) + } + + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("workspace refresh did not start") + } + eventually(t, func() bool { return authorizations.Load() == callers }) + for _, ctx := range contexts { + <-ctx.observed + } + if got := collector.refreshes.activeFlights(); got != 1 { + t.Fatalf("active refresh flights = %d, want 1", got) + } + close(release) + waiters.Wait() + close(results) + close(errors) + + for collectErr := range errors { + if collectErr != nil { + t.Fatalf("Collect() error = %v", collectErr) + } + } + for coverage := range results { + if coverage.Requested != 1 || coverage.Reachable != 1 { + t.Fatalf("shared coverage = %#v", coverage) + } + } + store.mu.Lock() + registeredCalls := store.registeredCalls + store.mu.Unlock() + if registeredCalls != 1 { + t.Fatalf("RegisteredSpokes() calls = %d, want 1", registeredCalls) + } + if got := collector.refreshes.activeFlights(); got != 0 { + t.Fatalf("completed refresh flights = %d, want 0", got) + } +} + +func TestCollectorRefusesCallerBeforeJoiningWorkspaceFlight(t *testing.T) { + t.Parallel() + + now := time.Date(2026, time.July, 16, 20, 0, 0, 0, time.UTC) + started := make(chan struct{}) + release := make(chan struct{}) + store := &memoryStore{ + spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, + snapshots: make(map[string]Snapshot), + failures: make(map[string]FailureKind), + } + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.HookFunc(func(_ context.Context, request pep.Request) (pep.Decision, error) { + if request.Actor == "user:denied" { + return pep.Decision{Verdict: pep.VerdictDeny, ReasonCode: "test-deny"}, nil + } + return pep.Decision{Verdict: pep.VerdictAllow, ReasonCode: "test-allow"}, nil + }), + Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { return nil }), + }) + if err != nil { + t.Fatal(err) + } + collector, err := NewCollector(CollectorConfig{ + LifecycleContext: t.Context(), + Store: store, + PEP: enforcer, + Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { + close(started) + <-release + return validSnapshot("spoke-a", now), nil + }), + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + + allowedResult := make(chan error, 1) + go func() { + _, collectErr := collector.Collect(context.Background(), readerScopeFor(t, "user:allowed", "workspace-a")) + allowedResult <- collectErr + }() + <-started + deniedContext := newObservedDoneContext(context.Background()) + if _, err := collector.Collect(deniedContext, readerScopeFor(t, "user:denied", "workspace-a")); err == nil { + t.Fatal("denied refresh unexpectedly joined the active workspace flight") + } + select { + case <-deniedContext.observed: + t.Fatal("denied refresh reached the workspace coordinator") + default: + } + close(release) + if err := <-allowedResult; err != nil { + t.Fatalf("authorized refresh error = %v", err) + } +} + +func TestRefreshCoordinatorKeepsWorkspacesIndependent(t *testing.T) { + t.Parallel() + + coordinator := newRefreshCoordinator(t.Context()) + releaseA := make(chan struct{}) + startedA := make(chan struct{}) + startedB := make(chan struct{}) + collect := func(_ context.Context, scope tenancy.Scope) (fleet.Coverage, error) { + switch scope.WorkspaceID() { + case "workspace-a": + close(startedA) + <-releaseA + case "workspace-b": + close(startedB) + default: + t.Fatalf("unexpected workspace %q", scope.WorkspaceID()) + } + return fleet.Coverage{Requested: 1, Reachable: 1}, nil + } + + resultA := make(chan error, 1) + go func() { + _, err := coordinator.collect(context.Background(), readerScope(t, "workspace-a"), collect) + resultA <- err + }() + <-startedA + resultB := make(chan error, 1) + go func() { + _, err := coordinator.collect(context.Background(), readerScope(t, "workspace-b"), collect) + resultB <- err + }() + select { + case <-startedB: + case <-time.After(5 * time.Second): + t.Fatal("workspace-b was serialized behind workspace-a") + } + if err := <-resultB; err != nil { + t.Fatalf("workspace-b collect error = %v", err) + } + close(releaseA) + if err := <-resultA; err != nil { + t.Fatalf("workspace-a collect error = %v", err) + } +} + +func TestRefreshCoordinatorDetachesLeaderAndWaiterCancellation(t *testing.T) { + t.Parallel() + + coordinator := newRefreshCoordinator(t.Context()) + started := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + collect := func(ctx context.Context, _ tenancy.Scope) (fleet.Coverage, error) { + calls.Add(1) + if ctx.Err() != nil { + t.Fatalf("detached refresh context error = %v", ctx.Err()) + } + close(started) + <-release + return fleet.Coverage{Requested: 2, Reachable: 2}, nil + } + + leaderContext, cancelLeader := context.WithCancel(context.Background()) + leaderResult := make(chan error, 1) + go func() { + _, err := coordinator.collect(leaderContext, readerScope(t, "workspace-a"), collect) + leaderResult <- err + }() + <-started + cancelLeader() + if err := <-leaderResult; !errors.Is(err, context.Canceled) { + t.Fatalf("leader cancellation error = %v, want context canceled", err) + } + + waiterContext, cancelWaiter := context.WithCancel(context.Background()) + observedWaiterContext := newObservedDoneContext(waiterContext) + waiterResult := make(chan error, 1) + go func() { + _, err := coordinator.collect(observedWaiterContext, readerScope(t, "workspace-a"), collect) + waiterResult <- err + }() + <-observedWaiterContext.observed + cancelWaiter() + if err := <-waiterResult; !errors.Is(err, context.Canceled) { + t.Fatalf("waiter cancellation error = %v, want context canceled", err) + } + + survivorResult := make(chan struct { + coverage fleet.Coverage + err error + }, 1) + survivorContext := newObservedDoneContext(context.Background()) + go func() { + coverage, err := coordinator.collect(survivorContext, readerScope(t, "workspace-a"), collect) + survivorResult <- struct { + coverage fleet.Coverage + err error + }{coverage: coverage, err: err} + }() + <-survivorContext.observed + close(release) + survivor := <-survivorResult + if survivor.err != nil || survivor.coverage.Reachable != 2 { + t.Fatalf("surviving waiter result = %#v, error = %v", survivor.coverage, survivor.err) + } + if calls.Load() != 1 || coordinator.activeFlights() != 0 { + t.Fatalf("refresh calls/flights = %d/%d, want 1/0", calls.Load(), coordinator.activeFlights()) + } +} + +func TestRefreshCoordinatorSharesFailuresAndRecoversAfterPanic(t *testing.T) { + t.Parallel() + + coordinator := newRefreshCoordinator(t.Context()) + sharedFailure := errors.New("closed store failure") + started := make(chan struct{}) + release := make(chan struct{}) + var calls atomic.Int32 + collectFailure := func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + calls.Add(1) + close(started) + <-release + return fleet.Coverage{Requested: 1, Unreachable: []string{"spoke-a"}}, sharedFailure + } + + type refreshResult struct { + coverage fleet.Coverage + err error + } + results := make(chan refreshResult, 2) + go func() { + coverage, err := coordinator.collect(context.Background(), readerScope(t, "workspace-a"), collectFailure) + results <- refreshResult{coverage: coverage, err: err} + }() + <-started + secondContext := newObservedDoneContext(context.Background()) + go func() { + coverage, err := coordinator.collect(secondContext, readerScope(t, "workspace-a"), collectFailure) + results <- refreshResult{coverage: coverage, err: err} + }() + <-secondContext.observed + close(release) + first := <-results + second := <-results + for _, result := range []refreshResult{first, second} { + if !errors.Is(result.err, sharedFailure) { + t.Fatalf("shared error = %v, want closed failure", result.err) + } + } + first.coverage.Unreachable[0] = "mutated-by-caller" + if second.coverage.Unreachable[0] != "spoke-a" { + t.Fatalf("shared coverage aliases caller memory: %#v", second.coverage) + } + if calls.Load() != 1 || coordinator.activeFlights() != 0 { + t.Fatalf("failure calls/flights = %d/%d, want 1/0", calls.Load(), coordinator.activeFlights()) + } + + if _, err := coordinator.collect(context.Background(), readerScope(t, "workspace-a"), func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + panic("credential-bearing panic must not escape") + }); !errors.Is(err, errRefreshFlightPanicked) { + t.Fatalf("panic result = %v, want closed panic error", err) + } + if coordinator.activeFlights() != 0 { + t.Fatalf("panic retained %d refresh flights", coordinator.activeFlights()) + } + + coverage, err := coordinator.collect(context.Background(), readerScope(t, "workspace-a"), func(context.Context, tenancy.Scope) (fleet.Coverage, error) { + return fleet.Coverage{Requested: 1, Reachable: 1}, nil + }) + if err != nil || coverage.Reachable != 1 { + t.Fatalf("post-panic refresh = %#v, error = %v", coverage, err) + } +} + +func TestRefreshCoordinatorStopsFlightsAtLifecycleBoundaryWithoutValues(t *testing.T) { + t.Parallel() + + type lifecycleValueKey struct{} + lifecycle, cancelLifecycle := context.WithCancel( + context.WithValue(context.Background(), lifecycleValueKey{}, "must-not-cross-refresh-boundary"), + ) + defer cancelLifecycle() + coordinator := newRefreshCoordinator(lifecycle) + started := make(chan struct{}) + result := make(chan error, 1) + go func() { + _, err := coordinator.collect(context.Background(), readerScope(t, "workspace-a"), func(ctx context.Context, _ tenancy.Scope) (fleet.Coverage, error) { + close(started) + if value := ctx.Value(lifecycleValueKey{}); value != nil { + return fleet.Coverage{}, fmt.Errorf("lifecycle value leaked into refresh: %v", value) + } + <-ctx.Done() + return fleet.Coverage{}, ctx.Err() + }) + result <- err + }() + <-started + cancelLifecycle() + if err := <-result; !errors.Is(err, context.Canceled) { + t.Fatalf("lifecycle cancellation error = %v, want context canceled", err) + } + if coordinator.activeFlights() != 0 { + t.Fatalf("lifecycle cancellation retained %d refresh flights", coordinator.activeFlights()) + } +} + +func eventually(t *testing.T, condition func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !condition() { + if time.Now().After(deadline) { + t.Fatal("condition did not become true") + } + time.Sleep(time.Millisecond) + } +} + +func readerScopeFor(t *testing.T, subject string, workspaceID tenancy.WorkspaceID) tenancy.Scope { + t.Helper() + principal, err := tenancy.NewPrincipal(subject, map[tenancy.WorkspaceID]tenancy.Role{workspaceID: tenancy.RoleReader}) + if err != nil { + t.Fatal(err) + } + scope, err := principal.Scope(workspaceID) + if err != nil { + t.Fatal(err) + } + return scope +} + +type observedDoneContext struct { + context.Context + once sync.Once + observed chan struct{} +} + +func newObservedDoneContext(ctx context.Context) *observedDoneContext { + return &observedDoneContext{Context: ctx, observed: make(chan struct{})} +} + +func (ctx *observedDoneContext) Done() <-chan struct{} { + ctx.once.Do(func() { close(ctx.observed) }) + return ctx.Context.Done() +} diff --git a/internal/hubfleet/source.go b/internal/hubfleet/source.go index 2fff97c..91b6fa9 100644 --- a/internal/hubfleet/source.go +++ b/internal/hubfleet/source.go @@ -5,6 +5,7 @@ package hubfleet import ( "context" "fmt" + "strings" "time" "github.com/ArdurAI/sith/internal/fleet" @@ -18,11 +19,92 @@ type FleetReader interface { ReadFleet(ctx context.Context, scope tenancy.Scope, freshness time.Duration, now time.Time) (fleet.FleetResult, error) } +// FleetReadOutcome is the bounded self-observability result of one authorized fleet read. It +// intentionally carries no workspace, spoke, resource, selector, principal, trace, or raw error. +type FleetReadOutcome string + +// Closed fleet-read outcomes. +const ( + FleetReadOutcomeComplete FleetReadOutcome = "complete" + FleetReadOutcomeDegraded FleetReadOutcome = "degraded" + FleetReadOutcomeEmpty FleetReadOutcome = "empty" + FleetReadOutcomeError FleetReadOutcome = "error" +) + +// Valid reports whether the outcome belongs to the closed fleet-read vocabulary. +func (outcome FleetReadOutcome) Valid() bool { + switch outcome { + case FleetReadOutcomeComplete, FleetReadOutcomeDegraded, FleetReadOutcomeEmpty, FleetReadOutcomeError: + return true + default: + return false + } +} + +// FleetFreshnessOutcome is the bounded request-time freshness result of one authorized fleet +// read. It intentionally describes only the aggregate returned view and carries no workspace, +// spoke, resource, selector, principal, trace, age, or raw error. +type FleetFreshnessOutcome string + +// Closed request-time fleet freshness outcomes. +const ( + FleetFreshnessOutcomeFresh FleetFreshnessOutcome = "fresh" + FleetFreshnessOutcomeStale FleetFreshnessOutcome = "stale" + FleetFreshnessOutcomeUnknown FleetFreshnessOutcome = "unknown" + FleetFreshnessOutcomeEmpty FleetFreshnessOutcome = "empty" + FleetFreshnessOutcomeError FleetFreshnessOutcome = "error" +) + +// Valid reports whether the freshness outcome belongs to the closed vocabulary. +func (outcome FleetFreshnessOutcome) Valid() bool { + switch outcome { + case FleetFreshnessOutcomeFresh, FleetFreshnessOutcomeStale, FleetFreshnessOutcomeUnknown, + FleetFreshnessOutcomeEmpty, FleetFreshnessOutcomeError: + return true + default: + return false + } +} + +// FleetReadObservation is one privacy-bounded pair of aggregate coverage and request-time +// freshness outcomes for an authorized fleet read. +type FleetReadObservation struct { + Outcome FleetReadOutcome + Freshness FleetFreshnessOutcome +} + +// Valid reports whether both dimensions belong to their closed vocabularies. +func (observation FleetReadObservation) Valid() bool { + if !observation.Outcome.Valid() || !observation.Freshness.Valid() { + return false + } + switch observation.Outcome { + case FleetReadOutcomeComplete: + return observation.Freshness == FleetFreshnessOutcomeFresh + case FleetReadOutcomeDegraded: + return observation.Freshness == FleetFreshnessOutcomeStale || + observation.Freshness == FleetFreshnessOutcomeUnknown + case FleetReadOutcomeEmpty: + return observation.Freshness == FleetFreshnessOutcomeEmpty + case FleetReadOutcomeError: + return observation.Freshness == FleetFreshnessOutcomeError + default: + return false + } +} + +// FleetReadObserver receives one passive result for an authorized fleet read. Implementations +// must not block or mutate read behavior; Source isolates observer panics defensively. +type FleetReadObserver interface { + ObserveFleetRead(FleetReadObservation) +} + // SourceConfig fixes a signed tenancy scope to a source-abstract hub reader. type SourceConfig struct { Reader FleetReader Scope tenancy.Scope PEP *pep.Enforcer + Observer FleetReadObserver Freshness time.Duration Now func() time.Time } @@ -32,6 +114,7 @@ type Source struct { reader FleetReader scope tenancy.Scope pep *pep.Enforcer + observer FleetReadObserver freshness time.Duration now func() time.Time } @@ -55,7 +138,10 @@ func NewSource(config SourceConfig) (*Source, error) { if config.Now == nil { config.Now = time.Now } - return &Source{reader: config.Reader, scope: config.Scope, pep: config.PEP, freshness: config.Freshness, now: config.Now}, nil + return &Source{ + reader: config.Reader, scope: config.Scope, pep: config.PEP, observer: config.Observer, + freshness: config.Freshness, now: config.Now, + }, nil } // Kind identifies OCM-brokered spoke snapshots at the common fleet.Source seam. @@ -76,7 +162,73 @@ func (source *Source) Fleet(ctx context.Context) (fleet.FleetResult, error) { } result, err := source.reader.ReadFleet(ctx, source.scope, source.freshness, source.now().UTC()) if err != nil { + source.observeFleetRead(FleetReadObservation{ + Outcome: FleetReadOutcomeError, Freshness: FleetFreshnessOutcomeError, + }) return fleet.FleetResult{}, fmt.Errorf("read OCM spoke fleet: %w", err) } + assessment := result.Coverage.Assessment() + observation := FleetReadObservation{Freshness: fleetFreshnessOutcome(result, assessment)} + switch { + case !assessment.Complete || len(result.Clusters) != result.Coverage.Requested: + observation.Outcome = FleetReadOutcomeDegraded + case result.Coverage.Requested == 0: + observation.Outcome = FleetReadOutcomeEmpty + default: + observation.Outcome = FleetReadOutcomeComplete + } + source.observeFleetRead(observation) return result, nil } + +func fleetFreshnessOutcome(result fleet.FleetResult, assessment fleet.CoverageAssessment) FleetFreshnessOutcome { + observed, validClusters := observedFleetScopes(result) + switch { + case assessment.Inconsistent || !validClusters: + return FleetFreshnessOutcomeUnknown + case len(assessment.Stale) != 0: + for _, scope := range assessment.Stale { + if observedAt, exists := observed[scope]; !exists || observedAt.IsZero() { + return FleetFreshnessOutcomeUnknown + } + } + return FleetFreshnessOutcomeStale + case result.Coverage.Requested == 0: + return FleetFreshnessOutcomeEmpty + case !assessment.Complete: + return FleetFreshnessOutcomeUnknown + } + for _, observedAt := range observed { + if observedAt.IsZero() { + return FleetFreshnessOutcomeUnknown + } + } + return FleetFreshnessOutcomeFresh +} + +func observedFleetScopes(result fleet.FleetResult) (map[string]time.Time, bool) { + if len(result.Clusters) != result.Coverage.Requested { + return nil, false + } + observed := make(map[string]time.Time, len(result.Clusters)) + for _, cluster := range result.Clusters { + if strings.TrimSpace(cluster.Name) == "" { + return nil, false + } + if _, exists := observed[cluster.Name]; exists { + return nil, false + } + observed[cluster.Name] = cluster.ObservedAt + } + return observed, true +} + +func (source *Source) observeFleetRead(observation FleetReadObservation) { + if source == nil || source.observer == nil || !observation.Valid() { + return + } + defer func() { + _ = recover() + }() + source.observer.ObserveFleetRead(observation) +} diff --git a/internal/hubfleet/source_observability_test.go b/internal/hubfleet/source_observability_test.go new file mode 100644 index 0000000..a4c9fea --- /dev/null +++ b/internal/hubfleet/source_observability_test.go @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubfleet + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestSourceObservesOneBoundedOutcomeAfterAuthorizedRead(t *testing.T) { + t.Parallel() + + readerFailure := errors.New("reader unavailable") + now := time.Date(2026, 7, 18, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + result fleet.FleetResult + readerErr error + want FleetReadObservation + wantErrIs error + }{ + { + name: "complete", result: fleet.FleetResult{ + Clusters: []fleet.Cluster{{Name: "spoke-a", ObservedAt: now.Add(-time.Minute)}, {Name: "spoke-b", ObservedAt: now.Add(-time.Minute)}}, + Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, + }, want: FleetReadObservation{Outcome: FleetReadOutcomeComplete, Freshness: FleetFreshnessOutcomeFresh}, + }, + { + name: "degraded stale", result: fleet.FleetResult{ + Clusters: []fleet.Cluster{{Name: "spoke-a", ObservedAt: now.Add(-time.Minute)}, {Name: "spoke-b", ObservedAt: now.Add(-10 * time.Minute)}}, + Coverage: fleet.Coverage{Requested: 2, Reachable: 2, Stale: []string{"spoke-b"}}, + }, want: FleetReadObservation{Outcome: FleetReadOutcomeDegraded, Freshness: FleetFreshnessOutcomeStale}, + }, + { + name: "degraded never observed", result: fleet.FleetResult{ + Clusters: []fleet.Cluster{{Name: "spoke-a", ObservedAt: now.Add(-time.Minute)}, {Name: "spoke-b"}}, + Coverage: fleet.Coverage{Requested: 2, Reachable: 1, Unreachable: []string{"spoke-b"}}, + }, want: FleetReadObservation{Outcome: FleetReadOutcomeDegraded, Freshness: FleetFreshnessOutcomeUnknown}, + }, + { + name: "degraded malformed empty", result: fleet.FleetResult{ + Coverage: fleet.Coverage{Requested: 0, Reachable: 1}, + }, want: FleetReadObservation{Outcome: FleetReadOutcomeDegraded, Freshness: FleetFreshnessOutcomeUnknown}, + }, + { + name: "degraded result coverage mismatch", result: fleet.FleetResult{ + Clusters: []fleet.Cluster{{Name: "spoke-a", ObservedAt: now.Add(-time.Minute)}}, + Coverage: fleet.Coverage{}, + }, want: FleetReadObservation{Outcome: FleetReadOutcomeDegraded, Freshness: FleetFreshnessOutcomeUnknown}, + }, + { + name: "degraded stale without retained observation", result: fleet.FleetResult{ + Clusters: []fleet.Cluster{{Name: "spoke-a", ObservedAt: now.Add(-time.Minute)}, {Name: "spoke-b"}}, + Coverage: fleet.Coverage{Requested: 2, Reachable: 2, Stale: []string{"spoke-b"}}, + }, want: FleetReadObservation{Outcome: FleetReadOutcomeDegraded, Freshness: FleetFreshnessOutcomeUnknown}, + }, + {name: "empty", result: fleet.FleetResult{Coverage: fleet.Coverage{}}, want: FleetReadObservation{ + Outcome: FleetReadOutcomeEmpty, Freshness: FleetFreshnessOutcomeEmpty, + }}, + {name: "error", readerErr: readerFailure, want: FleetReadObservation{ + Outcome: FleetReadOutcomeError, Freshness: FleetFreshnessOutcomeError, + }, wantErrIs: readerFailure}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + var observed []FleetReadObservation + source, err := NewSource(SourceConfig{ + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return test.result, test.readerErr + }), + Scope: readerScope(t, "workspace-a"), PEP: testReadPEP(t), + Observer: fleetReadObserverFunc(func(observation FleetReadObservation) { observed = append(observed, observation) }), + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + _, err = source.Fleet(context.Background()) + if !errors.Is(err, test.wantErrIs) || (test.wantErrIs == nil && err != nil) { + t.Fatalf("Fleet() error = %v, want errors.Is(%v)", err, test.wantErrIs) + } + if !slices.Equal(observed, []FleetReadObservation{test.want}) { + t.Fatalf("observed outcomes = %q, want %q", observed, test.want) + } + }) + } +} + +func TestSourceReadResultIsIndependentOfObserverFailure(t *testing.T) { + t.Parallel() + + readerFailure := errors.New("reader unavailable") + for _, test := range []struct { + name string + readerErr error + }{ + {name: "success"}, + {name: "reader error", readerErr: readerFailure}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + source, err := NewSource(SourceConfig{ + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{Coverage: fleet.Coverage{Requested: 1, Reachable: 1}}, test.readerErr + }), + Scope: readerScope(t, "workspace-a"), PEP: testReadPEP(t), + Observer: fleetReadObserverFunc(func(FleetReadObservation) { panic("observer failure") }), + }) + if err != nil { + t.Fatal(err) + } + result, err := source.Fleet(context.Background()) + if test.readerErr == nil && (err != nil || !result.Coverage.Complete()) { + t.Fatalf("successful read changed by observer panic: result %#v, error %v", result, err) + } + if test.readerErr != nil && !errors.Is(err, test.readerErr) { + t.Fatalf("reader error changed by observer panic: %v", err) + } + }) + } +} + +func TestFleetReadObservationRejectsContradictoryPairs(t *testing.T) { + t.Parallel() + + for _, observation := range []FleetReadObservation{ + {Outcome: FleetReadOutcomeComplete, Freshness: FleetFreshnessOutcomeStale}, + {Outcome: FleetReadOutcomeDegraded, Freshness: FleetFreshnessOutcomeFresh}, + {Outcome: FleetReadOutcomeEmpty, Freshness: FleetFreshnessOutcomeUnknown}, + {Outcome: FleetReadOutcomeError, Freshness: FleetFreshnessOutcomeEmpty}, + {Outcome: FleetReadOutcome("workspace-a"), Freshness: FleetFreshnessOutcomeFresh}, + } { + if observation.Valid() { + t.Fatalf("FleetReadObservation.Valid() accepted contradictory pair %#v", observation) + } + } +} + +type fleetReadObserverFunc func(FleetReadObservation) + +func (function fleetReadObserverFunc) ObserveFleetRead(observation FleetReadObservation) { + function(observation) +} diff --git a/internal/hubfleet/tracing_test.go b/internal/hubfleet/tracing_test.go index 90dd5cd..a047554 100644 --- a/internal/hubfleet/tracing_test.go +++ b/internal/hubfleet/tracing_test.go @@ -12,7 +12,7 @@ import ( "github.com/ArdurAI/sith/internal/tracing" ) -func TestCollectorPropagatesOneTraceToPEPAndSpokeTransport(t *testing.T) { +func TestCollectorSeparatesCallerAuthorizationTraceFromRefreshFlight(t *testing.T) { now := time.Date(2026, time.July, 14, 13, 0, 0, 0, time.UTC) var audits []pep.AuditEvent var events []tracing.Event @@ -29,12 +29,16 @@ func TestCollectorPropagatesOneTraceToPEPAndSpokeTransport(t *testing.T) { t.Fatal(err) } var transportTrace tracing.ID + type callerValueKey struct{} + var callerValueLeaked bool store := &memoryStore{ spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind), } collector, err := NewCollector(CollectorConfig{ - Store: store, + LifecycleContext: t.Context(), + Store: store, Transport: transportFunc(func(ctx context.Context, _ tenancy.WorkspaceID, spoke Spoke) (Snapshot, error) { + callerValueLeaked = ctx.Value(callerValueKey{}) != nil var ok bool transportTrace, ok = tracing.FromContext(ctx) if !ok { @@ -47,14 +51,19 @@ func TestCollectorPropagatesOneTraceToPEPAndSpokeTransport(t *testing.T) { if err != nil { t.Fatal(err) } - if coverage, err := collector.Collect(context.Background(), readerScope(t, "workspace-a")); err != nil || coverage.Reachable != 1 { + callerContext, callerTrace, err := tracing.Ensure(context.Background()) + if err != nil { + t.Fatal(err) + } + callerContext = context.WithValue(callerContext, callerValueKey{}, "request-only-value") + if coverage, err := collector.Collect(callerContext, readerScope(t, "workspace-a")); err != nil || coverage.Reachable != 1 { t.Fatalf("Collect() coverage = %#v, error = %v", coverage, err) } - if !transportTrace.Valid() || len(audits) != 1 || audits[0].TraceID != transportTrace { - t.Fatalf("trace/audit propagation = transport %q audits %#v", transportTrace, audits) + if callerValueLeaked || !transportTrace.Valid() || transportTrace == callerTrace || len(audits) != 1 || audits[0].TraceID != callerTrace { + t.Fatalf("trace isolation = caller %q transport %q audits %#v", callerTrace, transportTrace, audits) } if len(events) != 2 || events[0].Stage != tracing.StagePEPDecision || events[1].Stage != tracing.StageSpokeSnapshot || - events[0].TraceID != transportTrace || events[1].TraceID != transportTrace || events[1].Outcome != tracing.OutcomeSuccess { + events[0].TraceID != callerTrace || events[1].TraceID != transportTrace || events[1].Outcome != tracing.OutcomeSuccess { t.Fatalf("trace events = %#v", events) } } @@ -69,7 +78,8 @@ func TestCollectorSurvivesPanickingTraceObserver(t *testing.T) { t.Fatal(err) } collector, err := NewCollector(CollectorConfig{ - Store: &memoryStore{spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)}, + LifecycleContext: t.Context(), + Store: &memoryStore{spokes: []Spoke{{ID: "spoke-a", ManagedClusterRef: "ocm/spoke-a"}}, snapshots: make(map[string]Snapshot), failures: make(map[string]FailureKind)}, Transport: transportFunc(func(context.Context, tenancy.WorkspaceID, Spoke) (Snapshot, error) { return validSnapshot("spoke-a", now), nil }), diff --git a/internal/hubruntime/config.go b/internal/hubruntime/config.go index 7a59bf1..91dd5ff 100644 --- a/internal/hubruntime/config.go +++ b/internal/hubruntime/config.go @@ -11,13 +11,17 @@ import ( "fmt" "log/slog" "net" + "net/http" "os" "strconv" "strings" + "time" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "github.com/ArdurAI/sith/internal/auditdelivery" + "github.com/ArdurAI/sith/internal/buildinfo" "github.com/ArdurAI/sith/internal/hubauth" "github.com/ArdurAI/sith/internal/hubdb" "github.com/ArdurAI/sith/internal/hubfleet" @@ -32,6 +36,7 @@ const ( maxMountedKeyBytes = 64 * 1024 maxMountedCABundleBytes = 256 * 1024 maxMountedPublicKeyBytes = 16 * 1024 + maxMountedPrivateKeyBytes = 16 * 1024 ) // Runtime owns the configured application database while the hub server runs. @@ -69,7 +74,7 @@ func NewFromEnvironment(ctx context.Context, logger *slog.Logger) (*Runtime, err if err != nil { return nil, fmt.Errorf("construct hub runtime: session verification configuration is invalid") } - auditor, err := pep.NewSlogAuditor(logger) + processAuditor, err := pep.NewSlogAuditor(logger) if err != nil { return nil, fmt.Errorf("construct hub runtime: policy audit configuration is invalid") } @@ -77,15 +82,11 @@ func NewFromEnvironment(ctx context.Context, logger *slog.Logger) (*Runtime, err if err != nil { return nil, fmt.Errorf("construct hub runtime: trace configuration is invalid") } - authObserver, err := observability.NewSlogAuthObserver(logger) + info := buildinfo.Get() + metrics, err := observability.New(observability.Config{Version: info.Version, Commit: info.Commit}) if err != nil { - return nil, fmt.Errorf("construct hub runtime: authentication observability configuration is invalid") + return nil, fmt.Errorf("construct hub runtime: metrics configuration is invalid") } - enforcer, err := pep.NewEnforcer(pep.Config{Hook: pep.AllowReadHook{}, Auditor: auditor, TraceObserver: tracer}) - if err != nil { - return nil, fmt.Errorf("construct hub runtime: policy configuration is invalid") - } - inClusterConfig, err := rest.InClusterConfig() if err != nil { return nil, fmt.Errorf("construct hub runtime: in-cluster Kubernetes identity is required") @@ -111,8 +112,48 @@ func NewFromEnvironment(ctx context.Context, logger *slog.Logger) (*Runtime, err if err != nil { return nil, fmt.Errorf("construct hub runtime: database is unavailable") } - cleanup := database.Close - collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{Store: database, Transport: transport, PEP: enforcer, TraceObserver: tracer}) + cleanup := func() { database.Close() } + observedDatabaseAuditor, err := pep.NewObservedAuditor(pep.AuditSinkDurable, database, metrics) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: durable policy audit observation is invalid") + } + observedProcessAuditor, err := pep.NewObservedAuditor(pep.AuditSinkProcess, processAuditor, metrics) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: process policy audit observation is invalid") + } + durableAuditor, err := newOrderedPolicyAuditor(observedDatabaseAuditor, observedProcessAuditor) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: durable policy audit configuration is invalid") + } + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, Auditor: durableAuditor, Observer: metrics, TraceObserver: tracer, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: policy configuration is invalid") + } + processAuthObserver, err := auditdelivery.NewProcessObserver(auditdelivery.Config{Drops: metrics}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: process authentication audit delivery is unavailable") + } + authObserver, err := hubserver.NewAuthObserverFanout(processAuthObserver, metrics) + if err != nil { + _ = processAuthObserver.Close() + cleanup() + return nil, fmt.Errorf("construct hub runtime: authentication observation is invalid") + } + cleanup = func() { + _ = processAuthObserver.Close() + database.Close() + } + collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{ + LifecycleContext: ctx, Store: database, Transport: transport, PEP: enforcer, + Observer: metrics, TraceObserver: tracer, + }) if err != nil { cleanup() return nil, fmt.Errorf("construct hub runtime: collector configuration is invalid") @@ -127,27 +168,145 @@ func NewFromEnvironment(ctx context.Context, logger *slog.Logger) (*Runtime, err cleanup() return nil, fmt.Errorf("construct hub runtime: CVE search configuration is invalid") } - handler, err := hubserver.NewFleetHandler(hubserver.FleetHandlerConfig{ + fleetHandler, err := hubserver.NewFleetHandler(hubserver.FleetHandlerConfig{ Verifier: verifier, AuthObserver: authObserver, Collector: collector, Reader: database, ImageSearcher: imageSearcher, CVESearcher: cveSearcher, CVEIdentifierSearcher: cveSearcher, PEP: enforcer, + ReadObserver: metrics, }) if err != nil { cleanup() return nil, fmt.Errorf("construct hub runtime: HTTP handler configuration is invalid") } + auditExportHandler, err := hubserver.NewAuditExportHandler(hubserver.AuditExportHandlerConfig{ + Verifier: verifier, AuthObserver: authObserver, Exporter: database, PEP: enforcer, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: audit export handler configuration is invalid") + } + probeHandler, err := hubserver.NewProbeHandler(hubserver.ProbeHandlerConfig{Checker: database, Observer: metrics}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: probe handler configuration is invalid") + } + mux, err := newRuntimeMux(fleetHandler, auditExportHandler, probeHandler) + if err != nil { + cleanup() + return nil, err + } + if config.browserOIDC != nil { + privateKey, err := loadSessionPrivateKey(config.browserOIDC.sessionPrivateKeyFile) + if err != nil { + cleanup() + return nil, err + } + defer clear(privateKey) + if !publicKey.Equal(privateKey.Public()) { + cleanup() + return nil, fmt.Errorf("construct hub runtime: session signing key does not match the configured public key") + } + sessionIssuer, err := hubauth.NewSessionIssuer(hubauth.SessionIssuerConfig{ + Issuer: config.sessionIssuer, Audience: config.sessionAudience, KeyID: config.sessionKeyID, PrivateKey: privateKey, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: browser session issuer configuration is invalid") + } + oidcService, err := hubauth.NewOIDCService(hubauth.OIDCServiceConfig{ + Providers: []hubauth.OIDCProviderConfig{{Issuer: config.browserOIDC.issuer, Audience: config.browserOIDC.clientID}}, + Store: database, Issuer: sessionIssuer, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: browser OIDC configuration is invalid") + } + limiter, err := hubserver.NewAttemptLimiter(hubserver.AttemptLimiterConfig{Attempts: 20, Window: time.Minute, MaxKeys: 4096}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: browser OIDC rate limiting is invalid") + } + browserHandler, err := hubserver.NewBrowserOIDCHandler(hubserver.BrowserOIDCHandlerConfig{ + Service: oidcService, ProviderIssuer: config.browserOIDC.issuer, ClientID: config.browserOIDC.clientID, + RedirectURI: config.browserOIDC.redirectURI, Limiter: limiter, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: browser OIDC handler configuration is invalid") + } + consoleCorrelator, err := hubfleet.NewCorrelator(hubfleet.CorrelatorConfig{Querier: database, PEP: enforcer}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: console correlator configuration is invalid") + } + consoleInventory, err := hubfleet.NewInventorySearcher(hubfleet.InventorySearcherConfig{Querier: database, PEP: enforcer}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: console inventory configuration is invalid") + } + consoleCVE, err := hubfleet.NewCVESearcher(hubfleet.CVESearcherConfig{Querier: database, PEP: enforcer}) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: console CVE search configuration is invalid") + } + consoleHandler, err := hubserver.NewConsoleHandler(hubserver.ConsoleHandlerConfig{ + Verifier: verifier, AuthObserver: authObserver, Reader: database, Correlator: consoleCorrelator, Inventory: consoleInventory, CVE: consoleCVE, PEP: enforcer, + ReadObserver: metrics, + }) + if err != nil { + cleanup() + return nil, fmt.Errorf("construct hub runtime: console handler configuration is invalid") + } + mux.Handle("GET /v1/workspaces/{workspace}/console/login", http.HandlerFunc(browserHandler.Login)) + mux.Handle("GET "+browserHandler.CallbackPath(), http.HandlerFunc(browserHandler.Callback)) + mux.Handle("GET /v1/workspaces/{workspace}/console", http.HandlerFunc(consoleHandler.ServePage)) + mux.Handle("GET /v1/workspaces/{workspace}/console/fleet", http.HandlerFunc(consoleHandler.ServeFleet)) + mux.Handle("GET /v1/workspaces/{workspace}/console/correlate", http.HandlerFunc(consoleHandler.ServeCorrelation)) + mux.Handle("GET /v1/workspaces/{workspace}/console/inventory", http.HandlerFunc(consoleHandler.ServeInventory)) + mux.Handle("GET /v1/workspaces/{workspace}/console/cves", http.HandlerFunc(consoleHandler.ServeCVEIdentifier)) + mux.Handle("GET /v1/console/assets/console.css", http.HandlerFunc(consoleHandler.ServeCSS)) + mux.Handle("GET /v1/console/assets/console.js", http.HandlerFunc(consoleHandler.ServeJavaScript)) + } listener, err := net.Listen("tcp", config.listenAddress) if err != nil { cleanup() return nil, fmt.Errorf("construct hub runtime: listener is unavailable") } - server, err := NewServer(ServerConfig{Listener: listener, Handler: handler, TLSConfig: serverTLS}) + server, err := NewServer(ServerConfig{Listener: listener, Handler: mux, TLSConfig: serverTLS}) if err != nil { _ = listener.Close() cleanup() return nil, err } + metricsServer, err := newOptionalLoopbackMetricsServer(config.metricsListenAddress, metrics.Handler(), net.Listen) + if err != nil { + _ = listener.Close() + cleanup() + return nil, err + } + cleanup = func() { + _ = processAuthObserver.Close() + if metricsServer != nil { + _ = metricsServer.Close() + } + database.Close() + } return &Runtime{server: server, close: cleanup}, nil } +func newRuntimeMux(fleetHandler, auditExportHandler http.Handler, probeHandler *hubserver.ProbeHandler) (*http.ServeMux, error) { + if fleetHandler == nil || auditExportHandler == nil || probeHandler == nil { + return nil, fmt.Errorf("construct hub runtime: fleet, audit export, and probe handlers are required") + } + mux := http.NewServeMux() + // Register without a method qualifier so HEAD and every non-GET method reach the probe's exact + // fail-closed request validation instead of ServeMux's implicit GET-to-HEAD behavior. + mux.Handle(hubserver.LivenessPath, http.HandlerFunc(probeHandler.ServeLiveness)) + mux.Handle(hubserver.ReadinessPath, http.HandlerFunc(probeHandler.ServeReadiness)) + mux.Handle("/v1/workspaces/{workspace}/audit/export", auditExportHandler) + mux.Handle("/v1/workspaces/{workspace}/audit/export/pages", auditExportHandler) + mux.Handle("/", fleetHandler) + return mux, nil +} + // Run serves the configured hub and releases its application database pool on exit. func (runtime *Runtime) Run(ctx context.Context) error { if runtime == nil || runtime.server == nil || runtime.close == nil { @@ -172,6 +331,15 @@ type deploymentConfig struct { proxyCertFile string proxyKeyFile string kubeAPIServerName string + metricsListenAddress string + browserOIDC *browserOIDCDeploymentConfig +} + +type browserOIDCDeploymentConfig struct { + issuer string + clientID string + redirectURI string + sessionPrivateKeyFile string } func loadDeploymentConfig(lookup func(string) (string, bool)) (deploymentConfig, error) { @@ -207,6 +375,49 @@ func loadDeploymentConfig(lookup func(string) (string, bool)) (deploymentConfig, if err := validateListenAddress(config.listenAddress); err != nil { return deploymentConfig{}, fmt.Errorf("load hub configuration: listen address is invalid") } + config.metricsListenAddress, err = optionalLoopbackMetricsListenAddress(lookup) + if err != nil { + return deploymentConfig{}, err + } + browserOIDC, err := loadBrowserOIDCDeploymentConfig(lookup) + if err != nil { + return deploymentConfig{}, err + } + config.browserOIDC = browserOIDC + return config, nil +} + +func loadBrowserOIDCDeploymentConfig(lookup func(string) (string, bool)) (*browserOIDCDeploymentConfig, error) { + config := &browserOIDCDeploymentConfig{} + fields := []struct { + name string + target *string + }{ + {"SITH_HUB_BROWSER_OIDC_ISSUER", &config.issuer}, + {"SITH_HUB_BROWSER_OIDC_CLIENT_ID", &config.clientID}, + {"SITH_HUB_BROWSER_OIDC_REDIRECT_URI", &config.redirectURI}, + {"SITH_HUB_SESSION_PRIVATE_KEY_FILE", &config.sessionPrivateKeyFile}, + } + configured := 0 + for _, field := range fields { + value, present := lookup(field.name) + if present || value != "" { + configured++ + } + } + if configured == 0 { + return nil, nil + } + if configured != len(fields) { + return nil, fmt.Errorf("load hub configuration: browser OIDC inputs must be set together") + } + for _, field := range fields { + value, err := requiredEnvironment(lookup, field.name) + if err != nil { + return nil, err + } + *field.target = value + } return config, nil } @@ -218,6 +429,18 @@ func requiredEnvironment(lookup func(string) (string, bool), name string) (strin return value, nil } +func optionalLoopbackMetricsListenAddress(lookup func(string) (string, bool)) (string, error) { + const name = "SITH_HUB_METRICS_LISTEN_ADDR" + value, present := lookup(name) + if !present || value == "" { + return "", nil + } + if strings.TrimSpace(value) != value || len(value) > 4096 || validateLoopbackMetricsListenAddress(value) != nil { + return "", fmt.Errorf("load hub configuration: loopback metrics listen address is invalid") + } + return value, nil +} + func validateListenAddress(address string) error { host, port, err := net.SplitHostPort(address) if err != nil || host == "" { @@ -230,6 +453,18 @@ func validateListenAddress(address string) error { return nil } +func validateLoopbackMetricsListenAddress(address string) error { + host, port, err := net.SplitHostPort(address) + if err != nil || (host != "127.0.0.1" && host != "::1") { + return fmt.Errorf("metrics listen address must use an exact loopback IP and port") + } + value, err := strconv.ParseUint(port, 10, 16) + if err != nil || value == 0 { + return fmt.Errorf("metrics listen address must use a non-zero port") + } + return nil +} + func loadServerTLS(config deploymentConfig) (*tls.Config, error) { certificate, err := loadMountedCertificate("hub server certificate", config.serverCertFile, config.serverKeyFile) if err != nil { @@ -296,6 +531,33 @@ func loadSessionPublicKey(path string) (ed25519.PublicKey, error) { return append(ed25519.PublicKey(nil), key...), nil } +func loadSessionPrivateKey(path string) (ed25519.PrivateKey, error) { + encoded, err := readMountedFile("session private key", path, maxMountedPrivateKeyBytes) + if err != nil { + return nil, err + } + defer clear(encoded) + block, rest := pem.Decode(encoded) + if block == nil || block.Type != "PRIVATE KEY" || len(strings.TrimSpace(string(rest))) != 0 { + return nil, fmt.Errorf("load hub configuration: session private key is invalid") + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("load hub configuration: session private key is invalid") + } + key, ok := parsed.(ed25519.PrivateKey) + if !ok || len(key) != ed25519.PrivateKeySize { + return nil, fmt.Errorf("load hub configuration: session private key is not Ed25519") + } + return copyAndClearSessionPrivateKey(key), nil +} + +func copyAndClearSessionPrivateKey(key ed25519.PrivateKey) ed25519.PrivateKey { + copied := append(ed25519.PrivateKey(nil), key...) + clear(key) + return copied +} + func readMountedFile(label, path string, maxBytes int) ([]byte, error) { info, err := os.Stat(path) if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o222 != 0 { diff --git a/internal/hubruntime/metrics.go b/internal/hubruntime/metrics.go new file mode 100644 index 0000000..288797e --- /dev/null +++ b/internal/hubruntime/metrics.go @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "sync" + "time" +) + +const loopbackMetricsPath = "/metrics" + +type listenerFactory func(network, address string) (net.Listener, error) + +// loopbackMetricsServer owns the Hub's optional local-only metrics listener. The listener is +// deliberately separate from the tenant-authenticated TLS Hub API and serves only fixed metrics. +type loopbackMetricsServer struct { + server *http.Server + done chan error + shutdownTimeout time.Duration + closeOnce sync.Once + closeErr error +} + +type loopbackMetricsServerConfig struct { + Listener net.Listener + Handler http.Handler + ShutdownTimeout time.Duration +} + +// newOptionalLoopbackMetricsServer starts the strictly opt-in loopback endpoint. The registry may +// exist for internal self-observation while this listener remains disabled. +func newOptionalLoopbackMetricsServer(listenAddress string, handler http.Handler, listen listenerFactory) (*loopbackMetricsServer, error) { + if listenAddress == "" { + return nil, nil + } + if listen == nil { + return nil, fmt.Errorf("construct loopback metrics server: listener factory is required") + } + listener, err := listen("tcp", listenAddress) + if err != nil { + return nil, fmt.Errorf("construct hub runtime: loopback metrics listener is unavailable") + } + metricsServer, err := newLoopbackMetricsServer(loopbackMetricsServerConfig{Listener: listener, Handler: handler}) + if err != nil { + _ = listener.Close() + return nil, err + } + return metricsServer, nil +} + +func newLoopbackMetricsServer(config loopbackMetricsServerConfig) (*loopbackMetricsServer, error) { + if config.Listener == nil || config.Handler == nil { + return nil, fmt.Errorf("construct loopback metrics server: listener and handler are required") + } + if config.ShutdownTimeout == 0 { + config.ShutdownTimeout = defaultShutdownTimeout + } + if config.ShutdownTimeout < time.Second || config.ShutdownTimeout > time.Minute { + return nil, fmt.Errorf("construct loopback metrics server: shutdown timeout must be between 1s and 1m") + } + handler := http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet || request.URL.Path != loopbackMetricsPath || request.URL.RawQuery != "" { + http.NotFound(writer, request) + return + } + config.Handler.ServeHTTP(writer, request) + }) + metricsServer := &loopbackMetricsServer{ + server: &http.Server{ + Handler: handler, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: time.Minute, + MaxHeaderBytes: 16 * 1024, + }, + done: make(chan error, 1), + shutdownTimeout: config.ShutdownTimeout, + } + go func() { + metricsServer.done <- metricsServer.server.Serve(config.Listener) + }() + return metricsServer, nil +} + +// Close stops the listener within a fixed deadline. If graceful shutdown times out, it closes +// active connections and always waits for the Serve goroutine to return. +func (server *loopbackMetricsServer) Close() error { + if server == nil || server.server == nil || server.done == nil { + return nil + } + server.closeOnce.Do(func() { + shutdownContext, cancel := context.WithTimeout(context.Background(), server.shutdownTimeout) + shutdownErr := server.server.Shutdown(shutdownContext) + cancel() + if shutdownErr != nil { + _ = server.server.Close() + } + serveErr := <-server.done + if shutdownErr != nil { + server.closeErr = fmt.Errorf("stop loopback metrics server: %w", shutdownErr) + return + } + if !errors.Is(serveErr, http.ErrServerClosed) { + server.closeErr = fmt.Errorf("stop loopback metrics server: %w", serveErr) + } + }) + return server.closeErr +} diff --git a/internal/hubruntime/metrics_test.go b/internal/hubruntime/metrics_test.go new file mode 100644 index 0000000..4aff4b4 --- /dev/null +++ b/internal/hubruntime/metrics_test.go @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/observability" + "github.com/ArdurAI/sith/internal/pep" +) + +func TestLoopbackMetricsServerServesOnlyFixedMetricsRouteAndStops(t *testing.T) { + metrics, err := observability.New(observability.Config{Version: "v1.2.3", Commit: "0123456"}) + if err != nil { + t.Fatal(err) + } + metrics.ObserveDecision(pep.VerbFleetRead, pep.DecisionOutcomeAllow, time.Millisecond) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + endpoint := "http://" + listener.Addr().String() + server, err := newLoopbackMetricsServer(loopbackMetricsServerConfig{Listener: listener, Handler: metrics.Handler()}) + if err != nil { + _ = listener.Close() + t.Fatal(err) + } + client := &http.Client{Timeout: time.Second} + defer client.CloseIdleConnections() + + for _, test := range []struct { + method string + path string + statusCode int + contains string + }{ + {method: http.MethodGet, path: loopbackMetricsPath, statusCode: http.StatusOK, contains: "sith_policy_decisions_total"}, + {method: http.MethodHead, path: loopbackMetricsPath, statusCode: http.StatusNotFound}, + {method: http.MethodPost, path: loopbackMetricsPath, statusCode: http.StatusNotFound}, + {method: http.MethodGet, path: loopbackMetricsPath + "?workspace=workspace-a", statusCode: http.StatusNotFound}, + {method: http.MethodGet, path: loopbackMetricsPath + "/", statusCode: http.StatusNotFound}, + {method: http.MethodGet, path: "/", statusCode: http.StatusNotFound}, + {method: http.MethodGet, path: "/v1/workspaces/workspace-a/fleet", statusCode: http.StatusNotFound}, + } { + request, requestErr := http.NewRequest(test.method, endpoint+test.path, nil) + if requestErr != nil { + _ = server.Close() + t.Fatal(requestErr) + } + response, requestErr := client.Do(request) + if requestErr != nil { + _ = server.Close() + t.Fatalf("%s %s: %v", test.method, test.path, requestErr) + } + body, readErr := io.ReadAll(response.Body) + _ = response.Body.Close() + if readErr != nil { + _ = server.Close() + t.Fatal(readErr) + } + if response.StatusCode != test.statusCode || (test.contains != "" && !strings.Contains(string(body), test.contains)) { + _ = server.Close() + t.Fatalf("%s %s status/body = %d/%q", test.method, test.path, response.StatusCode, body) + } + } + if err := server.Close(); err != nil { + t.Fatal(err) + } + if connection, dialErr := net.DialTimeout("tcp", listener.Addr().String(), time.Second); dialErr == nil { + _ = connection.Close() + t.Fatal("loopback metrics listener remained reachable after Close") + } +} + +func TestOptionalLoopbackMetricsServerDoesNotBindWhenDisabled(t *testing.T) { + var calls int + metricsServer, err := newOptionalLoopbackMetricsServer("", http.NotFoundHandler(), func(string, string) (net.Listener, error) { + calls++ + return nil, errors.New("disabled metrics listener was called") + }) + if err != nil || metricsServer != nil || calls != 0 { + t.Fatalf("server/error/calls = %#v/%v/%d", metricsServer, err, calls) + } +} + +func TestNewLoopbackMetricsServerRejectsUnsafeConfiguration(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + for _, test := range []struct { + name string + config loopbackMetricsServerConfig + }{ + {name: "missing listener", config: loopbackMetricsServerConfig{Handler: http.NotFoundHandler()}}, + {name: "missing handler", config: loopbackMetricsServerConfig{Listener: listener}}, + {name: "short shutdown timeout", config: loopbackMetricsServerConfig{Listener: listener, Handler: http.NotFoundHandler(), ShutdownTimeout: 500 * time.Millisecond}}, + } { + t.Run(test.name, func(t *testing.T) { + if server, configErr := newLoopbackMetricsServer(test.config); configErr == nil { + _ = server.Close() + t.Fatal("newLoopbackMetricsServer accepted unsafe configuration") + } + }) + } +} diff --git a/internal/hubruntime/ocm_integration_test.go b/internal/hubruntime/ocm_integration_test.go index 35e7323..1cdeabc 100644 --- a/internal/hubruntime/ocm_integration_test.go +++ b/internal/hubruntime/ocm_integration_test.go @@ -10,7 +10,9 @@ import ( "fmt" "net" "net/http" + "net/http/httptest" "slices" + "strings" "sync" "testing" "time" @@ -23,6 +25,7 @@ import ( "github.com/ArdurAI/sith/internal/hubfleet" "github.com/ArdurAI/sith/internal/hubocm" "github.com/ArdurAI/sith/internal/hubserver" + "github.com/ArdurAI/sith/internal/observability" "github.com/ArdurAI/sith/internal/pep" "github.com/ArdurAI/sith/internal/tenancy" "github.com/ArdurAI/sith/tests/testutil/ocmlab" @@ -63,7 +66,9 @@ func TestHubRuntimeDirectClusterProxyM0(t *testing.T) { if err != nil { t.Fatal(err) } - collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{Store: store, Transport: transport, PEP: enforcer}) + collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{ + LifecycleContext: ctx, Store: store, Transport: transport, PEP: enforcer, + }) if err != nil { t.Fatal(err) } @@ -77,8 +82,12 @@ func TestHubRuntimeDirectClusterProxyM0(t *testing.T) { } now := time.Now().UTC() verifier, privateKey := m0RuntimeVerifier(t, now) + metrics, err := observability.New(observability.Config{}) + if err != nil { + t.Fatal(err) + } handler, err := hubserver.NewFleetHandler(hubserver.FleetHandlerConfig{ - Verifier: verifier, Collector: collector, Reader: store, ImageSearcher: imageSearcher, CVESearcher: cveSearcher, CVEIdentifierSearcher: cveSearcher, PEP: enforcer, + Verifier: verifier, Collector: collector, Reader: store, ImageSearcher: imageSearcher, CVESearcher: cveSearcher, CVEIdentifierSearcher: cveSearcher, PEP: enforcer, ReadObserver: metrics, }) if err != nil { t.Fatal(err) @@ -125,6 +134,11 @@ func TestHubRuntimeDirectClusterProxyM0(t *testing.T) { if len(result.Clusters) != 2 || result.Coverage.Requested != 2 || result.Coverage.Reachable != 2 { t.Fatalf("runtime direct fleet = %#v", result) } + metricsResponse := httptest.NewRecorder() + metrics.Handler().ServeHTTP(metricsResponse, httptest.NewRequest(http.MethodGet, "http://metrics.invalid/metrics", nil)) + if metricsResponse.Code != http.StatusOK || !strings.Contains(metricsResponse.Body.String(), `sith_federation_fleet_read_results_total{outcome="complete"} 1`) { + t.Fatalf("runtime fleet read metric status/body = %d/%q", metricsResponse.Code, metricsResponse.Body.String()) + } digest := m0RuntimeFixtureDigest(t, store) imageResponse := m0RuntimeRequest(t, ctx, client, http.MethodGet, endpoint+"/fleet/images/"+digest, token) defer imageResponse.Body.Close() diff --git a/internal/hubruntime/policy_audit.go b/internal/hubruntime/policy_audit.go new file mode 100644 index 0000000..25a86d5 --- /dev/null +++ b/internal/hubruntime/policy_audit.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "fmt" + + "github.com/ArdurAI/sith/internal/pep" +) + +type orderedPolicyAuditor struct { + durable pep.Auditor + process pep.Auditor +} + +func newOrderedPolicyAuditor(durable, process pep.Auditor) (pep.Auditor, error) { + if durable == nil || process == nil { + return nil, fmt.Errorf("construct ordered policy auditor: durable and process sinks are required") + } + return orderedPolicyAuditor{durable: durable, process: process}, nil +} + +func (auditor orderedPolicyAuditor) Record(ctx context.Context, event pep.AuditEvent) error { + if auditor.durable == nil || auditor.process == nil { + return fmt.Errorf("record ordered policy audit: sinks are required") + } + if err := auditor.durable.Record(ctx, event); err != nil { + return fmt.Errorf("record durable policy audit: %w", err) + } + if err := auditor.process.Record(ctx, event); err != nil { + return fmt.Errorf("record process policy audit: %w", err) + } + return nil +} diff --git a/internal/hubruntime/policy_audit_test.go b/internal/hubruntime/policy_audit_test.go new file mode 100644 index 0000000..f3ca88f --- /dev/null +++ b/internal/hubruntime/policy_audit_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubruntime + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/pep" +) + +type runtimeAuditObservation struct { + sink pep.AuditSink + outcome pep.AuditOutcome +} + +type runtimeAuditObserver struct { + observations []runtimeAuditObservation +} + +func (observer *runtimeAuditObserver) ObservePolicyAudit(sink pep.AuditSink, outcome pep.AuditOutcome, _ time.Duration) { + observer.observations = append(observer.observations, runtimeAuditObservation{sink: sink, outcome: outcome}) +} + +func TestOrderedPolicyAuditorRequiresDurabilityBeforeProcessLog(t *testing.T) { + t.Parallel() + + var calls []string + observer := &runtimeAuditObserver{} + durableFailure := errors.New("database unavailable") + durable, err := pep.NewObservedAuditor(pep.AuditSinkDurable, pep.AuditFunc(func(context.Context, pep.AuditEvent) error { + calls = append(calls, "durable") + return durableFailure + }), observer) + if err != nil { + t.Fatal(err) + } + process, err := pep.NewObservedAuditor(pep.AuditSinkProcess, pep.AuditFunc(func(context.Context, pep.AuditEvent) error { + calls = append(calls, "process") + return nil + }), observer) + if err != nil { + t.Fatal(err) + } + auditor, err := newOrderedPolicyAuditor( + durable, + process, + ) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), pep.AuditEvent{}); !errors.Is(err, durableFailure) { + t.Fatalf("Record() error = %v, want durable failure", err) + } + if !reflect.DeepEqual(calls, []string{"durable"}) { + t.Fatalf("sink calls = %v, want durable sink only", calls) + } + wantObservations := []runtimeAuditObservation{{sink: pep.AuditSinkDurable, outcome: pep.AuditOutcomeError}} + if !reflect.DeepEqual(observer.observations, wantObservations) { + t.Fatalf("audit observations = %#v, want %#v", observer.observations, wantObservations) + } +} + +func TestOrderedPolicyAuditorReportsProcessFailureAfterDurableAppend(t *testing.T) { + t.Parallel() + + var calls []string + observer := &runtimeAuditObserver{} + processFailure := errors.New("process sink unavailable") + durable, err := pep.NewObservedAuditor(pep.AuditSinkDurable, pep.AuditFunc(func(context.Context, pep.AuditEvent) error { + calls = append(calls, "durable") + return nil + }), observer) + if err != nil { + t.Fatal(err) + } + process, err := pep.NewObservedAuditor(pep.AuditSinkProcess, pep.AuditFunc(func(context.Context, pep.AuditEvent) error { + calls = append(calls, "process") + return processFailure + }), observer) + if err != nil { + t.Fatal(err) + } + auditor, err := newOrderedPolicyAuditor( + durable, + process, + ) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), pep.AuditEvent{}); !errors.Is(err, processFailure) { + t.Fatalf("Record() error = %v, want process failure", err) + } + if !reflect.DeepEqual(calls, []string{"durable", "process"}) { + t.Fatalf("sink calls = %v, want durable then process", calls) + } + wantObservations := []runtimeAuditObservation{ + {sink: pep.AuditSinkDurable, outcome: pep.AuditOutcomeSuccess}, + {sink: pep.AuditSinkProcess, outcome: pep.AuditOutcomeError}, + } + if !reflect.DeepEqual(observer.observations, wantObservations) { + t.Fatalf("audit observations = %#v, want %#v", observer.observations, wantObservations) + } +} + +func TestOrderedPolicyAuditorRejectsMissingSinks(t *testing.T) { + t.Parallel() + + if _, err := newOrderedPolicyAuditor(nil, nil); err == nil { + t.Fatal("missing policy audit sinks accepted") + } + if err := (orderedPolicyAuditor{}).Record(context.Background(), pep.AuditEvent{}); err == nil { + t.Fatal("zero-value ordered auditor accepted an event") + } +} diff --git a/internal/hubruntime/runtime_test.go b/internal/hubruntime/runtime_test.go index 81d0395..2fdd9e0 100644 --- a/internal/hubruntime/runtime_test.go +++ b/internal/hubruntime/runtime_test.go @@ -3,6 +3,7 @@ package hubruntime import ( + "bytes" "context" "crypto/ed25519" "crypto/rand" @@ -12,12 +13,19 @@ import ( "math/big" "net" "net/http" + "net/http/httptest" "os" "path/filepath" "testing" "time" + + "github.com/ArdurAI/sith/internal/hubserver" ) +type runtimeProbeChecker func(context.Context) error + +func (checker runtimeProbeChecker) Ping(ctx context.Context) error { return checker(ctx) } + func TestServerServesTLSAndStopsWithContext(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -95,12 +103,144 @@ func TestNewServerRejectsUnsafeConfiguration(t *testing.T) { } } +func TestRuntimeMuxMountsProbesOutsideAuthenticatedFleetFallback(t *testing.T) { + fallbackCalls := 0 + fallback := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + fallbackCalls++ + response.WriteHeader(http.StatusUnauthorized) + }) + auditCalls := 0 + audit := http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + auditCalls++ + response.WriteHeader(http.StatusTeapot) + }) + probes, err := hubserver.NewProbeHandler(hubserver.ProbeHandlerConfig{ + Checker: runtimeProbeChecker(func(context.Context) error { return nil }), + }) + if err != nil { + t.Fatal(err) + } + mux, err := newRuntimeMux(fallback, audit, probes) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + method string + target string + status int + calls int + }{ + {method: http.MethodGet, target: hubserver.LivenessPath, status: http.StatusNoContent}, + {method: http.MethodGet, target: hubserver.ReadinessPath, status: http.StatusNoContent}, + {method: http.MethodHead, target: hubserver.LivenessPath, status: http.StatusNotFound}, + {method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export", status: http.StatusTeapot}, + {method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export/pages", status: http.StatusTeapot}, + {method: http.MethodGet, target: "/v1/workspaces/workspace-a/fleet", status: http.StatusUnauthorized, calls: 1}, + } { + response := httptest.NewRecorder() + mux.ServeHTTP(response, httptest.NewRequest(test.method, test.target, nil)) + if response.Code != test.status || fallbackCalls != test.calls { + t.Fatalf("%s %s = status %d/fallback calls %d/audit calls %d, want %d/%d", test.method, test.target, response.Code, fallbackCalls, auditCalls, test.status, test.calls) + } + } + + if auditCalls != 2 { + t.Fatalf("audit route calls = %d, want 2", auditCalls) + } + if _, err := newRuntimeMux(nil, audit, probes); err == nil { + t.Fatal("newRuntimeMux accepted a missing fleet handler") + } + if _, err := newRuntimeMux(fallback, nil, probes); err == nil { + t.Fatal("newRuntimeMux accepted a missing audit export handler") + } + if _, err := newRuntimeMux(fallback, audit, nil); err == nil { + t.Fatal("newRuntimeMux accepted a missing probe handler") + } +} + func TestLoadDeploymentConfigRequiresEverySecurityInput(t *testing.T) { config, err := loadDeploymentConfig(func(string) (string, bool) { return "", false }) if err == nil || config != (deploymentConfig{}) { t.Fatalf("config/error = %#v/%v", config, err) } - values := map[string]string{ + values := deploymentConfigEnvironment() + config, err = loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }) + if err != nil || config.listenAddress != "127.0.0.1:8443" || config.proxyAddress != "proxy.sith.test:8090" { + t.Fatalf("config/error = %#v/%v", config, err) + } + values["SITH_HUB_LISTEN_ADDR"] = ":8443" + if _, err := loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }); err == nil { + t.Fatal("loadDeploymentConfig accepted an ambiguous listener") + } +} + +func TestLoadDeploymentConfigRequiresCompleteBrowserOIDCInputs(t *testing.T) { + values := deploymentConfigEnvironment() + lookup := func(name string) (string, bool) { value, ok := values[name]; return value, ok } + config, err := loadDeploymentConfig(lookup) + if err != nil || config.browserOIDC != nil { + t.Fatalf("default browser OIDC config/error = %#v/%v", config.browserOIDC, err) + } + values["SITH_HUB_BROWSER_OIDC_ISSUER"] = "https://idp.sith.test" + if _, err := loadDeploymentConfig(lookup); err == nil { + t.Fatal("partial browser OIDC configuration accepted") + } + values["SITH_HUB_BROWSER_OIDC_CLIENT_ID"] = "sith-browser" + values["SITH_HUB_BROWSER_OIDC_REDIRECT_URI"] = "https://hub.sith.test/v1/console/oidc/callback" + values["SITH_HUB_SESSION_PRIVATE_KEY_FILE"] = "/mnt/session/private.pem" + config, err = loadDeploymentConfig(lookup) + if err != nil || config.browserOIDC == nil || config.browserOIDC.clientID != "sith-browser" || config.browserOIDC.sessionPrivateKeyFile != "/mnt/session/private.pem" { + t.Fatalf("browser OIDC config/error = %#v/%v", config.browserOIDC, err) + } +} + +func TestLoadDeploymentConfigAllowsOnlyExactLoopbackMetricsAddresses(t *testing.T) { + for _, test := range []struct { + name string + value string + valid bool + }{ + {name: "disabled", value: "", valid: true}, + {name: "IPv4 loopback", value: "127.0.0.1:9464", valid: true}, + {name: "IPv6 loopback", value: "[::1]:9464", valid: true}, + {name: "hostname", value: "localhost:9464"}, + {name: "IPv4 wildcard", value: "0.0.0.0:9464"}, + {name: "IPv6 wildcard", value: "[::]:9464"}, + {name: "alternate loopback", value: "127.0.0.2:9464"}, + {name: "missing host", value: ":9464"}, + {name: "missing port", value: "127.0.0.1"}, + {name: "zero port", value: "127.0.0.1:0"}, + {name: "out of range port", value: "127.0.0.1:65536"}, + {name: "whitespace padded", value: " 127.0.0.1:9464"}, + } { + t.Run(test.name, func(t *testing.T) { + values := deploymentConfigEnvironment() + if test.value != "" { + values["SITH_HUB_METRICS_LISTEN_ADDR"] = test.value + } + config, err := loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }) + if test.valid { + if err != nil || config.metricsListenAddress != test.value { + t.Fatalf("config/error = %#v/%v", config, err) + } + return + } + if err == nil { + t.Fatal("loadDeploymentConfig accepted unsafe loopback metrics listener") + } + }) + } + values := deploymentConfigEnvironment() + values["SITH_HUB_METRICS_LISTEN_ADDR"] = "" + config, err := loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }) + if err != nil || config.metricsListenAddress != "" { + t.Fatalf("explicitly empty metrics configuration/error = %#v/%v", config, err) + } +} + +func deploymentConfigEnvironment() map[string]string { + return map[string]string{ "SITH_HUB_LISTEN_ADDR": "127.0.0.1:8443", "SITH_HUB_DATABASE_URL": "postgres://sith@db/sith?sslmode=require", "SITH_HUB_SESSION_ISSUER": "https://issuer.sith.test", @@ -116,14 +256,6 @@ func TestLoadDeploymentConfigRequiresEverySecurityInput(t *testing.T) { "SITH_HUB_PROXY_KEY_FILE": "/mnt/proxy/tls.key", "SITH_HUB_KUBE_API_SERVER_NAME": "kubernetes", } - config, err = loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }) - if err != nil || config.listenAddress != "127.0.0.1:8443" || config.proxyAddress != "proxy.sith.test:8090" { - t.Fatalf("config/error = %#v/%v", config, err) - } - values["SITH_HUB_LISTEN_ADDR"] = ":8443" - if _, err := loadDeploymentConfig(func(name string) (string, bool) { value, ok := values[name]; return value, ok }); err == nil { - t.Fatal("loadDeploymentConfig accepted an ambiguous listener") - } } func TestReadMountedFileRequiresReadOnlyRegularFile(t *testing.T) { @@ -147,6 +279,61 @@ func TestReadMountedFileRequiresReadOnlyRegularFile(t *testing.T) { } } +func TestLoadSessionPrivateKeyRequiresReadOnlyPKCS8Ed25519(t *testing.T) { + privateKey := ed25519.NewKeyFromSeed([]byte("01234567890123456789012345678901")) + der, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "session-private.pem") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o400); err != nil { + t.Fatal(err) + } + loaded, err := loadSessionPrivateKey(path) + if err != nil || string(loaded) != string(privateKey) { + t.Fatalf("private key/error = %x/%v", loaded, err) + } + if err := os.Chmod(path, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der}), 0o400); err != nil { + t.Fatal(err) + } + if err := os.Chmod(path, 0o400); err != nil { + t.Fatal(err) + } + if _, err := loadSessionPrivateKey(path); err == nil { + t.Fatal("non-PKCS8 private key accepted") + } +} + +func TestCopyAndClearSessionPrivateKeyTransfersOwnership(t *testing.T) { + publicKey, parsedKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + expected := append(ed25519.PrivateKey(nil), parsedKey...) + returned := copyAndClearSessionPrivateKey(parsedKey) + t.Cleanup(func() { + clear(expected) + clear(returned) + }) + + if &parsedKey[0] == &returned[0] { + t.Fatal("returned private key aliases the parser-owned key") + } + if !bytes.Equal(parsedKey, make([]byte, ed25519.PrivateKeySize)) { + t.Fatal("parser-owned private key was not cleared") + } + if !bytes.Equal(returned, expected) { + t.Fatal("returned private key changed during ownership transfer") + } + message := []byte("sith session key ownership transfer") + if !ed25519.Verify(publicKey, message, ed25519.Sign(returned, message)) { + t.Fatal("returned private key cannot produce a valid signature") + } +} + func runtimeTestTLS(t *testing.T) (*tls.Config, *tls.Config) { t.Helper() publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) diff --git a/internal/hubserver/audit_export.go b/internal/hubserver/audit_export.go new file mode 100644 index 0000000..7a0c960 --- /dev/null +++ b/internal/hubserver/audit_export.go @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/ArdurAI/sith/internal/auditrecord" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" + "github.com/ArdurAI/sith/internal/tracing" +) + +const ( + auditExportRoutePrefix = "/v1/workspaces/" + auditExportResource = "audit/export" + auditExportPagesResource = "audit/export/pages" + auditExportFilename = "sith-policy-audit.json" + auditExportPageFilename = "sith-policy-audit-page.json" + maxConcurrentAuditExports = 4 +) + +// PolicyAuditExporter reads complete or snapshot-paged verified retained chains after +// authorization succeeds. +type PolicyAuditExporter interface { + ExportPolicyAuditChain(context.Context, tenancy.Scope) (auditrecord.Export, error) + ExportPolicyAuditPage(context.Context, tenancy.Scope, auditrecord.PageRequest) (auditrecord.Page, error) +} + +// AuditExportHandlerConfig supplies the authenticated export boundary. +type AuditExportHandlerConfig struct { + Verifier Verifier + AuthObserver AuthObserver + Exporter PolicyAuditExporter + PEP *pep.Enforcer +} + +// NewAuditExportHandler constructs the exact bearer-authenticated audit download route. It bounds +// process concurrency before policy/database work; the exporter itself bounds retained entries and +// completes its database transaction before returning a value for HTTP encoding. +func NewAuditExportHandler(config AuditExportHandlerConfig) (http.Handler, error) { + if config.Verifier == nil || config.Exporter == nil || config.PEP == nil { + return nil, fmt.Errorf("construct audit export handler: verifier, exporter, and policy enforcer are required") + } + slots := make(chan struct{}, maxConcurrentAuditExports) + handler := http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + setNoStore(response.Header()) + response.Header().Set("X-Content-Type-Options", "nosniff") + route, ok := parseAuditExportRoute(request.URL) + if !ok { + writeAuditExportError(response, http.StatusNotFound, "not_found") + return + } + if request.Method != http.MethodGet { + response.Header().Set("Allow", http.MethodGet) + writeAuditExportError(response, http.StatusMethodNotAllowed, "method_not_allowed") + return + } + if requestHasBody(request) { + writeAuditExportError(response, http.StatusNotFound, "not_found") + return + } + if route.pages && len(request.Header.Values("Cookie")) != 0 { + writeAuditExportError(response, http.StatusNotFound, "not_found") + return + } + scope, err := ScopeFromContext(request.Context(), route.workspaceID) + if err != nil { + writeAuditExportError(response, http.StatusForbidden, "forbidden") + return + } + select { + case slots <- struct{}{}: + defer func() { <-slots }() + default: + response.Header().Set("Retry-After", "1") + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + + traceContext, _, err := tracing.Ensure(request.Context()) + if err != nil { + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + if err := config.PEP.AuthorizeAuditExport(traceContext, scope); err != nil { + if errors.Is(err, pep.ErrDenied) { + writeAuditExportError(response, http.StatusForbidden, "forbidden") + return + } + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + if route.pages { + pageRequest, err := auditrecord.FirstPage(scope.WorkspaceID()) + if route.cursor != "" { + pageRequest, err = auditrecord.ContinuePage(scope.WorkspaceID(), route.cursor) + } + if err != nil { + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + page, err := config.Exporter.ExportPolicyAuditPage(traceContext, scope, pageRequest) + if err != nil || page.VerifyForWorkspace(scope.WorkspaceID()) != nil || pageRequest.ValidatePage(page) != nil { + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + writeAuditExportDocument(response, page, auditExportPageFilename) + return + } + + exported, err := config.Exporter.ExportPolicyAuditChain(traceContext, scope) + if err != nil || exported.VerifyForWorkspace(scope.WorkspaceID()) != nil { + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + writeAuditExportDocument(response, exported, auditExportFilename) + }) + return AuthenticateWithObserver(config.Verifier, config.AuthObserver, handler) +} + +type auditExportRoute struct { + workspaceID tenancy.WorkspaceID + pages bool + cursor string +} + +func requestHasBody(request *http.Request) bool { + return request == nil || request.ContentLength != 0 || len(request.TransferEncoding) != 0 || + (request.Body != nil && request.Body != http.NoBody) +} + +func parseAuditExportRoute(requestURL *url.URL) (auditExportRoute, bool) { + if requestURL == nil || requestURL.ForceQuery || requestURL.Fragment != "" { + return auditExportRoute{}, false + } + escapedPath := requestURL.EscapedPath() + if !strings.HasPrefix(escapedPath, auditExportRoutePrefix) { + return auditExportRoute{}, false + } + workspaceSegment, resource, found := strings.Cut(strings.TrimPrefix(escapedPath, auditExportRoutePrefix), "/") + if !found || workspaceSegment == "" || (resource != auditExportResource && resource != auditExportPagesResource) { + return auditExportRoute{}, false + } + workspace, err := url.PathUnescape(workspaceSegment) + if err != nil || url.PathEscape(workspace) != workspaceSegment { + return auditExportRoute{}, false + } + workspaceID := tenancy.WorkspaceID(workspace) + if tenancy.ValidateWorkspaceID(workspaceID) != nil { + return auditExportRoute{}, false + } + route := auditExportRoute{workspaceID: workspaceID, pages: resource == auditExportPagesResource} + if !route.pages { + return route, requestURL.RawQuery == "" + } + if requestURL.RawQuery == "" { + return route, true + } + if len(requestURL.RawQuery) > len("cursor=")+auditrecord.PageCursorChars { + return auditExportRoute{}, false + } + values, err := url.ParseQuery(requestURL.RawQuery) + if err != nil || len(values) != 1 || len(values["cursor"]) != 1 || values.Get("cursor") == "" { + return auditExportRoute{}, false + } + route.cursor = values.Get("cursor") + if requestURL.RawQuery != "cursor="+route.cursor { + return auditExportRoute{}, false + } + return route, true +} + +func writeAuditExportDocument(response http.ResponseWriter, document any, filename string) { + encoded, err := json.Marshal(document) + if err != nil { + writeAuditExportError(response, http.StatusServiceUnavailable, "audit_export_unavailable") + return + } + encoded = append(encoded, '\n') + response.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`) + response.Header().Set("Content-Type", "application/json") + response.Header().Set("Content-Length", strconv.Itoa(len(encoded))) + response.WriteHeader(http.StatusOK) + _, _ = response.Write(encoded) +} + +func writeAuditExportError(response http.ResponseWriter, status int, code string) { + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(status) + _ = json.NewEncoder(response).Encode(struct { + Error string `json:"error"` + }{Error: code}) +} diff --git a/internal/hubserver/audit_export_test.go b/internal/hubserver/audit_export_test.go new file mode 100644 index 0000000..88b878d --- /dev/null +++ b/internal/hubserver/audit_export_test.go @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "go/parser" + "go/token" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/auditrecord" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +type policyAuditExporterFunc func(context.Context, tenancy.Scope) (auditrecord.Export, error) + +func (function policyAuditExporterFunc) ExportPolicyAuditChain(ctx context.Context, scope tenancy.Scope) (auditrecord.Export, error) { + return function(ctx, scope) +} + +func (function policyAuditExporterFunc) ExportPolicyAuditPage( + context.Context, + tenancy.Scope, + auditrecord.PageRequest, +) (auditrecord.Page, error) { + return auditrecord.Page{}, errors.New("paged export is unavailable") +} + +type policyAuditExporterStub struct { + complete func(context.Context, tenancy.Scope) (auditrecord.Export, error) + page func(context.Context, tenancy.Scope, auditrecord.PageRequest) (auditrecord.Page, error) +} + +func (stub policyAuditExporterStub) ExportPolicyAuditChain(ctx context.Context, scope tenancy.Scope) (auditrecord.Export, error) { + if stub.complete == nil { + return auditrecord.Export{}, errors.New("complete export is unavailable") + } + return stub.complete(ctx, scope) +} + +func (stub policyAuditExporterStub) ExportPolicyAuditPage( + ctx context.Context, + scope tenancy.Scope, + request auditrecord.PageRequest, +) (auditrecord.Page, error) { + if stub.page == nil { + return auditrecord.Page{}, errors.New("paged export is unavailable") + } + return stub.page(ctx, scope, request) +} + +func TestAuditExportHandlerAuthorizesBeforePortableDownload(t *testing.T) { + now := time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + var events []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + events = append(events, event) + return nil + }), + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + want := testPolicyAuditExport(now) + handler, err := NewAuditExportHandler(AuditExportHandlerConfig{ + Verifier: verifier, + Exporter: policyAuditExporterFunc(func(_ context.Context, scope tenancy.Scope) (auditrecord.Export, error) { + if len(events) != 1 || events[0].WorkspaceID != scope.WorkspaceID() || + events[0].Action != tenancy.ActionExportAudit || events[0].Verb != pep.VerbAuditExport || + events[0].Verdict != pep.VerdictAllow { + t.Fatalf("exporter observed policy events = %#v", events) + } + return want, nil + }), + PEP: enforcer, + }) + if err != nil { + t.Fatal(err) + } + request := authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin) + request.Header.Set("X-Workspace", "workspace-b") + request.Header.Set("X-Role", "reader") + request.Header.Set("Traceparent", "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01") + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + if response.Header().Get("Cache-Control") != "no-store" || response.Header().Get("Pragma") != "no-cache" || + response.Header().Get("X-Content-Type-Options") != "nosniff" || response.Header().Get("Content-Type") != "application/json" || + response.Header().Get("Content-Disposition") != `attachment; filename="sith-policy-audit.json"` || + response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) { + t.Fatalf("unsafe export headers = %#v", response.Header()) + } + var got auditrecord.Export + if err := json.NewDecoder(response.Body).Decode(&got); err != nil { + t.Fatal(err) + } + if got.Schema != want.Schema || got.WorkspaceID != "workspace-a" || got.Chain != want.Chain || len(got.Entries) != 1 || + got.Entries[0] != want.Entries[0] { + t.Fatalf("export = %#v, want %#v", got, want) + } + if len(events) != 1 || events[0].Actor != "user:alice" || events[0].Role != tenancy.RoleAdmin || + events[0].TraceID == "0123456789abcdef0123456789abcdef" { + t.Fatalf("policy events = %#v", events) + } +} + +func TestAuditExportHandlerAuthorizesSnapshotBoundPages(t *testing.T) { + now := time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + var events []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + events = append(events, event) + return nil + }), + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + initial := testPolicyAuditPage(now, 1, "sha256:"+strings.Repeat("0", 64)) + continuation := testPolicyAuditPage(now.Add(time.Microsecond), 513, "sha256:"+strings.Repeat("b", 64)) + cursor, err := auditrecord.EncodePageCursor( + "workspace-a", continuation.Snapshot.HeadSequence, continuation.Snapshot.HeadHash, + continuation.StartSequence, continuation.PreviousHash, + ) + if err != nil { + t.Fatal(err) + } + pageCalls := 0 + handler, err := NewAuditExportHandler(AuditExportHandlerConfig{ + Verifier: verifier, + Exporter: policyAuditExporterStub{page: func(_ context.Context, scope tenancy.Scope, request auditrecord.PageRequest) (auditrecord.Page, error) { + pageCalls++ + if len(events) != pageCalls || scope.WorkspaceID() != "workspace-a" { + t.Fatalf("page exporter observed events/scope = %#v/%#v", events, scope) + } + if pageCalls == 1 { + if !request.Initial() { + t.Fatal("initial page route produced a continuation request") + } + return initial, nil + } + if request.Initial() || request.HeadSequence() != 513 || request.NextSequence() != 513 || + request.PreviousHash() != continuation.PreviousHash { + t.Fatalf("continuation request = %#v", request) + } + return continuation, nil + }}, + PEP: enforcer, + }) + if err != nil { + t.Fatal(err) + } + + for index, target := range []string{ + "/v1/workspaces/workspace-a/audit/export/pages", + "/v1/workspaces/workspace-a/audit/export/pages?cursor=" + cursor, + } { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, target, privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusOK || response.Header().Get("Content-Disposition") != + `attachment; filename="sith-policy-audit-page.json"` || response.Header().Get("Content-Length") != strconv.Itoa(response.Body.Len()) { + t.Fatalf("page %d status/headers/body = %d/%#v/%q", index, response.Code, response.Header(), response.Body.String()) + } + var page auditrecord.Page + if err := json.NewDecoder(response.Body).Decode(&page); err != nil || page.Verify() != nil { + t.Fatalf("page %d decode/verify = %#v/%v", index, page, err) + } + } + if pageCalls != 2 || len(events) != 2 { + t.Fatalf("page calls/events = %d/%#v", pageCalls, events) + } + for _, event := range events { + if event.Action != tenancy.ActionExportAudit || event.Verb != pep.VerbAuditExport || + event.Verdict != pep.VerdictAllow { + t.Fatalf("page policy event = %#v", event) + } + } +} + +func TestAuditExportHandlerAuditsCanonicalButInvalidCursor(t *testing.T) { + now := time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + events := 0 + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { + events++ + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + handler := mustAuditExportHandler(t, verifier, policyAuditExporterStub{page: func( + context.Context, tenancy.Scope, auditrecord.PageRequest, + ) (auditrecord.Page, error) { + t.Fatal("invalid cursor reached exporter") + return auditrecord.Page{}, nil + }}, enforcer) + response := httptest.NewRecorder() + target := "/v1/workspaces/workspace-a/audit/export/pages?cursor=" + strings.Repeat("A", auditrecord.PageCursorChars) + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, target, privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusServiceUnavailable || events != 1 || + response.Body.String() != "{\"error\":\"audit_export_unavailable\"}\n" { + t.Fatalf("status/events/body = %d/%d/%q", response.Code, events, response.Body.String()) + } +} + +func TestAuditExportHandlerFailsClosedBeforeDisclosure(t *testing.T) { + now := time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + + t.Run("non-admin role", func(t *testing.T) { + var events []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + events = append(events, event) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + t.Fatal("non-admin request reached exporter") + return auditrecord.Export{}, nil + }), enforcer) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleReader)) + if response.Code != http.StatusForbidden || response.Body.String() != "{\"error\":\"forbidden\"}\n" || + len(events) != 1 || events[0].Verdict != pep.VerdictDeny || events[0].ReasonCode != "role-denied" { + t.Fatalf("status/body/events = %d/%q/%#v", response.Code, response.Body.String(), events) + } + }) + + t.Run("foreign workspace", func(t *testing.T) { + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + t.Fatal("foreign request reached exporter") + return auditrecord.Export{}, nil + }), fleetTestPEP(t, pep.AllowReadHook{})) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-b/audit/export", privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusForbidden { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + }) + + t.Run("policy audit failure", func(t *testing.T) { + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { return errors.New("token=secret") }), + }) + if err != nil { + t.Fatal(err) + } + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + t.Fatal("audit failure reached exporter") + return auditrecord.Export{}, nil + }), enforcer) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusServiceUnavailable || strings.Contains(response.Body.String(), "secret") { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + }) + + t.Run("export failure", func(t *testing.T) { + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + return auditrecord.Export{}, errors.New("database-url=secret") + }), fleetTestPEP(t, pep.AllowReadHook{})) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusServiceUnavailable || strings.Contains(response.Body.String(), "secret") { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + }) + + t.Run("foreign export object", func(t *testing.T) { + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + foreign := testPolicyAuditExport(now) + foreign.WorkspaceID = "workspace-b" + return foreign, nil + }), fleetTestPEP(t, pep.AllowReadHook{})) + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusServiceUnavailable || strings.Contains(response.Body.String(), "workspace-b") { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + }) +} + +func TestAuditExportHandlerRejectsMalformedSurfaceBeforePolicy(t *testing.T) { + now := time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + policyCalls := 0 + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.HookFunc(func(context.Context, pep.Request) (pep.Decision, error) { + policyCalls++ + return pep.Decision{Verdict: pep.VerdictAllow, ReasonCode: "phase-1-audit-export"}, nil + }), + Auditor: pep.AuditFunc(func(context.Context, pep.AuditEvent) error { return nil }), + }) + if err != nil { + t.Fatal(err) + } + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + t.Fatal("malformed request reached exporter") + return auditrecord.Export{}, nil + }), enforcer) + for _, test := range []struct { + name string + method string + target string + status int + }{ + {name: "method", method: http.MethodPost, target: "/v1/workspaces/workspace-a/audit/export", status: http.StatusMethodNotAllowed}, + {name: "query", method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export?after=1", status: http.StatusNotFound}, + {name: "page extra query", method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export/pages?cursor=a&limit=1", status: http.StatusNotFound}, + {name: "page escaped cursor", method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export/pages?cursor=%41", status: http.StatusNotFound}, + {name: "page trailing slash", method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export/pages/", status: http.StatusNotFound}, + {name: "trailing slash", method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export/", status: http.StatusNotFound}, + {name: "encoded workspace", method: http.MethodGet, target: "/v1/workspaces/workspace%2Da/audit/export", status: http.StatusNotFound}, + {name: "extra resource", method: http.MethodGet, target: "/v1/workspaces/workspace-a/audit/export/all", status: http.StatusNotFound}, + } { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, test.method, test.target, privateKey, now, tenancy.RoleAdmin)) + if response.Code != test.status { + t.Fatalf("status/body = %d/%q, want %d", response.Code, response.Body.String(), test.status) + } + }) + } + for _, body := range []struct { + name string + contentLength int64 + transferEncoding []string + body io.ReadCloser + }{ + {name: "content", contentLength: 12, body: io.NopCloser(strings.NewReader("token=secret"))}, + {name: "unknown length", contentLength: -1, body: io.NopCloser(strings.NewReader("token=secret"))}, + {name: "chunked", transferEncoding: []string{"chunked"}, body: io.NopCloser(strings.NewReader("token=secret"))}, + {name: "zero-length custom body", body: io.NopCloser(strings.NewReader(""))}, + } { + t.Run(body.name, func(t *testing.T) { + request := authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin) + request.ContentLength = body.contentLength + request.TransferEncoding = body.transferEncoding + request.Body = body.body + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + if response.Code != http.StatusNotFound { + t.Fatalf("body-framed status/body = %d/%q", response.Code, response.Body.String()) + } + }) + } + cookieRequest := authenticatedAuditExportRequest( + t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export/pages", privateKey, now, tenancy.RoleAdmin, + ) + cookieRequest.AddCookie(&http.Cookie{Name: "__Host-sith-session", Value: "ignored-browser-session"}) + cookieResponse := httptest.NewRecorder() + handler.ServeHTTP(cookieResponse, cookieRequest) + if cookieResponse.Code != http.StatusNotFound { + t.Fatalf("page cookie status/body = %d/%q", cookieResponse.Code, cookieResponse.Body.String()) + } + if policyCalls != 0 { + t.Fatalf("malformed requests reached policy %d times", policyCalls) + } + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/audit/export", nil)) + if response.Code != http.StatusUnauthorized { + t.Fatalf("unauthenticated status/body = %d/%q", response.Code, response.Body.String()) + } +} + +func TestAuditExportHandlerBoundsConcurrentDatabaseWork(t *testing.T) { + now := time.Date(2026, time.July, 18, 9, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + entered := make(chan struct{}, maxConcurrentAuditExports) + release := make(chan struct{}) + var releaseOnce sync.Once + releaseAll := func() { releaseOnce.Do(func() { close(release) }) } + handler := mustAuditExportHandler(t, verifier, policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { + entered <- struct{}{} + <-release + return testPolicyAuditExport(now), nil + }), fleetTestPEP(t, pep.AllowReadHook{})) + + var workers sync.WaitGroup + t.Cleanup(func() { + releaseAll() + workers.Wait() + }) + for range maxConcurrentAuditExports { + workers.Add(1) + go func() { + defer workers.Done() + response := httptest.NewRecorder() + handler.ServeHTTP(response, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin)) + if response.Code != http.StatusOK { + t.Errorf("worker status/body = %d/%q", response.Code, response.Body.String()) + } + }() + } + deadline := time.NewTimer(time.Second) + defer deadline.Stop() + for range maxConcurrentAuditExports { + select { + case <-entered: + case <-deadline.C: + t.Fatal("timed out waiting for audit exports to enter") + } + } + overflow := httptest.NewRecorder() + handler.ServeHTTP(overflow, authenticatedAuditExportRequest(t, http.MethodGet, "/v1/workspaces/workspace-a/audit/export", privateKey, now, tenancy.RoleAdmin)) + if overflow.Code != http.StatusServiceUnavailable || overflow.Header().Get("Retry-After") != "1" || + overflow.Body.String() != "{\"error\":\"audit_export_unavailable\"}\n" { + t.Fatalf("overflow status/headers/body = %d/%#v/%q", overflow.Code, overflow.Header(), overflow.Body.String()) + } + releaseAll() + workers.Wait() +} + +func TestNewAuditExportHandlerRejectsMissingDependencies(t *testing.T) { + verifier, _ := fleetTestVerifier(t, time.Now()) + exporter := policyAuditExporterFunc(func(context.Context, tenancy.Scope) (auditrecord.Export, error) { return auditrecord.Export{}, nil }) + enforcer := fleetTestPEP(t, pep.AllowReadHook{}) + for _, config := range []AuditExportHandlerConfig{ + {}, + {Verifier: verifier, Exporter: exporter}, + {Verifier: verifier, PEP: enforcer}, + {Exporter: exporter, PEP: enforcer}, + } { + if _, err := NewAuditExportHandler(config); err == nil { + t.Fatalf("unsafe config accepted: %#v", config) + } + } +} + +func TestAuditExportHandlerHasNoStorageConnectorOrExecutionCapability(t *testing.T) { + t.Parallel() + parsed, err := parser.ParseFile(token.NewFileSet(), "audit_export.go", nil, parser.ImportsOnly) + if err != nil { + t.Fatal(err) + } + for _, imported := range parsed.Imports { + path := strings.Trim(imported.Path.Value, `"`) + for _, forbidden := range []string{ + "os", "os/exec", "path", "path/filepath", "/hubdb", "/hubfleet", "/connector", "/localops", + "k8s.io/", "google.golang.org/grpc", + } { + if path == forbidden || strings.Contains(path, forbidden) { + t.Fatalf("audit export handler imports forbidden capability %q", path) + } + } + } +} + +func mustAuditExportHandler(t *testing.T, verifier Verifier, exporter PolicyAuditExporter, enforcer *pep.Enforcer) http.Handler { + t.Helper() + handler, err := NewAuditExportHandler(AuditExportHandlerConfig{Verifier: verifier, Exporter: exporter, PEP: enforcer}) + if err != nil { + t.Fatal(err) + } + return handler +} + +func authenticatedAuditExportRequest( + t *testing.T, + method, target string, + privateKey ed25519.PrivateKey, + now time.Time, + role tenancy.Role, +) *http.Request { + t.Helper() + claims := hubValidClaims(now) + claims.Memberships["workspace-a"] = role + request := httptest.NewRequest(method, "https://hub.sith.test"+target, nil) + request.Header.Set("Authorization", "Bearer "+signHubTestToken(t, claims, privateKey)) + return request +} + +func testPolicyAuditExport(now time.Time) auditrecord.Export { + exported := auditrecord.Export{ + Schema: auditrecord.SchemaV1, WorkspaceID: "workspace-a", + Chain: auditrecord.Chain{HashAlgorithm: auditrecord.HashAlgorithm, HeadSequence: 1}, + Entries: []auditrecord.Entry{{ + Sequence: 1, FormatVersion: 1, RecordedAt: now, TraceID: strings.Repeat("1", 32), Actor: "user:alice", + Role: "admin", Action: "export-audit", Verb: "audit.export", Verdict: "allow", + ReasonCode: "phase-1-audit-export", EventKind: "policy-decision", PreviousHash: "sha256:" + strings.Repeat("0", 64), + }}, + } + hash, err := auditrecord.RecomputeEntryHash("workspace-a", exported.Entries[0]) + if err != nil { + panic(err) + } + exported.Entries[0].EntryHash = hash + exported.Chain.HeadHash = hash + return exported +} + +func testPolicyAuditPage(now time.Time, sequence int64, previousHash string) auditrecord.Page { + entry := auditrecord.Entry{ + Sequence: sequence, FormatVersion: 1, RecordedAt: now, TraceID: strings.Repeat("1", 32), Actor: "user:alice", + Role: "admin", Action: "export-audit", Verb: "audit.export", Verdict: "allow", + ReasonCode: "phase-1-audit-export", EventKind: "policy-decision", PreviousHash: previousHash, + } + hash, err := auditrecord.RecomputeEntryHash("workspace-a", entry) + if err != nil { + panic(err) + } + entry.EntryHash = hash + return auditrecord.Page{ + Schema: auditrecord.PageSchemaV1, WorkspaceID: "workspace-a", + Snapshot: auditrecord.Chain{HashAlgorithm: auditrecord.HashAlgorithm, HeadSequence: sequence, HeadHash: hash}, + StartSequence: sequence, PreviousHash: previousHash, Entries: []auditrecord.Entry{entry}, + } +} diff --git a/internal/hubserver/auth.go b/internal/hubserver/auth.go index 67d91e7..8fb5201 100644 --- a/internal/hubserver/auth.go +++ b/internal/hubserver/auth.go @@ -27,9 +27,9 @@ func Authenticate(verifier Verifier, next http.Handler) (http.Handler, error) { return AuthenticateWithObserver(verifier, nil, next) } -// AuthenticateWithObserver constructs authentication middleware with one passive refusal -// observer. The observer is never given request metadata, credentials, verifier errors, or caller -// correlation values, and cannot alter the uniform unauthorized response. +// AuthenticateWithObserver constructs authentication middleware with one passive outcome observer. +// The observer is never given request metadata, credentials, verifier errors, or caller correlation +// values, and cannot alter the uniform unauthorized response or successful handler path. func AuthenticateWithObserver(verifier Verifier, observer AuthObserver, next http.Handler) (http.Handler, error) { if verifier == nil { return nil, fmt.Errorf("construct authentication middleware: verifier is required") @@ -53,6 +53,7 @@ func AuthenticateWithObserver(verifier Verifier, observer AuthObserver, next htt refuseAuthentication(observer, response) return } + ObserveAuth(observer, AuthEvent{Outcome: AuthOutcomeAccepted}) ctx := context.WithValue(cloned.Context(), principalContextKey{}, principal) next.ServeHTTP(response, cloned.WithContext(ctx)) }), nil diff --git a/internal/hubserver/auth_observability.go b/internal/hubserver/auth_observability.go index 628afd9..c048e21 100644 --- a/internal/hubserver/auth_observability.go +++ b/internal/hubserver/auth_observability.go @@ -4,12 +4,17 @@ package hubserver import "fmt" -// AuthOutcome is the closed self-observability result of one pre-principal -// authentication attempt. It intentionally does not distinguish credential failure modes. +// AuthOutcome is the closed self-observability result of one completed authentication verifier +// decision. It intentionally does not distinguish credential modes or failure reasons. type AuthOutcome string -// AuthOutcomeRefused is emitted for every request the bearer-token middleware rejects. -const AuthOutcomeRefused AuthOutcome = "refused" +const ( + // AuthOutcomeAccepted is emitted after a bearer token or browser session verifies successfully, + // before any workspace authorization decision. + AuthOutcomeAccepted AuthOutcome = "accepted" + // AuthOutcomeRefused is emitted for every request the authentication boundary rejects. + AuthOutcomeRefused AuthOutcome = "refused" +) // AuthEvent is one passive, sanitized authentication observation. It deliberately has no // request, credential, verifier-error, principal, path, network, or correlation fields: none are @@ -20,7 +25,7 @@ type AuthEvent struct { // Validate rejects unsupported outcome values before an observer can emit them. func (event AuthEvent) Validate() error { - if event.Outcome != AuthOutcomeRefused { + if event.Outcome != AuthOutcomeAccepted && event.Outcome != AuthOutcomeRefused { return fmt.Errorf("authentication event outcome is unsupported") } return nil @@ -40,6 +45,33 @@ func (function AuthObserverFunc) ObserveAuth(event AuthEvent) { function(event) } +type authObserverFanout struct { + observers []AuthObserver +} + +// NewAuthObserverFanout composes two or more required authentication observers. Each destination +// remains independently panic-isolated, so a faulty observer cannot suppress later destinations. +func NewAuthObserverFanout(observers ...AuthObserver) (AuthObserver, error) { + if len(observers) < 2 { + return nil, fmt.Errorf("construct authentication observer fanout: at least two observers are required") + } + for _, observer := range observers { + if observer == nil { + return nil, fmt.Errorf("construct authentication observer fanout: observers are required") + } + } + return &authObserverFanout{observers: append([]AuthObserver(nil), observers...)}, nil +} + +func (fanout *authObserverFanout) ObserveAuth(event AuthEvent) { + if fanout == nil { + return + } + for _, observer := range fanout.observers { + ObserveAuth(observer, event) + } +} + // ObserveAuth sends a valid event to a passive observer. Invalid events and observer panics are // intentionally ignored so observability cannot alter the uniform unauthorized response. func ObserveAuth(observer AuthObserver, event AuthEvent) { diff --git a/internal/hubserver/auth_observability_test.go b/internal/hubserver/auth_observability_test.go index 8264eb9..a9e21ce 100644 --- a/internal/hubserver/auth_observability_test.go +++ b/internal/hubserver/auth_observability_test.go @@ -62,7 +62,7 @@ func TestAuthenticateWithObserverRecordsOnlyUniformRefusals(t *testing.T) { } } -func TestAuthenticateWithObserverIsSilentAfterValidAuthentication(t *testing.T) { +func TestAuthenticateWithObserverRecordsAcceptedAfterValidAuthentication(t *testing.T) { now := time.Date(2026, 7, 14, 13, 0, 0, 0, time.UTC) publicKey, privateKey := hubTestKeyPair() verifier, err := hubauth.NewJWTVerifier(hubauth.JWTConfig{ @@ -82,11 +82,24 @@ func TestAuthenticateWithObserverIsSilentAfterValidAuthentication(t *testing.T) request.Header.Set("Authorization", "Bearer "+signHubTestToken(t, hubValidClaims(now), privateKey)) response := httptest.NewRecorder() handler.ServeHTTP(response, request) - if response.Code != http.StatusNoContent || len(events) != 0 { + if response.Code != http.StatusNoContent || len(events) != 1 || events[0] != (AuthEvent{Outcome: AuthOutcomeAccepted}) { t.Fatalf("status = %d events = %#v", response.Code, events) } } +func TestAuthEventValidationUsesOnlyClosedOutcomes(t *testing.T) { + for _, outcome := range []AuthOutcome{AuthOutcomeAccepted, AuthOutcomeRefused} { + if err := (AuthEvent{Outcome: outcome}).Validate(); err != nil { + t.Fatalf("Validate(%q) = %v", outcome, err) + } + } + for _, outcome := range []AuthOutcome{"", "token=secret", "forbidden"} { + if err := (AuthEvent{Outcome: outcome}).Validate(); err == nil { + t.Fatalf("Validate(%q) accepted an unsupported outcome", outcome) + } + } +} + func TestObserveAuthRejectsUnsafeEventsAndContainsObserverPanics(t *testing.T) { called := false ObserveAuth(AuthObserverFunc(func(AuthEvent) { called = true }), AuthEvent{Outcome: "token=secret"}) @@ -95,6 +108,7 @@ func TestObserveAuthRejectsUnsafeEventsAndContainsObserverPanics(t *testing.T) { } ObserveAuth(AuthObserverFunc(func(AuthEvent) { panic("observer fault") }), AuthEvent{Outcome: AuthOutcomeRefused}) + ObserveAuth(AuthObserverFunc(func(AuthEvent) { panic("observer fault") }), AuthEvent{Outcome: AuthOutcomeAccepted}) handler, err := AuthenticateWithObserver(authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { return tenancy.Principal{}, errors.New("invalid") @@ -109,4 +123,61 @@ func TestObserveAuthRejectsUnsafeEventsAndContainsObserverPanics(t *testing.T) { if response.Code != http.StatusUnauthorized || response.Body.String() != "{\"error\":\"unauthorized\"}\n" { t.Fatalf("status = %d, body = %q", response.Code, response.Body.String()) } + + successful, err := AuthenticateWithObserver(authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { + return tenancy.Principal{}, nil + }), AuthObserverFunc(func(AuthEvent) { panic("observer fault") }), http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.WriteHeader(http.StatusNoContent) + })) + if err != nil { + t.Fatal(err) + } + successRequest := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/api", nil) + successRequest.Header.Set("Authorization", "Bearer valid") + successResponse := httptest.NewRecorder() + successful.ServeHTTP(successResponse, successRequest) + if successResponse.Code != http.StatusNoContent { + t.Fatalf("successful authentication with panicking observer status = %d", successResponse.Code) + } +} + +func TestAuthObserverFanoutIsolatesEachRequiredDestination(t *testing.T) { + var deliveries []string + first := AuthObserverFunc(func(AuthEvent) { + deliveries = append(deliveries, "first") + panic("observer fault") + }) + second := AuthObserverFunc(func(AuthEvent) { deliveries = append(deliveries, "second") }) + observers := []AuthObserver{first, second} + fanout, err := NewAuthObserverFanout(observers...) + if err != nil { + t.Fatal(err) + } + observers[1] = AuthObserverFunc(func(AuthEvent) { t.Fatal("fanout retained caller-owned slice") }) + + ObserveAuth(fanout, AuthEvent{Outcome: AuthOutcomeRefused}) + ObserveAuth(fanout, AuthEvent{Outcome: AuthOutcomeAccepted}) + if len(deliveries) != 4 || deliveries[0] != "first" || deliveries[1] != "second" || + deliveries[2] != "first" || deliveries[3] != "second" { + t.Fatalf("fanout deliveries = %#v, want independently isolated order", deliveries) + } + + deliveries = nil + ObserveAuth(fanout, AuthEvent{Outcome: "token=secret"}) + if len(deliveries) != 0 { + t.Fatalf("unsafe event reached fanout destinations: %#v", deliveries) + } +} + +func TestNewAuthObserverFanoutRejectsIncompleteConfiguration(t *testing.T) { + observer := AuthObserverFunc(func(AuthEvent) {}) + for _, observers := range [][]AuthObserver{ + nil, + {observer}, + {observer, nil}, + } { + if fanout, err := NewAuthObserverFanout(observers...); err == nil || fanout != nil { + t.Fatalf("NewAuthObserverFanout(%#v) = %#v, %v", observers, fanout, err) + } + } } diff --git a/internal/hubserver/browser_oidc.go b/internal/hubserver/browser_oidc.go new file mode 100644 index 0000000..99ef3cb --- /dev/null +++ b/internal/hubserver/browser_oidc.go @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/ArdurAI/sith/internal/hubauth" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + browserOIDCTransactionCookie = "__Host-sith-oidc-tx" + browserOIDCSessionCookie = "__Host-sith-session" + browserOIDCRandomBytes = 32 + defaultBrowserOIDCTTL = 5 * time.Minute + defaultBrowserOIDCMaxSessions = 256 + maximumBrowserOIDCSessions = 4096 +) + +// BrowserOIDCService is the server-side OIDC code-flow boundary. It never returns a token to a +// browser response; BrowserOIDCHandler stores the resulting Sith session only in an HttpOnly cookie. +type BrowserOIDCService interface { + BrowserAuthorizationURL(context.Context, hubauth.OIDCBrowserAuthorizationRequest) (string, error) + ExchangeAuthorizationCode(context.Context, hubauth.OIDCBrowserCodeExchange) (string, error) + ExchangeWithNonce(context.Context, tenancy.WorkspaceID, string, string) (hubauth.IssuedSession, error) +} + +// BrowserOIDCHandlerConfig fixes one OIDC issuer, public client, and callback URL for a Hub. +type BrowserOIDCHandlerConfig struct { + Service BrowserOIDCService + ProviderIssuer string + ClientID string + RedirectURI string + Limiter *AttemptLimiter + TransactionTTL time.Duration + MaxTransactions int + Now func() time.Time + Random io.Reader +} + +// BrowserOIDCHandler owns bounded, restart-ephemeral authorization-code transactions. +type BrowserOIDCHandler struct { + service BrowserOIDCService + providerIssuer string + clientID string + redirectURI string + callbackURL *url.URL + limiter *AttemptLimiter + transactionTTL time.Duration + maxSessions int + now func() time.Time + random io.Reader + + mu sync.Mutex + transactions map[string]browserOIDCTransaction +} + +type browserOIDCTransaction struct { + workspaceID tenancy.WorkspaceID + state string + nonce string + codeVerifier string + expiresAt time.Time +} + +// NewBrowserOIDCHandler constructs explicit login and callback handlers without a cookie-auth API. +func NewBrowserOIDCHandler(config BrowserOIDCHandlerConfig) (*BrowserOIDCHandler, error) { + if config.Service == nil || config.Limiter == nil || config.ProviderIssuer == "" || strings.TrimSpace(config.ProviderIssuer) != config.ProviderIssuer || + config.ClientID == "" || strings.TrimSpace(config.ClientID) != config.ClientID || len(config.ClientID) > 256 { + return nil, fmt.Errorf("construct browser OIDC handler: service, limiter, issuer, and client ID are required") + } + callbackURL, err := parseBrowserOIDCRedirectURI(config.RedirectURI) + if err != nil { + return nil, fmt.Errorf("construct browser OIDC handler: callback URL is invalid") + } + if config.TransactionTTL == 0 { + config.TransactionTTL = defaultBrowserOIDCTTL + } + if config.TransactionTTL < time.Second || config.TransactionTTL > 10*time.Minute { + return nil, fmt.Errorf("construct browser OIDC handler: transaction lifetime must be between one second and ten minutes") + } + if config.MaxTransactions == 0 { + config.MaxTransactions = defaultBrowserOIDCMaxSessions + } + if config.MaxTransactions < 1 || config.MaxTransactions > maximumBrowserOIDCSessions { + return nil, fmt.Errorf("construct browser OIDC handler: transaction capacity is invalid") + } + if config.Now == nil { + config.Now = time.Now + } + if config.Random == nil { + config.Random = rand.Reader + } + return &BrowserOIDCHandler{ + service: config.Service, providerIssuer: config.ProviderIssuer, clientID: config.ClientID, redirectURI: config.RedirectURI, + callbackURL: callbackURL, limiter: config.Limiter, transactionTTL: config.TransactionTTL, maxSessions: config.MaxTransactions, + now: config.Now, random: config.Random, transactions: make(map[string]browserOIDCTransaction), + }, nil +} + +// CallbackPath is the exact path that must be registered with the Hub's HTTPS mux. +func (handler *BrowserOIDCHandler) CallbackPath() string { + if handler == nil || handler.callbackURL == nil { + return "" + } + return handler.callbackURL.Path +} + +// Login starts one code+PKCE transaction for the workspace encoded in the fixed login route. +func (handler *BrowserOIDCHandler) Login(response http.ResponseWriter, request *http.Request) { + if !handler.acceptRequest(response, request, true) { + return + } + workspaceID, ok := browserOIDCWorkspace(request.URL.Path) + if !ok { + browserOIDCFailure(response, http.StatusNotFound) + return + } + transactionID, err := handler.randomValue() + if err != nil { + browserOIDCFailure(response, http.StatusServiceUnavailable) + return + } + state, err := handler.randomValue() + if err != nil { + browserOIDCFailure(response, http.StatusServiceUnavailable) + return + } + nonce, err := handler.randomValue() + if err != nil { + browserOIDCFailure(response, http.StatusServiceUnavailable) + return + } + codeVerifier, err := handler.randomValue() + if err != nil { + browserOIDCFailure(response, http.StatusServiceUnavailable) + return + } + expiresAt := handler.now().UTC().Add(handler.transactionTTL) + if !handler.storeTransaction(transactionID, browserOIDCTransaction{ + workspaceID: workspaceID, state: state, nonce: nonce, codeVerifier: codeVerifier, expiresAt: expiresAt, + }) { + browserOIDCFailure(response, http.StatusTooManyRequests) + return + } + challenge := sha256.Sum256([]byte(codeVerifier)) + authorizationURL, err := handler.service.BrowserAuthorizationURL(request.Context(), hubauth.OIDCBrowserAuthorizationRequest{ + Issuer: handler.providerIssuer, ClientID: handler.clientID, RedirectURI: handler.redirectURI, + State: state, Nonce: nonce, CodeChallenge: base64.RawURLEncoding.EncodeToString(challenge[:]), + }) + if err != nil { + handler.deleteTransaction(transactionID) + browserOIDCFailure(response, http.StatusServiceUnavailable) + return + } + browserOIDCHeaders(response.Header()) + http.SetCookie(response, &http.Cookie{ + Name: browserOIDCTransactionCookie, Value: transactionID, Path: "/", Secure: true, HttpOnly: true, + SameSite: http.SameSiteLaxMode, Expires: expiresAt, MaxAge: max(1, int(handler.transactionTTL.Seconds())), + }) + response.Header().Set("Location", authorizationURL) + response.WriteHeader(http.StatusFound) +} + +// Callback consumes one transaction and sets the final host-only session cookie on success. +// Lax is required for the safe top-level redirect whose navigation began at the external IdP. +func (handler *BrowserOIDCHandler) Callback(response http.ResponseWriter, request *http.Request) { + if !handler.acceptRequest(response, request, false) { + return + } + query, err := url.ParseQuery(request.URL.RawQuery) + if err != nil || len(query["state"]) != 1 || query.Get("state") == "" { + handler.clearTransactionCookie(response) + browserOIDCFailure(response, http.StatusUnauthorized) + return + } + cookie, err := request.Cookie(browserOIDCTransactionCookie) + if err != nil || cookie.Value == "" { + handler.clearTransactionCookie(response) + browserOIDCFailure(response, http.StatusUnauthorized) + return + } + transaction, ok := handler.takeTransaction(cookie.Value, query.Get("state")) + if !ok { + handler.clearTransactionCookie(response) + browserOIDCFailure(response, http.StatusUnauthorized) + return + } + if len(query) != 2 || len(query["code"]) != 1 || query.Get("code") == "" { + handler.clearTransactionCookie(response) + browserOIDCFailure(response, http.StatusUnauthorized) + return + } + rawToken, err := handler.service.ExchangeAuthorizationCode(request.Context(), hubauth.OIDCBrowserCodeExchange{ + Issuer: handler.providerIssuer, ClientID: handler.clientID, RedirectURI: handler.redirectURI, + Code: query.Get("code"), CodeVerifier: transaction.codeVerifier, + }) + if err != nil { + handler.clearTransactionCookie(response) + browserOIDCFailure(response, http.StatusUnauthorized) + return + } + session, err := handler.service.ExchangeWithNonce(request.Context(), transaction.workspaceID, rawToken, transaction.nonce) + if err != nil || session.TokenType != "Bearer" || session.AccessToken == "" || !session.ExpiresAt.After(handler.now()) { + handler.clearTransactionCookie(response) + browserOIDCFailure(response, http.StatusUnauthorized) + return + } + handler.clearTransactionCookie(response) + browserOIDCHeaders(response.Header()) + remaining := int(session.ExpiresAt.Sub(handler.now()).Seconds()) + http.SetCookie(response, &http.Cookie{ + Name: browserOIDCSessionCookie, Value: session.AccessToken, Path: "/", Secure: true, HttpOnly: true, + SameSite: http.SameSiteLaxMode, Expires: session.ExpiresAt.UTC(), MaxAge: max(1, remaining), + }) + response.Header().Set("Location", browserConsolePath(transaction.workspaceID)) + response.WriteHeader(http.StatusSeeOther) +} + +func browserConsolePath(workspaceID tenancy.WorkspaceID) string { + return "/v1/workspaces/" + url.PathEscape(string(workspaceID)) + "/console" +} + +func (handler *BrowserOIDCHandler) acceptRequest(response http.ResponseWriter, request *http.Request, login bool) bool { + if handler == nil || request == nil || request.Method != http.MethodGet || request.URL == nil || request.URL.Fragment != "" || + !strings.EqualFold(request.Host, handler.callbackURL.Host) || (login && request.URL.RawQuery != "") || (!login && request.URL.Path != handler.callbackURL.Path) { + browserOIDCFailure(response, http.StatusNotFound) + return false + } + if !handler.limiter.Allow(browserOIDCClientAddress(request.RemoteAddr)) { + browserOIDCFailure(response, http.StatusTooManyRequests) + return false + } + return true +} + +func (handler *BrowserOIDCHandler) randomValue() (string, error) { + value := make([]byte, browserOIDCRandomBytes) + if _, err := io.ReadFull(handler.random, value); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(value), nil +} + +func (handler *BrowserOIDCHandler) storeTransaction(identifier string, transaction browserOIDCTransaction) bool { + handler.mu.Lock() + defer handler.mu.Unlock() + now := handler.now().UTC() + for candidate, stored := range handler.transactions { + if !now.Before(stored.expiresAt) { + delete(handler.transactions, candidate) + } + } + if _, exists := handler.transactions[identifier]; exists || len(handler.transactions) >= handler.maxSessions { + return false + } + handler.transactions[identifier] = transaction + return true +} + +func (handler *BrowserOIDCHandler) takeTransaction(identifier, state string) (browserOIDCTransaction, bool) { + handler.mu.Lock() + defer handler.mu.Unlock() + transaction, exists := handler.transactions[identifier] + if !exists { + return browserOIDCTransaction{}, false + } + delete(handler.transactions, identifier) + if !handler.now().UTC().Before(transaction.expiresAt) || subtle.ConstantTimeCompare([]byte(transaction.state), []byte(state)) != 1 { + return browserOIDCTransaction{}, false + } + return transaction, true +} + +func (handler *BrowserOIDCHandler) deleteTransaction(identifier string) { + handler.mu.Lock() + defer handler.mu.Unlock() + delete(handler.transactions, identifier) +} + +func (handler *BrowserOIDCHandler) clearTransactionCookie(response http.ResponseWriter) { + http.SetCookie(response, &http.Cookie{ + Name: browserOIDCTransactionCookie, Value: "", Path: "/", Secure: true, HttpOnly: true, + SameSite: http.SameSiteLaxMode, MaxAge: -1, Expires: time.Unix(1, 0).UTC(), + }) +} + +func parseBrowserOIDCRedirectURI(rawURI string) (*url.URL, error) { + parsed, err := url.Parse(rawURI) + if err != nil || len(rawURI) > 2048 || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || + parsed.RawQuery != "" || parsed.Fragment != "" || parsed.Path == "" || parsed.Path == "/" || parsed.EscapedPath() != parsed.Path { + return nil, fmt.Errorf("invalid callback URL") + } + return parsed, nil +} + +func browserOIDCWorkspace(path string) (tenancy.WorkspaceID, bool) { + const prefix = "/v1/workspaces/" + const suffix = "/console/login" + if !strings.HasPrefix(path, prefix) || !strings.HasSuffix(path, suffix) { + return "", false + } + segment := strings.TrimSuffix(strings.TrimPrefix(path, prefix), suffix) + if segment == "" || strings.Contains(segment, "/") { + return "", false + } + workspace, err := url.PathUnescape(segment) + if err != nil || url.PathEscape(workspace) != segment || tenancy.ValidateWorkspaceID(tenancy.WorkspaceID(workspace)) != nil { + return "", false + } + return tenancy.WorkspaceID(workspace), true +} + +func browserOIDCClientAddress(remoteAddress string) string { + host, _, err := net.SplitHostPort(remoteAddress) + if err != nil || host == "" { + return "unknown" + } + return host +} + +func browserOIDCHeaders(header http.Header) { + header.Set("Cache-Control", "no-store") + header.Set("Pragma", "no-cache") + header.Set("Referrer-Policy", "no-referrer") + header.Set("X-Content-Type-Options", "nosniff") + header.Set("X-Frame-Options", "DENY") + header.Set("Content-Security-Policy", "default-src 'none'; base-uri 'none'; frame-ancestors 'none'") +} + +func browserOIDCFailure(response http.ResponseWriter, status int) { + browserOIDCHeaders(response.Header()) + response.Header().Set("Content-Type", "application/json; charset=utf-8") + response.WriteHeader(status) + _, _ = response.Write([]byte("{\"error\":\"oidc_login_failed\"}\n")) +} diff --git a/internal/hubserver/browser_oidc_test.go b/internal/hubserver/browser_oidc_test.go new file mode 100644 index 0000000..9625ac1 --- /dev/null +++ b/internal/hubserver/browser_oidc_test.go @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/hubauth" + "github.com/ArdurAI/sith/internal/tenancy" +) + +type browserOIDCServiceStub struct { + authorization hubauth.OIDCBrowserAuthorizationRequest + exchange hubauth.OIDCBrowserCodeExchange + workspaceID tenancy.WorkspaceID + nonce string + authorizeCalls int + exchangeCalls int + sessionCalls int + rawToken string + session hubauth.IssuedSession + authorizationErr error +} + +func (stub *browserOIDCServiceStub) BrowserAuthorizationURL(_ context.Context, request hubauth.OIDCBrowserAuthorizationRequest) (string, error) { + stub.authorizeCalls++ + stub.authorization = request + if stub.authorizationErr != nil { + return "", stub.authorizationErr + } + return "https://idp.sith.test/authorize?provider=fixed", nil +} + +func (stub *browserOIDCServiceStub) ExchangeAuthorizationCode(_ context.Context, request hubauth.OIDCBrowserCodeExchange) (string, error) { + stub.exchangeCalls++ + stub.exchange = request + return stub.rawToken, nil +} + +func (stub *browserOIDCServiceStub) ExchangeWithNonce( + _ context.Context, + workspaceID tenancy.WorkspaceID, + rawToken, nonce string, +) (hubauth.IssuedSession, error) { + stub.sessionCalls++ + stub.workspaceID = workspaceID + stub.nonce = nonce + if rawToken != stub.rawToken { + return hubauth.IssuedSession{}, errors.New("unexpected raw token") + } + return stub.session, nil +} + +func TestBrowserOIDCHandlerKeepsTokensOutOfBrowserPayloads(t *testing.T) { + now := time.Date(2026, 7, 15, 14, 0, 0, 0, time.UTC) + stub := &browserOIDCServiceStub{ + rawToken: "upstream.id.token", session: hubauth.IssuedSession{ + AccessToken: "sith.signed.session", TokenType: "Bearer", ExpiresAt: now.Add(15 * time.Minute), + }, + } + handler := newBrowserOIDCTestHandler(t, stub, &now) + login := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/login", nil) + login.RemoteAddr = "192.0.2.10:8443" + loginResponse := httptest.NewRecorder() + handler.Login(loginResponse, login) + if loginResponse.Code != http.StatusFound || loginResponse.Header().Get("Location") != "https://idp.sith.test/authorize?provider=fixed" { + t.Fatalf("login status/location = %d/%q", loginResponse.Code, loginResponse.Header().Get("Location")) + } + transactionCookie := requiredBrowserCookie(t, loginResponse.Result().Cookies(), browserOIDCTransactionCookie) + if !transactionCookie.Secure || !transactionCookie.HttpOnly || transactionCookie.Path != "/" || transactionCookie.Domain != "" || transactionCookie.SameSite != http.SameSiteLaxMode { + t.Fatalf("transaction cookie = %#v", transactionCookie) + } + state := browserOIDCTestValue(0x02) + nonce := browserOIDCTestValue(0x03) + verifier := browserOIDCTestValue(0x04) + challenge := sha256.Sum256([]byte(verifier)) + if stub.authorizeCalls != 1 || stub.authorization.Issuer != "https://issuer.sith.test" || stub.authorization.ClientID != "sith-browser" || + stub.authorization.RedirectURI != "https://hub.sith.test/v1/console/oidc/callback" || stub.authorization.State != state || + stub.authorization.Nonce != nonce || stub.authorization.CodeChallenge != base64.RawURLEncoding.EncodeToString(challenge[:]) { + t.Fatalf("authorization request = %#v", stub.authorization) + } + if strings.Contains(loginResponse.Body.String(), stub.rawToken) || strings.Contains(loginResponse.Body.String(), stub.session.AccessToken) { + t.Fatalf("login body leaked a token: %q", loginResponse.Body.String()) + } + + callbackURL := "https://hub.sith.test/v1/console/oidc/callback?code=provider-code&state=" + url.QueryEscape(state) + callback := httptest.NewRequest(http.MethodGet, callbackURL, nil) + callback.RemoteAddr = login.RemoteAddr + callback.AddCookie(transactionCookie) + callbackResponse := httptest.NewRecorder() + handler.Callback(callbackResponse, callback) + if callbackResponse.Code != http.StatusSeeOther || callbackResponse.Header().Get("Location") != "/v1/workspaces/workspace-a/console" || callbackResponse.Body.Len() != 0 { + t.Fatalf("callback status/location/body = %d/%q/%q", callbackResponse.Code, callbackResponse.Header().Get("Location"), callbackResponse.Body.String()) + } + sessionCookie := requiredBrowserCookie(t, callbackResponse.Result().Cookies(), browserOIDCSessionCookie) + if !sessionCookie.Secure || !sessionCookie.HttpOnly || sessionCookie.Path != "/" || sessionCookie.Domain != "" || sessionCookie.SameSite != http.SameSiteLaxMode || sessionCookie.Value != stub.session.AccessToken { + t.Fatalf("session cookie = %#v", sessionCookie) + } + if stub.exchangeCalls != 1 || stub.exchange.Code != "provider-code" || stub.exchange.CodeVerifier != verifier || + stub.workspaceID != "workspace-a" || stub.nonce != nonce || stub.sessionCalls != 1 { + t.Fatalf("exchange/session = %#v/%q/%q/%d", stub.exchange, stub.workspaceID, stub.nonce, stub.sessionCalls) + } + if strings.Contains(callbackResponse.Body.String(), stub.rawToken) || strings.Contains(callbackResponse.Body.String(), stub.session.AccessToken) || + strings.Contains(callbackResponse.Header().Get("Location"), stub.rawToken) || strings.Contains(callbackResponse.Header().Get("Location"), stub.session.AccessToken) { + t.Fatal("callback response leaked a raw token outside its HttpOnly cookie") + } + + replay := httptest.NewRequest(http.MethodGet, callbackURL, nil) + replay.RemoteAddr = login.RemoteAddr + replay.AddCookie(transactionCookie) + replayResponse := httptest.NewRecorder() + handler.Callback(replayResponse, replay) + if replayResponse.Code != http.StatusUnauthorized || stub.exchangeCalls != 1 || stub.sessionCalls != 1 { + t.Fatalf("replayed callback status/calls = %d/%d/%d", replayResponse.Code, stub.exchangeCalls, stub.sessionCalls) + } +} + +func TestBrowserOIDCHandlerRejectsUnsafeMethods(t *testing.T) { + now := time.Date(2026, 7, 22, 18, 0, 0, 0, time.UTC) + stub := &browserOIDCServiceStub{} + handler := newBrowserOIDCTestHandler(t, stub, &now) + + for _, target := range []string{ + "https://hub.sith.test/v1/workspaces/workspace-a/console/login", + "https://hub.sith.test/v1/console/oidc/callback?code=provider-code&state=provider-state", + } { + request := httptest.NewRequest(http.MethodPost, target, nil) + request.RemoteAddr = "192.0.2.21:8443" + response := httptest.NewRecorder() + if request.URL.Path == handler.callbackURL.Path { + handler.Callback(response, request) + } else { + handler.Login(response, request) + } + if response.Code != http.StatusNotFound { + t.Fatalf("POST %s status = %d, want %d", request.URL.Path, response.Code, http.StatusNotFound) + } + } + if stub.authorizeCalls != 0 || stub.exchangeCalls != 0 || stub.sessionCalls != 0 || len(handler.transactions) != 0 { + t.Fatalf("unsafe method calls/transactions = %d/%d/%d/%d", stub.authorizeCalls, stub.exchangeCalls, stub.sessionCalls, len(handler.transactions)) + } +} + +func TestBrowserOIDCHandlerCountsEachRequestOnceAtRateLimit(t *testing.T) { + now := time.Date(2026, 7, 16, 15, 0, 0, 0, time.UTC) + limiter, err := NewAttemptLimiter(AttemptLimiterConfig{ + Attempts: 20, Window: time.Minute, MaxKeys: 1, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatal(err) + } + stub := &browserOIDCServiceStub{ + rawToken: "upstream.id.token", + session: hubauth.IssuedSession{ + AccessToken: "sith.signed.session", TokenType: "Bearer", ExpiresAt: now.Add(15 * time.Minute), + }, + } + handler, err := NewBrowserOIDCHandler(BrowserOIDCHandlerConfig{ + Service: stub, ProviderIssuer: "https://issuer.sith.test", ClientID: "sith-browser", + RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", Limiter: limiter, + Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x01}, 10*4*browserOIDCRandomBytes)), + }) + if err != nil { + t.Fatal(err) + } + + for flow := 1; flow <= 10; flow++ { + login := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/login", nil) + login.RemoteAddr = "192.0.2.20:8443" + loginResponse := httptest.NewRecorder() + handler.Login(loginResponse, login) + if loginResponse.Code != http.StatusFound { + t.Fatalf("flow %d login status = %d", flow, loginResponse.Code) + } + if attempts := browserOIDCAttemptCount(limiter, "192.0.2.20"); attempts != 2*flow-1 { + t.Fatalf("flow %d login attempt count = %d, want %d", flow, attempts, 2*flow-1) + } + transactionCookie := requiredBrowserCookie(t, loginResponse.Result().Cookies(), browserOIDCTransactionCookie) + callback := httptest.NewRequest( + http.MethodGet, + "https://hub.sith.test/v1/console/oidc/callback?code=provider-code&state="+url.QueryEscape(stub.authorization.State), + nil, + ) + callback.RemoteAddr = login.RemoteAddr + callback.AddCookie(transactionCookie) + callbackResponse := httptest.NewRecorder() + handler.Callback(callbackResponse, callback) + if callbackResponse.Code != http.StatusSeeOther || callbackResponse.Header().Get("Location") != "/v1/workspaces/workspace-a/console" { + t.Fatalf("flow %d callback status/location = %d/%q", flow, callbackResponse.Code, callbackResponse.Header().Get("Location")) + } + if attempts := browserOIDCAttemptCount(limiter, "192.0.2.20"); attempts != 2*flow { + t.Fatalf("flow %d callback attempt count = %d, want %d", flow, attempts, 2*flow) + } + } + + rejected := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/login", nil) + rejected.RemoteAddr = "192.0.2.20:8443" + rejectedResponse := httptest.NewRecorder() + handler.Login(rejectedResponse, rejected) + if rejectedResponse.Code != http.StatusTooManyRequests || len(handler.transactions) != 0 || + stub.authorizeCalls != 10 || stub.exchangeCalls != 10 || stub.sessionCalls != 10 { + t.Fatalf( + "rejected status/transactions/calls = %d/%d/%d/%d/%d", + rejectedResponse.Code, len(handler.transactions), stub.authorizeCalls, stub.exchangeCalls, stub.sessionCalls, + ) + } +} + +func browserOIDCAttemptCount(limiter *AttemptLimiter, key string) int { + limiter.mu.Lock() + defer limiter.mu.Unlock() + return limiter.entries[key].count +} + +func TestBrowserOIDCHandlerFailsClosedAndDoesNotAuthenticateBearerRoutes(t *testing.T) { + now := time.Date(2026, 7, 15, 14, 0, 0, 0, time.UTC) + stub := &browserOIDCServiceStub{rawToken: "upstream.id.token", session: hubauth.IssuedSession{AccessToken: "sith.signed.session", TokenType: "Bearer", ExpiresAt: now.Add(time.Minute)}} + handler := newBrowserOIDCTestHandler(t, stub, &now) + login := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/login", nil) + login.RemoteAddr = "192.0.2.11:8443" + loginResponse := httptest.NewRecorder() + handler.Login(loginResponse, login) + transactionCookie := requiredBrowserCookie(t, loginResponse.Result().Cookies(), browserOIDCTransactionCookie) + state := browserOIDCTestValue(0x02) + wrongState := browserOIDCTestValue(0x09) + wrongCallback := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/console/oidc/other?code=provider-code&state="+state, nil) + wrongCallback.RemoteAddr = login.RemoteAddr + wrongCallback.AddCookie(transactionCookie) + wrongCallbackResponse := httptest.NewRecorder() + handler.Callback(wrongCallbackResponse, wrongCallback) + if wrongCallbackResponse.Code != http.StatusNotFound || stub.exchangeCalls != 0 { + t.Fatalf("wrong callback status/calls = %d/%d", wrongCallbackResponse.Code, stub.exchangeCalls) + } + wrong := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/console/oidc/callback?code=provider-code&state="+wrongState, nil) + wrong.RemoteAddr = login.RemoteAddr + wrong.AddCookie(transactionCookie) + wrongResponse := httptest.NewRecorder() + handler.Callback(wrongResponse, wrong) + if wrongResponse.Code != http.StatusUnauthorized || stub.exchangeCalls != 0 { + t.Fatalf("wrong-state callback status/calls = %d/%d", wrongResponse.Code, stub.exchangeCalls) + } + if browserOIDCTestValue(0x01) == state { + t.Fatal("test transaction values unexpectedly collided") + } + + protected, err := Authenticate(authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { + t.Fatal("cookie unexpectedly reached bearer verifier") + return tenancy.Principal{}, nil + }), http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("cookie unexpectedly reached bearer route") + })) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/fleet", nil) + request.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: stub.session.AccessToken}) + response := httptest.NewRecorder() + protected.ServeHTTP(response, request) + if response.Code != http.StatusUnauthorized { + t.Fatalf("cookie-only bearer route status = %d", response.Code) + } + + failing := &browserOIDCServiceStub{authorizationErr: errors.New("provider unavailable")} + failingHandler := newBrowserOIDCTestHandler(t, failing, &now) + failingLogin := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/login", nil) + failingLogin.RemoteAddr = "192.0.2.12:8443" + failingResponse := httptest.NewRecorder() + failingHandler.Login(failingResponse, failingLogin) + if failingResponse.Code != http.StatusServiceUnavailable || len(failingHandler.transactions) != 0 { + t.Fatalf("provider outage status/transactions = %d/%d", failingResponse.Code, len(failingHandler.transactions)) + } + + expiring := &browserOIDCServiceStub{rawToken: "upstream.id.token", session: stub.session} + expiringHandler := newBrowserOIDCTestHandler(t, expiring, &now) + expiringLogin := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/login", nil) + expiringLogin.RemoteAddr = "192.0.2.13:8443" + expiringLoginResponse := httptest.NewRecorder() + expiringHandler.Login(expiringLoginResponse, expiringLogin) + expiringCookie := requiredBrowserCookie(t, expiringLoginResponse.Result().Cookies(), browserOIDCTransactionCookie) + now = now.Add(6 * time.Minute) + expired := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/console/oidc/callback?code=provider-code&state="+state, nil) + expired.RemoteAddr = expiringLogin.RemoteAddr + expired.AddCookie(expiringCookie) + expiredResponse := httptest.NewRecorder() + expiringHandler.Callback(expiredResponse, expired) + if expiredResponse.Code != http.StatusUnauthorized || expiring.exchangeCalls != 0 { + t.Fatalf("expired callback status/calls = %d/%d", expiredResponse.Code, expiring.exchangeCalls) + } +} + +func TestNewBrowserOIDCHandlerRejectsUnsafeConfiguration(t *testing.T) { + limiter, err := NewAttemptLimiter(AttemptLimiterConfig{Attempts: 2, Window: time.Minute, MaxKeys: 2}) + if err != nil { + t.Fatal(err) + } + for _, config := range []BrowserOIDCHandlerConfig{ + {}, + {Service: &browserOIDCServiceStub{}, Limiter: limiter, ProviderIssuer: "https://issuer.sith.test", ClientID: "client", RedirectURI: "http://hub.sith.test/callback"}, + {Service: &browserOIDCServiceStub{}, Limiter: limiter, ProviderIssuer: "https://issuer.sith.test", ClientID: "client", RedirectURI: "https://hub.sith.test/callback?query=forbidden"}, + {Service: &browserOIDCServiceStub{}, Limiter: limiter, ProviderIssuer: "https://issuer.sith.test", ClientID: "client", RedirectURI: "https://hub.sith.test/callback", MaxTransactions: maximumBrowserOIDCSessions + 1}, + } { + if _, err := NewBrowserOIDCHandler(config); err == nil { + t.Errorf("unsafe browser OIDC configuration accepted: %#v", config) + } + } +} + +func newBrowserOIDCTestHandler(t *testing.T, service BrowserOIDCService, now *time.Time) *BrowserOIDCHandler { + t.Helper() + limiter, err := NewAttemptLimiter(AttemptLimiterConfig{Attempts: 8, Window: time.Minute, MaxKeys: 8, Now: func() time.Time { return *now }}) + if err != nil { + t.Fatal(err) + } + random := bytes.NewReader(append(append(append( + bytes.Repeat([]byte{0x01}, browserOIDCRandomBytes), + bytes.Repeat([]byte{0x02}, browserOIDCRandomBytes)...), + bytes.Repeat([]byte{0x03}, browserOIDCRandomBytes)...), + bytes.Repeat([]byte{0x04}, browserOIDCRandomBytes)...)) + handler, err := NewBrowserOIDCHandler(BrowserOIDCHandlerConfig{ + Service: service, ProviderIssuer: "https://issuer.sith.test", ClientID: "sith-browser", RedirectURI: "https://hub.sith.test/v1/console/oidc/callback", + Limiter: limiter, Now: func() time.Time { return *now }, Random: random, + }) + if err != nil { + t.Fatal(err) + } + return handler +} + +func browserOIDCTestValue(fill byte) string { + return base64.RawURLEncoding.EncodeToString(bytes.Repeat([]byte{fill}, browserOIDCRandomBytes)) +} + +func requiredBrowserCookie(t *testing.T, cookies []*http.Cookie, name string) *http.Cookie { + t.Helper() + for _, cookie := range cookies { + if cookie.Name == name { + return cookie + } + } + t.Fatalf("missing %s cookie in %#v", name, cookies) + return nil +} diff --git a/internal/hubserver/console.go b/internal/hubserver/console.go new file mode 100644 index 0000000..75f00dd --- /dev/null +++ b/internal/hubserver/console.go @@ -0,0 +1,960 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "bytes" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "embed" + "encoding/base64" + "encoding/json" + "fmt" + "html/template" + "io" + "io/fs" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + "unicode" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + consoleCSRFHeader = "X-Sith-CSRF" + consoleCSRFVersion = byte(1) + consoleCSRFTimeBytes = 10 + consoleCSRFRandomBytes = 32 + consoleCSRFKeyBytes = 32 + consoleCSRFDigestBytes = sha256.Size + consoleCSRFPayloadBytes = 1 + consoleCSRFTimeBytes + consoleCSRFRandomBytes + consoleCSRFTokenBytes = consoleCSRFPayloadBytes + consoleCSRFDigestBytes + consoleFleetCSRFDomain = "sith-hub-console-fleet/v1" + consoleCorrelationDomain = "sith-hub-console-correlation/v1" + consoleInventoryDomain = "sith-hub-console-inventory/v1" + consoleCVEIdentifierDomain = "sith-hub-console-cve-identifier/v1" + consoleCorrelationReadLimit = 257 + consoleCorrelationMaxMatches = 256 + consoleCorrelationMaxCoverageScopes = 1_000 + consoleInventoryReadLimit = 257 + consoleInventoryMaxRecords = 256 + consoleCVEReadLimit = 257 + consoleCVEMaxRecords = 256 + consoleMaximumSafeInteger = int64(9_007_199_254_740_991) + defaultConsoleCSRFLifetime = 5 * time.Minute + maximumConsoleCSRFLifetime = 10 * time.Minute +) + +//go:embed console_assets/* +var embeddedConsoleAssets embed.FS + +// ConsoleHandlerConfig supplies only the authenticated read dependencies for the Hub console. +type ConsoleHandlerConfig struct { + Verifier Verifier + AuthObserver AuthObserver + Reader hubfleet.FleetReader + Correlator *hubfleet.Correlator + Inventory *hubfleet.InventorySearcher + CVE *hubfleet.CVESearcher + PEP *pep.Enforcer + ReadObserver hubfleet.FleetReadObserver + CSRFLifetime time.Duration + Now func() time.Time + Random io.Reader +} + +// ConsoleHandler owns the Hub-only cookie/session adapter and its embedded read-only frontend. +type ConsoleHandler struct { + verifier Verifier + authObserver AuthObserver + reader hubfleet.FleetReader + correlator *hubfleet.Correlator + inventory *hubfleet.InventorySearcher + cve *hubfleet.CVESearcher + pep *pep.Enforcer + readObserver hubfleet.FleetReadObserver + csrfLifetime time.Duration + now func() time.Time + random io.Reader + randomMu sync.Mutex + csrfKey [consoleCSRFKeyBytes]byte + page *template.Template + assets fs.FS +} + +type consolePageData struct { + Workspace string + FleetCSRFToken string + CorrelationCSRFToken string + InventoryCSRFToken string + CVECSRFToken string +} + +type consoleFleetResponse struct { + Fleet fleet.FleetResult `json:"fleet"` + Assessment fleet.CoverageAssessment `json:"assessment"` +} + +type consoleHealthMatch struct { + Scope string `json:"scope"` + ResourceKind string `json:"resource_kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + Health string `json:"health"` + ObservedAt time.Time `json:"observed_at"` + Stale bool `json:"stale"` + StaleFor string `json:"stale_for,omitempty"` +} + +type consoleCorrelationResponse struct { + Matches []consoleHealthMatch `json:"matches"` + Coverage fleet.Coverage `json:"coverage"` + Assessment fleet.CoverageAssessment `json:"assessment"` +} + +type consoleInventoryRecord struct { + Scope string `json:"scope"` + ResourceKind string `json:"resource_kind"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name"` + ObservedAt time.Time `json:"observed_at"` + Stale bool `json:"stale"` + StaleFor string `json:"stale_for,omitempty"` + Replicas *int64 `json:"replicas,omitempty"` + AvailableReplicas *int64 `json:"available_replicas,omitempty"` + Ready *bool `json:"ready,omitempty"` + Generation int64 `json:"generation"` +} + +type consoleInventoryResponse struct { + Records []consoleInventoryRecord `json:"records"` + Coverage fleet.Coverage `json:"coverage"` + Assessment fleet.CoverageAssessment `json:"assessment"` +} + +type consoleCVERecord struct { + Scope string `json:"scope"` + ImageDigest string `json:"image_digest"` + Identifier string `json:"identifier"` + Severity string `json:"severity"` + ObservedAt time.Time `json:"observed_at"` + Stale bool `json:"stale"` + StaleFor string `json:"stale_for,omitempty"` +} + +type consoleCVEResponse struct { + Records []consoleCVERecord `json:"records"` + Coverage fleet.Coverage `json:"coverage"` + Assessment fleet.CoverageAssessment `json:"assessment"` +} + +// NewConsoleHandler constructs a separate cookie-authenticated read surface. It does not alter +// bearer API authentication and exposes no refresh, connector, local-operation, or write seam. +func NewConsoleHandler(config ConsoleHandlerConfig) (*ConsoleHandler, error) { + if config.Verifier == nil || config.Reader == nil || config.Correlator == nil || config.Inventory == nil || config.CVE == nil || config.PEP == nil { + return nil, fmt.Errorf("construct Hub console: verifier, reader, correlator, inventory searcher, CVE searcher, and policy enforcer are required") + } + if config.CSRFLifetime == 0 { + config.CSRFLifetime = defaultConsoleCSRFLifetime + } + if config.CSRFLifetime < time.Minute || config.CSRFLifetime > maximumConsoleCSRFLifetime { + return nil, fmt.Errorf("construct Hub console: CSRF lifetime must be between one and ten minutes") + } + if config.Now == nil { + config.Now = time.Now + } + if config.Random == nil { + config.Random = rand.Reader + } + assets, err := fs.Sub(embeddedConsoleAssets, "console_assets") + if err != nil { + return nil, fmt.Errorf("construct Hub console: embedded assets are unavailable") + } + page, err := template.ParseFS(assets, "console.html") + if err != nil { + return nil, fmt.Errorf("construct Hub console: page template is invalid") + } + handler := &ConsoleHandler{ + verifier: config.Verifier, authObserver: config.AuthObserver, reader: config.Reader, correlator: config.Correlator, inventory: config.Inventory, cve: config.CVE, pep: config.PEP, + readObserver: config.ReadObserver, csrfLifetime: config.CSRFLifetime, now: config.Now, random: config.Random, page: page, assets: assets, + } + if _, err := io.ReadFull(handler.random, handler.csrfKey[:]); err != nil { + return nil, fmt.Errorf("construct Hub console: CSRF key generation failed") + } + return handler, nil +} + +// ServePage returns the authenticated shell and purpose-separated session/workspace-bound proofs. +func (handler *ConsoleHandler) ServePage(response http.ResponseWriter, request *http.Request) { + setConsoleSecurityHeaders(response.Header()) + workspaceID, ok := canonicalConsoleRequest(request, "/console") + if !ok { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + scope, rawSession, ok := handler.authorize(response, request, workspaceID) + if !ok { + return + } + fleetCSRFToken, err := handler.newCSRFToken(rawSession, scope.WorkspaceID(), consoleFleetCSRFDomain) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "console_unavailable") + return + } + correlationCSRFToken, err := handler.newCSRFToken(rawSession, scope.WorkspaceID(), consoleCorrelationDomain) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "console_unavailable") + return + } + inventoryCSRFToken, err := handler.newCSRFToken(rawSession, scope.WorkspaceID(), consoleInventoryDomain) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "console_unavailable") + return + } + cveCSRFToken, err := handler.newCSRFToken(rawSession, scope.WorkspaceID(), consoleCVEIdentifierDomain) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "console_unavailable") + return + } + var rendered bytes.Buffer + if err := handler.page.Execute(&rendered, consolePageData{ + Workspace: string(scope.WorkspaceID()), FleetCSRFToken: fleetCSRFToken, CorrelationCSRFToken: correlationCSRFToken, InventoryCSRFToken: inventoryCSRFToken, CVECSRFToken: cveCSRFToken, + }); err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "console_unavailable") + return + } + response.Header().Set("Content-Type", "text/html; charset=utf-8") + response.WriteHeader(http.StatusOK) + _, _ = response.Write(rendered.Bytes()) +} + +// ServeFleet returns one persisted tenant-scoped snapshot through the existing PEP read path. +func (handler *ConsoleHandler) ServeFleet(response http.ResponseWriter, request *http.Request) { + setConsoleSecurityHeaders(response.Header()) + workspaceID, ok := canonicalConsoleRequest(request, "/console/fleet") + if !ok { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + scope, rawSession, ok := handler.authorize(response, request, workspaceID) + if !ok { + return + } + if !sameOriginFetch(request.Header.Values("Sec-Fetch-Site")) || + !handler.validCSRFToken(request.Header.Values(consoleCSRFHeader), rawSession, scope.WorkspaceID(), consoleFleetCSRFDomain) { + writeConsoleError(response, http.StatusForbidden, "forbidden") + return + } + source, err := hubfleet.NewSource(hubfleet.SourceConfig{ + Reader: handler.reader, Scope: scope, PEP: handler.pep, Observer: handler.readObserver, + }) + if err != nil { + writeConsoleError(response, http.StatusForbidden, "forbidden") + return + } + result, err := source.Fleet(request.Context()) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "fleet_unavailable") + return + } + writeConsoleJSON(response, http.StatusOK, consoleFleetResponse{Fleet: result, Assessment: result.Coverage.Assessment()}) +} + +// ServeCorrelation answers one explicit exact-resource health question through the existing +// tenant-scoped PEP correlator, then projects stored facts to a deliberately minimal response. +func (handler *ConsoleHandler) ServeCorrelation(response http.ResponseWriter, request *http.Request) { + setConsoleSecurityHeaders(response.Header()) + workspaceID, correlationRequest, ok := canonicalConsoleCorrelationRequest(request) + if !ok { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + scope, rawSession, ok := handler.authorize(response, request, workspaceID) + if !ok { + return + } + if !sameOriginFetch(request.Header.Values("Sec-Fetch-Site")) || + !handler.validCSRFToken(request.Header.Values(consoleCSRFHeader), rawSession, scope.WorkspaceID(), consoleCorrelationDomain) { + writeConsoleError(response, http.StatusForbidden, "forbidden") + return + } + result, err := handler.correlator.Correlate(request.Context(), scope, correlationRequest) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "correlation_unavailable") + return + } + projected, err := projectConsoleCorrelation(result, scope.WorkspaceID(), correlationRequest) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "correlation_unavailable") + return + } + writeConsoleJSON(response, http.StatusOK, projected) +} + +// ServeInventory answers one explicit bounded inventory selection through the dedicated +// tenant-scoped PEP service, then projects stored facts to a closed browser response. +func (handler *ConsoleHandler) ServeInventory(response http.ResponseWriter, request *http.Request) { + setConsoleSecurityHeaders(response.Header()) + workspaceID, inventoryRequest, ok := canonicalConsoleInventoryRequest(request) + if !ok { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + scope, rawSession, ok := handler.authorize(response, request, workspaceID) + if !ok { + return + } + if !sameOriginFetch(request.Header.Values("Sec-Fetch-Site")) || + !handler.validCSRFToken(request.Header.Values(consoleCSRFHeader), rawSession, scope.WorkspaceID(), consoleInventoryDomain) { + writeConsoleError(response, http.StatusForbidden, "forbidden") + return + } + result, err := handler.inventory.Search(request.Context(), scope, inventoryRequest) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "inventory_unavailable") + return + } + projected, err := projectConsoleInventory(result, scope.WorkspaceID(), inventoryRequest) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "inventory_unavailable") + return + } + writeConsoleJSON(response, http.StatusOK, projected) +} + +// ServeCVEIdentifier answers one explicit canonical CVE question through the existing +// tenant-scoped PEP service, then projects immutable runtime-image evidence to a closed response. +func (handler *ConsoleHandler) ServeCVEIdentifier(response http.ResponseWriter, request *http.Request) { + setConsoleSecurityHeaders(response.Header()) + workspaceID, cveRequest, ok := canonicalConsoleCVEIdentifierRequest(request) + if !ok { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + scope, rawSession, ok := handler.authorize(response, request, workspaceID) + if !ok { + return + } + if !sameOriginFetch(request.Header.Values("Sec-Fetch-Site")) || + !handler.validCSRFToken(request.Header.Values(consoleCSRFHeader), rawSession, scope.WorkspaceID(), consoleCVEIdentifierDomain) { + writeConsoleError(response, http.StatusForbidden, "forbidden") + return + } + result, err := handler.cve.SearchByIdentifier(request.Context(), scope, cveRequest) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "cve_evidence_unavailable") + return + } + projected, err := projectConsoleCVEIdentifier(result, scope.WorkspaceID(), cveRequest) + if err != nil { + writeConsoleError(response, http.StatusServiceUnavailable, "cve_evidence_unavailable") + return + } + writeConsoleJSON(response, http.StatusOK, projected) +} + +// ServeCSS returns the fixed embedded stylesheet and never inspects a session cookie. +func (handler *ConsoleHandler) ServeCSS(response http.ResponseWriter, request *http.Request) { + handler.serveAsset(response, request, "/v1/console/assets/console.css", "console.css", "text/css; charset=utf-8") +} + +// ServeJavaScript returns the fixed embedded renderer and never inspects a session cookie. +func (handler *ConsoleHandler) ServeJavaScript(response http.ResponseWriter, request *http.Request) { + handler.serveAsset(response, request, "/v1/console/assets/console.js", "console.js", "text/javascript; charset=utf-8") +} + +func (handler *ConsoleHandler) serveAsset(response http.ResponseWriter, request *http.Request, expectedPath, name, contentType string) { + setConsoleSecurityHeaders(response.Header()) + if handler == nil || request == nil || request.Method != http.MethodGet || request.URL == nil || request.URL.RawQuery != "" || + request.URL.EscapedPath() != expectedPath { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + payload, err := fs.ReadFile(handler.assets, name) + if err != nil { + writeConsoleError(response, http.StatusNotFound, "not_found") + return + } + response.Header().Set("Content-Type", contentType) + response.WriteHeader(http.StatusOK) + _, _ = response.Write(payload) +} + +func (handler *ConsoleHandler) authorize( + response http.ResponseWriter, + request *http.Request, + workspaceID tenancy.WorkspaceID, +) (tenancy.Scope, string, bool) { + if handler == nil || handler.verifier == nil || request == nil || len(request.Header.Values("Authorization")) != 0 { + refuseAuthentication(handlerAuthObserver(handler), response) + return tenancy.Scope{}, "", false + } + rawSession, ok := exactConsoleSession(request) + if !ok { + refuseAuthentication(handler.authObserver, response) + return tenancy.Scope{}, "", false + } + principal, err := handler.verifier.Verify(request.Context(), rawSession) + if err != nil { + refuseAuthentication(handler.authObserver, response) + return tenancy.Scope{}, "", false + } + ObserveAuth(handler.authObserver, AuthEvent{Outcome: AuthOutcomeAccepted}) + scope, err := principal.Scope(workspaceID) + if err != nil { + writeConsoleError(response, http.StatusForbidden, "forbidden") + return tenancy.Scope{}, "", false + } + return scope, rawSession, true +} + +func handlerAuthObserver(handler *ConsoleHandler) AuthObserver { + if handler == nil { + return nil + } + return handler.authObserver +} + +func exactConsoleSession(request *http.Request) (string, bool) { + if request == nil { + return "", false + } + var session string + count := 0 + for _, cookie := range request.Cookies() { + if cookie.Name != browserOIDCSessionCookie { + continue + } + count++ + session = cookie.Value + } + return session, count == 1 && session != "" && len(session) <= maxBearerTokenBytes && strings.TrimSpace(session) == session +} + +func canonicalConsoleRequest(request *http.Request, suffix string) (tenancy.WorkspaceID, bool) { + if request == nil || request.Method != http.MethodGet || request.URL == nil || request.URL.RawQuery != "" || request.URL.Fragment != "" { + return "", false + } + workspace := request.PathValue("workspace") + workspaceID := tenancy.WorkspaceID(workspace) + if tenancy.ValidateWorkspaceID(workspaceID) != nil { + return "", false + } + wantPath := "/v1/workspaces/" + url.PathEscape(workspace) + suffix + return workspaceID, request.URL.EscapedPath() == wantPath +} + +func canonicalConsoleCorrelationRequest(request *http.Request) (tenancy.WorkspaceID, hubfleet.CorrelationRequest, bool) { + if request == nil || request.Method != http.MethodGet || request.URL == nil || request.URL.Fragment != "" || + request.URL.RawQuery == "" || len(request.URL.RawQuery) > 1_024 { + return "", hubfleet.CorrelationRequest{}, false + } + workspaceID := tenancy.WorkspaceID(request.PathValue("workspace")) + if tenancy.ValidateWorkspaceID(workspaceID) != nil { + return "", hubfleet.CorrelationRequest{}, false + } + wantPath := "/v1/workspaces/" + url.PathEscape(string(workspaceID)) + "/console/correlate" + if request.URL.EscapedPath() != wantPath { + return "", hubfleet.CorrelationRequest{}, false + } + values, err := url.ParseQuery(request.URL.RawQuery) + if err != nil || len(values) < 2 || len(values) > 3 || values.Encode() != request.URL.RawQuery { + return "", hubfleet.CorrelationRequest{}, false + } + for key, entries := range values { + if (key != "kind" && key != "name" && key != "namespace") || len(entries) != 1 { + return "", hubfleet.CorrelationRequest{}, false + } + } + if len(values["kind"]) != 1 || len(values["name"]) != 1 || + (len(values["namespace"]) == 1 && values.Get("namespace") == "") { + return "", hubfleet.CorrelationRequest{}, false + } + correlationRequest := hubfleet.CorrelationRequest{ + ResourceKind: values.Get("kind"), Name: values.Get("name"), Namespace: values.Get("namespace"), + HealthNot: "Healthy", Limit: consoleCorrelationReadLimit, + } + if correlationRequest.Validate() != nil { + return "", hubfleet.CorrelationRequest{}, false + } + return workspaceID, correlationRequest, true +} + +func canonicalConsoleInventoryRequest(request *http.Request) (tenancy.WorkspaceID, hubfleet.InventorySearchRequest, bool) { + if request == nil || request.Method != http.MethodGet || request.URL == nil || request.URL.Fragment != "" || + request.URL.RawQuery == "" || len(request.URL.RawQuery) > 1_024 { + return "", hubfleet.InventorySearchRequest{}, false + } + workspaceID := tenancy.WorkspaceID(request.PathValue("workspace")) + if tenancy.ValidateWorkspaceID(workspaceID) != nil { + return "", hubfleet.InventorySearchRequest{}, false + } + wantPath := "/v1/workspaces/" + url.PathEscape(string(workspaceID)) + "/console/inventory" + if request.URL.EscapedPath() != wantPath { + return "", hubfleet.InventorySearchRequest{}, false + } + values, err := url.ParseQuery(request.URL.RawQuery) + if err != nil || len(values) < 1 || len(values) > 3 || values.Encode() != request.URL.RawQuery { + return "", hubfleet.InventorySearchRequest{}, false + } + for key, entries := range values { + if (key != "kind" && key != "name" && key != "namespace") || len(entries) != 1 { + return "", hubfleet.InventorySearchRequest{}, false + } + } + if len(values["kind"]) != 1 || values.Get("kind") == "" || + (len(values["name"]) == 1 && values.Get("name") == "") || + (len(values["namespace"]) == 1 && values.Get("namespace") == "") { + return "", hubfleet.InventorySearchRequest{}, false + } + inventoryRequest := hubfleet.InventorySearchRequest{ + ResourceKind: values.Get("kind"), Namespace: values.Get("namespace"), Name: values.Get("name"), Limit: consoleInventoryReadLimit, + } + if inventoryRequest.Validate() != nil { + return "", hubfleet.InventorySearchRequest{}, false + } + return workspaceID, inventoryRequest, true +} + +func canonicalConsoleCVEIdentifierRequest(request *http.Request) (tenancy.WorkspaceID, hubfleet.CVEIdentifierSearchRequest, bool) { + if request == nil || request.Method != http.MethodGet || request.URL == nil || request.URL.Fragment != "" || + request.URL.RawQuery == "" || len(request.URL.RawQuery) > 96 { + return "", hubfleet.CVEIdentifierSearchRequest{}, false + } + workspaceID := tenancy.WorkspaceID(request.PathValue("workspace")) + if tenancy.ValidateWorkspaceID(workspaceID) != nil { + return "", hubfleet.CVEIdentifierSearchRequest{}, false + } + wantPath := "/v1/workspaces/" + url.PathEscape(string(workspaceID)) + "/console/cves" + if request.URL.EscapedPath() != wantPath { + return "", hubfleet.CVEIdentifierSearchRequest{}, false + } + values, err := url.ParseQuery(request.URL.RawQuery) + if err != nil || len(values) != 1 || len(values["identifier"]) != 1 || values.Encode() != request.URL.RawQuery { + return "", hubfleet.CVEIdentifierSearchRequest{}, false + } + identifier := values.Get("identifier") + if len(identifier) > 64 { + return "", hubfleet.CVEIdentifierSearchRequest{}, false + } + canonical, err := fleet.NormalizeCVEIdentifier(identifier) + if err != nil || canonical != identifier { + return "", hubfleet.CVEIdentifierSearchRequest{}, false + } + return workspaceID, hubfleet.CVEIdentifierSearchRequest{Identifier: identifier, Limit: consoleCVEReadLimit}, true +} + +func projectConsoleCorrelation( + result fleet.QueryResult, + workspaceID tenancy.WorkspaceID, + request hubfleet.CorrelationRequest, +) (consoleCorrelationResponse, error) { + if workspaceID == "" || len(result.Facts) > consoleCorrelationMaxMatches { + return consoleCorrelationResponse{}, fmt.Errorf("project console correlation: invalid result bounds") + } + coverage, err := projectConsoleCoverage(result.Coverage) + if err != nil { + return consoleCorrelationResponse{}, err + } + matches := make([]consoleHealthMatch, 0, len(result.Facts)) + seen := make(map[string]struct{}, len(result.Facts)) + for _, fact := range result.Facts { + if fact.Workspace != string(workspaceID) || fact.Kind != fleet.FactHealth || fact.Ref.SourceKind != hubfleet.SourceKind || + fact.Source != fact.Ref.Scope || fact.Ref.Kind != request.ResourceKind || fact.Ref.Name != request.Name || + fact.Ref.Namespace != request.Namespace || fact.ObservedAt.IsZero() || + validateConsoleProjectionText(fact.Ref.Scope, 256, false) != nil || + validateConsoleProjectionText(fact.Ref.Kind, 128, false) != nil || + validateConsoleProjectionText(fact.Ref.Name, 256, false) != nil || + validateConsoleProjectionText(fact.Ref.Namespace, 256, true) != nil || + validateConsoleProjectionText(fact.StaleFor, 128, true) != nil || !validConsoleStaleFor(fact.Stale, fact.StaleFor) { + return consoleCorrelationResponse{}, fmt.Errorf("project console correlation: invalid stored fact") + } + health, err := decodeConsoleHealth(fact.Observed) + if err != nil || health == request.HealthNot { + return consoleCorrelationResponse{}, fmt.Errorf("project console correlation: invalid stored health") + } + identity := strings.Join([]string{fact.Ref.Scope, fact.Ref.Kind, fact.Ref.Namespace, fact.Ref.Name}, "\x00") + if _, exists := seen[identity]; exists { + return consoleCorrelationResponse{}, fmt.Errorf("project console correlation: duplicate stored fact") + } + seen[identity] = struct{}{} + matches = append(matches, consoleHealthMatch{ + Scope: fact.Ref.Scope, ResourceKind: fact.Ref.Kind, Namespace: fact.Ref.Namespace, Name: fact.Ref.Name, + Health: health, ObservedAt: fact.ObservedAt.UTC(), Stale: fact.Stale, StaleFor: fact.StaleFor, + }) + } + return consoleCorrelationResponse{Matches: matches, Coverage: coverage, Assessment: coverage.Assessment()}, nil +} + +func projectConsoleInventory( + result fleet.QueryResult, + workspaceID tenancy.WorkspaceID, + request hubfleet.InventorySearchRequest, +) (consoleInventoryResponse, error) { + if workspaceID == "" || len(result.Facts) > consoleInventoryMaxRecords { + return consoleInventoryResponse{}, fmt.Errorf("project console inventory: invalid result bounds") + } + coverage, err := projectConsoleCoverage(result.Coverage) + if err != nil { + return consoleInventoryResponse{}, err + } + records := make([]consoleInventoryRecord, 0, len(result.Facts)) + seen := make(map[string]struct{}, len(result.Facts)) + for _, fact := range result.Facts { + if fact.Workspace != string(workspaceID) || fact.Kind != fleet.FactInventory || fact.Ref.SourceKind != hubfleet.SourceKind || + fact.Source != fact.Ref.Scope || fact.Ref.Kind != request.ResourceKind || + (request.Namespace != "" && fact.Ref.Namespace != request.Namespace) || + (request.Name != "" && fact.Ref.Name != request.Name) || fact.ObservedAt.IsZero() || + len(fact.Ref.Attributes) != 0 || len(fact.Display) != 0 || + fact.Provenance.Adapter != hubfleet.SourceKind || fact.Provenance.ProtocolV != "1.0.0" || + fact.Provenance.NativeID != "" || fact.Provenance.DeepLink != "" || fact.Provenance.Collector != "" || + validateConsoleProjectionText(fact.Ref.Scope, 256, false) != nil || + validateConsoleProjectionText(fact.Ref.Kind, 128, false) != nil || + validateConsoleProjectionText(fact.Ref.Name, 256, false) != nil || + validateConsoleProjectionText(fact.Ref.Namespace, 256, true) != nil || + validateConsoleProjectionText(fact.StaleFor, 128, true) != nil || !validConsoleStaleFor(fact.Stale, fact.StaleFor) { + return consoleInventoryResponse{}, fmt.Errorf("project console inventory: invalid stored fact") + } + identity := strings.Join([]string{fact.Ref.Scope, fact.Ref.Kind, fact.Ref.Namespace, fact.Ref.Name}, "\x00") + if _, exists := seen[identity]; exists { + return consoleInventoryResponse{}, fmt.Errorf("project console inventory: duplicate stored fact") + } + seen[identity] = struct{}{} + record, err := decodeConsoleInventory(fact.Observed, fact.Ref.Kind) + if err != nil { + return consoleInventoryResponse{}, fmt.Errorf("project console inventory: invalid stored observation") + } + record.Scope = fact.Ref.Scope + record.ResourceKind = fact.Ref.Kind + record.Namespace = fact.Ref.Namespace + record.Name = fact.Ref.Name + record.ObservedAt = fact.ObservedAt.UTC() + record.Stale = fact.Stale + record.StaleFor = fact.StaleFor + records = append(records, record) + } + return consoleInventoryResponse{Records: records, Coverage: coverage, Assessment: coverage.Assessment()}, nil +} + +func projectConsoleCVEIdentifier( + result fleet.QueryResult, + workspaceID tenancy.WorkspaceID, + request hubfleet.CVEIdentifierSearchRequest, +) (consoleCVEResponse, error) { + if workspaceID == "" || len(result.Facts) > consoleCVEMaxRecords { + return consoleCVEResponse{}, fmt.Errorf("project console CVE evidence: invalid result bounds") + } + coverage, err := projectConsoleCoverage(result.Coverage) + if err != nil { + return consoleCVEResponse{}, err + } + records := make([]consoleCVERecord, 0, len(result.Facts)) + seen := make(map[string]struct{}, len(result.Facts)) + for _, fact := range result.Facts { + if fact.Workspace != string(workspaceID) || fact.Kind != fleet.FactCVE || fact.Ref.SourceKind != hubfleet.SourceKind || + fact.Source != fact.Ref.Scope || fact.Ref.Kind != "Image" || fact.Ref.Namespace != "" || + len(fact.Ref.Attributes) != 0 || len(fact.Display) != 0 || fact.ObservedAt.IsZero() || + fact.Provenance.Adapter != hubfleet.SourceKind || fact.Provenance.ProtocolV != "1.0.0" || + fact.Provenance.NativeID != "" || fact.Provenance.DeepLink != "" || fact.Provenance.Collector != "" || + validateConsoleProjectionText(fact.Ref.Scope, 256, false) != nil || + validateConsoleProjectionText(fact.StaleFor, 128, true) != nil || !validConsoleStaleFor(fact.Stale, fact.StaleFor) { + return consoleCVEResponse{}, fmt.Errorf("project console CVE evidence: invalid stored fact") + } + if err := rejectConsoleDuplicateJSONMembers(fact.Observed); err != nil { + return consoleCVEResponse{}, fmt.Errorf("project console CVE evidence: invalid stored observation") + } + decoder := json.NewDecoder(bytes.NewReader(fact.Observed)) + decoder.DisallowUnknownFields() + var observation fleet.CVEObservation + if err := decoder.Decode(&observation); err != nil || decoder.Decode(&struct{}{}) != io.EOF || + fleet.ValidateCVEObservation(observation) != nil || observation.Image != fact.Ref.Name { + return consoleCVEResponse{}, fmt.Errorf("project console CVE evidence: invalid stored observation") + } + found := false + for _, identifier := range observation.IDs { + if identifier == request.Identifier { + found = true + break + } + } + if !found { + return consoleCVEResponse{}, fmt.Errorf("project console CVE evidence: stored observation does not match selector") + } + identity := fact.Ref.Scope + "\x00" + observation.Image + "\x00" + request.Identifier + if _, exists := seen[identity]; exists { + return consoleCVEResponse{}, fmt.Errorf("project console CVE evidence: duplicate stored fact") + } + seen[identity] = struct{}{} + records = append(records, consoleCVERecord{ + Scope: fact.Ref.Scope, ImageDigest: observation.Image, Identifier: request.Identifier, Severity: observation.Severity, + ObservedAt: fact.ObservedAt.UTC(), Stale: fact.Stale, StaleFor: fact.StaleFor, + }) + } + return consoleCVEResponse{Records: records, Coverage: coverage, Assessment: coverage.Assessment()}, nil +} + +func decodeConsoleInventory(payload json.RawMessage, kind string) (consoleInventoryRecord, error) { + if err := rejectConsoleDuplicateJSONMembers(payload); err != nil { + return consoleInventoryRecord{}, err + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + var observation struct { + Resource string `json:"resource"` + Replicas *int64 `json:"replicas"` + AvailableReplicas *int64 `json:"available_replicas"` + Ready *int64 `json:"ready"` + Generation *int64 `json:"generation"` + ImageDigests json.RawMessage `json:"image_digests"` + } + if err := decoder.Decode(&observation); err != nil { + return consoleInventoryRecord{}, err + } + if err := decoder.Decode(&struct{}{}); err != io.EOF || observation.Resource != kind || observation.Generation == nil || + *observation.Generation < 0 || *observation.Generation > consoleMaximumSafeInteger { + return consoleInventoryRecord{}, fmt.Errorf("inventory observation has an invalid envelope") + } + record := consoleInventoryRecord{Generation: *observation.Generation} + switch kind { + case "Deployment", "Rollout": + if observation.Replicas == nil || observation.AvailableReplicas == nil || observation.Ready != nil || len(observation.ImageDigests) != 0 || + !validConsoleCount(*observation.Replicas) || !validConsoleCount(*observation.AvailableReplicas) { + return consoleInventoryRecord{}, fmt.Errorf("workload inventory observation is invalid") + } + replicas, available := *observation.Replicas, *observation.AvailableReplicas + record.Replicas = &replicas + record.AvailableReplicas = &available + case "Pod": + if observation.Replicas != nil || observation.AvailableReplicas != nil || observation.Ready == nil || + (*observation.Ready != 0 && *observation.Ready != 1) { + return consoleInventoryRecord{}, fmt.Errorf("pod inventory observation is invalid") + } + ready := *observation.Ready == 1 + record.Ready = &ready + if len(observation.ImageDigests) != 0 { + var digests []string + if err := json.Unmarshal(observation.ImageDigests, &digests); err != nil || len(digests) == 0 || len(digests) > 64 { + return consoleInventoryRecord{}, fmt.Errorf("pod inventory image digests are invalid") + } + for index, digest := range digests { + if fleet.ValidateImageDigest(digest) != nil || (index > 0 && digests[index-1] >= digest) { + return consoleInventoryRecord{}, fmt.Errorf("pod inventory image digests are invalid") + } + } + } + default: + return consoleInventoryRecord{}, fmt.Errorf("inventory resource kind is unsupported") + } + return record, nil +} + +func rejectConsoleDuplicateJSONMembers(payload json.RawMessage) error { + decoder := json.NewDecoder(bytes.NewReader(payload)) + first, err := decoder.Token() + if err != nil { + return err + } + if delimiter, ok := first.(json.Delim); !ok || delimiter != '{' { + return fmt.Errorf("inventory observation must be an object") + } + seen := make(map[string]struct{}) + for decoder.More() { + member, err := decoder.Token() + if err != nil { + return err + } + name, ok := member.(string) + if !ok { + return fmt.Errorf("inventory observation member is invalid") + } + if _, exists := seen[name]; exists { + return fmt.Errorf("inventory observation contains a duplicate member") + } + seen[name] = struct{}{} + var value json.RawMessage + if err := decoder.Decode(&value); err != nil { + return err + } + } + if _, err := decoder.Token(); err != nil { + return err + } + if _, err := decoder.Token(); err != io.EOF { + return fmt.Errorf("inventory observation contains trailing data") + } + return nil +} + +func validConsoleCount(value int64) bool { + return value >= 0 && value <= consoleMaximumSafeInteger +} + +func validConsoleStaleFor(stale bool, value string) bool { + if !stale { + return value == "" + } + if value == "collection failed" { + return true + } + duration, err := time.ParseDuration(value) + return err == nil && duration >= time.Second && duration.String() == value +} + +func projectConsoleCoverage(coverage fleet.Coverage) (fleet.Coverage, error) { + if coverage.Requested < 0 || coverage.Reachable < 0 { + return fleet.Coverage{}, fmt.Errorf("project console correlation: invalid coverage counts") + } + count := len(coverage.Unreachable) + len(coverage.Stale) + len(coverage.Truncated) + if count > consoleCorrelationMaxCoverageScopes { + return fleet.Coverage{}, fmt.Errorf("project console correlation: coverage exceeds response bound") + } + unreachable, err := projectConsoleCoverageScopes(coverage.Unreachable) + if err != nil { + return fleet.Coverage{}, err + } + stale, err := projectConsoleCoverageScopes(coverage.Stale) + if err != nil { + return fleet.Coverage{}, err + } + truncated, err := projectConsoleCoverageScopes(coverage.Truncated) + if err != nil { + return fleet.Coverage{}, err + } + return fleet.Coverage{ + Requested: coverage.Requested, Reachable: coverage.Reachable, + Unreachable: unreachable, Stale: stale, Truncated: truncated, + }, nil +} + +func projectConsoleCoverageScopes(scopes []string) ([]string, error) { + projected := make([]string, 0, len(scopes)) + for _, scope := range scopes { + if err := validateConsoleProjectionText(scope, 256, false); err != nil { + return nil, fmt.Errorf("project console correlation: invalid coverage scope") + } + projected = append(projected, scope) + } + return projected, nil +} + +func decodeConsoleHealth(payload json.RawMessage) (string, error) { + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + var observation struct { + Status string `json:"status"` + } + if err := decoder.Decode(&observation); err != nil { + return "", err + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return "", fmt.Errorf("health observation contains trailing data") + } + switch observation.Status { + case "Healthy", "Degraded", "Progressing", "Unknown": + return observation.Status, nil + default: + return "", fmt.Errorf("health observation contains an unsupported status") + } +} + +func validateConsoleProjectionText(value string, maximum int, allowEmpty bool) error { + if value == "" && allowEmpty { + return nil + } + if value == "" || len(value) > maximum || strings.TrimSpace(value) != value { + return fmt.Errorf("invalid projected text") + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("invalid projected text") + } + } + return nil +} + +func sameOriginFetch(values []string) bool { + return len(values) == 0 || (len(values) == 1 && values[0] == "same-origin") +} + +func (handler *ConsoleHandler) newCSRFToken(rawSession string, workspaceID tenancy.WorkspaceID, domain string) (string, error) { + payload := make([]byte, consoleCSRFPayloadBytes) + payload[0] = consoleCSRFVersion + expiresAt := strconv.FormatInt(handler.now().UTC().Add(handler.csrfLifetime).Unix(), 10) + if len(expiresAt) != consoleCSRFTimeBytes { + return "", fmt.Errorf("generate console CSRF token") + } + copy(payload[1:1+consoleCSRFTimeBytes], expiresAt) + handler.randomMu.Lock() + _, err := io.ReadFull(handler.random, payload[1+consoleCSRFTimeBytes:]) + handler.randomMu.Unlock() + if err != nil { + return "", fmt.Errorf("generate console CSRF token") + } + mac := handler.csrfMAC(payload, rawSession, workspaceID, domain) + return base64.RawURLEncoding.EncodeToString(append(payload, mac...)), nil +} + +func (handler *ConsoleHandler) validCSRFToken(values []string, rawSession string, workspaceID tenancy.WorkspaceID, domain string) bool { + if handler == nil || len(values) != 1 || values[0] == "" || strings.TrimSpace(values[0]) != values[0] { + return false + } + decoded, err := base64.RawURLEncoding.DecodeString(values[0]) + if err != nil || len(decoded) != consoleCSRFTokenBytes || decoded[0] != consoleCSRFVersion { + return false + } + expiresUnix, err := strconv.ParseInt(string(decoded[1:1+consoleCSRFTimeBytes]), 10, 64) + if err != nil { + return false + } + now := handler.now().UTC() + expiresAt := time.Unix(expiresUnix, 0).UTC() + if !expiresAt.After(now) || expiresAt.After(now.Add(handler.csrfLifetime)) { + return false + } + payload := decoded[:consoleCSRFPayloadBytes] + expected := handler.csrfMAC(payload, rawSession, workspaceID, domain) + return subtle.ConstantTimeCompare(decoded[consoleCSRFPayloadBytes:], expected) == 1 +} + +func (handler *ConsoleHandler) csrfMAC(payload []byte, rawSession string, workspaceID tenancy.WorkspaceID, domain string) []byte { + sessionDigest := sha256.Sum256([]byte(rawSession)) + mac := hmac.New(sha256.New, handler.csrfKey[:]) + _, _ = mac.Write([]byte(domain)) + _, _ = mac.Write([]byte{0}) + _, _ = mac.Write(payload) + _, _ = mac.Write([]byte{0}) + _, _ = mac.Write(sessionDigest[:]) + _, _ = mac.Write([]byte{0}) + _, _ = mac.Write([]byte(workspaceID)) + return mac.Sum(nil) +} + +func setConsoleSecurityHeaders(header http.Header) { + setNoStore(header) + header.Set("Content-Security-Policy", "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self'; font-src 'none'; object-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; media-src 'none'; worker-src 'none'; manifest-src 'none'") + header.Set("Permissions-Policy", "camera=(), geolocation=(), microphone=(), payment=(), usb=()") + header.Set("Referrer-Policy", "no-referrer") + header.Set("X-Content-Type-Options", "nosniff") + header.Set("X-Frame-Options", "DENY") +} + +func writeConsoleJSON(response http.ResponseWriter, status int, value any) { + response.Header().Set("Content-Type", "application/json; charset=utf-8") + response.WriteHeader(status) + _ = json.NewEncoder(response).Encode(value) +} + +func writeConsoleError(response http.ResponseWriter, status int, code string) { + writeConsoleJSON(response, status, struct { + Error string `json:"error"` + }{Error: code}) +} diff --git a/internal/hubserver/console_assets/console.css b/internal/hubserver/console_assets/console.css new file mode 100644 index 0000000..28fc93a --- /dev/null +++ b/internal/hubserver/console_assets/console.css @@ -0,0 +1,980 @@ +:root { + color: #111823; + background: #e9f0f4; + font-family: "Avenir Next", "Segoe UI", sans-serif; + font-synthesis: none; + --basalt: #111823; + --frost: #e9f0f4; + --paper: #f7fafb; + --oxide: #2d6a73; + --amber: #d99a2b; + --rust: #c4583c; + --slate: #71808d; + --hairline: rgba(17, 24, 35, 0.18); +} + +* { + box-sizing: border-box; +} + +body { + min-width: 18rem; + margin: 0; + background: + linear-gradient(90deg, rgba(45, 106, 115, 0.06) 1px, transparent 1px) 0 0 / 4.5rem 4.5rem, + var(--frost); +} + +button, +a { + font: inherit; +} + +button:focus-visible, +a:focus-visible, +input:focus-visible, +select:focus-visible { + outline: 3px solid var(--amber); + outline-offset: 3px; +} + +.skip-link { + position: fixed; + z-index: 10; + top: 0.75rem; + left: 0.75rem; + padding: 0.65rem 0.85rem; + color: var(--paper); + background: var(--basalt); + transform: translateY(-180%); +} + +.skip-link:focus { + transform: translateY(0); +} + +.console-shell { + width: min(76rem, calc(100% - 2rem)); + margin: 0 auto; + padding: 2rem 0 5rem; +} + +.masthead { + display: grid; + grid-template-columns: minmax(0, 1.6fr) minmax(15rem, 0.7fr); + gap: 3rem; + align-items: end; + min-height: 20rem; + padding: 2.5rem 0 2rem; + border-bottom: 1px solid var(--basalt); +} + +.eyebrow, +.snapshot-time, +.cluster-source, +.cluster-observed, +.resource-address, +.workspace-stamp dt, +.workspace-stamp dd, +.rail-legend, +.cluster-sequence { + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.eyebrow { + margin: 0 0 0.8rem; + color: var(--oxide); + font-weight: 700; +} + +h1, +h2, +p { + margin-top: 0; +} + +h1 { + max-width: 13ch; + margin-bottom: 0; + font-size: clamp(3rem, 7vw, 6.8rem); + font-weight: 500; + letter-spacing: -0.065em; + line-height: 0.88; +} + +h2 { + margin-bottom: 0; + font-size: clamp(1.55rem, 3vw, 2.45rem); + font-weight: 550; + letter-spacing: -0.035em; +} + +.workspace-stamp { + margin: 0; + border-top: 1px solid var(--basalt); +} + +.workspace-stamp div { + display: grid; + grid-template-columns: 6rem 1fr; + gap: 1rem; + padding: 0.8rem 0; + border-bottom: 1px solid var(--hairline); +} + +.workspace-stamp dt { + color: var(--slate); +} + +.workspace-stamp dd { + margin: 0; + overflow-wrap: anywhere; + font-weight: 700; +} + +.coverage-panel, +.inventory-panel, +.cve-panel, +.correlation-panel, +.ledger { + padding: 3rem 0; + border-bottom: 1px solid var(--basalt); +} + +.section-heading { + display: flex; + gap: 2rem; + align-items: end; + justify-content: space-between; +} + +.reload-button { + min-height: 2.75rem; + padding: 0.65rem 1rem; + border: 1px solid var(--basalt); + border-radius: 0; + color: var(--paper); + background: var(--basalt); + cursor: pointer; +} + +.reload-button:hover:not(:disabled) { + color: var(--basalt); + background: var(--amber); +} + +.reload-button:disabled { + cursor: wait; + opacity: 0.6; +} + +.correlation-panel { + background: linear-gradient(115deg, rgba(247, 250, 251, 0.72), rgba(217, 154, 43, 0.08)); + padding-inline: clamp(1rem, 3vw, 2.25rem); + border-inline: 1px solid var(--hairline); +} + +.cve-panel { + margin-top: 3rem; + padding-inline: clamp(1rem, 3vw, 2.25rem); + border: 1px solid var(--basalt); + background: + linear-gradient(120deg, rgba(196, 88, 60, 0.11), transparent 52%), + var(--paper); +} + +.cve-panel .eyebrow { + color: #8f321f; +} + +.inventory-panel { + margin-top: 3rem; + padding-inline: clamp(1rem, 3vw, 2.25rem); + color: var(--paper); + background: + linear-gradient(130deg, rgba(45, 106, 115, 0.3), transparent 48%), + var(--basalt); + border-bottom: 0; +} + +.inventory-panel .eyebrow { + color: #7fc1c8; +} + +.inventory-panel .query-contract, +.inventory-panel .snapshot-time { + color: #aab6bf; +} + +.inventory-intro { + max-width: 55rem; + margin: 1.5rem 0 0; + color: #cfdae0; + font-size: 1.05rem; + line-height: 1.55; +} + +.inventory-form { + display: grid; + grid-template-columns: minmax(10rem, 0.8fr) minmax(12rem, 1fr) minmax(12rem, 1fr) auto; + gap: 0.8rem; + align-items: end; + margin-top: 1.8rem; +} + +.inventory-form label { + display: grid; + gap: 0.45rem; +} + +.inventory-form label span { + color: #c9d5dc; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.inventory-form small { + color: #96a6b1; + font: inherit; + font-weight: 400; +} + +.inventory-form input, +.inventory-form select { + width: 100%; + min-height: 2.75rem; + padding: 0.65rem 0.75rem; + border: 1px solid #7b8993; + border-radius: 0; + color: var(--paper); + background: #192431; + font: inherit; +} + +.inventory-form input::placeholder { + color: #8e9ba5; +} + +.inventory-button { + min-height: 2.75rem; + padding: 0.65rem 1rem; + border: 1px solid var(--amber); + border-radius: 0; + color: var(--basalt); + background: var(--amber); + cursor: pointer; + font-weight: 700; +} + +.inventory-button:hover:not(:disabled) { + color: var(--paper); + background: transparent; +} + +.inventory-button:disabled { + cursor: wait; + opacity: 0.6; +} + +.inventory-state { + border-left-color: #7fc1c8; + background: rgba(247, 250, 251, 0.08); +} + +.inventory-answer { + margin-top: 1.5rem; +} + +.inventory-answer[hidden] { + display: none; +} + +.inventory-panel .inventory-gaps { + color: var(--basalt); + background: var(--paper); +} + +.inventory-list { + margin: 1.5rem 0 0; + padding: 0; + border-top: 1px solid #72808b; + list-style: none; +} + +.inventory-row { + display: grid; + grid-template-columns: 3rem minmax(13rem, 1.35fr) minmax(11rem, 0.9fr) minmax(12rem, 1fr) minmax(8rem, 0.7fr); + gap: 1.25rem; + align-items: center; + min-height: 6.25rem; + padding: 1rem 0; + border-bottom: 1px solid rgba(247, 250, 251, 0.2); +} + +.inventory-row .cluster-name { + color: var(--paper); +} + +.inventory-row .cluster-source, +.inventory-row .cluster-observed, +.inventory-row .resource-address { + color: #b8c6ce; +} + +.inventory-metrics { + display: grid; + gap: 0.25rem; + margin: 0; +} + +.inventory-metrics div { + display: flex; + gap: 0.7rem; + justify-content: space-between; +} + +.inventory-metrics dt, +.inventory-metrics dd, +.inventory-freshness { + margin: 0; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + letter-spacing: 0.05em; + text-transform: uppercase; +} + +.inventory-metrics dt { + color: #92a2ad; +} + +.inventory-metrics dd { + color: var(--paper); + font-weight: 700; +} + +.inventory-freshness { + padding: 0.45rem 0.65rem; + border: 1px solid #7fc1c8; + color: #a9dde1; + text-align: center; + overflow-wrap: anywhere; +} + +.inventory-freshness.stale { + border-color: var(--amber); + color: #f0bf67; +} + +.query-contract, +.correlation-form label span, +.cve-form label span, +.match-freshness { + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.query-contract { + margin: 0 0 0.2rem; + color: var(--slate); +} + +.correlation-intro { + max-width: 52rem; + margin: 1.5rem 0 0; + color: #384652; + font-size: 1.05rem; + line-height: 1.55; +} + +.cve-intro { + max-width: 55rem; + margin: 1.5rem 0 0; + color: #384652; + font-size: 1.05rem; + line-height: 1.55; +} + +.cve-form { + display: grid; + grid-template-columns: minmax(15rem, 1fr) minmax(14rem, 1fr) auto; + gap: 0.8rem; + align-items: end; + margin-top: 1.8rem; +} + +.cve-form label { + display: grid; + gap: 0.45rem; +} + +.cve-form label span { + color: #46535f; + font-weight: 700; +} + +.cve-form input { + width: 100%; + min-height: 2.75rem; + padding: 0.65rem 0.75rem; + border: 1px solid var(--basalt); + border-radius: 0; + color: var(--basalt); + background: #fff; + font: inherit; +} + +.input-help { + margin: 0 0 0.7rem; + color: var(--slate); + font-size: 0.85rem; +} + +.cve-button { + min-height: 2.75rem; + padding: 0.65rem 1rem; + border: 1px solid var(--basalt); + border-radius: 0; + color: var(--paper); + background: #8f321f; + cursor: pointer; + font-weight: 700; +} + +.cve-button:hover:not(:disabled) { + color: var(--basalt); + background: var(--amber); +} + +.cve-button:disabled { + cursor: wait; + opacity: 0.6; +} + +.cve-answer { + margin-top: 1.5rem; +} + +.cve-answer[hidden] { + display: none; +} + +.cve-list { + margin: 1.5rem 0 0; + padding: 0; + border-top: 1px solid var(--basalt); + list-style: none; +} + +.cve-row { + display: grid; + grid-template-columns: 3rem minmax(18rem, 1.6fr) minmax(11rem, 1fr) auto minmax(9rem, 0.7fr); + gap: 1.25rem; + align-items: center; + min-height: 6.5rem; + padding: 1rem 0; + border-bottom: 1px solid var(--hairline); +} + +.digest-address { + margin: 0.2rem 0 0; + color: #46535f; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + overflow-wrap: anywhere; +} + +.severity-badge { + min-width: 6.5rem; + padding: 0.45rem 0.65rem; + border: 1px solid currentColor; + color: #46535f; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.06em; + text-align: center; + text-transform: uppercase; +} + +.severity-badge.critical, +.severity-badge.high { + color: #8f321f; +} + +.severity-badge.medium { + color: #80560e; +} + +.severity-badge.low { + color: var(--oxide); +} + +.correlation-form { + display: grid; + grid-template-columns: minmax(9rem, 0.8fr) minmax(12rem, 1.2fr) minmax(12rem, 1fr) auto; + gap: 0.8rem; + align-items: end; + margin-top: 1.8rem; +} + +.correlation-form label { + display: grid; + gap: 0.45rem; +} + +.correlation-form label span { + color: #46535f; + font-weight: 700; +} + +.correlation-form small { + color: var(--slate); + font: inherit; + font-weight: 400; +} + +.correlation-form input { + width: 100%; + min-height: 2.75rem; + padding: 0.65rem 0.75rem; + border: 1px solid var(--basalt); + border-radius: 0; + color: var(--basalt); + background: var(--paper); + font: inherit; +} + +.correlation-form input::placeholder { + color: #7a8791; +} + +.correlation-button { + min-height: 2.75rem; + padding: 0.65rem 1rem; + border: 1px solid var(--basalt); + border-radius: 0; + color: var(--basalt); + background: var(--amber); + cursor: pointer; + font-weight: 700; +} + +.correlation-button:hover:not(:disabled) { + color: var(--paper); + background: var(--basalt); +} + +.correlation-button:disabled { + cursor: wait; + opacity: 0.6; +} + +.correlation-state { + margin-bottom: 0; +} + +.visually-hidden { + position: absolute !important; + width: 1px !important; + height: 1px !important; + padding: 0 !important; + border: 0 !important; + margin: -1px !important; + overflow: hidden !important; + clip: rect(0 0 0 0) !important; + white-space: nowrap !important; +} + +.correlation-answer { + margin-top: 1.5rem; +} + +.correlation-answer[hidden] { + display: none; +} + +.answer-heading { + display: flex; + gap: 2rem; + align-items: baseline; + justify-content: space-between; +} + +.correlation-summary { + max-width: 52rem; + margin-bottom: 0; + font-size: clamp(1.1rem, 2vw, 1.45rem); + line-height: 1.45; +} + +.correlation-list { + margin: 1.5rem 0 0; + padding: 0; + border-top: 1px solid var(--basalt); + list-style: none; +} + +.correlation-row { + display: grid; + grid-template-columns: 3rem minmax(13rem, 1.4fr) minmax(11rem, 1fr) auto minmax(9rem, 0.7fr); + gap: 1.25rem; + align-items: center; + min-height: 6rem; + padding: 1rem 0; + border-bottom: 1px solid var(--hairline); +} + +.health-badge { + min-width: 7.5rem; + padding: 0.45rem 0.65rem; + border: 1px solid currentColor; + color: #8f321f; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.06em; + text-align: center; + text-transform: uppercase; +} + +.health-badge.progressing { + color: #80560e; +} + +.health-badge.unknown { + color: #46535f; +} + +.match-freshness { + margin: 0; + color: #46535f; + line-height: 1.4; +} + +.coverage-summary { + max-width: 48rem; + margin: 2.4rem 0 1rem; + font-size: clamp(1.05rem, 2vw, 1.35rem); + line-height: 1.5; +} + +.coverage-rail { + display: flex; + gap: 0.22rem; + min-height: 4.5rem; + padding: 0.35rem; + border: 1px solid var(--basalt); + background: var(--paper); + overflow: hidden; +} + +.rail-segment { + flex: 1 1 0; + min-width: 0.22rem; + background: var(--oxide); + transform-origin: bottom; + animation: rail-arrive 320ms ease-out both; +} + +.rail-segment.stale { + background: repeating-linear-gradient(135deg, var(--amber) 0 0.55rem, #f3c66f 0.55rem 1.1rem); +} + +.rail-segment.truncated { + background: repeating-linear-gradient(90deg, var(--oxide) 0 0.35rem, var(--paper) 0.35rem 0.55rem); +} + +.rail-segment.unreachable { + background: var(--rust); +} + +.rail-segment.unknown { + background: var(--slate); +} + +.rail-legend { + display: flex; + flex-wrap: wrap; + gap: 0.75rem 1.5rem; + margin: 0.85rem 0 0; + padding: 0; + color: #46535f; + list-style: none; +} + +.coverage-details { + display: grid; + gap: 0.35rem; + margin: 1.25rem 0 0; + padding: 1rem 1rem 1rem 2.25rem; + border-left: 0.35rem solid var(--amber); + background: rgba(247, 250, 251, 0.78); + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.78rem; + line-height: 1.45; +} + +.rail-legend li { + display: inline-flex; + gap: 0.45rem; + align-items: center; +} + +.legend-mark { + width: 0.85rem; + height: 0.85rem; + border: 1px solid rgba(17, 24, 35, 0.3); + background: var(--oxide); +} + +.legend-mark.stale { background: var(--amber); } +.legend-mark.truncated { background: linear-gradient(90deg, var(--oxide) 55%, var(--paper) 55%); } +.legend-mark.unreachable { background: var(--rust); } +.legend-mark.unknown { background: var(--slate); } + +.ledger-heading { + padding-bottom: 1rem; +} + +.snapshot-time { + margin-bottom: 0.15rem; + color: var(--slate); +} + +.console-state { + margin-top: 1.5rem; + padding: 1.25rem; + border-left: 0.35rem solid var(--oxide); + background: rgba(247, 250, 251, 0.78); +} + +.console-state p { + margin: 0; + line-height: 1.5; +} + +.console-state.error { + border-left-color: var(--rust); +} + +.console-state[hidden] { + display: none; +} + +.cluster-list { + margin: 1.5rem 0 0; + padding: 0; + border-top: 1px solid var(--basalt); + list-style: none; +} + +.cluster-row { + display: grid; + grid-template-columns: 3rem minmax(10rem, 1.2fr) minmax(8rem, 0.8fr) minmax(11rem, 1fr) auto; + gap: 1.5rem; + align-items: center; + min-height: 5.5rem; + padding: 1rem 0; + border-bottom: 1px solid var(--hairline); +} + +.cluster-sequence { + color: var(--slate); +} + +.cluster-name { + margin: 0; + overflow-wrap: anywhere; + font-size: 1.2rem; + font-weight: 650; +} + +.cluster-source, +.cluster-observed, +.resource-address { + margin: 0; + color: #46535f; + overflow-wrap: anywhere; +} + +.resource-address { + margin-top: 0.18rem; +} + +.state-badge { + min-width: 7.5rem; + padding: 0.45rem 0.65rem; + border: 1px solid currentColor; + color: var(--oxide); + background: transparent; + font-family: "SFMono-Regular", Consolas, monospace; + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.06em; + text-align: center; + text-transform: uppercase; +} + +.state-badge.stale, +.state-badge.truncated { + color: #80560e; +} + +.state-badge.unreachable { + color: #8f321f; +} + +.state-badge.unknown { + color: #46535f; +} + +.session-link { + color: var(--basalt); + font-weight: 700; +} + +@keyframes rail-arrive { + from { transform: scaleY(0.2); opacity: 0; } + to { transform: scaleY(1); opacity: 1; } +} + +@media (max-width: 60rem) { + .console-shell { + width: min(100% - 1.25rem, 76rem); + padding-top: 0.5rem; + } + + .masthead { + grid-template-columns: 1fr; + gap: 2rem; + min-height: auto; + padding-top: 3rem; + } + + h1 { + font-size: clamp(2.8rem, 15vw, 4.4rem); + } + + .section-heading { + display: grid; + align-items: start; + } + + .reload-button { + width: 100%; + } + + .correlation-form { + grid-template-columns: 1fr; + } + + .cve-form { + grid-template-columns: 1fr; + } + + .inventory-form { + grid-template-columns: 1fr; + } + + .correlation-button { + width: 100%; + } + + .cve-button { + width: 100%; + } + + .inventory-button { + width: 100%; + } + + .answer-heading { + display: grid; + gap: 0.75rem; + } + + .correlation-row { + grid-template-columns: 2rem 1fr auto; + gap: 0.7rem 1rem; + align-items: start; + } + + .cve-row { + grid-template-columns: 2rem minmax(0, 1fr) auto; + gap: 0.7rem 1rem; + align-items: start; + } + + .cve-row .cluster-observed, + .cve-row .match-freshness { + grid-column: 2 / -1; + } + + .cve-row .severity-badge { + grid-column: 3; + grid-row: 1; + min-width: auto; + } + + .inventory-row { + grid-template-columns: 2rem minmax(0, 1fr); + gap: 0.7rem 1rem; + align-items: start; + } + + .inventory-row .cluster-observed, + .inventory-row .inventory-metrics, + .inventory-row .inventory-freshness { + grid-column: 2; + } + + .correlation-row .cluster-observed, + .correlation-row .match-freshness { + grid-column: 2 / -1; + } + + .correlation-row .health-badge { + grid-column: 3; + grid-row: 1; + min-width: auto; + } + + .coverage-rail { + min-height: 3.5rem; + } + + .cluster-row { + grid-template-columns: 2rem 1fr auto; + gap: 0.7rem 1rem; + align-items: start; + } + + .cluster-source, + .cluster-observed { + grid-column: 2 / -1; + } + + .state-badge { + grid-column: 3; + grid-row: 1; + min-width: auto; + } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + scroll-behavior: auto !important; + animation-duration: 0.01ms !important; + animation-iteration-count: 1 !important; + } +} diff --git a/internal/hubserver/console_assets/console.html b/internal/hubserver/console_assets/console.html new file mode 100644 index 0000000..b6dd3db --- /dev/null +++ b/internal/hubserver/console_assets/console.html @@ -0,0 +1,179 @@ + + + + + + + + + + + + Sith fleet trust console + + + + + +
+
+
+

Sith / fleet evidence

+

Trust the coverage,
then trust the view.

+
+
+
+
Workspace
+
{{.Workspace}}
+
+
+
Mode
+
Hub · read only
+
+
+
+ +
+
+
+

Evidence boundary

+

Coverage rail

+
+ +
+

Reading the signed workspace snapshot…

+ +
    +
  • Current
    • +
  • Stale
    • +
  • Partial
    • +
  • Unreachable
    • +
  • Unaccounted
    • +
+
    +
    + +
    +
    +
    +

    Normalized inventory ledger

    +

    What has each spoke observed?

    +
    +

    Maximum 256 records · no raw payload

    +
    +

    Choose one normalized resource kind and optionally narrow by exact namespace or name. Counts are typed projections; image digests and source metadata stay behind the evidence boundary.

    +
    + + + + +
    +
    +

    No inventory selection has been read. No completeness claim is active.

    +
    + +
    + +
    +
    +
    +

    Runtime vulnerability evidence

    +

    Where was this CVE observed?

    +
    +

    Maximum 256 records · immutable digests only

    +
    +

    Enter one canonical CVE identifier. Sith reports only persisted observations tied to immutable runtime image digests and keeps named coverage gaps beside the evidence.

    +
    + +

    Use uppercase canonical form, for example CVE-2026-0001.

    + +
    +
    +

    No CVE evidence has been read. No absence claim is active.

    +
    + +
    + +
    +
    +
    +

    Exact health correlation

    +

    Where is this resource not Healthy?

    +
    +

    One persisted read · no refresh

    +
    +

    Name one exact non-Secret resource. Sith reports only observed non-Healthy matches and keeps coverage gaps beside the answer.

    +
    + + + + +
    +
    +

    No correlation has been run. No health claim is active.

    +
    + +
    + +
    +
    +
    +

    Signed workspace only

    +

    Cluster ledger

    +
    +

    No snapshot read yet

    +
    +
    +

    Establishing the evidence boundary.

    +
    +
      +
      +
      + + diff --git a/internal/hubserver/console_assets/console.js b/internal/hubserver/console_assets/console.js new file mode 100644 index 0000000..d0c96e8 --- /dev/null +++ b/internal/hubserver/console_assets/console.js @@ -0,0 +1,771 @@ +(() => { + "use strict"; + + const workspace = document.querySelector('meta[name="sith-workspace"]')?.content || ""; + const csrfToken = document.querySelector('meta[name="sith-csrf"]')?.content || ""; + const correlationCSRFToken = document.querySelector('meta[name="sith-correlation-csrf"]')?.content || ""; + const inventoryCSRFToken = document.querySelector('meta[name="sith-inventory-csrf"]')?.content || ""; + const cveCSRFToken = document.querySelector('meta[name="sith-cve-csrf"]')?.content || ""; + const rail = document.getElementById("coverage-rail"); + const summary = document.getElementById("coverage-summary"); + const details = document.getElementById("coverage-details"); + const state = document.getElementById("console-state"); + const list = document.getElementById("cluster-list"); + const snapshotTime = document.getElementById("snapshot-time"); + const reload = document.getElementById("reload-fleet"); + const correlationForm = document.getElementById("correlation-form"); + const correlationKind = document.getElementById("correlation-kind"); + const correlationName = document.getElementById("correlation-name"); + const correlationNamespace = document.getElementById("correlation-namespace"); + const correlationButton = document.getElementById("run-correlation"); + const correlationState = document.getElementById("correlation-state"); + const correlationAnswer = document.getElementById("correlation-answer"); + const correlationSummary = document.getElementById("correlation-summary"); + const correlationTime = document.getElementById("correlation-time"); + const correlationGaps = document.getElementById("correlation-gaps"); + const correlationList = document.getElementById("correlation-list"); + const inventoryForm = document.getElementById("inventory-form"); + const inventoryKind = document.getElementById("inventory-kind"); + const inventoryNamespace = document.getElementById("inventory-namespace"); + const inventoryName = document.getElementById("inventory-name"); + const inventoryButton = document.getElementById("run-inventory"); + const inventoryState = document.getElementById("inventory-state"); + const inventoryAnswer = document.getElementById("inventory-answer"); + const inventorySummary = document.getElementById("inventory-summary"); + const inventoryTime = document.getElementById("inventory-time"); + const inventoryGaps = document.getElementById("inventory-gaps"); + const inventoryList = document.getElementById("inventory-list"); + const cveForm = document.getElementById("cve-form"); + const cveIdentifier = document.getElementById("cve-identifier"); + const cveButton = document.getElementById("run-cve"); + const cveState = document.getElementById("cve-state"); + const cveAnswer = document.getElementById("cve-answer"); + const cveSummary = document.getElementById("cve-summary"); + const cveTime = document.getElementById("cve-time"); + const cveGaps = document.getElementById("cve-gaps"); + const cveList = document.getElementById("cve-list"); + + const setState = (message, error = false) => { + state.hidden = false; + state.classList.toggle("error", error); + state.replaceChildren(document.createElement("p")); + state.firstElementChild.textContent = message; + }; + + const setCoverageDetailsMessage = (message) => { + details.replaceChildren(document.createElement("li")); + details.firstElementChild.textContent = message; + }; + + const clearCorrelation = () => { + correlationAnswer.hidden = true; + correlationSummary.textContent = ""; + correlationTime.textContent = "No correlation read yet"; + correlationGaps.replaceChildren(); + correlationList.replaceChildren(); + }; + + const clearInventory = () => { + inventoryAnswer.hidden = true; + inventorySummary.textContent = ""; + inventoryTime.textContent = "No inventory read yet"; + inventoryGaps.replaceChildren(); + inventoryList.replaceChildren(); + }; + + const clearCVE = () => { + cveAnswer.hidden = true; + cveSummary.textContent = ""; + cveTime.textContent = "No CVE read yet"; + cveGaps.replaceChildren(); + cveList.replaceChildren(); + }; + + const setCorrelationState = (message, error = false, visuallyHidden = false) => { + correlationState.hidden = false; + correlationState.classList.toggle("error", error); + correlationState.classList.toggle("visually-hidden", visuallyHidden); + correlationState.replaceChildren(document.createElement("p")); + correlationState.firstElementChild.textContent = message; + }; + + const setInventoryState = (message, error = false, visuallyHidden = false) => { + inventoryState.hidden = false; + inventoryState.classList.toggle("error", error); + inventoryState.classList.toggle("visually-hidden", visuallyHidden); + inventoryState.replaceChildren(document.createElement("p")); + inventoryState.firstElementChild.textContent = message; + }; + + const setCVEState = (message, error = false, visuallyHidden = false) => { + cveState.hidden = false; + cveState.classList.toggle("error", error); + cveState.classList.toggle("visually-hidden", visuallyHidden); + cveState.replaceChildren(document.createElement("p")); + cveState.firstElementChild.textContent = message; + }; + + const setSessionExpired = () => { + snapshotTime.textContent = "No valid snapshot"; + rail.replaceChildren(); + list.replaceChildren(); + setCoverageDetailsMessage("Named gaps are unavailable because the session expired."); + summary.textContent = "The session expired. No current coverage claim can be made."; + rail.setAttribute("aria-label", summary.textContent); + state.hidden = false; + state.classList.add("error"); + const message = document.createElement("p"); + message.append("The session expired. "); + const login = document.createElement("a"); + login.className = "session-link"; + login.href = `/v1/workspaces/${encodeURIComponent(workspace)}/console/login`; + login.textContent = "Sign in again"; + message.append(login, "."); + state.replaceChildren(message); + clearCorrelation(); + setCorrelationState("The session expired. Sign in again before running a correlation.", true); + clearInventory(); + setInventoryState("The session expired. Sign in again before reading inventory.", true); + clearCVE(); + setCVEState("The session expired. Sign in again before reading CVE evidence.", true); + }; + + const setProofExpired = () => { + snapshotTime.textContent = "No valid snapshot"; + rail.replaceChildren(); + list.replaceChildren(); + setCoverageDetailsMessage("Named gaps are unavailable until the authenticated console is reloaded."); + summary.textContent = "The page proof expired or workspace access changed. No current coverage claim can be made."; + rail.setAttribute("aria-label", summary.textContent); + state.hidden = false; + state.classList.add("error"); + const message = document.createElement("p"); + message.append("Reload the authenticated console before reading again. "); + const reloadPage = document.createElement("a"); + reloadPage.className = "session-link"; + reloadPage.href = window.location.pathname; + reloadPage.textContent = "Reload console"; + message.append(reloadPage, "."); + state.replaceChildren(message); + }; + + const stringSet = (values) => new Set(Array.isArray(values) ? values.filter((value) => typeof value === "string") : []); + + const hasOnlyKeys = (value, required, optional = []) => { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const keys = Object.keys(value); + return required.every((key) => Object.hasOwn(value, key)) && + keys.every((key) => required.includes(key) || optional.includes(key)); + }; + + const validProjectedText = (value, maximum) => typeof value === "string" && value.length > 0 && + value.length <= maximum && value.trim() === value && !/[\u0000-\u001f\u007f]/.test(value); + + const validCoverageScopes = (values) => { + if (values === undefined) return []; + if (!Array.isArray(values) || values.length > 1000 || values.some((value) => !validProjectedText(value, 256))) return null; + const sorted = [...values].sort((left, right) => left.localeCompare(right)); + return new Set(sorted).size === sorted.length ? sorted : null; + }; + + const expectedCoverageAssessment = (coverage) => { + if (!hasOnlyKeys(coverage, ["requested", "reachable"], ["unreachable", "stale", "truncated"]) || + !Number.isSafeInteger(coverage.requested) || coverage.requested < 0 || + !Number.isSafeInteger(coverage.reachable) || coverage.reachable < 0) return null; + const unreachable = validCoverageScopes(coverage.unreachable); + const stale = validCoverageScopes(coverage.stale); + const truncated = validCoverageScopes(coverage.truncated); + if (!unreachable || !stale || !truncated || unreachable.length + stale.length + truncated.length > 1000) return null; + let inconsistent = coverage.reachable > coverage.requested || stale.length > coverage.requested || + truncated.length > coverage.reachable || unreachable.some((scope) => truncated.includes(scope)); + let unaccounted = 0; + if (!inconsistent) { + const remaining = coverage.requested - coverage.reachable; + if (unreachable.length > remaining) inconsistent = true; + else if (unreachable.length < remaining) unaccounted = remaining - unreachable.length; + } + const gaps = []; + if (inconsistent) gaps.push("inconsistent"); + if (unreachable.length > 0) gaps.push("unreachable"); + if (stale.length > 0) gaps.push("stale"); + if (truncated.length > 0) gaps.push("truncated"); + if (unaccounted > 0) gaps.push("unaccounted"); + return { complete: gaps.length === 0, gaps, unreachable, stale, truncated, unaccounted, inconsistent }; + }; + + const validCVEResponse = (payload, expectedIdentifier) => { + if (!hasOnlyKeys(payload, ["records", "coverage", "assessment"]) || !Array.isArray(payload.records) || + payload.records.length > 256 || !/^CVE-[0-9]{4}-[0-9]{4,}$/.test(expectedIdentifier)) return false; + const identities = new Set(); + for (const record of payload.records) { + if (!hasOnlyKeys(record, ["scope", "image_digest", "identifier", "severity", "observed_at", "stale"], ["stale_for"]) || + !validProjectedText(record.scope, 256) || !/^sha256:[0-9a-f]{64}$/.test(record.image_digest) || + record.identifier !== expectedIdentifier || !["critical", "high", "medium", "low", "unknown"].includes(record.severity) || + typeof record.observed_at !== "string" || Number.isNaN(Date.parse(record.observed_at)) || typeof record.stale !== "boolean" || + (record.stale ? !validProjectedText(record.stale_for, 128) : record.stale_for !== undefined)) return false; + const identity = `${record.scope}\u0000${record.image_digest}\u0000${record.identifier}`; + if (identities.has(identity)) return false; + identities.add(identity); + } + const expected = expectedCoverageAssessment(payload.coverage); + if (!expected || !hasOnlyKeys(payload.assessment, ["complete", "inconsistent"], ["gaps", "unreachable", "stale", "truncated", "unaccounted"]) || + typeof payload.assessment.complete !== "boolean" || typeof payload.assessment.inconsistent !== "boolean") return false; + const actual = { + complete: payload.assessment.complete, + gaps: payload.assessment.gaps === undefined ? [] : payload.assessment.gaps, + unreachable: payload.assessment.unreachable === undefined ? [] : payload.assessment.unreachable, + stale: payload.assessment.stale === undefined ? [] : payload.assessment.stale, + truncated: payload.assessment.truncated === undefined ? [] : payload.assessment.truncated, + unaccounted: payload.assessment.unaccounted === undefined ? 0 : payload.assessment.unaccounted, + inconsistent: payload.assessment.inconsistent, + }; + return JSON.stringify(actual) === JSON.stringify(expected); + }; + + const clusterState = (cluster, coverage) => { + const name = typeof cluster?.name === "string" ? cluster.name : ""; + if (!cluster?.reachable || stringSet(coverage?.unreachable).has(name)) return "unreachable"; + if (stringSet(coverage?.stale).has(name)) return "stale"; + if (stringSet(coverage?.truncated).has(name)) return "truncated"; + return "current"; + }; + + const appendRailSegment = (kind, label) => { + const segment = document.createElement("span"); + segment.className = `rail-segment ${kind}`; + segment.title = label; + rail.append(segment); + }; + + const renderAssessmentDetails = (target, assessment, emptyMessage) => { + target.replaceChildren(); + const appendDetail = (message) => { + const item = document.createElement("li"); + item.textContent = message; + target.append(item); + }; + if (assessment?.inconsistent) appendDetail("Coverage metadata is internally inconsistent."); + [ + ["Unreachable", assessment?.unreachable], + ["Stale", assessment?.stale], + ["Partial", assessment?.truncated], + ].forEach(([label, values]) => { + const names = [...stringSet(values)].sort((left, right) => left.localeCompare(right)); + if (names.length > 0) appendDetail(`${label}: ${names.join(", ")}`); + }); + if (Number.isSafeInteger(assessment?.unaccounted) && assessment.unaccounted > 0) { + appendDetail(`Unaccounted requested scopes: ${assessment.unaccounted}`); + } + if (target.childElementCount === 0) appendDetail(emptyMessage); + }; + + const renderCoverageDetails = (assessment) => renderAssessmentDetails(details, assessment, "No named coverage gaps."); + + const renderRail = (fleet, assessment) => { + rail.replaceChildren(); + const coverage = fleet.coverage || {}; + const clusters = Array.isArray(fleet.clusters) ? fleet.clusters : []; + const represented = new Set(); + clusters.forEach((cluster) => { + const name = typeof cluster?.name === "string" && cluster.name ? cluster.name : "Unnamed cluster"; + represented.add(name); + appendRailSegment(clusterState(cluster, coverage), name); + }); + stringSet(coverage.unreachable).forEach((name) => { + if (!represented.has(name)) appendRailSegment("unreachable", name); + }); + const requested = Number.isSafeInteger(coverage.requested) && coverage.requested > 0 ? coverage.requested : 0; + const missing = Math.max(0, requested - rail.childElementCount); + for (let index = 0; index < missing; index += 1) appendRailSegment("unknown", "Unaccounted scope"); + if (rail.childElementCount === 0) appendRailSegment("unknown", "No requested scopes"); + + const reachable = Number.isSafeInteger(coverage.reachable) ? coverage.reachable : 0; + const gaps = Array.isArray(assessment?.gaps) ? assessment.gaps.join(", ") : "unknown"; + summary.textContent = requested === 0 + ? "No cluster scopes were requested. No fleet-health claim can be made." + : assessment?.complete + ? `${reachable} of ${requested} requested clusters are reachable with complete current coverage.` + : `${reachable} of ${requested} requested clusters answered. Coverage is incomplete: ${gaps || "unclassified gap"}.`; + rail.setAttribute("aria-label", summary.textContent); + renderCoverageDetails(assessment); + }; + + const observedLabel = (value) => { + if (typeof value !== "string" || !value) return "No observation time"; + const observed = new Date(value); + if (Number.isNaN(observed.getTime())) return "Invalid observation time"; + return `Observed ${observed.toISOString()}`; + }; + + const renderClusters = (fleet) => { + list.replaceChildren(); + const clusters = Array.isArray(fleet.clusters) ? [...fleet.clusters] : []; + clusters.sort((left, right) => String(left?.name || "").localeCompare(String(right?.name || ""))); + if (clusters.length === 0) { + setState("No cluster observations are available for this signed workspace. This is not a healthy-fleet claim."); + return; + } + state.hidden = true; + clusters.forEach((cluster, index) => { + const status = clusterState(cluster, fleet.coverage || {}); + const row = document.createElement("li"); + row.className = "cluster-row"; + + const sequence = document.createElement("span"); + sequence.className = "cluster-sequence"; + sequence.textContent = String(index + 1).padStart(2, "0"); + + const name = document.createElement("h3"); + name.className = "cluster-name"; + name.textContent = typeof cluster?.name === "string" && cluster.name ? cluster.name : "Unnamed cluster"; + + const source = document.createElement("p"); + source.className = "cluster-source"; + source.textContent = `Source ${typeof cluster?.source_kind === "string" && cluster.source_kind ? cluster.source_kind : "unknown"}`; + + const observed = document.createElement("p"); + observed.className = "cluster-observed"; + observed.textContent = observedLabel(cluster?.observed_at); + + const badge = document.createElement("span"); + badge.className = `state-badge ${status}`; + badge.textContent = status; + + row.append(sequence, name, source, observed, badge); + list.append(row); + }); + }; + + const setInventoryProofExpired = () => { + clearInventory(); + inventoryState.hidden = false; + inventoryState.classList.add("error"); + inventoryState.classList.remove("visually-hidden"); + const message = document.createElement("p"); + message.append("The inventory proof expired or workspace access changed. "); + const reloadPage = document.createElement("a"); + reloadPage.className = "session-link"; + reloadPage.href = window.location.pathname; + reloadPage.textContent = "Reload console"; + message.append(reloadPage, " before reading inventory again."); + inventoryState.replaceChildren(message); + }; + + const appendInventoryMetric = (metrics, label, value) => { + const item = document.createElement("div"); + const term = document.createElement("dt"); + const description = document.createElement("dd"); + term.textContent = label; + description.textContent = value; + item.append(term, description); + metrics.append(item); + }; + + const renderInventory = (payload) => { + const records = Array.isArray(payload.records) ? payload.records : []; + const coverage = payload.coverage || {}; + const assessment = payload.assessment || {}; + inventoryList.replaceChildren(); + records.forEach((record, index) => { + const row = document.createElement("li"); + row.className = "inventory-row"; + + const sequence = document.createElement("span"); + sequence.className = "cluster-sequence"; + sequence.textContent = String(index + 1).padStart(2, "0"); + + const identity = document.createElement("div"); + const resource = document.createElement("h3"); + resource.className = "cluster-name"; + resource.textContent = typeof record?.resource_kind === "string" ? record.resource_kind : "Resource"; + const namespace = typeof record?.namespace === "string" && record.namespace ? `${record.namespace}/` : ""; + const address = document.createElement("p"); + address.className = "resource-address"; + address.textContent = `${namespace}${typeof record?.name === "string" ? record.name : ""}`; + const scope = document.createElement("p"); + scope.className = "cluster-source"; + scope.textContent = `Cluster ${typeof record?.scope === "string" ? record.scope : "unknown"}`; + identity.append(resource, address, scope); + + const observed = document.createElement("p"); + observed.className = "cluster-observed"; + observed.textContent = observedLabel(record?.observed_at); + + const metrics = document.createElement("dl"); + metrics.className = "inventory-metrics"; + appendInventoryMetric(metrics, "Generation", Number.isSafeInteger(record?.generation) ? String(record.generation) : "Unavailable"); + if (Number.isSafeInteger(record?.replicas) && Number.isSafeInteger(record?.available_replicas)) { + appendInventoryMetric(metrics, "Available", `${record.available_replicas} / ${record.replicas}`); + } + if (typeof record?.ready === "boolean") appendInventoryMetric(metrics, "Ready", record.ready ? "Yes" : "No"); + + const freshness = document.createElement("p"); + freshness.className = `inventory-freshness${record?.stale ? " stale" : ""}`; + freshness.textContent = record?.stale + ? `Stale · ${typeof record?.stale_for === "string" ? record.stale_for : "age unavailable"}` + : "Current"; + + row.append(sequence, identity, observed, metrics, freshness); + inventoryList.append(row); + }); + + const requested = Number.isSafeInteger(coverage.requested) ? coverage.requested : 0; + const reachable = Number.isSafeInteger(coverage.reachable) ? coverage.reachable : 0; + const gaps = Array.isArray(assessment.gaps) ? assessment.gaps.join(", ") : "unknown"; + if (requested === 0) { + inventorySummary.textContent = "No cluster scopes were requested. No inventory-completeness claim can be made."; + } else if (assessment.complete && records.length === 0) { + inventorySummary.textContent = `No matching inventory record was observed across ${reachable} of ${requested} clusters with complete current coverage.`; + } else if (assessment.complete) { + inventorySummary.textContent = `${records.length} normalized inventory record${records.length === 1 ? "" : "s"} observed across complete current coverage (${reachable} of ${requested} clusters).`; + } else { + inventorySummary.textContent = `${records.length} normalized inventory record${records.length === 1 ? "" : "s"} observed; ${reachable} of ${requested} clusters answered. This is not a complete inventory: ${gaps || "unclassified gap"}.`; + } + renderAssessmentDetails(inventoryGaps, assessment, "No named inventory coverage gaps."); + inventoryTime.textContent = `Read ${new Date().toISOString()}`; + setInventoryState(`Inventory read complete. ${inventorySummary.textContent}`, false, true); + inventoryAnswer.hidden = false; + }; + + const runInventory = async (event) => { + event.preventDefault(); + clearInventory(); + if (!inventoryForm.checkValidity()) { + inventoryForm.reportValidity(); + setInventoryState("Choose a supported resource kind. No inventory read was run.", true); + return; + } + const kind = inventoryKind.value; + const namespace = inventoryNamespace.value; + const name = inventoryName.value; + if (!["Deployment", "Pod", "Rollout"].includes(kind) || namespace.trim() !== namespace || name.trim() !== name) { + setInventoryState("Use a supported kind and trimmed exact values. No inventory read was run.", true); + return; + } + const parameters = new URLSearchParams(); + parameters.set("kind", kind); + if (name) parameters.set("name", name); + if (namespace) parameters.set("namespace", namespace); + inventoryButton.disabled = true; + setInventoryState("Reading one persisted, tenant-scoped inventory selection."); + try { + const response = await fetch(`${window.location.pathname}/inventory?${parameters.toString()}`, { + method: "GET", + credentials: "same-origin", + cache: "no-store", + headers: { "X-Sith-CSRF": inventoryCSRFToken }, + }); + if (response.status === 401) { + setSessionExpired(); + return; + } + if (response.status === 403) { + setInventoryProofExpired(); + return; + } + if (!response.ok) throw new Error("inventory request refused"); + const payload = await response.json(); + if (!payload || typeof payload !== "object" || !Array.isArray(payload.records) || !payload.coverage || !payload.assessment) { + throw new Error("invalid inventory response"); + } + renderInventory(payload); + } catch (_error) { + clearInventory(); + setInventoryState("The persisted inventory could not be read. No connector refresh, local operation, or write was attempted.", true); + } finally { + inventoryButton.disabled = false; + } + }; + + const setCVEProofExpired = () => { + clearCVE(); + cveState.hidden = false; + cveState.classList.add("error"); + cveState.classList.remove("visually-hidden"); + const message = document.createElement("p"); + message.append("The CVE evidence proof expired or workspace access changed. "); + const reloadPage = document.createElement("a"); + reloadPage.className = "session-link"; + reloadPage.href = window.location.pathname; + reloadPage.textContent = "Reload console"; + message.append(reloadPage, " before reading again."); + cveState.replaceChildren(message); + }; + + const renderCVE = (payload) => { + const records = Array.isArray(payload.records) ? payload.records : []; + const coverage = payload.coverage || {}; + const assessment = payload.assessment || {}; + cveList.replaceChildren(); + records.forEach((record, index) => { + const row = document.createElement("li"); + row.className = "cve-row"; + + const sequence = document.createElement("span"); + sequence.className = "cluster-sequence"; + sequence.textContent = String(index + 1).padStart(2, "0"); + + const identity = document.createElement("div"); + const identifier = document.createElement("h3"); + identifier.className = "cluster-name"; + identifier.textContent = typeof record?.identifier === "string" ? record.identifier : "CVE evidence"; + const digest = document.createElement("p"); + digest.className = "digest-address"; + digest.textContent = typeof record?.image_digest === "string" ? record.image_digest : "Digest unavailable"; + const scope = document.createElement("p"); + scope.className = "cluster-source"; + scope.textContent = `Cluster ${typeof record?.scope === "string" ? record.scope : "unknown"}`; + identity.append(identifier, digest, scope); + + const observed = document.createElement("p"); + observed.className = "cluster-observed"; + observed.textContent = observedLabel(record?.observed_at); + + const severity = document.createElement("span"); + const severityValue = typeof record?.severity === "string" ? record.severity : "unknown"; + severity.className = `severity-badge ${["critical", "high", "medium", "low", "unknown"].includes(severityValue) ? severityValue : "unknown"}`; + severity.textContent = severityValue; + + const freshness = document.createElement("p"); + freshness.className = "match-freshness"; + freshness.textContent = record?.stale + ? `Stale · ${typeof record?.stale_for === "string" ? record.stale_for : "age unavailable"}` + : "Current observation"; + + row.append(sequence, identity, observed, severity, freshness); + cveList.append(row); + }); + + const requested = Number.isSafeInteger(coverage.requested) ? coverage.requested : 0; + const reachable = Number.isSafeInteger(coverage.reachable) ? coverage.reachable : 0; + const gaps = Array.isArray(assessment.gaps) ? assessment.gaps.join(", ") : "unknown"; + if (requested === 0) { + cveSummary.textContent = "No cluster scopes were requested. No CVE-evidence claim can be made."; + } else if (assessment.complete && records.length === 0) { + cveSummary.textContent = `No matching runtime CVE observation was found across ${reachable} of ${requested} clusters with complete current coverage.`; + } else if (assessment.complete) { + cveSummary.textContent = `${records.length} runtime CVE observation${records.length === 1 ? "" : "s"} found across complete current coverage (${reachable} of ${requested} clusters).`; + } else { + cveSummary.textContent = `${records.length} runtime CVE observation${records.length === 1 ? "" : "s"} found; ${reachable} of ${requested} clusters answered. This is not complete evidence: ${gaps || "unclassified gap"}.`; + } + renderAssessmentDetails(cveGaps, assessment, "No named CVE evidence coverage gaps."); + cveTime.textContent = `Read ${new Date().toISOString()}`; + setCVEState(`CVE evidence read complete. ${cveSummary.textContent}`, false, true); + cveAnswer.hidden = false; + }; + + const runCVE = async (event) => { + event.preventDefault(); + clearCVE(); + const identifier = cveIdentifier.value; + if (!cveForm.checkValidity() || !/^CVE-[0-9]{4}-[0-9]{4,}$/.test(identifier)) { + cveIdentifier.setCustomValidity("Enter a canonical uppercase CVE identifier, for example CVE-2026-0001."); + cveForm.reportValidity(); + cveIdentifier.setCustomValidity(""); + setCVEState("Enter one canonical uppercase CVE identifier. No evidence read was run.", true); + return; + } + const parameters = new URLSearchParams(); + parameters.set("identifier", identifier); + cveButton.disabled = true; + setCVEState("Reading persisted, tenant-scoped runtime CVE evidence."); + try { + const response = await fetch(`${window.location.pathname}/cves?${parameters.toString()}`, { + method: "GET", + credentials: "same-origin", + cache: "no-store", + headers: { "X-Sith-CSRF": cveCSRFToken }, + }); + if (response.status === 401) { + setSessionExpired(); + return; + } + if (response.status === 403) { + setCVEProofExpired(); + return; + } + if (!response.ok) throw new Error("CVE evidence request refused"); + const payload = await response.json(); + if (!validCVEResponse(payload, identifier)) { + throw new Error("invalid CVE evidence response"); + } + renderCVE(payload); + } catch (_error) { + clearCVE(); + setCVEState("The persisted CVE evidence could not be read. No scanner refresh, spoke request, or write was attempted.", true); + } finally { + cveButton.disabled = false; + } + }; + + const setCorrelationProofExpired = () => { + clearCorrelation(); + correlationState.hidden = false; + correlationState.classList.add("error"); + correlationState.classList.remove("visually-hidden"); + const message = document.createElement("p"); + message.append("The correlation proof expired or workspace access changed. "); + const reloadPage = document.createElement("a"); + reloadPage.className = "session-link"; + reloadPage.href = window.location.pathname; + reloadPage.textContent = "Reload console"; + message.append(reloadPage, " before asking again."); + correlationState.replaceChildren(message); + }; + + const renderCorrelation = (payload) => { + const matches = Array.isArray(payload.matches) ? payload.matches : []; + const coverage = payload.coverage || {}; + const assessment = payload.assessment || {}; + correlationList.replaceChildren(); + matches.forEach((match, index) => { + const row = document.createElement("li"); + row.className = "correlation-row"; + + const sequence = document.createElement("span"); + sequence.className = "cluster-sequence"; + sequence.textContent = String(index + 1).padStart(2, "0"); + + const identity = document.createElement("div"); + const resource = document.createElement("h3"); + resource.className = "cluster-name"; + const namespace = typeof match?.namespace === "string" && match.namespace ? `${match.namespace}/` : ""; + resource.textContent = typeof match?.resource_kind === "string" ? match.resource_kind : "Resource"; + const address = document.createElement("p"); + address.className = "resource-address"; + address.textContent = `${namespace}${typeof match?.name === "string" ? match.name : ""}`; + const scope = document.createElement("p"); + scope.className = "cluster-source"; + scope.textContent = `Cluster ${typeof match?.scope === "string" ? match.scope : "unknown"}`; + identity.append(resource, address, scope); + + const observed = document.createElement("p"); + observed.className = "cluster-observed"; + observed.textContent = observedLabel(match?.observed_at); + + const health = document.createElement("span"); + const status = typeof match?.health === "string" ? match.health : "Unknown"; + health.className = `health-badge ${status.toLowerCase()}`; + health.textContent = status; + + const freshness = document.createElement("p"); + freshness.className = "match-freshness"; + freshness.textContent = match?.stale + ? `Stale · ${typeof match?.stale_for === "string" ? match.stale_for : "age unavailable"}` + : "Current observation"; + + row.append(sequence, identity, observed, health, freshness); + correlationList.append(row); + }); + + const requested = Number.isSafeInteger(coverage.requested) ? coverage.requested : 0; + const reachable = Number.isSafeInteger(coverage.reachable) ? coverage.reachable : 0; + const gaps = Array.isArray(assessment.gaps) ? assessment.gaps.join(", ") : "unknown"; + if (requested === 0) { + correlationSummary.textContent = "No cluster scopes were requested. No resource-health claim can be made."; + } else if (assessment.complete && matches.length === 0) { + correlationSummary.textContent = `No non-Healthy observation was found across ${reachable} of ${requested} clusters with complete current coverage.`; + } else if (assessment.complete) { + correlationSummary.textContent = `${matches.length} non-Healthy observation${matches.length === 1 ? "" : "s"} found across complete current coverage (${reachable} of ${requested} clusters).`; + } else { + correlationSummary.textContent = `${matches.length} non-Healthy observation${matches.length === 1 ? "" : "s"} found; ${reachable} of ${requested} clusters answered. This is not a complete answer: ${gaps || "unclassified gap"}.`; + } + renderAssessmentDetails(correlationGaps, assessment, "No named correlation coverage gaps."); + correlationTime.textContent = `Read ${new Date().toISOString()}`; + setCorrelationState(`Correlation complete. ${correlationSummary.textContent}`, false, true); + correlationAnswer.hidden = false; + }; + + const runCorrelation = async (event) => { + event.preventDefault(); + clearCorrelation(); + if (!correlationForm.checkValidity()) { + correlationForm.reportValidity(); + setCorrelationState("Resource kind and exact name are required. No correlation was run.", true); + return; + } + const kind = correlationKind.value; + const name = correlationName.value; + const namespace = correlationNamespace.value; + if (kind.trim() !== kind || name.trim() !== name || namespace.trim() !== namespace || kind.toLowerCase() === "secret") { + setCorrelationState("Use trimmed values and a non-Secret resource kind. No correlation was run.", true); + return; + } + const parameters = new URLSearchParams(); + parameters.set("kind", kind); + parameters.set("name", name); + if (namespace) parameters.set("namespace", namespace); + correlationButton.disabled = true; + setCorrelationState("Reading one persisted, tenant-scoped health correlation."); + try { + const response = await fetch(`${window.location.pathname}/correlate?${parameters.toString()}`, { + method: "GET", + credentials: "same-origin", + cache: "no-store", + headers: { "X-Sith-CSRF": correlationCSRFToken }, + }); + if (response.status === 401) { + setSessionExpired(); + return; + } + if (response.status === 403) { + setCorrelationProofExpired(); + return; + } + if (!response.ok) throw new Error("correlation request refused"); + const payload = await response.json(); + if (!payload || typeof payload !== "object" || !Array.isArray(payload.matches) || !payload.coverage || !payload.assessment) { + throw new Error("invalid correlation response"); + } + renderCorrelation(payload); + } catch (_error) { + clearCorrelation(); + setCorrelationState("The persisted correlation could not be read. No connector refresh, local operation, or write was attempted.", true); + } finally { + correlationButton.disabled = false; + } + }; + + const readFleet = async () => { + reload.disabled = true; + setState("Reading the persisted workspace snapshot."); + try { + const response = await fetch(`${window.location.pathname}/fleet`, { + method: "GET", + credentials: "same-origin", + cache: "no-store", + headers: { "X-Sith-CSRF": csrfToken }, + }); + if (response.status === 401) { + setSessionExpired(); + return; + } + if (response.status === 403) { + setProofExpired(); + return; + } + if (!response.ok) throw new Error("fleet request refused"); + const payload = await response.json(); + if (!payload || typeof payload !== "object" || !payload.fleet || !payload.assessment) throw new Error("invalid fleet response"); + renderRail(payload.fleet, payload.assessment); + renderClusters(payload.fleet); + snapshotTime.textContent = `Read ${new Date().toISOString()}`; + } catch (_error) { + snapshotTime.textContent = "No snapshot available"; + rail.replaceChildren(); + list.replaceChildren(); + setCoverageDetailsMessage("Named gaps are unavailable because the snapshot could not be read."); + summary.textContent = "The fleet snapshot is unavailable. No coverage claim can be made."; + rail.setAttribute("aria-label", summary.textContent); + setState("The persisted fleet view could not be read. Try again; no connector refresh or write was attempted.", true); + } finally { + reload.disabled = false; + } + }; + + reload.addEventListener("click", readFleet); + inventoryForm.addEventListener("submit", runInventory); + cveForm.addEventListener("submit", runCVE); + correlationForm.addEventListener("submit", runCorrelation); + readFleet(); +})(); diff --git a/internal/hubserver/console_boundary_test.go b/internal/hubserver/console_boundary_test.go new file mode 100644 index 0000000..6490969 --- /dev/null +++ b/internal/hubserver/console_boundary_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestConsoleProductionBoundaryIsReadOnlyAndStructurallyPinned(t *testing.T) { + entries, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + var productionFiles []string + for _, entry := range entries { + name := entry.Name() + if !entry.IsDir() && strings.HasPrefix(name, "console") && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { + productionFiles = append(productionFiles, name) + } + } + if len(productionFiles) != 1 || productionFiles[0] != "console.go" { + t.Fatalf("console production files = %v, want [console.go]", productionFiles) + } + + parsed, err := parser.ParseFile(token.NewFileSet(), "console.go", nil, 0) + if err != nil { + t.Fatal(err) + } + allowedImports := map[string]bool{ + "bytes": true, "crypto/hmac": true, "crypto/rand": true, "crypto/sha256": true, "crypto/subtle": true, + "embed": true, "encoding/base64": true, "encoding/json": true, "fmt": true, + "html/template": true, "io": true, "io/fs": true, "net/http": true, "net/url": true, "strconv": true, "strings": true, + "sync": true, "time": true, "unicode": true, + "github.com/ArdurAI/sith/internal/fleet": true, "github.com/ArdurAI/sith/internal/hubfleet": true, + "github.com/ArdurAI/sith/internal/pep": true, "github.com/ArdurAI/sith/internal/tenancy": true, + } + seenImports := make(map[string]bool, len(parsed.Imports)) + for _, declaration := range parsed.Imports { + path, err := strconv.Unquote(declaration.Path.Value) + if err != nil { + t.Fatal(err) + } + if !allowedImports[path] { + t.Fatalf("console.go imports unreviewed capability %q", path) + } + seenImports[path] = true + } + if len(seenImports) != len(allowedImports) { + t.Fatalf("console.go import set drifted: got %v", seenImports) + } + + forbiddenCalls := map[string]bool{ + "Collect": true, "SyncOnce": true, "SyncKinds": true, "Exec": true, + "Apply": true, "Edit": true, "PortForward": true, "WriteFile": true, "Create": true, + } + ast.Inspect(parsed, func(node ast.Node) bool { + selector, ok := node.(*ast.SelectorExpr) + if ok && forbiddenCalls[selector.Sel.Name] { + t.Errorf("console.go calls forbidden capability %s", selector.Sel.Name) + } + return true + }) + + assetEntries, err := os.ReadDir("console_assets") + if err != nil { + t.Fatal(err) + } + var assets []string + for _, entry := range assetEntries { + if entry.IsDir() { + t.Fatalf("console asset directory contains nested directory %q", entry.Name()) + } + assets = append(assets, filepath.ToSlash(entry.Name())) + } + wantAssets := []string{"console.css", "console.html", "console.js"} + if strings.Join(assets, "|") != strings.Join(wantAssets, "|") { + t.Fatalf("console assets = %v, want %v", assets, wantAssets) + } + + runtimeSource, err := os.ReadFile(filepath.Join("..", "hubruntime", "config.go")) + if err != nil { + t.Fatal(err) + } + for _, dependency := range []string{ + `hubfleet.NewCorrelator(hubfleet.CorrelatorConfig{Querier: database, PEP: enforcer})`, + `Correlator: consoleCorrelator`, + `hubfleet.NewInventorySearcher(hubfleet.InventorySearcherConfig{Querier: database, PEP: enforcer})`, + `Inventory: consoleInventory`, + `consoleCVE, err := hubfleet.NewCVESearcher(hubfleet.CVESearcherConfig{Querier: database, PEP: enforcer})`, + `CVE: consoleCVE`, + } { + if strings.Count(string(runtimeSource), dependency) != 1 { + t.Fatalf("Hub runtime does not compose exact console dependency %q once", dependency) + } + } + for _, route := range []string{ + `mux.Handle("GET /v1/workspaces/{workspace}/console", http.HandlerFunc(consoleHandler.ServePage))`, + `mux.Handle("GET /v1/workspaces/{workspace}/console/fleet", http.HandlerFunc(consoleHandler.ServeFleet))`, + `mux.Handle("GET /v1/workspaces/{workspace}/console/correlate", http.HandlerFunc(consoleHandler.ServeCorrelation))`, + `mux.Handle("GET /v1/workspaces/{workspace}/console/inventory", http.HandlerFunc(consoleHandler.ServeInventory))`, + `mux.Handle("GET /v1/workspaces/{workspace}/console/cves", http.HandlerFunc(consoleHandler.ServeCVEIdentifier))`, + `mux.Handle("GET /v1/console/assets/console.css", http.HandlerFunc(consoleHandler.ServeCSS))`, + `mux.Handle("GET /v1/console/assets/console.js", http.HandlerFunc(consoleHandler.ServeJavaScript))`, + } { + if strings.Count(string(runtimeSource), route) != 1 { + t.Fatalf("Hub runtime does not mount exact console route %q once", route) + } + } + for _, forbidden := range []string{`mux.Handle("POST /v1/workspaces/{workspace}/console`, `mux.Handle("PUT /v1/workspaces/{workspace}/console`, `mux.Handle("DELETE /v1/workspaces/{workspace}/console`} { + if strings.Contains(string(runtimeSource), forbidden) { + t.Fatalf("Hub runtime mounts forbidden console mutation route %q", forbidden) + } + } +} diff --git a/internal/hubserver/console_correlation_test.go b/internal/hubserver/console_correlation_test.go new file mode 100644 index 0000000..6d3098d --- /dev/null +++ b/internal/hubserver/console_correlation_test.go @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +type consoleFleetQuerierFunc func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) + +func (function consoleFleetQuerierFunc) QueryFleet( + ctx context.Context, + scope tenancy.Scope, + query fleet.Query, + freshness time.Duration, + now time.Time, +) (fleet.QueryResult, error) { + return function(ctx, scope, query, freshness, now) +} + +func noopConsoleCorrelator(t *testing.T) *hubfleet.Correlator { + t.Helper() + return consoleTestCorrelator(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, nil + }), fleetTestPEP(t, pep.AllowReadHook{})) +} + +func consoleTestCorrelator(t *testing.T, querier hubfleet.FleetQuerier, enforcer *pep.Enforcer) *hubfleet.Correlator { + t.Helper() + correlator, err := hubfleet.NewCorrelator(hubfleet.CorrelatorConfig{Querier: querier, PEP: enforcer}) + if err != nil { + t.Fatal(err) + } + return correlator +} + +func noopConsoleInventory(t *testing.T) *hubfleet.InventorySearcher { + t.Helper() + return consoleTestInventory(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, nil + }), fleetTestPEP(t, pep.AllowReadHook{})) +} + +func consoleTestInventory(t *testing.T, querier hubfleet.FleetQuerier, enforcer *pep.Enforcer) *hubfleet.InventorySearcher { + t.Helper() + searcher, err := hubfleet.NewInventorySearcher(hubfleet.InventorySearcherConfig{Querier: querier, PEP: enforcer}) + if err != nil { + t.Fatal(err) + } + return searcher +} + +func noopConsoleCVE(t *testing.T) *hubfleet.CVESearcher { + t.Helper() + return consoleTestCVE(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, nil + }), fleetTestPEP(t, pep.AllowReadHook{})) +} + +func consoleTestCVE(t *testing.T, querier hubfleet.FleetQuerier, enforcer *pep.Enforcer) *hubfleet.CVESearcher { + t.Helper() + searcher, err := hubfleet.NewCVESearcher(hubfleet.CVESearcherConfig{Querier: querier, PEP: enforcer}) + if err != nil { + t.Fatal(err) + } + return searcher +} + +func TestConsoleCorrelationUsesPurposeProofPEPAndMinimalProjection(t *testing.T) { + now := time.Date(2026, 7, 17, 15, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + var audits []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + audits = append(audits, event) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + queryCalls := 0 + correlator := consoleTestCorrelator(t, consoleFleetQuerierFunc(func( + _ context.Context, scope tenancy.Scope, query fleet.Query, freshness time.Duration, gotNow time.Time, + ) (fleet.QueryResult, error) { + queryCalls++ + if scope.WorkspaceID() != "workspace-a" || scope.Subject() != "user:alice" || freshness != 5*time.Minute || gotNow.IsZero() || + query.Limit != consoleCorrelationReadLimit || len(query.Kinds) != 1 || query.Kinds[0] != fleet.FactHealth || + query.Selector.ResourceKind != "Deployment" || query.Selector.Name != "payments" || + query.Selector.Namespace != "payments" || query.Selector.HealthNot != "Healthy" { + t.Fatalf("correlation scope/query/freshness/now = %#v/%#v/%s/%s", scope, query, freshness, gotNow) + } + return fleet.QueryResult{ + Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Deployment", Namespace: "payments", Name: "payments", + Attributes: map[string]string{"credential": "must-not-leak"}}, + Kind: fleet.FactHealth, Observed: json.RawMessage(`{"status":"Degraded"}`), ObservedAt: now.Add(-6 * time.Minute), + Source: "cluster-a", Provenance: fleet.Provenance{NativeID: "private-native-id", DeepLink: "https://private.example"}, + }, + Workspace: "workspace-a", Stale: true, StaleFor: "6m0s", + }}, + Coverage: fleet.Coverage{Requested: 3, Reachable: 2, Unreachable: []string{"cluster-c"}, Stale: []string{"cluster-a"}}, + }, nil + }), enforcer) + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: correlator, Inventory: noopConsoleInventory(t), CVE: noopConsoleCVE(t), PEP: enforcer, Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x72}, 192)), + }) + if err != nil { + t.Fatal(err) + } + + page, fleetProof := authenticatedConsolePage(t, handler, session, "workspace-a") + correlationProof := consolePageProof(t, page.Body.String(), "sith-correlation-csrf") + if correlationProof == "" || correlationProof == fleetProof || strings.Contains(page.Body.String(), session) { + t.Fatalf("purpose proof separation/session leak = %q/%q", fleetProof, correlationProof) + } + request := validConsoleCorrelationRequest(session, correlationProof) + response := httptest.NewRecorder() + handler.ServeCorrelation(response, request) + if response.Code != http.StatusOK || queryCalls != 1 || len(audits) != 1 || audits[0].Verb != pep.VerbFleetCorrelate { + t.Fatalf("status/query calls/audits/body = %d/%d/%#v/%q", response.Code, queryCalls, audits, response.Body.String()) + } + assertConsoleSecurityHeaders(t, response.Header()) + for _, forbidden := range []string{"must-not-leak", "private-native-id", "private.example", `"observed"`, `"attributes"`, `"provenance"`, `"workspace"`, `"source_kind"`, `"deep_link"`} { + if strings.Contains(response.Body.String(), forbidden) { + t.Fatalf("correlation response leaked %q: %s", forbidden, response.Body.String()) + } + } + var payload consoleCorrelationResponse + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if len(payload.Matches) != 1 || payload.Matches[0].Scope != "cluster-a" || payload.Matches[0].Health != "Degraded" || + !payload.Matches[0].Stale || payload.Matches[0].StaleFor != "6m0s" || payload.Assessment.Complete || payload.Assessment.Unaccounted != 0 { + t.Fatalf("projected correlation = %#v", payload) + } +} + +func TestConsoleCorrelationRejectsUnsafeRequestsBeforeQuery(t *testing.T) { + now := time.Date(2026, 7, 17, 16, 0, 0, 0, time.UTC) + consoleNow := now + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + queryCalls := 0 + enforcer := fleetTestPEP(t, pep.AllowReadHook{}) + correlator := consoleTestCorrelator(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + queryCalls++ + return fleet.QueryResult{}, nil + }), enforcer) + newHandler := func(fill byte) *ConsoleHandler { + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: correlator, Inventory: noopConsoleInventory(t), CVE: noopConsoleCVE(t), PEP: enforcer, Now: func() time.Time { return consoleNow }, + Random: bytes.NewReader(bytes.Repeat([]byte{fill}, 192)), + }) + if err != nil { + t.Fatal(err) + } + return handler + } + handler := newHandler(0x31) + page, fleetProof := authenticatedConsolePage(t, handler, session, "workspace-a") + correlationProof := consolePageProof(t, page.Body.String(), "sith-correlation-csrf") + + tests := []struct { + name string + request func() *http.Request + }{ + {name: "fleet proof is not reusable", request: func() *http.Request { return validConsoleCorrelationRequest(session, fleetProof) }}, + {name: "missing proof", request: func() *http.Request { + request := validConsoleCorrelationRequest(session, correlationProof) + request.Header.Del(consoleCSRFHeader) + return request + }}, + {name: "duplicate proof", request: func() *http.Request { + request := validConsoleCorrelationRequest(session, correlationProof) + request.Header.Add(consoleCSRFHeader, correlationProof) + return request + }}, + {name: "cross site", request: func() *http.Request { + request := validConsoleCorrelationRequest(session, correlationProof) + request.Header.Set("Sec-Fetch-Site", "cross-site") + return request + }}, + {name: "bearer fallback", request: func() *http.Request { + request := validConsoleCorrelationRequest(session, correlationProof) + request.Header.Set("Authorization", "Bearer "+session) + return request + }}, + {name: "duplicate cookie", request: func() *http.Request { + request := validConsoleCorrelationRequest(session, correlationProof) + request.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: session}) + return request + }}, + {name: "unknown query", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "kind=Deployment&name=payments&selector=all") + }}, + {name: "duplicate query", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "kind=Deployment&kind=Pod&name=payments") + }}, + {name: "condition override", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "health_not=Unknown&kind=Deployment&name=payments") + }}, + {name: "Secret kind", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "kind=Secret&name=payments") + }}, + {name: "blank namespace", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "kind=Deployment&name=payments&namespace=") + }}, + {name: "noncanonical order", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "name=payments&kind=Deployment") + }}, + {name: "untrimmed name", request: func() *http.Request { + return consoleCorrelationRequest(session, correlationProof, "kind=Deployment&name=+payments") + }}, + {name: "method", request: func() *http.Request { + request := validConsoleCorrelationRequest(session, correlationProof) + request.Method = http.MethodPost + return request + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeCorrelation(response, test.request()) + if response.Code == http.StatusOK { + t.Fatalf("unsafe request succeeded: %q", response.Body.String()) + } + }) + } + if queryCalls != 0 { + t.Fatalf("unsafe requests reached query %d times", queryCalls) + } + + consoleNow = now.Add(defaultConsoleCSRFLifetime + time.Second) + expired := httptest.NewRecorder() + handler.ServeCorrelation(expired, validConsoleCorrelationRequest(session, correlationProof)) + if expired.Code != http.StatusForbidden || queryCalls != 0 { + t.Fatalf("expired proof status/query calls = %d/%d", expired.Code, queryCalls) + } + consoleNow = now + restarted := newHandler(0x77) + replayed := httptest.NewRecorder() + restarted.ServeCorrelation(replayed, validConsoleCorrelationRequest(session, correlationProof)) + if replayed.Code != http.StatusForbidden || queryCalls != 0 { + t.Fatalf("restarted proof status/query calls = %d/%d", replayed.Code, queryCalls) + } +} + +func TestProjectConsoleCorrelationFailsClosedOnStoredShape(t *testing.T) { + now := time.Date(2026, 7, 17, 17, 0, 0, 0, time.UTC) + request := hubfleet.CorrelationRequest{ResourceKind: "Deployment", Name: "payments", Namespace: "payments", HealthNot: "Healthy", Limit: consoleCorrelationReadLimit} + valid := fleet.QueryResult{ + Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Deployment", Namespace: "payments", Name: "payments"}, + Kind: fleet.FactHealth, Observed: json.RawMessage(`{"status":"Unknown"}`), ObservedAt: now, Source: "cluster-a", + }, + Workspace: "workspace-a", + }}, + Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + } + if projected, err := projectConsoleCorrelation(valid, "workspace-a", request); err != nil || len(projected.Matches) != 1 || !projected.Assessment.Complete { + t.Fatalf("valid projection = %#v, %v", projected, err) + } + + tests := []struct { + name string + mutate func(*fleet.QueryResult) + }{ + {name: "foreign workspace", mutate: func(result *fleet.QueryResult) { result.Facts[0].Workspace = "workspace-b" }}, + {name: "unexpected observation field", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"status":"Unknown","token":"secret"}`) + }}, + {name: "unsupported health", mutate: func(result *fleet.QueryResult) { result.Facts[0].Observed = json.RawMessage(`{"status":"Broken"}`) }}, + {name: "healthy mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].Observed = json.RawMessage(`{"status":"Healthy"}`) }}, + {name: "source mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].Source = "cluster-b" }}, + {name: "stale mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].StaleFor = "collection failed" }}, + {name: "untrusted stale text", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Stale = true + result.Facts[0].StaleFor = "credential=must-not-leak" + }}, + {name: "duplicate fact", mutate: func(result *fleet.QueryResult) { result.Facts = append(result.Facts, result.Facts[0]) }}, + {name: "oversized coverage", mutate: func(result *fleet.QueryResult) { + result.Coverage.Stale = make([]string, consoleCorrelationMaxCoverageScopes+1) + for index := range result.Coverage.Stale { + result.Coverage.Stale[index] = "cluster" + } + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + candidate := valid + candidate.Facts = append([]fleet.Fact(nil), valid.Facts...) + test.mutate(&candidate) + if _, err := projectConsoleCorrelation(candidate, "workspace-a", request); err == nil { + t.Fatal("unsafe stored shape was projected") + } + }) + } + + tooMany := valid + tooMany.Facts = make([]fleet.Fact, consoleCorrelationMaxMatches+1) + if _, err := projectConsoleCorrelation(tooMany, "workspace-a", request); err == nil { + t.Fatal("sentinel result above response bound was projected") + } + inconsistent := valid + inconsistent.Coverage = fleet.Coverage{Requested: 1, Reachable: 2} + projected, err := projectConsoleCorrelation(inconsistent, "workspace-a", request) + if err != nil || !projected.Assessment.Inconsistent || projected.Assessment.Complete { + t.Fatalf("inconsistent coverage was not preserved honestly: %#v, %v", projected, err) + } +} + +func validConsoleCorrelationRequest(session, proof string) *http.Request { + return consoleCorrelationRequest(session, proof, "kind=Deployment&name=payments&namespace=payments") +} + +func consoleCorrelationRequest(session, proof, rawQuery string) *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console/correlate?"+rawQuery, "workspace-a", session) + request.Header.Set(consoleCSRFHeader, proof) + request.Header.Set("Sec-Fetch-Site", "same-origin") + return request +} + +func consolePageProof(t *testing.T, page, name string) string { + t.Helper() + marker := ``) + if !found || proof == "" { + t.Fatalf("console page emitted malformed %s proof", name) + } + return proof +} diff --git a/internal/hubserver/console_cve_test.go b/internal/hubserver/console_cve_test.go new file mode 100644 index 0000000..7a7e5d5 --- /dev/null +++ b/internal/hubserver/console_cve_test.go @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestConsoleCVEUsesPurposeProofPEPAndMinimalProjection(t *testing.T) { + now := time.Date(2026, 7, 17, 22, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + var audits []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + audits = append(audits, event) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + digest := "sha256:" + strings.Repeat("a", 64) + queryCalls := 0 + cve := consoleTestCVE(t, consoleFleetQuerierFunc(func( + _ context.Context, scope tenancy.Scope, query fleet.Query, freshness time.Duration, gotNow time.Time, + ) (fleet.QueryResult, error) { + queryCalls++ + if scope.WorkspaceID() != "workspace-a" || scope.Subject() != "user:alice" || freshness != 5*time.Minute || gotNow.IsZero() || + query.Limit != consoleCVEReadLimit || len(query.Kinds) != 1 || query.Kinds[0] != fleet.FactCVE || + query.Selector.ResourceKind != "Image" || query.Selector.CVE != "CVE-2026-0001" || query.Selector.Image != "" { + t.Fatalf("CVE scope/query/freshness/now = %#v/%#v/%s/%s", scope, query, freshness, gotNow) + } + return consoleCVEQueryResult(now, digest), nil + }), enforcer) + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), Inventory: noopConsoleInventory(t), CVE: cve, PEP: enforcer, Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x74}, 192)), + }) + if err != nil { + t.Fatal(err) + } + + page, fleetProof := authenticatedConsolePage(t, handler, session, "workspace-a") + cveProof := consolePageProof(t, page.Body.String(), "sith-cve-csrf") + correlationProof := consolePageProof(t, page.Body.String(), "sith-correlation-csrf") + inventoryProof := consolePageProof(t, page.Body.String(), "sith-inventory-csrf") + if cveProof == "" || cveProof == fleetProof || cveProof == correlationProof || cveProof == inventoryProof || strings.Contains(page.Body.String(), session) { + t.Fatalf("purpose proof separation/session leak = %q/%q/%q/%q", fleetProof, correlationProof, inventoryProof, cveProof) + } + response := httptest.NewRecorder() + handler.ServeCVEIdentifier(response, validConsoleCVERequest(session, cveProof)) + if response.Code != http.StatusOK || queryCalls != 1 || len(audits) != 1 || audits[0].Verb != pep.VerbFleetCVEIdentifierSearch { + t.Fatalf("status/query calls/audits/body = %d/%d/%#v/%q", response.Code, queryCalls, audits, response.Body.String()) + } + assertConsoleSecurityHeaders(t, response.Header()) + for _, forbidden := range []string{`"observed"`, `"ids"`, `"attributes"`, `"provenance"`, `"workspace"`, `"source_kind"`, `"native_id"`, `"deep_link"`} { + if strings.Contains(response.Body.String(), forbidden) { + t.Fatalf("CVE response leaked %q: %s", forbidden, response.Body.String()) + } + } + var payload consoleCVEResponse + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if len(payload.Records) != 1 || payload.Records[0].Scope != "cluster-a" || payload.Records[0].ImageDigest != digest || + payload.Records[0].Identifier != "CVE-2026-0001" || payload.Records[0].Severity != "high" || + !payload.Records[0].ObservedAt.Equal(now.Add(-6*time.Minute)) || !payload.Records[0].Stale || payload.Records[0].StaleFor != "6m0s" || + payload.Assessment.Complete || !containsCoverageGap(payload.Assessment.Gaps, fleet.CoverageGapStale) || + !containsCoverageGap(payload.Assessment.Gaps, fleet.CoverageGapUnreachable) || + len(payload.Assessment.Stale) != 1 || payload.Assessment.Stale[0] != "cluster-a" || + len(payload.Assessment.Unreachable) != 1 || payload.Assessment.Unreachable[0] != "cluster-c" { + t.Fatalf("projected CVE evidence = %#v", payload) + } +} + +func TestConsoleCVERejectsUnsafeRequestsBeforeQuery(t *testing.T) { + now := time.Date(2026, 7, 17, 22, 30, 0, 0, time.UTC) + consoleNow := now + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + queryCalls := 0 + enforcer := fleetTestPEP(t, pep.AllowReadHook{}) + cve := consoleTestCVE(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + queryCalls++ + return fleet.QueryResult{}, nil + }), enforcer) + newHandler := func(fill byte) *ConsoleHandler { + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), Inventory: noopConsoleInventory(t), CVE: cve, PEP: enforcer, Now: func() time.Time { return consoleNow }, + Random: bytes.NewReader(bytes.Repeat([]byte{fill}, 192)), + }) + if err != nil { + t.Fatal(err) + } + return handler + } + handler := newHandler(0x36) + page, fleetProof := authenticatedConsolePage(t, handler, session, "workspace-a") + cveProof := consolePageProof(t, page.Body.String(), "sith-cve-csrf") + inventoryProof := consolePageProof(t, page.Body.String(), "sith-inventory-csrf") + correlationProof := consolePageProof(t, page.Body.String(), "sith-correlation-csrf") + + tests := []struct { + name string + request func() *http.Request + }{ + {name: "fleet proof", request: func() *http.Request { return validConsoleCVERequest(session, fleetProof) }}, + {name: "inventory proof", request: func() *http.Request { return validConsoleCVERequest(session, inventoryProof) }}, + {name: "correlation proof", request: func() *http.Request { return validConsoleCVERequest(session, correlationProof) }}, + {name: "missing proof", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.Header.Del(consoleCSRFHeader) + return request + }}, + {name: "duplicate proof", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.Header.Add(consoleCSRFHeader, cveProof) + return request + }}, + {name: "cross site", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.Header.Set("Sec-Fetch-Site", "cross-site") + return request + }}, + {name: "foreign workspace", request: func() *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-b/console/cves?identifier=CVE-2026-0001", "workspace-b", session) + request.Header.Set(consoleCSRFHeader, cveProof) + request.Header.Set("Sec-Fetch-Site", "same-origin") + return request + }}, + {name: "bearer fallback", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.Header.Set("Authorization", "Bearer "+session) + return request + }}, + {name: "duplicate cookie", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: session}) + return request + }}, + {name: "lowercase", request: func() *http.Request { return consoleCVERequest(session, cveProof, "identifier=cve-2026-0001") }}, + {name: "wildcard", request: func() *http.Request { return consoleCVERequest(session, cveProof, "identifier=CVE-2026-0001%2A") }}, + {name: "duplicate query", request: func() *http.Request { + return consoleCVERequest(session, cveProof, "identifier=CVE-2026-0001&identifier=CVE-2026-0002") + }}, + {name: "unknown query", request: func() *http.Request { return consoleCVERequest(session, cveProof, "identifier=CVE-2026-0001&limit=1") }}, + {name: "malformed query", request: func() *http.Request { return consoleCVERequest(session, cveProof, "identifier=%zz") }}, + {name: "noncanonical encoding", request: func() *http.Request { return consoleCVERequest(session, cveProof, "identifier=CVE%2D2026%2D0001") }}, + {name: "oversized identifier", request: func() *http.Request { + return consoleCVERequest(session, cveProof, "identifier=CVE-2026-"+strings.Repeat("1", 56)) + }}, + {name: "fragment", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.URL.Fragment = "private" + return request + }}, + {name: "path", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.URL.Path += "/" + return request + }}, + {name: "method", request: func() *http.Request { + request := validConsoleCVERequest(session, cveProof) + request.Method = http.MethodPost + return request + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeCVEIdentifier(response, test.request()) + if response.Code == http.StatusOK { + t.Fatalf("unsafe request succeeded: %q", response.Body.String()) + } + }) + } + if queryCalls != 0 { + t.Fatalf("unsafe requests reached query %d times", queryCalls) + } + + consoleNow = now.Add(defaultConsoleCSRFLifetime + time.Second) + expired := httptest.NewRecorder() + handler.ServeCVEIdentifier(expired, validConsoleCVERequest(session, cveProof)) + if expired.Code != http.StatusForbidden || queryCalls != 0 { + t.Fatalf("expired proof status/query calls = %d/%d", expired.Code, queryCalls) + } + consoleNow = now + restarted := newHandler(0x78) + replayed := httptest.NewRecorder() + restarted.ServeCVEIdentifier(replayed, validConsoleCVERequest(session, cveProof)) + if replayed.Code != http.StatusForbidden || queryCalls != 0 { + t.Fatalf("restarted proof status/query calls = %d/%d", replayed.Code, queryCalls) + } +} + +func TestConsoleCVEHidesServiceErrors(t *testing.T) { + now := time.Date(2026, 7, 17, 23, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + enforcer := fleetTestPEP(t, pep.AllowReadHook{}) + cve := consoleTestCVE(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, errors.New("database password=secret topology=private") + }), enforcer) + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), Inventory: noopConsoleInventory(t), CVE: cve, PEP: enforcer, Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x66}, 192)), + }) + if err != nil { + t.Fatal(err) + } + page, _ := authenticatedConsolePage(t, handler, session, "workspace-a") + proof := consolePageProof(t, page.Body.String(), "sith-cve-csrf") + response := httptest.NewRecorder() + handler.ServeCVEIdentifier(response, validConsoleCVERequest(session, proof)) + if response.Code != http.StatusServiceUnavailable || response.Body.String() != "{\"error\":\"cve_evidence_unavailable\"}\n" || strings.Contains(response.Body.String(), "secret") { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } +} + +func TestProjectConsoleCVEFailsClosedOnStoredShape(t *testing.T) { + now := time.Date(2026, 7, 17, 23, 30, 0, 0, time.UTC) + digest := "sha256:" + strings.Repeat("a", 64) + request := hubfleet.CVEIdentifierSearchRequest{Identifier: "CVE-2026-0001", Limit: consoleCVEReadLimit} + valid := consoleCVEQueryResult(now, digest) + projected, err := projectConsoleCVEIdentifier(valid, "workspace-a", request) + if err != nil || len(projected.Records) != 1 || projected.Records[0].ImageDigest != digest || projected.Records[0].Severity != "high" { + t.Fatalf("valid projection = %#v, %v", projected, err) + } + + tests := []struct { + name string + mutate func(*fleet.QueryResult) + }{ + {name: "foreign workspace", mutate: func(result *fleet.QueryResult) { result.Facts[0].Workspace = "workspace-b" }}, + {name: "wrong kind", mutate: func(result *fleet.QueryResult) { result.Facts[0].Kind = fleet.FactInventory }}, + {name: "source mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].Source = "cluster-b" }}, + {name: "tag instead of digest", mutate: func(result *fleet.QueryResult) { result.Facts[0].Ref.Name = "registry.example/api:latest" }}, + {name: "attributes", mutate: func(result *fleet.QueryResult) { result.Facts[0].Ref.Attributes = map[string]string{"token": "secret"} }}, + {name: "provenance", mutate: func(result *fleet.QueryResult) { result.Facts[0].Provenance.NativeID = "private" }}, + {name: "display", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Display = []fleet.DisplayField{{Name: "token", Value: "secret"}} + }}, + {name: "selector mismatch", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ids":["CVE-2026-0002"],"severity":"high"}`) + }}, + {name: "digest mismatch", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"image":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","ids":["CVE-2026-0001"],"severity":"high"}`) + }}, + {name: "unknown field", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ids":["CVE-2026-0001"],"severity":"high","token":"secret"}`) + }}, + {name: "duplicate field", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ids":["CVE-2026-0001"],"ids":["CVE-2026-0002"],"severity":"high"}`) + }}, + {name: "untrusted stale text", mutate: func(result *fleet.QueryResult) { result.Facts[0].StaleFor = "credential=must-not-leak" }}, + {name: "duplicate fact", mutate: func(result *fleet.QueryResult) { result.Facts = append(result.Facts, result.Facts[0]) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + candidate := valid + candidate.Facts = append([]fleet.Fact(nil), valid.Facts...) + test.mutate(&candidate) + if _, err := projectConsoleCVEIdentifier(candidate, "workspace-a", request); err == nil { + t.Fatal("unsafe stored shape was projected") + } + }) + } + + tooMany := valid + tooMany.Facts = make([]fleet.Fact, consoleCVEMaxRecords+1) + for index := range tooMany.Facts { + distinctDigest := fmt.Sprintf("sha256:%064x", index+1) + fact := valid.Facts[0] + fact.Ref.Scope = fmt.Sprintf("cluster-%03d", index+1) + fact.Ref.Name = distinctDigest + fact.Source = fact.Ref.Scope + fact.Observed = json.RawMessage(`{"image":"` + distinctDigest + `","ids":["CVE-2026-0001"],"severity":"high"}`) + tooMany.Facts[index] = fact + } + if _, err := projectConsoleCVEIdentifier(tooMany, "workspace-a", request); err == nil { + t.Fatal("sentinel result above response bound was projected") + } + inconsistent := valid + inconsistent.Coverage = fleet.Coverage{Requested: 1, Reachable: 2} + projected, err = projectConsoleCVEIdentifier(inconsistent, "workspace-a", request) + if err != nil || !projected.Assessment.Inconsistent || projected.Assessment.Complete { + t.Fatalf("inconsistent coverage was not preserved honestly: %#v, %v", projected, err) + } +} + +func FuzzProjectConsoleCVEObserved(f *testing.F) { + f.Add([]byte(`{"image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ids":["CVE-2026-0001"],"severity":"high"}`)) + f.Add([]byte(`{"image":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","ids":["CVE-2026-0001"],"ids":["CVE-2026-0002"],"severity":"high"}`)) + f.Fuzz(func(t *testing.T, observed []byte) { + now := time.Date(2026, 7, 17, 23, 45, 0, 0, time.UTC) + digest := "sha256:" + strings.Repeat("a", 64) + result := consoleCVEQueryResult(now, digest) + result.Facts[0].Observed = observed + projected, err := projectConsoleCVEIdentifier(result, "workspace-a", hubfleet.CVEIdentifierSearchRequest{Identifier: "CVE-2026-0001", Limit: consoleCVEReadLimit}) + if err != nil { + return + } + encoded, err := json.Marshal(projected) + if err != nil || len(projected.Records) != 1 || strings.Contains(string(encoded), `"observed"`) || strings.Contains(string(encoded), `"ids"`) { + t.Fatalf("successful projection violated closed response: %#v, %s, %v", projected, encoded, err) + } + }) +} + +func consoleCVEQueryResult(now time.Time, digest string) fleet.QueryResult { + return fleet.QueryResult{ + Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Image", Name: digest}, + Kind: fleet.FactCVE, Observed: json.RawMessage(`{"image":"` + digest + `","ids":["CVE-2026-0001","CVE-2026-0002"],"severity":"high"}`), + ObservedAt: now.Add(-6 * time.Minute), Source: "cluster-a", + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, + Workspace: "workspace-a", Stale: true, StaleFor: "6m0s", + }}, + Coverage: fleet.Coverage{Requested: 3, Reachable: 2, Unreachable: []string{"cluster-c"}, Stale: []string{"cluster-a"}}, + } +} + +func validConsoleCVERequest(session, proof string) *http.Request { + return consoleCVERequest(session, proof, "identifier=CVE-2026-0001") +} + +func consoleCVERequest(session, proof, rawQuery string) *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console/cves?"+rawQuery, "workspace-a", session) + request.Header.Set(consoleCSRFHeader, proof) + request.Header.Set("Sec-Fetch-Site", "same-origin") + return request +} diff --git a/internal/hubserver/console_inventory_test.go b/internal/hubserver/console_inventory_test.go new file mode 100644 index 0000000..a5df475 --- /dev/null +++ b/internal/hubserver/console_inventory_test.go @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestConsoleInventoryUsesPurposeProofPEPAndMinimalProjection(t *testing.T) { + now := time.Date(2026, 7, 17, 19, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + var audits []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + audits = append(audits, event) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + queryCalls := 0 + inventory := consoleTestInventory(t, consoleFleetQuerierFunc(func( + _ context.Context, scope tenancy.Scope, query fleet.Query, freshness time.Duration, gotNow time.Time, + ) (fleet.QueryResult, error) { + queryCalls++ + if scope.WorkspaceID() != "workspace-a" || scope.Subject() != "user:alice" || freshness != 5*time.Minute || gotNow.IsZero() || + query.Limit != consoleInventoryReadLimit || len(query.Kinds) != 1 || query.Kinds[0] != fleet.FactInventory || + query.Selector.ResourceKind != "Deployment" || query.Selector.Name != "payments" || query.Selector.Namespace != "payments" || + query.Selector.NamePrefix != "" || len(query.Selector.Labels) != 0 { + t.Fatalf("inventory scope/query/freshness/now = %#v/%#v/%s/%s", scope, query, freshness, gotNow) + } + return fleet.QueryResult{ + Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Deployment", Namespace: "payments", Name: "payments"}, + Kind: fleet.FactInventory, Observed: json.RawMessage(`{"resource":"Deployment","replicas":4,"available_replicas":3,"generation":9}`), + ObservedAt: now.Add(-6 * time.Minute), Source: "cluster-a", + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, + Workspace: "workspace-a", Stale: true, StaleFor: "6m0s", + }}, + Coverage: fleet.Coverage{Requested: 3, Reachable: 2, Unreachable: []string{"cluster-c"}, Stale: []string{"cluster-a"}}, + }, nil + }), enforcer) + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), Inventory: inventory, CVE: noopConsoleCVE(t), PEP: enforcer, Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x73}, 192)), + }) + if err != nil { + t.Fatal(err) + } + + page, fleetProof := authenticatedConsolePage(t, handler, session, "workspace-a") + correlationProof := consolePageProof(t, page.Body.String(), "sith-correlation-csrf") + inventoryProof := consolePageProof(t, page.Body.String(), "sith-inventory-csrf") + if inventoryProof == "" || inventoryProof == fleetProof || inventoryProof == correlationProof || strings.Contains(page.Body.String(), session) { + t.Fatalf("purpose proof separation/session leak = %q/%q/%q", fleetProof, correlationProof, inventoryProof) + } + response := httptest.NewRecorder() + handler.ServeInventory(response, validConsoleInventoryRequest(session, inventoryProof)) + if response.Code != http.StatusOK || queryCalls != 1 || len(audits) != 1 || audits[0].Verb != pep.VerbFleetInventorySearch { + t.Fatalf("status/query calls/audits/body = %d/%d/%#v/%q", response.Code, queryCalls, audits, response.Body.String()) + } + assertConsoleSecurityHeaders(t, response.Header()) + for _, forbidden := range []string{`"observed"`, `"image_digests"`, `"attributes"`, `"provenance"`, `"workspace"`, `"source_kind"`, `"native_id"`, `"deep_link"`} { + if strings.Contains(response.Body.String(), forbidden) { + t.Fatalf("inventory response leaked %q: %s", forbidden, response.Body.String()) + } + } + var payload consoleInventoryResponse + if err := json.NewDecoder(response.Body).Decode(&payload); err != nil { + t.Fatal(err) + } + if len(payload.Records) != 1 || payload.Records[0].Scope != "cluster-a" || payload.Records[0].Replicas == nil || *payload.Records[0].Replicas != 4 || + payload.Records[0].AvailableReplicas == nil || *payload.Records[0].AvailableReplicas != 3 || payload.Records[0].Generation != 9 || + !payload.Records[0].Stale || payload.Records[0].StaleFor != "6m0s" || payload.Assessment.Complete { + t.Fatalf("projected inventory = %#v", payload) + } +} + +func TestConsoleInventoryRejectsUnsafeRequestsBeforeQuery(t *testing.T) { + now := time.Date(2026, 7, 17, 20, 0, 0, 0, time.UTC) + consoleNow := now + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + queryCalls := 0 + enforcer := fleetTestPEP(t, pep.AllowReadHook{}) + inventory := consoleTestInventory(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + queryCalls++ + return fleet.QueryResult{}, nil + }), enforcer) + newHandler := func(fill byte) *ConsoleHandler { + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), Inventory: inventory, CVE: noopConsoleCVE(t), PEP: enforcer, Now: func() time.Time { return consoleNow }, + Random: bytes.NewReader(bytes.Repeat([]byte{fill}, 192)), + }) + if err != nil { + t.Fatal(err) + } + return handler + } + handler := newHandler(0x35) + page, fleetProof := authenticatedConsolePage(t, handler, session, "workspace-a") + inventoryProof := consolePageProof(t, page.Body.String(), "sith-inventory-csrf") + correlationProof := consolePageProof(t, page.Body.String(), "sith-correlation-csrf") + + tests := []struct { + name string + request func() *http.Request + }{ + {name: "fleet proof", request: func() *http.Request { return validConsoleInventoryRequest(session, fleetProof) }}, + {name: "correlation proof", request: func() *http.Request { return validConsoleInventoryRequest(session, correlationProof) }}, + {name: "missing proof", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.Header.Del(consoleCSRFHeader) + return request + }}, + {name: "duplicate proof", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.Header.Add(consoleCSRFHeader, inventoryProof) + return request + }}, + {name: "cross site", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.Header.Set("Sec-Fetch-Site", "cross-site") + return request + }}, + {name: "foreign workspace", request: func() *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-b/console/inventory?kind=Deployment", "workspace-b", session) + request.Header.Set(consoleCSRFHeader, inventoryProof) + request.Header.Set("Sec-Fetch-Site", "same-origin") + return request + }}, + {name: "bearer fallback", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.Header.Set("Authorization", "Bearer "+session) + return request + }}, + {name: "duplicate cookie", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: session}) + return request + }}, + {name: "unknown query", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=Pod&selector=all") }}, + {name: "malformed query", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=%zz") }}, + {name: "duplicate query", request: func() *http.Request { + return consoleInventoryRequest(session, inventoryProof, "kind=Deployment&kind=Pod") + }}, + {name: "Secret", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=Secret") }}, + {name: "unsupported kind", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=Service") }}, + {name: "blank name", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=Pod&name=") }}, + {name: "blank namespace", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=Pod&namespace=") }}, + {name: "noncanonical order", request: func() *http.Request { + return consoleInventoryRequest(session, inventoryProof, "namespace=payments&kind=Pod") + }}, + {name: "untrimmed name", request: func() *http.Request { return consoleInventoryRequest(session, inventoryProof, "kind=Pod&name=+api") }}, + {name: "oversized kind", request: func() *http.Request { + return consoleInventoryRequest(session, inventoryProof, "kind="+strings.Repeat("a", 129)) + }}, + {name: "fragment", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.URL.Fragment = "private" + return request + }}, + {name: "path", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.URL.Path += "/" + return request + }}, + {name: "method", request: func() *http.Request { + request := validConsoleInventoryRequest(session, inventoryProof) + request.Method = http.MethodPost + return request + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeInventory(response, test.request()) + if response.Code == http.StatusOK { + t.Fatalf("unsafe request succeeded: %q", response.Body.String()) + } + }) + } + if queryCalls != 0 { + t.Fatalf("unsafe requests reached query %d times", queryCalls) + } + + consoleNow = now.Add(defaultConsoleCSRFLifetime + time.Second) + expired := httptest.NewRecorder() + handler.ServeInventory(expired, validConsoleInventoryRequest(session, inventoryProof)) + if expired.Code != http.StatusForbidden || queryCalls != 0 { + t.Fatalf("expired proof status/query calls = %d/%d", expired.Code, queryCalls) + } + consoleNow = now + restarted := newHandler(0x79) + replayed := httptest.NewRecorder() + restarted.ServeInventory(replayed, validConsoleInventoryRequest(session, inventoryProof)) + if replayed.Code != http.StatusForbidden || queryCalls != 0 { + t.Fatalf("restarted proof status/query calls = %d/%d", replayed.Code, queryCalls) + } +} + +func TestConsoleInventoryHidesServiceErrors(t *testing.T) { + now := time.Date(2026, 7, 17, 20, 30, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + enforcer := fleetTestPEP(t, pep.AllowReadHook{}) + inventory := consoleTestInventory(t, consoleFleetQuerierFunc(func(context.Context, tenancy.Scope, fleet.Query, time.Duration, time.Time) (fleet.QueryResult, error) { + return fleet.QueryResult{}, errors.New("database password=secret topology=private") + }), enforcer) + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), Inventory: inventory, CVE: noopConsoleCVE(t), PEP: enforcer, Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x65}, 192)), + }) + if err != nil { + t.Fatal(err) + } + page, _ := authenticatedConsolePage(t, handler, session, "workspace-a") + proof := consolePageProof(t, page.Body.String(), "sith-inventory-csrf") + response := httptest.NewRecorder() + handler.ServeInventory(response, validConsoleInventoryRequest(session, proof)) + if response.Code != http.StatusServiceUnavailable || response.Body.String() != "{\"error\":\"inventory_unavailable\"}\n" || strings.Contains(response.Body.String(), "secret") { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } +} + +func TestProjectConsoleInventoryFailsClosedOnStoredShape(t *testing.T) { + now := time.Date(2026, 7, 17, 21, 0, 0, 0, time.UTC) + request := hubfleet.InventorySearchRequest{ResourceKind: "Pod", Namespace: "payments", Limit: consoleInventoryReadLimit} + valid := fleet.QueryResult{ + Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Pod", Namespace: "payments", Name: "api-0"}, + Kind: fleet.FactInventory, + Observed: json.RawMessage(`{"resource":"Pod","ready":1,"generation":3,"image_digests":["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}`), + ObservedAt: now, Source: "cluster-a", Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, + Workspace: "workspace-a", + }}, + Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + } + projected, err := projectConsoleInventory(valid, "workspace-a", request) + if err != nil || len(projected.Records) != 1 || projected.Records[0].Ready == nil || !*projected.Records[0].Ready || !projected.Assessment.Complete { + t.Fatalf("valid projection = %#v, %v", projected, err) + } + encoded, err := json.Marshal(projected) + if err != nil || strings.Contains(string(encoded), "sha256:") || strings.Contains(string(encoded), "image_digests") { + t.Fatalf("projected Pod leaked digest: %s, %v", encoded, err) + } + + tests := []struct { + name string + mutate func(*fleet.QueryResult) + }{ + {name: "foreign workspace", mutate: func(result *fleet.QueryResult) { result.Facts[0].Workspace = "workspace-b" }}, + {name: "unexpected field", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"resource":"Pod","ready":1,"generation":3,"token":"secret"}`) + }}, + {name: "duplicate field", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"resource":"Pod","ready":1,"ready":0,"generation":3}`) + }}, + {name: "wrong resource", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"resource":"Deployment","ready":1,"generation":3}`) + }}, + {name: "fractional count", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"resource":"Pod","ready":0.5,"generation":3}`) + }}, + {name: "bad digest", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Observed = json.RawMessage(`{"resource":"Pod","ready":1,"generation":3,"image_digests":["latest"]}`) + }}, + {name: "source mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].Source = "cluster-b" }}, + {name: "attributes", mutate: func(result *fleet.QueryResult) { result.Facts[0].Ref.Attributes = map[string]string{"token": "secret"} }}, + {name: "provenance", mutate: func(result *fleet.QueryResult) { result.Facts[0].Provenance.NativeID = "private" }}, + {name: "display", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Display = []fleet.DisplayField{{Name: "token", Value: "secret"}} + }}, + {name: "selector mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].Ref.Namespace = "other" }}, + {name: "stale mismatch", mutate: func(result *fleet.QueryResult) { result.Facts[0].StaleFor = "collection failed" }}, + {name: "untrusted stale text", mutate: func(result *fleet.QueryResult) { + result.Facts[0].Stale = true + result.Facts[0].StaleFor = "credential=must-not-leak" + }}, + {name: "duplicate fact", mutate: func(result *fleet.QueryResult) { result.Facts = append(result.Facts, result.Facts[0]) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + candidate := valid + candidate.Facts = append([]fleet.Fact(nil), valid.Facts...) + test.mutate(&candidate) + if _, err := projectConsoleInventory(candidate, "workspace-a", request); err == nil { + t.Fatal("unsafe stored shape was projected") + } + }) + } + + tooMany := valid + tooMany.Facts = make([]fleet.Fact, consoleInventoryMaxRecords+1) + if _, err := projectConsoleInventory(tooMany, "workspace-a", request); err == nil { + t.Fatal("sentinel result above response bound was projected") + } + inconsistent := valid + inconsistent.Coverage = fleet.Coverage{Requested: 1, Reachable: 2} + projected, err = projectConsoleInventory(inconsistent, "workspace-a", request) + if err != nil || !projected.Assessment.Inconsistent || projected.Assessment.Complete { + t.Fatalf("inconsistent coverage was not preserved honestly: %#v, %v", projected, err) + } +} + +func FuzzProjectConsoleInventoryObserved(f *testing.F) { + f.Add([]byte(`{"resource":"Pod","ready":1,"generation":3}`)) + f.Add([]byte(`{"resource":"Pod","ready":1,"ready":0,"generation":3}`)) + f.Add([]byte(`{"resource":"Pod","ready":1,"generation":3,"image_digests":["sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"]}`)) + f.Fuzz(func(t *testing.T, observed []byte) { + now := time.Date(2026, 7, 17, 21, 30, 0, 0, time.UTC) + result := fleet.QueryResult{ + Facts: []fleet.Fact{{ + Evidence: fleet.Evidence{ + Ref: fleet.ResourceRef{SourceKind: hubfleet.SourceKind, Scope: "cluster-a", Kind: "Pod", Namespace: "payments", Name: "api-0"}, + Kind: fleet.FactInventory, Observed: observed, ObservedAt: now, Source: "cluster-a", + Provenance: fleet.Provenance{Adapter: hubfleet.SourceKind, ProtocolV: "1.0.0"}, + }, + Workspace: "workspace-a", + }}, + Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + } + projected, err := projectConsoleInventory(result, "workspace-a", hubfleet.InventorySearchRequest{ResourceKind: "Pod", Namespace: "payments", Limit: consoleInventoryReadLimit}) + if err != nil { + return + } + encoded, err := json.Marshal(projected) + if err != nil || len(projected.Records) != 1 || strings.Contains(string(encoded), "image_digests") || strings.Contains(string(encoded), `"observed"`) { + t.Fatalf("successful projection violated the closed response: %#v, %s, %v", projected, encoded, err) + } + }) +} + +func validConsoleInventoryRequest(session, proof string) *http.Request { + return consoleInventoryRequest(session, proof, "kind=Deployment&name=payments&namespace=payments") +} + +func consoleInventoryRequest(session, proof, rawQuery string) *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console/inventory?"+rawQuery, "workspace-a", session) + request.Header.Set(consoleCSRFHeader, proof) + request.Header.Set("Sec-Fetch-Site", "same-origin") + return request +} diff --git a/internal/hubserver/console_test.go b/internal/hubserver/console_test.go new file mode 100644 index 0000000..dce3659 --- /dev/null +++ b/internal/hubserver/console_test.go @@ -0,0 +1,540 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "html/template" + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/fstest" + "time" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/pep" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestConsoleHandlerReadsOnlySignedWorkspaceWithCSRFAndPEP(t *testing.T) { + now := time.Date(2026, 7, 17, 9, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + var audits []pep.AuditEvent + enforcer, err := pep.NewEnforcer(pep.Config{ + Hook: pep.AllowReadHook{}, + Auditor: pep.AuditFunc(func(_ context.Context, event pep.AuditEvent) error { + audits = append(audits, event) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + readerCalled := 0 + var observed []hubfleet.FleetReadObservation + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Correlator: noopConsoleCorrelator(t), + Inventory: noopConsoleInventory(t), + CVE: noopConsoleCVE(t), + Reader: fleetReaderFunc(func(_ context.Context, scope tenancy.Scope, freshness time.Duration, gotNow time.Time) (fleet.FleetResult, error) { + readerCalled++ + if scope.WorkspaceID() != "workspace-a" || scope.Subject() != "user:alice" || freshness != 5*time.Minute || gotNow.IsZero() { + t.Fatalf("scope/freshness/now = %#v/%s/%s", scope, freshness, gotNow) + } + return fleet.FleetResult{ + Clusters: []fleet.Cluster{ + {Name: "spoke-a", SourceKind: hubfleet.SourceKind, Reachable: true, ObservedAt: now.Add(-time.Minute)}, + {Name: "spoke-b", SourceKind: hubfleet.SourceKind, Reachable: true, ObservedAt: now.Add(-10 * time.Minute)}, + {Name: "spoke-c", SourceKind: hubfleet.SourceKind, Reachable: false}, + }, + Coverage: fleet.Coverage{Requested: 4, Reachable: 2, Unreachable: []string{"spoke-c"}, Stale: []string{"spoke-b"}}, + }, nil + }), + PEP: enforcer, ReadObserver: fleetReadObserverFunc(func(observation hubfleet.FleetReadObservation) { + observed = append(observed, observation) + }), + Now: func() time.Time { return now }, Random: bytes.NewReader(bytes.Repeat([]byte{0x5a}, 192)), + }) + if err != nil { + t.Fatal(err) + } + + pageResponse, csrfToken := authenticatedConsolePage(t, handler, session, "workspace-a") + if pageResponse.Code != http.StatusOK || strings.Contains(pageResponse.Body.String(), session) || csrfToken == "" { + t.Fatalf("page status/token/body = %d/%q/%q", pageResponse.Code, csrfToken, pageResponse.Body.String()) + } + for _, forbidden := range []string{"`) + if !found { + t.Fatal("ServeMux page emitted malformed CSRF token") + } + + fleetRequest := httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/workspaces/workspace-a/console/fleet", nil) + fleetRequest.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: session}) + fleetRequest.Header.Set(consoleCSRFHeader, csrfToken) + fleetRequest.Header.Set("Sec-Fetch-Site", "same-origin") + fleetResponse := httptest.NewRecorder() + mux.ServeHTTP(fleetResponse, fleetRequest) + if fleetResponse.Code != http.StatusOK || readerCalled != 1 { + t.Fatalf("ServeMux fleet status/reader/body = %d/%d/%q", fleetResponse.Code, readerCalled, fleetResponse.Body.String()) + } + + mutation := httptest.NewRequest(http.MethodPost, "https://hub.sith.test/v1/workspaces/workspace-a/console/fleet", nil) + mutationResponse := httptest.NewRecorder() + mux.ServeHTTP(mutationResponse, mutation) + if mutationResponse.Code != http.StatusMethodNotAllowed || readerCalled != 1 { + t.Fatalf("ServeMux mutation status/reader = %d/%d", mutationResponse.Code, readerCalled) + } +} + +func TestConsoleHandlerForwardsUniformAuthRefusalsToConfiguredObserver(t *testing.T) { + now := time.Date(2026, 7, 18, 8, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + var events []AuthEvent + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, AuthObserver: AuthObserverFunc(func(event AuthEvent) { + events = append(events, event) + }), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + t.Fatal("unauthenticated request reached reader") + return fleet.FleetResult{}, nil + }), + Correlator: noopConsoleCorrelator(t), + Inventory: noopConsoleInventory(t), + CVE: noopConsoleCVE(t), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), + Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x7a}, 192)), + }) + if err != nil { + t.Fatal(err) + } + + for _, session := range []string{"", "token=secret"} { + response := httptest.NewRecorder() + handler.ServePage(response, consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console", "workspace-a", session)) + if response.Code != http.StatusUnauthorized || response.Body.String() != "{\"error\":\"unauthorized\"}\n" { + t.Fatalf("session %q status/body = %d/%q", session, response.Code, response.Body.String()) + } + } + foreign := consoleTestRequest( + http.MethodGet, + "/v1/workspaces/workspace-b/console", + "workspace-b", + signHubTestToken(t, hubValidClaims(now), privateKey), + ) + foreignResponse := httptest.NewRecorder() + handler.ServePage(foreignResponse, foreign) + if foreignResponse.Code != http.StatusForbidden || foreignResponse.Body.String() != "{\"error\":\"forbidden\"}\n" { + t.Fatalf("foreign workspace status/body = %d/%q", foreignResponse.Code, foreignResponse.Body.String()) + } + if len(events) != 3 || events[0] != (AuthEvent{Outcome: AuthOutcomeRefused}) || events[1] != events[0] || + events[2] != (AuthEvent{Outcome: AuthOutcomeAccepted}) { + t.Fatalf("console authentication events = %#v, want two refusals then accepted before authorization", events) + } +} + +func TestConsoleHandlerFailsClosedBeforeReader(t *testing.T) { + verifierNow := time.Date(2026, 7, 17, 10, 0, 0, 0, time.UTC) + consoleNow := verifierNow + verifier, privateKey := fleetTestVerifier(t, verifierNow) + session := signHubTestToken(t, hubValidClaims(verifierNow), privateKey) + readerCalled := 0 + newHandler := func(fill byte) *ConsoleHandler { + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Correlator: noopConsoleCorrelator(t), + Inventory: noopConsoleInventory(t), + CVE: noopConsoleCVE(t), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + readerCalled++ + return fleet.FleetResult{Coverage: fleet.Coverage{}}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), Now: func() time.Time { return consoleNow }, + Random: bytes.NewReader(bytes.Repeat([]byte{fill}, 192)), + }) + if err != nil { + t.Fatal(err) + } + return handler + } + handler := newHandler(0x31) + _, csrfToken := authenticatedConsolePage(t, handler, session, "workspace-a") + + tests := []struct { + name string + request func() *http.Request + wantCode int + wantError string + }{ + {name: "missing session", request: func() *http.Request { + return consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console/fleet", "workspace-a", "") + }, wantCode: http.StatusUnauthorized, wantError: "unauthorized"}, + {name: "authorization fallback refused", request: func() *http.Request { + request := validConsoleFleetRequest(session, csrfToken) + request.Header.Set("Authorization", "Bearer "+session) + return request + }, wantCode: http.StatusUnauthorized, wantError: "unauthorized"}, + {name: "duplicate session cookie", request: func() *http.Request { + request := validConsoleFleetRequest(session, csrfToken) + request.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: session}) + return request + }, wantCode: http.StatusUnauthorized, wantError: "unauthorized"}, + {name: "foreign workspace", request: func() *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-b/console/fleet", "workspace-b", session) + request.Header.Set(consoleCSRFHeader, csrfToken) + return request + }, wantCode: http.StatusForbidden, wantError: "forbidden"}, + {name: "missing CSRF", request: func() *http.Request { + request := validConsoleFleetRequest(session, csrfToken) + request.Header.Del(consoleCSRFHeader) + return request + }, wantCode: http.StatusForbidden, wantError: "forbidden"}, + {name: "wrong CSRF", request: func() *http.Request { + request := validConsoleFleetRequest(session, csrfToken) + request.Header.Set(consoleCSRFHeader, strings.Repeat("x", len(csrfToken))) + return request + }, wantCode: http.StatusForbidden, wantError: "forbidden"}, + {name: "cross-site metadata", request: func() *http.Request { + request := validConsoleFleetRequest(session, csrfToken) + request.Header.Set("Sec-Fetch-Site", "cross-site") + return request + }, wantCode: http.StatusForbidden, wantError: "forbidden"}, + {name: "query rejected", request: func() *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console/fleet?refresh=true", "workspace-a", session) + request.Header.Set(consoleCSRFHeader, csrfToken) + return request + }, wantCode: http.StatusNotFound, wantError: "not_found"}, + {name: "method rejected", request: func() *http.Request { + request := consoleTestRequest(http.MethodPost, "/v1/workspaces/workspace-a/console/fleet", "workspace-a", session) + request.Header.Set(consoleCSRFHeader, csrfToken) + return request + }, wantCode: http.StatusNotFound, wantError: "not_found"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := httptest.NewRecorder() + handler.ServeFleet(response, test.request()) + if response.Code != test.wantCode || !strings.Contains(response.Body.String(), test.wantError) { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + }) + } + if readerCalled != 0 { + t.Fatalf("invalid requests reached reader %d times", readerCalled) + } + + consoleNow = consoleNow.Add(defaultConsoleCSRFLifetime + time.Second) + expired := validConsoleFleetRequest(session, csrfToken) + expiredResponse := httptest.NewRecorder() + handler.ServeFleet(expiredResponse, expired) + if expiredResponse.Code != http.StatusForbidden || readerCalled != 0 { + t.Fatalf("expired CSRF status/reader = %d/%d", expiredResponse.Code, readerCalled) + } + + consoleNow = verifierNow + restarted := newHandler(0x77) + replay := validConsoleFleetRequest(session, csrfToken) + replayResponse := httptest.NewRecorder() + restarted.ServeFleet(replayResponse, replay) + if replayResponse.Code != http.StatusForbidden || readerCalled != 0 { + t.Fatalf("post-restart replay status/reader = %d/%d", replayResponse.Code, readerCalled) + } +} + +func TestConsoleHandlerHidesReaderErrorsAndServesFixedAssets(t *testing.T) { + now := time.Date(2026, 7, 17, 11, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + var observed []hubfleet.FleetReadObservation + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Correlator: noopConsoleCorrelator(t), + Inventory: noopConsoleInventory(t), + CVE: noopConsoleCVE(t), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, errors.New("database=secret topology=private") + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), ReadObserver: fleetReadObserverFunc(func(observation hubfleet.FleetReadObservation) { + observed = append(observed, observation) + }), + Now: func() time.Time { return now }, Random: bytes.NewReader(bytes.Repeat([]byte{0x41}, 192)), + }) + if err != nil { + t.Fatal(err) + } + _, csrfToken := authenticatedConsolePage(t, handler, session, "workspace-a") + response := httptest.NewRecorder() + handler.ServeFleet(response, validConsoleFleetRequest(session, csrfToken)) + if response.Code != http.StatusServiceUnavailable || response.Body.String() != "{\"error\":\"fleet_unavailable\"}\n" || strings.Contains(response.Body.String(), "secret") { + t.Fatalf("status/body = %d/%q", response.Code, response.Body.String()) + } + if len(observed) != 1 || observed[0] != (hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeError, Freshness: hubfleet.FleetFreshnessOutcomeError, + }) { + t.Fatalf("console error observations = %#v, want error/error", observed) + } + + for _, asset := range []struct { + name string + serve func(http.ResponseWriter, *http.Request) + contentType string + required string + }{ + {name: "CSS", serve: handler.ServeCSS, contentType: "text/css; charset=utf-8", required: "prefers-reduced-motion"}, + {name: "JavaScript", serve: handler.ServeJavaScript, contentType: "text/javascript; charset=utf-8", required: "Inventory read complete."}, + } { + t.Run(asset.name, func(t *testing.T) { + path := "/v1/console/assets/console.css" + if asset.name == "JavaScript" { + path = "/v1/console/assets/console.js" + } + request := httptest.NewRequest(http.MethodGet, "https://hub.sith.test"+path, nil) + response := httptest.NewRecorder() + asset.serve(response, request) + body := response.Body.String() + if response.Code != http.StatusOK || response.Header().Get("Content-Type") != asset.contentType || !strings.Contains(body, asset.required) { + t.Fatalf("status/type/body = %d/%q/%q", response.Code, response.Header().Get("Content-Type"), body) + } + for _, forbidden := range []string{"localStorage", "sessionStorage", "innerHTML", "setInterval", "fleet:refresh", "/api/v1/exec", "/api/v1/edit"} { + if strings.Contains(body, forbidden) { + t.Fatalf("asset contains forbidden capability %q", forbidden) + } + } + }) + } +} + +func TestNewConsoleHandlerRejectsUnsafeConfiguration(t *testing.T) { + for _, config := range []ConsoleHandlerConfig{ + {}, + {Verifier: authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { return tenancy.Principal{}, nil })}, + {Verifier: authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { return tenancy.Principal{}, nil }), Reader: fleetReaderFunc(nil), PEP: nil}, + {Verifier: authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { return tenancy.Principal{}, nil }), Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), PEP: fleetTestPEP(t, pep.AllowReadHook{})}, + {Verifier: authVerifierFunc(func(context.Context, string) (tenancy.Principal, error) { return tenancy.Principal{}, nil }), Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), Correlator: noopConsoleCorrelator(t), Inventory: noopConsoleInventory(t), CVE: noopConsoleCVE(t), PEP: fleetTestPEP(t, pep.AllowReadHook{}), CSRFLifetime: 30 * time.Second}, + } { + if _, err := NewConsoleHandler(config); err == nil { + t.Fatalf("NewConsoleHandler accepted unsafe config %#v", config) + } + } + verifier, _ := fleetTestVerifier(t, time.Date(2026, 7, 17, 12, 0, 0, 0, time.UTC)) + if _, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, + Correlator: noopConsoleCorrelator(t), + Inventory: noopConsoleInventory(t), + CVE: noopConsoleCVE(t), + Reader: fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }), + PEP: fleetTestPEP(t, pep.AllowReadHook{}), Random: bytes.NewReader(nil), + }); err == nil { + t.Fatal("NewConsoleHandler accepted failed random source") + } +} + +func TestConsolePageAndAssetFailuresRemainGeneric(t *testing.T) { + now := time.Date(2026, 7, 17, 13, 0, 0, 0, time.UTC) + verifier, privateKey := fleetTestVerifier(t, now) + session := signHubTestToken(t, hubValidClaims(now), privateKey) + reader := fleetReaderFunc(func(context.Context, tenancy.Scope, time.Duration, time.Time) (fleet.FleetResult, error) { + return fleet.FleetResult{}, nil + }) + + shortRandom, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, Reader: reader, Correlator: noopConsoleCorrelator(t), Inventory: noopConsoleInventory(t), CVE: noopConsoleCVE(t), PEP: fleetTestPEP(t, pep.AllowReadHook{}), Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x51}, consoleCSRFKeyBytes)), + }) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + shortRandom.ServePage(response, consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console", "workspace-a", session)) + if response.Code != http.StatusServiceUnavailable || response.Body.String() != "{\"error\":\"console_unavailable\"}\n" { + t.Fatalf("random failure status/body = %d/%q", response.Code, response.Body.String()) + } + + handler, err := NewConsoleHandler(ConsoleHandlerConfig{ + Verifier: verifier, Reader: reader, Correlator: noopConsoleCorrelator(t), Inventory: noopConsoleInventory(t), CVE: noopConsoleCVE(t), PEP: fleetTestPEP(t, pep.AllowReadHook{}), Now: func() time.Time { return now }, + Random: bytes.NewReader(bytes.Repeat([]byte{0x61}, 192)), + }) + if err != nil { + t.Fatal(err) + } + handler.page = template.Must(template.New("console.html").Parse(`{{call .Workspace}}`)) + response = httptest.NewRecorder() + handler.ServePage(response, consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console", "workspace-a", session)) + if response.Code != http.StatusServiceUnavailable || response.Body.String() != "{\"error\":\"console_unavailable\"}\n" { + t.Fatalf("template failure status/body = %d/%q", response.Code, response.Body.String()) + } + + handler.assets = fstest.MapFS{} + assetResponse := httptest.NewRecorder() + handler.ServeCSS(assetResponse, httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/console/assets/console.css", nil)) + if assetResponse.Code != http.StatusNotFound || assetResponse.Body.String() != "{\"error\":\"not_found\"}\n" { + t.Fatalf("missing asset status/body = %d/%q", assetResponse.Code, assetResponse.Body.String()) + } + wrongPathResponse := httptest.NewRecorder() + handler.ServeJavaScript(wrongPathResponse, httptest.NewRequest(http.MethodGet, "https://hub.sith.test/v1/console/assets/other.js", nil)) + if wrongPathResponse.Code != http.StatusNotFound { + t.Fatalf("wrong asset path status = %d", wrongPathResponse.Code) + } + + invalidPage := httptest.NewRecorder() + handler.ServePage(invalidPage, consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console?next=evil", "workspace-a", session)) + if invalidPage.Code != http.StatusNotFound { + t.Fatalf("queried page status = %d", invalidPage.Code) + } + + var nilHandler *ConsoleHandler + nilResponse := httptest.NewRecorder() + nilHandler.ServePage(nilResponse, consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console", "workspace-a", session)) + if nilResponse.Code != http.StatusUnauthorized { + t.Fatalf("nil handler page status = %d", nilResponse.Code) + } + if _, ok := exactConsoleSession(nil); ok { + t.Fatal("nil request produced a console session") + } + if _, ok := canonicalConsoleRequest(nil, "/console"); ok { + t.Fatal("nil request produced a console workspace") + } +} + +func authenticatedConsolePage(t *testing.T, handler *ConsoleHandler, session, workspace string) (*httptest.ResponseRecorder, string) { + t.Helper() + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/"+workspace+"/console", workspace, session) + response := httptest.NewRecorder() + handler.ServePage(response, request) + const marker = ``) + if !found { + t.Fatalf("console page emitted malformed CSRF token") + } + return response, token +} + +func validConsoleFleetRequest(session, csrfToken string) *http.Request { + request := consoleTestRequest(http.MethodGet, "/v1/workspaces/workspace-a/console/fleet", "workspace-a", session) + request.Header.Set(consoleCSRFHeader, csrfToken) + request.Header.Set("Sec-Fetch-Site", "same-origin") + return request +} + +func consoleTestRequest(method, target, workspace, session string) *http.Request { + request := httptest.NewRequest(method, "https://hub.sith.test"+target, nil) + request.SetPathValue("workspace", workspace) + if session != "" { + request.AddCookie(&http.Cookie{Name: browserOIDCSessionCookie, Value: session}) + } + return request +} + +func assertConsoleSecurityHeaders(t *testing.T, header http.Header) { + t.Helper() + if header.Get("Cache-Control") != "no-store" || header.Get("Pragma") != "no-cache" || header.Get("X-Frame-Options") != "DENY" || + header.Get("Referrer-Policy") != "no-referrer" || !strings.Contains(header.Get("Content-Security-Policy"), "default-src 'none'") || + !strings.Contains(header.Get("Content-Security-Policy"), "connect-src 'self'") { + t.Fatalf("console security headers = %#v", header) + } +} + +func containsCoverageGap(gaps []fleet.CoverageGap, wanted fleet.CoverageGap) bool { + for _, gap := range gaps { + if gap == wanted { + return true + } + } + return false +} diff --git a/internal/hubserver/fleet.go b/internal/hubserver/fleet.go index 936c2eb..54b9449 100644 --- a/internal/hubserver/fleet.go +++ b/internal/hubserver/fleet.go @@ -49,6 +49,7 @@ type FleetHandlerConfig struct { CVESearcher FleetCVESearcher CVEIdentifierSearcher FleetCVEIdentifierSearcher PEP *pep.Enforcer + ReadObserver hubfleet.FleetReadObserver } // NewFleetHandler constructs the fixed, authenticated hub read surface. @@ -87,9 +88,7 @@ func NewFleetHandler(config FleetHandlerConfig) (http.Handler, error) { return } source, err := hubfleet.NewSource(hubfleet.SourceConfig{ - Reader: config.Reader, - Scope: scope, - PEP: config.PEP, + Reader: config.Reader, Scope: scope, PEP: config.PEP, Observer: config.ReadObserver, }) if err != nil { writeFleetError(response, http.StatusForbidden, "forbidden") diff --git a/internal/hubserver/fleet_test.go b/internal/hubserver/fleet_test.go index bcf9dad..caa202e 100644 --- a/internal/hubserver/fleet_test.go +++ b/internal/hubserver/fleet_test.go @@ -64,10 +64,17 @@ var _ FleetImageSearcher = fleetImageSearcherFunc(nil) var _ FleetCVESearcher = fleetImageSearcherFunc(nil) var _ FleetCVEIdentifierSearcher = fleetCVEIdentifierSearcherFunc(nil) +type fleetReadObserverFunc func(hubfleet.FleetReadObservation) + +func (function fleetReadObserverFunc) ObserveFleetRead(observation hubfleet.FleetReadObservation) { + function(observation) +} + func TestFleetHandlerRefreshUsesOnlySignedWorkspaceScope(t *testing.T) { now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) verifier, privateKey := fleetTestVerifier(t, now) called := false + var observed []hubfleet.FleetReadObservation handler, err := NewFleetHandler(FleetHandlerConfig{ Verifier: verifier, Collector: fleetRefresherFunc(func(_ context.Context, scope tenancy.Scope) (fleet.Coverage, error) { @@ -89,6 +96,9 @@ func TestFleetHandlerRefreshUsesOnlySignedWorkspaceScope(t *testing.T) { return fleet.QueryResult{}, nil }), PEP: fleetTestPEP(t, pep.AllowReadHook{}), + ReadObserver: fleetReadObserverFunc(func(observation hubfleet.FleetReadObservation) { + observed = append(observed, observation) + }), }) if err != nil { t.Fatal(err) @@ -103,6 +113,9 @@ func TestFleetHandlerRefreshUsesOnlySignedWorkspaceScope(t *testing.T) { if !called { t.Fatal("refresh collector was not called") } + if len(observed) != 0 { + t.Fatalf("refresh emitted fleet-read observations = %q", observed) + } if response.Header().Get("Cache-Control") != "no-store" || response.Header().Get("Pragma") != "no-cache" { t.Fatalf("refresh response caching headers = Cache-Control %q, Pragma %q", response.Header().Get("Cache-Control"), response.Header().Get("Pragma")) } @@ -182,6 +195,7 @@ func TestFleetHandlerReadConstructsRequestScopedSource(t *testing.T) { now := time.Date(2026, 7, 14, 7, 0, 0, 0, time.UTC) verifier, privateKey := fleetTestVerifier(t, now) called := false + var observed []hubfleet.FleetReadObservation handler, err := NewFleetHandler(FleetHandlerConfig{ Verifier: verifier, Collector: fleetRefresherFunc(func(context.Context, tenancy.Scope) (fleet.Coverage, error) { @@ -199,13 +213,18 @@ func TestFleetHandlerReadConstructsRequestScopedSource(t *testing.T) { if gotNow.IsZero() { t.Fatal("fleet source supplied a zero observation time") } - return fleet.FleetResult{Clusters: []fleet.Cluster{{Name: "spoke-a", SourceKind: hubfleet.SourceKind, Reachable: true}}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}}, nil + return fleet.FleetResult{Clusters: []fleet.Cluster{{ + Name: "spoke-a", SourceKind: hubfleet.SourceKind, Reachable: true, ObservedAt: gotNow.Add(-time.Minute), + }}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}}, nil }), ImageSearcher: fleetImageSearcherFunc(func(context.Context, tenancy.Scope, hubfleet.ImageSearchRequest) (fleet.QueryResult, error) { t.Fatal("fleet read reached image searcher") return fleet.QueryResult{}, nil }), PEP: fleetTestPEP(t, pep.AllowReadHook{}), + ReadObserver: fleetReadObserverFunc(func(observation hubfleet.FleetReadObservation) { + observed = append(observed, observation) + }), }) if err != nil { t.Fatal(err) @@ -218,6 +237,11 @@ func TestFleetHandlerReadConstructsRequestScopedSource(t *testing.T) { if !called { t.Fatal("fleet reader was not called") } + if len(observed) != 1 || observed[0] != (hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeComplete, Freshness: hubfleet.FleetFreshnessOutcomeFresh, + }) { + t.Fatalf("fleet read observations = %#v, want complete/fresh", observed) + } var result fleet.FleetResult if err := json.NewDecoder(response.Body).Decode(&result); err != nil { t.Fatal(err) diff --git a/internal/hubserver/probes.go b/internal/hubserver/probes.go new file mode 100644 index 0000000..a787ab1 --- /dev/null +++ b/internal/hubserver/probes.go @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "fmt" + "net/http" + "time" +) + +const ( + // LivenessPath is the fixed dependency-free process probe route. + LivenessPath = "/healthz" + // ReadinessPath is the fixed application-database readiness route. + ReadinessPath = "/readyz" + + readinessTimeout = time.Second +) + +// ReadinessChecker verifies one required Hub dependency. Implementations must honor context +// cancellation and must not return sensitive details to the caller; ProbeHandler discards errors. +type ReadinessChecker interface { + Ping(context.Context) error +} + +// ReadinessOutcome is the closed self-observability result of one completed database readiness +// check. It deliberately does not distinguish dependency failure classes or carry request data. +type ReadinessOutcome string + +const ( + // ReadinessOutcomeReady means the application database responded inside the server deadline. + ReadinessOutcomeReady ReadinessOutcome = "ready" + // ReadinessOutcomeUnavailable collapses errors, cancellation, timeout, and checker panic. + ReadinessOutcomeUnavailable ReadinessOutcome = "unavailable" +) + +// Valid reports whether the outcome belongs to the fixed readiness vocabulary. +func (outcome ReadinessOutcome) Valid() bool { + return outcome == ReadinessOutcomeReady || outcome == ReadinessOutcomeUnavailable +} + +// ReadinessObserver records one completed valid readiness dependency check. Implementations must +// not affect probe behavior; ProbeHandler recovers observer panics at the instrumentation seam. +type ReadinessObserver interface { + ObserveReadiness(ReadinessOutcome, time.Duration) +} + +// ProbeHandlerConfig supplies the required dependency and optional self-observability sink. +type ProbeHandlerConfig struct { + Checker ReadinessChecker + Observer ReadinessObserver +} + +// ProbeHandler exposes fixed, body-free liveness and readiness responses. It deliberately does +// not authenticate because the kubelet has no Hub identity, and it never reveals dependency or +// error details. +type ProbeHandler struct { + checker ReadinessChecker + observer ReadinessObserver + timeout time.Duration +} + +// NewProbeHandler constructs the Hub probe boundary over its required application database. +func NewProbeHandler(config ProbeHandlerConfig) (*ProbeHandler, error) { + if config.Checker == nil { + return nil, fmt.Errorf("construct hub probes: readiness checker is required") + } + return &ProbeHandler{checker: config.Checker, observer: config.Observer, timeout: readinessTimeout}, nil +} + +// ServeLiveness reports only whether the Hub HTTP process can serve this exact request. Required +// dependencies intentionally do not participate so a dependency outage cannot cause a restart +// storm. +func (handler *ProbeHandler) ServeLiveness(response http.ResponseWriter, request *http.Request) { + if handler == nil || !validProbeRequest(request, LivenessPath) { + writeProbeStatus(response, http.StatusNotFound) + return + } + writeProbeStatus(response, http.StatusNoContent) +} + +// ServeReadiness reports whether the application database responds inside the fixed deadline. +// Errors and panics collapse to one body-free unavailable response. +func (handler *ProbeHandler) ServeReadiness(response http.ResponseWriter, request *http.Request) { + if handler == nil || handler.checker == nil || handler.timeout <= 0 || handler.timeout > time.Second || + !validProbeRequest(request, ReadinessPath) { + writeProbeStatus(response, http.StatusNotFound) + return + } + ctx, cancel := context.WithTimeout(request.Context(), handler.timeout) + defer cancel() + started := time.Now() + ready := dependencyReady(ctx, handler.checker) + outcome := ReadinessOutcomeUnavailable + status := http.StatusServiceUnavailable + if ready { + outcome = ReadinessOutcomeReady + status = http.StatusNoContent + } + observeReadiness(handler.observer, outcome, time.Since(started)) + writeProbeStatus(response, status) +} + +func validProbeRequest(request *http.Request, path string) bool { + return request != nil && request.Method == http.MethodGet && request.URL != nil && + request.URL.Path == path && request.URL.EscapedPath() == path && request.URL.RawQuery == "" +} + +func dependencyReady(ctx context.Context, checker ReadinessChecker) (ready bool) { + defer func() { + if recover() != nil { + ready = false + } + }() + return checker.Ping(ctx) == nil +} + +func observeReadiness(observer ReadinessObserver, outcome ReadinessOutcome, duration time.Duration) { + if observer == nil { + return + } + defer func() { + _ = recover() + }() + observer.ObserveReadiness(outcome, duration) +} + +func writeProbeStatus(response http.ResponseWriter, status int) { + response.Header().Set("Cache-Control", "no-store") + response.Header().Set("Content-Length", "0") + response.Header().Set("X-Content-Type-Options", "nosniff") + response.WriteHeader(status) +} diff --git a/internal/hubserver/probes_test.go b/internal/hubserver/probes_test.go new file mode 100644 index 0000000..8c89d36 --- /dev/null +++ b/internal/hubserver/probes_test.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 + +package hubserver + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +type probeChecker func(context.Context) error + +func (checker probeChecker) Ping(ctx context.Context) error { return checker(ctx) } + +type readinessObservation struct { + outcome ReadinessOutcome + duration time.Duration +} + +type readinessObserverFunc func(ReadinessOutcome, time.Duration) + +func (observer readinessObserverFunc) ObserveReadiness(outcome ReadinessOutcome, duration time.Duration) { + observer(outcome, duration) +} + +type recordingReadinessObserver struct { + events []readinessObservation +} + +func (observer *recordingReadinessObserver) ObserveReadiness(outcome ReadinessOutcome, duration time.Duration) { + observer.events = append(observer.events, readinessObservation{outcome: outcome, duration: duration}) +} + +func TestProbeHandlerSeparatesLivenessFromDatabaseReadiness(t *testing.T) { + checker := probeChecker(func(context.Context) error { return errors.New("database endpoint secret") }) + observer := &recordingReadinessObserver{} + handler, err := NewProbeHandler(ProbeHandlerConfig{ + Checker: checker, Observer: observer, + }) + if err != nil { + t.Fatal(err) + } + + live := serveProbe(t, handler.ServeLiveness, http.MethodGet, LivenessPath) + assertProbeResponse(t, live, http.StatusNoContent) + ready := serveProbe(t, handler.ServeReadiness, http.MethodGet, ReadinessPath) + assertProbeResponse(t, ready, http.StatusServiceUnavailable) + if ready.Body.String() != "" { + t.Fatalf("readiness body leaked dependency state: %q", ready.Body.String()) + } + if len(observer.events) != 1 || observer.events[0].outcome != ReadinessOutcomeUnavailable || observer.events[0].duration < 0 { + t.Fatalf("readiness observations = %#v", observer.events) + } +} + +func TestProbeHandlerReportsReadyAndBoundsDependencyCheck(t *testing.T) { + t.Run("ready", func(t *testing.T) { + observer := &recordingReadinessObserver{} + handler, err := NewProbeHandler(ProbeHandlerConfig{ + Checker: probeChecker(func(context.Context) error { return nil }), Observer: observer, + }) + if err != nil { + t.Fatal(err) + } + assertProbeResponse(t, serveProbe(t, handler.ServeReadiness, http.MethodGet, ReadinessPath), http.StatusNoContent) + assertReadinessObservation(t, observer, ReadinessOutcomeReady) + }) + + t.Run("timeout", func(t *testing.T) { + var deadline time.Time + observer := &recordingReadinessObserver{} + handler, err := NewProbeHandler(ProbeHandlerConfig{Checker: probeChecker(func(ctx context.Context) error { + var ok bool + deadline, ok = ctx.Deadline() + if !ok { + t.Fatal("readiness checker received no deadline") + } + <-ctx.Done() + return ctx.Err() + }), Observer: observer}) + if err != nil { + t.Fatal(err) + } + handler.timeout = 5 * time.Millisecond + started := time.Now() + response := serveProbe(t, handler.ServeReadiness, http.MethodGet, ReadinessPath) + assertProbeResponse(t, response, http.StatusServiceUnavailable) + if deadline.IsZero() || time.Since(started) > 250*time.Millisecond { + t.Fatalf("readiness deadline/elapsed = %s/%s", deadline, time.Since(started)) + } + assertReadinessObservation(t, observer, ReadinessOutcomeUnavailable) + }) + + t.Run("caller cancellation", func(t *testing.T) { + checker := probeChecker(func(ctx context.Context) error { + <-ctx.Done() + return ctx.Err() + }) + observer := &recordingReadinessObserver{} + handler, err := NewProbeHandler(ProbeHandlerConfig{Checker: checker, Observer: observer}) + if err != nil { + t.Fatal(err) + } + request := httptest.NewRequest(http.MethodGet, ReadinessPath, nil) + ctx, cancel := context.WithCancel(request.Context()) + cancel() + response := httptest.NewRecorder() + handler.ServeReadiness(response, request.WithContext(ctx)) + assertProbeResponse(t, response, http.StatusServiceUnavailable) + assertReadinessObservation(t, observer, ReadinessOutcomeUnavailable) + }) +} + +func TestProbeHandlerFailsClosedOnPanicAndInvalidRequests(t *testing.T) { + observations := make([]ReadinessOutcome, 0, 1) + checkerCalls := 0 + handler, err := NewProbeHandler(ProbeHandlerConfig{ + Checker: probeChecker(func(context.Context) error { + checkerCalls++ + panic("database endpoint secret") + }), + Observer: readinessObserverFunc(func(outcome ReadinessOutcome, _ time.Duration) { + observations = append(observations, outcome) + }), + }) + if err != nil { + t.Fatal(err) + } + assertProbeResponse(t, serveProbe(t, handler.ServeReadiness, http.MethodGet, ReadinessPath), http.StatusServiceUnavailable) + if checkerCalls != 1 || len(observations) != 1 || observations[0] != ReadinessOutcomeUnavailable { + t.Fatalf("checker calls/observations = %d/%v", checkerCalls, observations) + } + + for _, test := range []struct { + name string + serve func(http.ResponseWriter, *http.Request) + method string + target string + }{ + {name: "liveness post", serve: handler.ServeLiveness, method: http.MethodPost, target: LivenessPath}, + {name: "liveness query", serve: handler.ServeLiveness, method: http.MethodGet, target: LivenessPath + "?detail=1"}, + {name: "liveness path", serve: handler.ServeLiveness, method: http.MethodGet, target: LivenessPath + "/"}, + {name: "readiness head", serve: handler.ServeReadiness, method: http.MethodHead, target: ReadinessPath}, + {name: "readiness query", serve: handler.ServeReadiness, method: http.MethodGet, target: ReadinessPath + "?detail=1"}, + {name: "readiness encoded path", serve: handler.ServeReadiness, method: http.MethodGet, target: "/%72eadyz"}, + } { + t.Run(test.name, func(t *testing.T) { + assertProbeResponse(t, serveProbe(t, test.serve, test.method, test.target), http.StatusNotFound) + }) + } + if checkerCalls != 1 || len(observations) != 1 { + t.Fatalf("invalid requests reached checker/observer: calls=%d observations=%v", checkerCalls, observations) + } + + if _, err := NewProbeHandler(ProbeHandlerConfig{}); err == nil { + t.Fatal("NewProbeHandler accepted a missing readiness checker") + } + var nilHandler *ProbeHandler + assertProbeResponse(t, serveProbe(t, nilHandler.ServeLiveness, http.MethodGet, LivenessPath), http.StatusNotFound) +} + +func TestProbeHandlerRecoversReadinessObserverPanic(t *testing.T) { + checkerCalls := 0 + handler, err := NewProbeHandler(ProbeHandlerConfig{ + Checker: probeChecker(func(context.Context) error { + checkerCalls++ + return nil + }), + Observer: readinessObserverFunc(func(ReadinessOutcome, time.Duration) { panic("metrics fault") }), + }) + if err != nil { + t.Fatal(err) + } + assertProbeResponse(t, serveProbe(t, handler.ServeReadiness, http.MethodGet, ReadinessPath), http.StatusNoContent) + if checkerCalls != 1 { + t.Fatalf("readiness checker calls = %d, want 1", checkerCalls) + } +} + +func serveProbe(t *testing.T, serve func(http.ResponseWriter, *http.Request), method, target string) *httptest.ResponseRecorder { + t.Helper() + response := httptest.NewRecorder() + serve(response, httptest.NewRequest(method, target, nil)) + return response +} + +func assertReadinessObservation(t *testing.T, observer *recordingReadinessObserver, want ReadinessOutcome) { + t.Helper() + if len(observer.events) != 1 || observer.events[0].outcome != want || observer.events[0].duration < 0 { + t.Fatalf("readiness observations = %#v, want one %q", observer.events, want) + } +} + +func assertProbeResponse(t *testing.T, response *httptest.ResponseRecorder, wantStatus int) { + t.Helper() + if response.Code != wantStatus || response.Body.Len() != 0 || response.Header().Get("Content-Length") != "0" || + response.Header().Get("Cache-Control") != "no-store" || response.Header().Get("X-Content-Type-Options") != "nosniff" || response.Header().Get("Content-Type") != "" { + t.Fatalf("probe response = status %d headers %#v body %q, want body-free %d", response.Code, response.Header(), response.Body.String(), wantStatus) + } +} diff --git a/internal/hydrate/hydrator.go b/internal/hydrate/hydrator.go index 90bd931..14dc138 100644 --- a/internal/hydrate/hydrator.go +++ b/internal/hydrate/hydrator.go @@ -253,27 +253,31 @@ func waitForRetry(ctx context.Context, delay time.Duration) bool { } } -func (hydrator *Hydrator) sync(ctx context.Context, kinds []string) error { - if hydrator.store.Paused() { +func (hydrator *Hydrator) sync(ctx context.Context, kinds []string) (syncErr error) { + if hydrator.store.Paused(fleet.LocalWorkspace) { return ErrPaused } - if !hydrator.store.BeginSync(kinds...) { - if hydrator.store.Paused() { + if !hydrator.store.BeginSync(fleet.LocalWorkspace, kinds...) { + if hydrator.store.Paused(fleet.LocalWorkspace) { return ErrPaused } return ErrSyncInProgress } - var syncErr error defer func() { - hydrator.store.EndSync(syncErr) + if endErr := hydrator.store.EndSync(fleet.LocalWorkspace, syncErr); endErr != nil { + syncErr = errors.Join(syncErr, endErr) + } }() discovery, err := hydrator.reader.Discover(ctx) if err != nil { syncErr = fmt.Errorf("discover hydration scopes: %w", err) return syncErr } - hydrator.store.SetDiscovery(fleet.LocalWorkspace, discovery) + if err := hydrator.store.SetDiscovery(fleet.LocalWorkspace, discovery); err != nil { + syncErr = err + return syncErr + } errorsByKind := make([]error, len(kinds)) workers := min(hydrator.limit, len(kinds)) @@ -293,7 +297,7 @@ func (hydrator *Hydrator) sync(ctx context.Context, kinds []string) error { errorsByKind[index] = fmt.Errorf("query %s cache: %w", kind, queryErr) continue } - if replaceErr := hydrator.store.Replace(kind, result); replaceErr != nil { + if replaceErr := hydrator.store.Replace(fleet.LocalWorkspace, kind, result); replaceErr != nil { errorsByKind[index] = fmt.Errorf("replace %s cache: %w", kind, replaceErr) } } diff --git a/internal/hydrate/hydrator_test.go b/internal/hydrate/hydrator_test.go index fec25c1..9e32158 100644 --- a/internal/hydrate/hydrator_test.go +++ b/internal/hydrate/hydrator_test.go @@ -138,7 +138,7 @@ func TestSyncOnceRejectsPauseAndDuplicateRun(t *testing.T) { t.Fatalf("first SyncOnce() error = %v", err) } - store.SetPaused(true) + store.SetPaused(fleet.LocalWorkspace, true) if err := hydrator.SyncOnce(context.Background()); !errors.Is(err, ErrPaused) { t.Fatalf("paused SyncOnce() error = %v, want ErrPaused", err) } @@ -165,7 +165,7 @@ func TestRunAppliesWatchDeltasAndAddsGenericKinds(t *testing.T) { now := time.Now().UTC() reader.events <- connector.WatchEvent{ - Type: connector.WatchUpsert, Kind: "Pod", Scope: "alpha", + Type: connector.WatchUpsert, Workspace: fleet.LocalWorkspace, Kind: "Pod", Scope: "alpha", Fact: fakeFact("Pod", "alpha", now), ObservedAt: now, } waitForCondition(t, func() bool { @@ -179,7 +179,8 @@ func TestRunAppliesWatchDeltasAndAddsGenericKinds(t *testing.T) { t.Fatalf("updated watch kinds = %v, want Widgets", kinds) } reader.events <- connector.WatchEvent{ - Type: connector.WatchError, Kind: "Pod", Scope: "beta", Err: errors.New("connection reset"), + Type: connector.WatchError, Workspace: fleet.LocalWorkspace, + Kind: "Pod", Scope: "beta", Err: errors.New("connection reset"), } waitForCondition(t, func() bool { return slices.Contains(store.Query(fleet.LocalWorkspace, fleetcache.Query{Kind: "Pod"}).Coverage.Unreachable, "beta") @@ -236,11 +237,12 @@ func (*fakeReader) Capabilities() []connector.Capability { func (reader *fakeReader) Descriptor() connector.Descriptor { return connector.Descriptor{ - Kind: reader.Kind(), - ConnKind: connector.KindReadAdapter, - ProtocolV: "1.0.0", - Owner: "test", - Capabilities: reader.Capabilities(), + Kind: reader.Kind(), + ConnKind: connector.KindReadAdapter, + WireVersions: []connector.WireVersion{connector.CurrentWireVersion()}, + AdapterVersion: "1.0.0", + Owner: "test", + Capabilities: reader.Capabilities(), } } diff --git a/internal/intent/boundary_test.go b/internal/intent/boundary_test.go new file mode 100644 index 0000000..7b1035c --- /dev/null +++ b/internal/intent/boundary_test.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 + +package intent + +import ( + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestVocabularyHasNoSideEffectImports(t *testing.T) { + t.Parallel() + + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read intent package: %v", err) + } + allowed := map[string]bool{"encoding/json": true, "fmt": true, "sort": true} + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), entry.Name(), nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("decode import in %s: %v", entry.Name(), err) + } + if !allowed[path] { + t.Fatalf("intent vocabulary imports unreviewed package %q", path) + } + } + } +} diff --git a/internal/intent/verb.go b/internal/intent/verb.go new file mode 100644 index 0000000..78f37fb --- /dev/null +++ b/internal/intent/verb.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package intent defines the fail-closed vocabulary for governed Sith actions. +package intent + +import ( + "encoding/json" + "fmt" + "sort" +) + +// Verb names one reviewed governed action. Unknown strings are never verbs. +type Verb string + +// The initial action vocabulary is locked by ADR-0004. Adding a verb is an ADR-level change. +const ( + VerbArgoCDSync Verb = "argocd.sync" + VerbArgoCDRollback Verb = "argocd.rollback" + VerbRolloutPromote Verb = "rollout.promote" + VerbRolloutAbort Verb = "rollout.abort" + VerbDeploymentScale Verb = "deployment.scale" + VerbDeploymentRestart Verb = "deployment.restart" + VerbGitOpsOpenPR Verb = "gitops.open-pr" +) + +// Class separates the proposal-only first write from live cluster mutations. +type Class string + +const ( + // ClassProposal names an action whose output still requires a separate human merge gate. + ClassProposal Class = "proposal" + // ClassLiveMutation names a P3 action that can directly change managed state. + ClassLiveMutation Class = "live-mutation" +) + +// Definition is immutable classification metadata for one reviewed verb. +type Definition struct { + Verb Verb `json:"verb"` + Class Class `json:"class"` +} + +type definitionWire struct { + Verb Verb `json:"verb"` + Class Class `json:"class"` +} + +var definitions = map[Verb]Definition{ + VerbArgoCDSync: {Verb: VerbArgoCDSync, Class: ClassLiveMutation}, + VerbArgoCDRollback: {Verb: VerbArgoCDRollback, Class: ClassLiveMutation}, + VerbRolloutPromote: {Verb: VerbRolloutPromote, Class: ClassLiveMutation}, + VerbRolloutAbort: {Verb: VerbRolloutAbort, Class: ClassLiveMutation}, + VerbDeploymentScale: {Verb: VerbDeploymentScale, Class: ClassLiveMutation}, + VerbDeploymentRestart: {Verb: VerbDeploymentRestart, Class: ClassLiveMutation}, + VerbGitOpsOpenPR: {Verb: VerbGitOpsOpenPR, Class: ClassProposal}, +} + +// ParseVerb accepts only an exact member of the reviewed vocabulary. +func ParseVerb(value string) (Verb, error) { + verb := Verb(value) + if !verb.Valid() { + return "", fmt.Errorf("unknown action verb") + } + return verb, nil +} + +// Valid reports whether verb belongs to the reviewed vocabulary. +func (verb Verb) Valid() bool { + _, exists := definitions[verb] + return exists +} + +// Definition returns the immutable classification for verb. +func (verb Verb) Definition() (Definition, bool) { + definition, exists := definitions[verb] + return definition, exists +} + +// Valid reports whether definition exactly matches the canonical classification table. +func (definition Definition) Valid() bool { + canonical, exists := definitions[definition.Verb] + return exists && definition == canonical +} + +// MarshalJSON prevents forged or inconsistent classification metadata from crossing a wire boundary. +func (definition Definition) MarshalJSON() ([]byte, error) { + if !definition.Valid() { + return nil, fmt.Errorf("encode action definition: classification mismatch") + } + return json.Marshal(definitionWire(definition)) +} + +// UnmarshalJSON replaces accepted input with the canonical definition and rejects mismatches. +func (definition *Definition) UnmarshalJSON(payload []byte) error { + if definition == nil { + return fmt.Errorf("decode action definition: destination is nil") + } + var decoded definitionWire + if err := json.Unmarshal(payload, &decoded); err != nil { + return fmt.Errorf("decode action definition: %w", err) + } + canonical, exists := definitions[decoded.Verb] + if !exists || decoded.Class != canonical.Class { + return fmt.Errorf("decode action definition: classification mismatch") + } + *definition = canonical + return nil +} + +// Vocabulary returns deterministic copies of all reviewed definitions. +func Vocabulary() []Definition { + result := make([]Definition, 0, len(definitions)) + for _, definition := range definitions { + result = append(result, definition) + } + sort.Slice(result, func(left, right int) bool { + return result[left].Verb < result[right].Verb + }) + return result +} + +// MarshalJSON prevents an invalid programmatic value from crossing a wire boundary. +func (verb Verb) MarshalJSON() ([]byte, error) { + if !verb.Valid() { + return nil, fmt.Errorf("encode action verb: unknown action verb") + } + return json.Marshal(string(verb)) +} + +// UnmarshalJSON rejects unknown wire strings at the type boundary. +func (verb *Verb) UnmarshalJSON(payload []byte) error { + if verb == nil { + return fmt.Errorf("decode action verb: destination is nil") + } + var value string + if err := json.Unmarshal(payload, &value); err != nil { + return fmt.Errorf("decode action verb: %w", err) + } + parsed, err := ParseVerb(value) + if err != nil { + return fmt.Errorf("decode action verb: %w", err) + } + *verb = parsed + return nil +} diff --git a/internal/intent/verb_test.go b/internal/intent/verb_test.go new file mode 100644 index 0000000..ae61376 --- /dev/null +++ b/internal/intent/verb_test.go @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: Apache-2.0 + +package intent + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestVocabularyIsExactAndClassified(t *testing.T) { + t.Parallel() + + want := []Definition{ + {Verb: VerbArgoCDRollback, Class: ClassLiveMutation}, + {Verb: VerbArgoCDSync, Class: ClassLiveMutation}, + {Verb: VerbDeploymentRestart, Class: ClassLiveMutation}, + {Verb: VerbDeploymentScale, Class: ClassLiveMutation}, + {Verb: VerbGitOpsOpenPR, Class: ClassProposal}, + {Verb: VerbRolloutAbort, Class: ClassLiveMutation}, + {Verb: VerbRolloutPromote, Class: ClassLiveMutation}, + } + if got := Vocabulary(); !reflect.DeepEqual(got, want) { + t.Fatalf("Vocabulary() = %#v, want %#v", got, want) + } + + got := Vocabulary() + got[0] = Definition{Verb: "shell.exec", Class: ClassProposal} + if second := Vocabulary(); !reflect.DeepEqual(second, want) { + t.Fatalf("Vocabulary() exposed mutable state: %#v", second) + } +} + +func TestVerbParsingAndJSONFailClosed(t *testing.T) { + t.Parallel() + + for _, definition := range Vocabulary() { + parsed, err := ParseVerb(string(definition.Verb)) + if err != nil || parsed != definition.Verb { + t.Fatalf("ParseVerb(%q) = %q, %v", definition.Verb, parsed, err) + } + payload, err := json.Marshal(definition.Verb) + if err != nil { + t.Fatalf("Marshal(%q) error = %v", definition.Verb, err) + } + var decoded Verb + if err := json.Unmarshal(payload, &decoded); err != nil || decoded != definition.Verb { + t.Fatalf("Unmarshal(%q) = %q, %v", payload, decoded, err) + } + } + + for _, value := range []string{"", " shell.exec", "shell.exec", "kubectl.apply", "secret.read", "rbac.mutate", "GITOPS.OPEN-PR"} { + if parsed, err := ParseVerb(value); err == nil || parsed.Valid() { + t.Fatalf("ParseVerb(%q) = %q, %v; want refusal", value, parsed, err) + } + if _, err := json.Marshal(Verb(value)); err == nil { + t.Fatalf("Marshal(%q) accepted unknown verb", value) + } + payload, err := json.Marshal(value) + if err != nil { + t.Fatalf("Marshal(%q) error = %v", value, err) + } + var decoded Verb + if err := json.Unmarshal(payload, &decoded); err == nil { + t.Fatalf("Unmarshal(%q) accepted unknown verb", payload) + } + } + + for _, payload := range [][]byte{[]byte("null"), []byte("1"), []byte("{}"), []byte(`"argocd.sync" "rollout.abort"`)} { + var decoded Verb + if err := json.Unmarshal(payload, &decoded); err == nil { + t.Fatalf("Unmarshal(%q) accepted malformed verb", payload) + } + } +} + +func TestDefinitionsAreInternallyConsistent(t *testing.T) { + t.Parallel() + + for _, definition := range Vocabulary() { + got, ok := definition.Verb.Definition() + if !ok || got != definition { + t.Fatalf("Definition(%q) = %#v/%t", definition.Verb, got, ok) + } + if definition.Verb == VerbGitOpsOpenPR { + if definition.Class != ClassProposal { + t.Fatalf("first write class = %q, want proposal", definition.Class) + } + continue + } + if definition.Class != ClassLiveMutation { + t.Fatalf("live verb %q class = %q", definition.Verb, definition.Class) + } + } +} + +func TestDefinitionJSONRejectsForgedClassification(t *testing.T) { + t.Parallel() + + for _, definition := range Vocabulary() { + payload, err := json.Marshal(definition) + if err != nil { + t.Fatalf("Marshal(%#v) error = %v", definition, err) + } + var decoded Definition + if err := json.Unmarshal(payload, &decoded); err != nil || decoded != definition { + t.Fatalf("Unmarshal(%q) = %#v, %v", payload, decoded, err) + } + } + + for _, definition := range []Definition{ + {Verb: VerbArgoCDSync, Class: ClassProposal}, + {Verb: VerbGitOpsOpenPR, Class: ClassLiveMutation}, + {Verb: "shell.exec", Class: ClassProposal}, + {}, + } { + if definition.Valid() { + t.Fatalf("inconsistent definition is valid: %#v", definition) + } + if _, err := json.Marshal(definition); err == nil { + t.Fatalf("Marshal(%#v) accepted inconsistent definition", definition) + } + } + + for _, payload := range [][]byte{ + []byte(`{"verb":"argocd.sync","class":"proposal"}`), + []byte(`{"verb":"gitops.open-pr","class":"live-mutation"}`), + []byte(`{"verb":"shell.exec","class":"proposal"}`), + []byte(`{"verb":"argocd.sync"}`), + } { + var decoded Definition + if err := json.Unmarshal(payload, &decoded); err == nil { + t.Fatalf("Unmarshal(%q) accepted forged definition", payload) + } + } +} diff --git a/internal/intentargs/boundary_test.go b/internal/intentargs/boundary_test.go new file mode 100644 index 0000000..7f0d0c2 --- /dev/null +++ b/internal/intentargs/boundary_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package intentargs + +import ( + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestPackageHasNoExecutionImports(t *testing.T) { + t.Parallel() + + allowed := map[string]bool{ + "bytes": true, "encoding/json": true, "errors": true, "fmt": true, "io": true, + "math/big": true, "strconv": true, "strings": true, "unicode": true, "unicode/utf8": true, + "github.com/google/jsonschema-go/jsonschema": true, + } + entries, err := os.ReadDir(".") + if err != nil { + t.Fatalf("read intentargs package: %v", err) + } + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".go" || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(token.NewFileSet(), entry.Name(), nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("parse %s: %v", entry.Name(), err) + } + for _, imported := range file.Imports { + path, err := strconv.Unquote(imported.Path.Value) + if err != nil { + t.Fatalf("decode import in %s: %v", entry.Name(), err) + } + if !allowed[path] { + t.Fatalf("intentargs imports unreviewed package %q", path) + } + } + } +} diff --git a/internal/intentargs/schema.go b/internal/intentargs/schema.go new file mode 100644 index 0000000..8f5b02f --- /dev/null +++ b/internal/intentargs/schema.go @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package intentargs compiles and validates bounded, exact argument schemas for governed intents. +package intentargs + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/google/jsonschema-go/jsonschema" +) + +const ( + // Draft2020 is the only schema dialect accepted for governed intent arguments. + Draft2020 = "https://json-schema.org/draft/2020-12/schema" + // MaxDocumentBytes bounds both registered schemas and untrusted argument documents. + MaxDocumentBytes = 64 * 1024 + // MaxNestingDepth prevents deeply nested JSON from exhausting parser stacks. + MaxNestingDepth = 32 + maxNumberBytes = 128 + maxExponent = 1000 + maxSchemaNodes = 256 + maxSchemaDepth = 16 + maxAnnotationBytes = 512 + maxAnnotationBytesTotal = 4 * 1024 +) + +// ErrInvalidArgs is returned without echoing untrusted argument values. +var ErrInvalidArgs = errors.New("intent args are invalid") + +// Schema is an immutable, locally resolved Draft 2020-12 argument schema. +type Schema struct { + raw json.RawMessage + resolved *jsonschema.Resolved +} + +// Compile validates and locally resolves one exact object schema. Remote references and schema +// features that this validator does not enforce are rejected rather than silently ignored. +func Compile(document json.RawMessage) (*Schema, error) { + if _, err := decodeExactObject(document); err != nil { + return nil, fmt.Errorf("compile intent args schema: %w", err) + } + + var parsed jsonschema.Schema + if err := json.Unmarshal(document, &parsed); err != nil { + return nil, fmt.Errorf("compile intent args schema: decode schema: %w", err) + } + if parsed.Schema != Draft2020 { + return nil, fmt.Errorf("compile intent args schema: schema dialect must be Draft 2020-12") + } + if parsed.Type != "object" || len(parsed.Types) != 0 || parsed.Ref != "" || parsed.DynamicRef != "" { + return nil, fmt.Errorf("compile intent args schema: root must be a direct object schema") + } + budget := schemaBudget{} + if err := validateSchemaPolicy(&parsed, true, 1, &budget); err != nil { + return nil, fmt.Errorf("compile intent args schema: %w", err) + } + + resolved, err := parsed.Resolve(nil) + if err != nil { + return nil, fmt.Errorf("compile intent args schema: resolve local schema: %w", err) + } + return &Schema{raw: append(json.RawMessage(nil), document...), resolved: resolved}, nil +} + +// Validate strictly parses and validates one argument object. Error text never contains input. +func (schema *Schema) Validate(document json.RawMessage) error { + if schema == nil || schema.resolved == nil { + return fmt.Errorf("%w: schema is unavailable", ErrInvalidArgs) + } + value, err := decodeExactObject(document) + if err != nil { + return fmt.Errorf("%w: malformed document", ErrInvalidArgs) + } + if err := schema.resolved.Validate(value); err != nil { + return fmt.Errorf("%w: schema mismatch", ErrInvalidArgs) + } + return nil +} + +// JSON returns a defensive copy of the registered schema document. +func (schema *Schema) JSON() json.RawMessage { + if schema == nil { + return nil + } + return append(json.RawMessage(nil), schema.raw...) +} + +func decodeExactObject(document json.RawMessage) (map[string]any, error) { + if len(document) == 0 { + return nil, fmt.Errorf("JSON document is empty") + } + if len(document) > MaxDocumentBytes { + return nil, fmt.Errorf("JSON document exceeds %d bytes", MaxDocumentBytes) + } + if !utf8.Valid(document) { + return nil, fmt.Errorf("JSON document is not valid UTF-8") + } + + decoder := json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + token, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("read JSON root: %w", err) + } + if delimiter, ok := token.(json.Delim); !ok || delimiter != '{' { + return nil, fmt.Errorf("JSON root must be an object") + } + if err := inspectObject(decoder, 1); err != nil { + return nil, err + } + if token, err := decoder.Token(); err != io.EOF || token != nil { + return nil, fmt.Errorf("JSON document contains trailing data") + } + + decoder = json.NewDecoder(bytes.NewReader(document)) + decoder.UseNumber() + var value map[string]any + if err := decoder.Decode(&value); err != nil { + return nil, fmt.Errorf("decode JSON object: %w", err) + } + if _, err := normalizeNumbers(value); err != nil { + return nil, err + } + return value, nil +} + +func normalizeNumbers(value any) (any, error) { + switch typed := value.(type) { + case json.Number: + number, ok := new(big.Rat).SetString(typed.String()) + if !ok || !number.IsInt() || !number.Num().IsInt64() { + return nil, fmt.Errorf("JSON number must be an exact signed 64-bit integer") + } + return number.Num().Int64(), nil + case []any: + for index, child := range typed { + normalized, err := normalizeNumbers(child) + if err != nil { + return nil, err + } + typed[index] = normalized + } + case map[string]any: + for name, child := range typed { + normalized, err := normalizeNumbers(child) + if err != nil { + return nil, err + } + typed[name] = normalized + } + } + return value, nil +} + +func inspectObject(decoder *json.Decoder, depth int) error { + seen := make(map[string]struct{}) + for decoder.More() { + token, err := decoder.Token() + if err != nil { + return fmt.Errorf("read JSON object member: %w", err) + } + name, ok := token.(string) + if !ok { + return fmt.Errorf("JSON object member name is invalid") + } + if _, duplicate := seen[name]; duplicate { + return fmt.Errorf("JSON object contains a duplicate member") + } + seen[name] = struct{}{} + if err := inspectValue(decoder, depth); err != nil { + return err + } + } + return closeContainer(decoder, '}') +} + +func inspectArray(decoder *json.Decoder, depth int) error { + for decoder.More() { + if err := inspectValue(decoder, depth); err != nil { + return err + } + } + return closeContainer(decoder, ']') +} + +func inspectValue(decoder *json.Decoder, depth int) error { + token, err := decoder.Token() + if err != nil { + return fmt.Errorf("read JSON value: %w", err) + } + if number, ok := token.(json.Number); ok { + return validateNumber(number) + } + delimiter, ok := token.(json.Delim) + if !ok { + return nil + } + if depth >= MaxNestingDepth { + return fmt.Errorf("JSON nesting exceeds %d levels", MaxNestingDepth) + } + switch delimiter { + case '{': + return inspectObject(decoder, depth+1) + case '[': + return inspectArray(decoder, depth+1) + default: + return fmt.Errorf("JSON value contains an invalid delimiter") + } +} + +func closeContainer(decoder *json.Decoder, want json.Delim) error { + token, err := decoder.Token() + if err != nil { + return fmt.Errorf("close JSON container: %w", err) + } + delimiter, ok := token.(json.Delim) + if !ok || delimiter != want { + return fmt.Errorf("JSON container has an invalid closing delimiter") + } + return nil +} + +func validateNumber(number json.Number) error { + text := string(number) + if len(text) > maxNumberBytes { + return fmt.Errorf("JSON number exceeds %d bytes", maxNumberBytes) + } + index := strings.IndexAny(text, "eE") + if index < 0 { + return nil + } + exponent, err := strconv.Atoi(text[index+1:]) + if err != nil || exponent < -maxExponent || exponent > maxExponent { + return fmt.Errorf("JSON number exponent is out of bounds") + } + return nil +} + +type schemaBudget struct { + nodes int + annotationBytes int +} + +func validateSchemaPolicy(schema *jsonschema.Schema, root bool, depth int, budget *schemaBudget) error { + if schema == nil { + return nil + } + budget.nodes++ + if budget.nodes > maxSchemaNodes { + return fmt.Errorf("schema exceeds %d nodes", maxSchemaNodes) + } + if depth > maxSchemaDepth { + return fmt.Errorf("schema exceeds %d levels", maxSchemaDepth) + } + if schema.ID != "" { + return fmt.Errorf("schema IDs are not allowed") + } + if schema.Ref != "" && !strings.HasPrefix(schema.Ref, "#") { + return fmt.Errorf("remote schema references are not allowed") + } + if schema.DynamicRef != "" && !strings.HasPrefix(schema.DynamicRef, "#") { + return fmt.Errorf("remote dynamic schema references are not allowed") + } + if schema.Format != "" || schema.ContentEncoding != "" || schema.ContentMediaType != "" || schema.ContentSchema != nil { + return fmt.Errorf("non-enforcing format and content keywords are not allowed") + } + if len(schema.Extra) != 0 { + return fmt.Errorf("unsupported schema keywords are not allowed") + } + if !root && schema.Schema != "" { + return fmt.Errorf("nested schema dialect declarations are not allowed") + } + if len(schema.Vocabulary) != 0 { + return fmt.Errorf("custom schema vocabularies are not allowed") + } + if schema.Comment != "" || len(schema.Default) != 0 || len(schema.Examples) != 0 { + return fmt.Errorf("schema comments, defaults, and examples are not allowed") + } + if err := validateAnnotation("title", schema.Title, budget); err != nil { + return err + } + if err := validateAnnotation("description", schema.Description, budget); err != nil { + return err + } + if schemaMayMatchObject(schema) && !isFalseSchema(schema.AdditionalProperties) { + return fmt.Errorf("every object schema must set additionalProperties to false") + } + + for _, child := range childSchemas(schema) { + if err := validateSchemaPolicy(child, false, depth+1, budget); err != nil { + return err + } + } + return nil +} + +func validateAnnotation(label, value string, budget *schemaBudget) error { + if value == "" { + return nil + } + budget.annotationBytes += len(value) + if len(value) > maxAnnotationBytes || budget.annotationBytes > maxAnnotationBytesTotal || + !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return fmt.Errorf("schema %s is not a bounded annotation", label) + } + for _, character := range value { + if unicode.IsControl(character) || strings.ContainsRune("<>{}", character) { + return fmt.Errorf("schema %s contains unsafe annotation content", label) + } + } + return nil +} + +func schemaMayMatchObject(schema *jsonschema.Schema) bool { + if schema.Type == "object" || len(schema.Properties) != 0 || len(schema.PatternProperties) != 0 || + schema.AdditionalProperties != nil || schema.PropertyNames != nil || len(schema.DependentSchemas) != 0 || + len(schema.DependentRequired) != 0 || schema.MinProperties != nil || schema.MaxProperties != nil { + return true + } + for _, schemaType := range schema.Types { + if schemaType == "object" { + return true + } + } + return false +} + +func isFalseSchema(schema *jsonschema.Schema) bool { + want := &jsonschema.Schema{Not: &jsonschema.Schema{}} + return schema != nil && jsonschema.Equal(schema, want) +} + +func childSchemas(schema *jsonschema.Schema) []*jsonschema.Schema { + children := make([]*jsonschema.Schema, 0) + appendMap := func(values map[string]*jsonschema.Schema) { + for _, child := range values { + children = append(children, child) + } + } + appendMap(schema.Defs) + appendMap(schema.Definitions) + appendMap(schema.DependencySchemas) + appendMap(schema.Properties) + appendMap(schema.PatternProperties) + appendMap(schema.DependentSchemas) + children = append(children, schema.PrefixItems...) + children = append(children, schema.Items) + children = append(children, schema.ItemsArray...) + children = append(children, schema.AdditionalItems, schema.Contains, schema.UnevaluatedItems) + children = append(children, schema.AdditionalProperties, schema.PropertyNames, schema.UnevaluatedProperties) + children = append(children, schema.AllOf...) + children = append(children, schema.AnyOf...) + children = append(children, schema.OneOf...) + children = append(children, schema.Not, schema.If, schema.Then, schema.Else, schema.ContentSchema) + return children +} diff --git a/internal/intentargs/schema_test.go b/internal/intentargs/schema_test.go new file mode 100644 index 0000000..48366fb --- /dev/null +++ b/internal/intentargs/schema_test.go @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: Apache-2.0 + +package intentargs + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "testing" +) + +var scaleSchema = json.RawMessage(`{ + "$schema":"https://json-schema.org/draft/2020-12/schema", + "type":"object", + "description":"Sets a bounded desired replica count.", + "properties":{"replicas":{"type":"integer","minimum":0,"maximum":100,"description":"Desired replicas from zero through one hundred."}}, + "required":["replicas"], + "additionalProperties":false +}`) + +func TestSchemaValidatesExactObject(t *testing.T) { + t.Parallel() + + schema, err := Compile(scaleSchema) + if err != nil { + t.Fatalf("Compile() error = %v", err) + } + for _, document := range []json.RawMessage{ + json.RawMessage(`{"replicas":0}`), + json.RawMessage(`{"replicas":100}`), + json.RawMessage(`{"replicas":1e2}`), + } { + if err := schema.Validate(document); err != nil { + t.Fatalf("Validate(%s) error = %v", document, err) + } + } + + invalid := []json.RawMessage{ + json.RawMessage(`{}`), + json.RawMessage(`{"replicas":-1}`), + json.RawMessage(`{"replicas":101}`), + json.RawMessage(`{"replicas":1.5}`), + json.RawMessage(`{"replicas":"3"}`), + json.RawMessage(`{"replicas":"SITH-SENSITIVE-MARKER-7f4a"}`), + json.RawMessage(`{"replicas":3,"force":true}`), + } + for _, document := range invalid { + err := schema.Validate(document) + if !errors.Is(err, ErrInvalidArgs) { + t.Fatalf("Validate(%s) error = %v, want ErrInvalidArgs", document, err) + } + if strings.Contains(err.Error(), string(document)) { + t.Fatalf("Validate() leaked input in error %q", err) + } + if strings.Contains(err.Error(), "SITH-SENSITIVE-MARKER-7f4a") { + t.Fatalf("Validate() leaked sensitive marker in error %q", err) + } + } +} + +func TestSchemaRejectsMalformedDocuments(t *testing.T) { + t.Parallel() + + schema, err := Compile(scaleSchema) + if err != nil { + t.Fatalf("Compile() error = %v", err) + } + nestedDuplicate := json.RawMessage(`{"replicas":3,"nested":{"key":1,"key":2}}`) + tooLarge := json.RawMessage(`{"replicas":3,"padding":"` + strings.Repeat("a", MaxDocumentBytes) + `"}`) + tooDeep := json.RawMessage(strings.Repeat(`{"x":`, MaxNestingDepth) + `{"replicas":3}` + strings.Repeat(`}`, MaxNestingDepth)) + tooLongNumber := json.RawMessage(`{"replicas":` + strings.Repeat("1", maxNumberBytes+1) + `}`) + + tests := []json.RawMessage{ + nil, + {'{', '"', 'x', '"', ':', '"', 0xff, '"', '}'}, + json.RawMessage(`null`), + json.RawMessage(`[]`), + json.RawMessage(`"object"`), + json.RawMessage(`1`), + json.RawMessage(`{"replicas":3,"replicas":4}`), + nestedDuplicate, + json.RawMessage(`{"replicas":3} {}`), + json.RawMessage(`{"replicas":3`), + json.RawMessage(`{"replicas":1e1001}`), + tooLongNumber, + tooLarge, + tooDeep, + } + for _, document := range tests { + if err := schema.Validate(document); !errors.Is(err, ErrInvalidArgs) { + t.Fatalf("Validate(%q) error = %v, want ErrInvalidArgs", document, err) + } + } +} + +func TestCompileRejectsUnsafeSchemas(t *testing.T) { + t.Parallel() + + tooManyNodes := make([]string, 0, maxSchemaNodes) + tooManyDescriptions := make([]string, 0, 10) + for index := 0; index < maxSchemaNodes; index++ { + tooManyNodes = append(tooManyNodes, fmt.Sprintf(`"p%d":{"type":"string"}`, index)) + } + for index := 0; index < 10; index++ { + tooManyDescriptions = append(tooManyDescriptions, fmt.Sprintf( + `"p%d":{"type":"string","description":"%s"}`, index, strings.Repeat("a", maxAnnotationBytes), + )) + } + tooDeep := `{"type":"string"}` + for range maxSchemaDepth { + tooDeep = `{"type":"array","items":` + tooDeep + `}` + } + + tests := map[string]string{ + "empty": "", + "non-object document": `[]`, + "duplicate keyword": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","type":"array","additionalProperties":false}`, + "wrong draft": `{"$schema":"http://json-schema.org/draft-07/schema#","type":"object","additionalProperties":false}`, + "implicit root": `{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"additionalProperties":false}`, + "open object": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object"}`, + "open nested object": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"nested":{"type":"object"}},"additionalProperties":false}`, + "remote ref": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"$ref":"https://example.invalid/schema"}},"additionalProperties":false}`, + "remote dynamic ref": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"$dynamicRef":"other.json#value"}},"additionalProperties":false}`, + "schema id": `{"$schema":"https://json-schema.org/draft/2020-12/schema","$id":"https://example.invalid/schema","type":"object","additionalProperties":false}`, + "format is not enforced": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"url":{"type":"string","format":"uri"}},"additionalProperties":false}`, + "content is not enforced": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"type":"string","contentEncoding":"base64"}},"additionalProperties":false}`, + "unsupported keyword": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","madeUp":true,"additionalProperties":false}`, + "nested dialect": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"string"}},"additionalProperties":false}`, + "custom vocabulary": `{"$schema":"https://json-schema.org/draft/2020-12/schema","$vocabulary":{"https://example.invalid/vocab":true},"type":"object","additionalProperties":false}`, + "invalid local reference": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"$ref":"#/$defs/missing"}},"additionalProperties":false}`, + "invalid regular expression": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"type":"string","pattern":"["}},"additionalProperties":false}`, + "root markup injection": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","description":"Unsafe annotation.","additionalProperties":false}`, + "nested markup injection": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"type":"string","description":""}},"additionalProperties":false}`, + "control in description": "{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"type\":\"object\",\"description\":\"line one\\nline two\",\"additionalProperties\":false}", + "oversized description": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","description":"` + strings.Repeat("a", maxAnnotationBytes+1) + `","additionalProperties":false}`, + "total description budget": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{` + strings.Join(tooManyDescriptions, ",") + `},"additionalProperties":false}`, + "schema node budget": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{` + strings.Join(tooManyNodes, ",") + `},"additionalProperties":false}`, + "schema depth budget": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":` + tooDeep + `},"additionalProperties":false}`, + "comment metadata": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","$comment":"private note","additionalProperties":false}`, + "default metadata": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"value":{"type":"string","default":"private"}},"additionalProperties":false}`, + "example metadata": `{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","examples":[{"private":"value"}],"additionalProperties":false}`, + } + for name, document := range tests { + name, document := name, document + t.Run(name, func(t *testing.T) { + t.Parallel() + if _, err := Compile(json.RawMessage(document)); err == nil { + t.Fatal("Compile() error = nil, want rejection") + } + }) + } +} + +func TestCompileSupportsClosedLocalDefinitions(t *testing.T) { + t.Parallel() + + document := json.RawMessage(`{ + "$schema":"https://json-schema.org/draft/2020-12/schema", + "type":"object", + "$defs":{"options":{"type":"object","properties":{"safe":{"type":"boolean"}},"required":["safe"],"additionalProperties":false}}, + "properties":{"options":{"$ref":"#/$defs/options"}}, + "required":["options"], + "additionalProperties":false + }`) + schema, err := Compile(document) + if err != nil { + t.Fatalf("Compile() error = %v", err) + } + if err := schema.Validate(json.RawMessage(`{"options":{"safe":true}}`)); err != nil { + t.Fatalf("Validate() error = %v", err) + } + if err := schema.Validate(json.RawMessage(`{"options":{"safe":true,"extra":1}}`)); !errors.Is(err, ErrInvalidArgs) { + t.Fatalf("Validate(extra) error = %v, want ErrInvalidArgs", err) + } +} + +func TestSchemaJSONIsDefensive(t *testing.T) { + t.Parallel() + + schema, err := Compile(scaleSchema) + if err != nil { + t.Fatalf("Compile() error = %v", err) + } + first := schema.JSON() + first[0] = '!' + second := schema.JSON() + if len(second) == 0 || second[0] == '!' { + t.Fatal("JSON() exposed mutable schema storage") + } + var nilSchema *Schema + if got := nilSchema.JSON(); got != nil { + t.Fatalf("nil Schema.JSON() = %q, want nil", got) + } + if err := nilSchema.Validate(json.RawMessage(`{}`)); !errors.Is(err, ErrInvalidArgs) { + t.Fatalf("nil Schema.Validate() error = %v, want ErrInvalidArgs", err) + } +} + +func TestSchemaValidationIsConcurrent(t *testing.T) { + t.Parallel() + + schema, err := Compile(scaleSchema) + if err != nil { + t.Fatalf("Compile() error = %v", err) + } + var wait sync.WaitGroup + errorsSeen := make(chan error, 32) + for index := 0; index < 32; index++ { + wait.Add(1) + go func(value int) { + defer wait.Done() + document := json.RawMessage(fmt.Sprintf(`{"replicas":%d}`, value)) + if err := schema.Validate(document); err != nil { + errorsSeen <- err + } + }(index) + } + wait.Wait() + close(errorsSeen) + for err := range errorsSeen { + t.Fatalf("concurrent Validate() error = %v", err) + } +} + +func FuzzSchemaValidateNeverPanics(f *testing.F) { + schema, err := Compile(scaleSchema) + if err != nil { + f.Fatalf("Compile() error = %v", err) + } + for _, seed := range [][]byte{ + []byte(`{"replicas":3}`), + []byte(`{"replicas":3,"replicas":4}`), + []byte(`null`), + []byte(`{"nested":[{"value":1}]}`), + } { + f.Add(seed) + } + f.Fuzz(func(_ *testing.T, document []byte) { + _ = schema.Validate(json.RawMessage(document)) + }) +} diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index 6db8d90..6a6993f 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -254,10 +254,12 @@ func mcpStore(t *testing.T) *fleetcache.Store { }}) local := mcpPodFact(t, fleet.LocalWorkspace, "alpha", "local-api", now) other := mcpPodFact(t, "other", "beta", "other-api", now) - if err := store.Replace("Pod", fleet.QueryResult{ - Facts: []fleet.Fact{local, other}, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, - }); err != nil { - t.Fatal(err) + for workspace, fact := range map[string]fleet.Fact{fleet.LocalWorkspace: local, "other": other} { + if err := store.Replace(workspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{fact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatal(err) + } } cvePayload, err := json.Marshal(fleet.CVEObservation{ Image: "registry.example/payments:v4", IDs: []string{"CVE-2026-1234"}, Severity: "high", @@ -269,7 +271,7 @@ func mcpStore(t *testing.T) *fleetcache.Store { Ref: fleet.ResourceRef{SourceKind: "scanner", Scope: "alpha", Kind: "Image", Name: "payments-v4"}, Kind: fleet.FactCVE, Observed: cvePayload, ObservedAt: now, Source: "scanner", }, Workspace: fleet.LocalWorkspace} - if err := store.Replace("CVE", fleet.QueryResult{ + if err := store.Replace(fleet.LocalWorkspace, "CVE", fleet.QueryResult{ Facts: []fleet.Fact{cve}, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}, }); err != nil { t.Fatal(err) diff --git a/internal/observability/alert_rules_contract_test.go b/internal/observability/alert_rules_contract_test.go new file mode 100644 index 0000000..52951a4 --- /dev/null +++ b/internal/observability/alert_rules_contract_test.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: Apache-2.0 + +package observability + +import ( + "os" + "strings" + "testing" + + "go.yaml.in/yaml/v3" +) + +type alertRuleFile struct { + Groups []alertRuleGroup `yaml:"groups"` +} + +type alertRuleGroup struct { + Name string `yaml:"name"` + Interval string `yaml:"interval"` + Limit int `yaml:"limit"` + Rules []alertRule `yaml:"rules"` +} + +type alertRule struct { + Alert string `yaml:"alert"` + Expr string `yaml:"expr"` + For string `yaml:"for"` + Labels map[string]string `yaml:"labels"` + Annotations map[string]string `yaml:"annotations"` +} + +func TestPortableAlertRulesStayBoundedAndStatic(t *testing.T) { + t.Parallel() + + contents, err := os.ReadFile("../../monitoring/sith-hub.rules.yml") + if err != nil { + t.Fatalf("read portable alert rules: %v", err) + } + var file alertRuleFile + if err := yaml.Unmarshal(contents, &file); err != nil { + t.Fatalf("decode portable alert rules: %v", err) + } + if len(file.Groups) != 1 { + t.Fatalf("rule groups = %d, want 1", len(file.Groups)) + } + group := file.Groups[0] + if group.Name != "sith-hub.failure-signals" || group.Interval != "1m" || group.Limit != 9 { + t.Errorf("rule group contract = %#v", group) + } + if len(group.Rules) != 9 { + t.Fatalf("alert rules = %d, want 9", len(group.Rules)) + } + + want := map[string]struct { + severity string + hold string + expr string + summary string + description string + }{ + "SithHubPolicyAuditFailure": { + severity: "critical", hold: "2m", + expr: `sum(increase(sith_policy_audit_attempts_total{outcome="error"}[5m])) > 0`, + }, + "SithHubPolicyDecisionErrorRatioHigh": { + severity: "warning", hold: "10m", + expr: `( sum(increase(sith_policy_decisions_total{outcome="error"}[15m])) / clamp_min(sum(increase(sith_policy_decisions_total{outcome=~"allow|deny|require-approval|error"}[15m])), 1) ) > 0.05 and sum(increase(sith_policy_decisions_total{outcome=~"allow|deny|require-approval|error"}[15m])) >= 20`, + summary: "Sith hub policy decisions are returning sustained errors", + description: "More than five percent of at least twenty aggregate eligible policy decisions ended in error over fifteen minutes.", + }, + "SithHubAuthRefusalDeliveryDrop": { + severity: "warning", hold: "5m", + expr: `sum(increase(sith_auth_refusal_delivery_drops_total[10m])) > 0`, + }, + "SithHubAuthenticationRefusalOnly": { + severity: "warning", hold: "10m", + expr: `sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 and sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0`, + summary: "Sith hub authentication traffic is persistently refusal-only", + description: "At least twenty aggregate authentication attempts were refused and none were accepted over fifteen minutes; a recent scraped sample from the preinitialized accepted-outcome series was present during the last ten minutes.", + }, + "SithHubFederationSnapshotFailureRatioHigh": { + severity: "warning", hold: "10m", + expr: `( sum(increase(sith_federation_spoke_snapshot_attempts_total{outcome!="success"}[15m])) / clamp_min(sum(increase(sith_federation_spoke_snapshot_attempts_total[15m])), 1) ) > 0.05 and sum(increase(sith_federation_spoke_snapshot_attempts_total[15m])) >= 20`, + }, + "SithHubFleetReadCoverageDegradationHigh": { + severity: "warning", hold: "10m", + expr: `( sum(increase(sith_federation_fleet_read_results_total{outcome=~"degraded|error"}[15m])) / clamp_min(sum(increase(sith_federation_fleet_read_results_total{outcome=~"complete|degraded|error"}[15m])), 1) ) > 0.05 and sum(increase(sith_federation_fleet_read_results_total{outcome=~"complete|degraded|error"}[15m])) >= 20`, + }, + "SithHubFleetReadStalenessHigh": { + severity: "warning", hold: "10m", + expr: `( sum(increase(sith_federation_fleet_read_freshness_total{outcome="stale"}[15m])) / clamp_min(sum(increase(sith_federation_fleet_read_freshness_total{outcome=~"fresh|stale"}[15m])), 1) ) > 0.05 and sum(increase(sith_federation_fleet_read_freshness_total{outcome=~"fresh|stale"}[15m])) >= 20`, + }, + "SithHubDatabaseReadinessDegradationHigh": { + severity: "warning", hold: "10m", + expr: `( sum(increase(sith_hub_readiness_checks_total{outcome="unavailable"}[15m])) / clamp_min(sum(increase(sith_hub_readiness_checks_total{outcome=~"ready|unavailable"}[15m])), 1) ) > 0.05 and sum(increase(sith_hub_readiness_checks_total{outcome=~"ready|unavailable"}[15m])) >= 20`, + }, + "SithHubTelemetryMissing": { + severity: "warning", hold: "5m", + expr: `absent_over_time(sith_build_info[10m])`, + summary: "Sith hub telemetry is absent from the rule evaluator", + description: "No Sith build-info sample reached the rule evaluator during the last ten minutes.", + }, + } + for _, rule := range group.Rules { + expected, ok := want[rule.Alert] + if !ok { + t.Errorf("unexpected alert %q", rule.Alert) + continue + } + if rule.For != expected.hold { + t.Errorf("%s hold = %q, want %q", rule.Alert, rule.For, expected.hold) + } + if len(rule.Labels) != 2 || rule.Labels["component"] != "sith-hub" || rule.Labels["severity"] != expected.severity { + t.Errorf("%s labels = %#v", rule.Alert, rule.Labels) + } + wantRunbook := "https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#" + + strings.ToLower(rule.Alert) + if len(rule.Annotations) != 3 || rule.Annotations["summary"] == "" || + rule.Annotations["description"] == "" || rule.Annotations["runbook_url"] != wantRunbook { + t.Errorf("%s annotations = %#v", rule.Alert, rule.Annotations) + } + if expected.summary != "" && (rule.Annotations["summary"] != expected.summary || + rule.Annotations["description"] != expected.description) { + t.Errorf("%s fixed annotation contract = %#v", rule.Alert, rule.Annotations) + } + for key, value := range rule.Annotations { + if strings.Contains(value, "{{") || strings.Contains(value, "$labels") || strings.ContainsAny(value, "\r\n") { + t.Errorf("%s annotation %q is dynamic or multiline", rule.Alert, key) + } + } + if normalized := strings.Join(strings.Fields(rule.Expr), " "); normalized != expected.expr { + t.Errorf("%s normalized expression = %q, want %q", rule.Alert, normalized, expected.expr) + } + delete(want, rule.Alert) + } + if len(want) != 0 { + t.Errorf("missing alert contracts: %#v", want) + } +} diff --git a/internal/observability/auth.go b/internal/observability/auth.go index 7e15ff6..70ff2b6 100644 --- a/internal/observability/auth.go +++ b/internal/observability/auth.go @@ -9,9 +9,11 @@ import ( "github.com/ArdurAI/sith/internal/hubserver" ) -// NewSlogAuthObserver constructs the local structured authentication-refusal recorder used by -// the hub runtime. It serializes only the closed AuthEvent contract; it owns no listener, -// exporter, queue, persistence, telemetry backend, or network path. +// NewSlogAuthObserver constructs a synchronous structured authentication-refusal recorder for a +// controlled embedding. The Hub runtime uses the process-supervised auditdelivery observer so an +// arbitrary local slog handler cannot delay a governed response. This adapter serializes only the +// closed AuthEvent contract; it owns no listener, exporter, queue, persistence, telemetry backend, +// or network path. func NewSlogAuthObserver(logger *slog.Logger) (hubserver.AuthObserver, error) { if logger == nil { return nil, fmt.Errorf("construct authentication observer: logger is required") @@ -24,7 +26,7 @@ type slogAuthObserver struct { } func (observer slogAuthObserver) ObserveAuth(event hubserver.AuthEvent) { - if observer.logger == nil || event.Validate() != nil { + if observer.logger == nil || event.Validate() != nil || event.Outcome != hubserver.AuthOutcomeRefused { return } observer.logger.Warn( diff --git a/internal/observability/auth_test.go b/internal/observability/auth_test.go index f0a0814..dae8a94 100644 --- a/internal/observability/auth_test.go +++ b/internal/observability/auth_test.go @@ -19,6 +19,7 @@ func TestSlogAuthObserverEmitsOnlyValidatedFixedFields(t *testing.T) { t.Fatal(err) } observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + observer.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeAccepted}) observer.ObserveAuth(hubserver.AuthEvent{Outcome: "token=secret"}) lines := strings.Split(strings.TrimSpace(output.String()), "\n") diff --git a/internal/observability/metrics.go b/internal/observability/metrics.go index 7833693..6075055 100644 --- a/internal/observability/metrics.go +++ b/internal/observability/metrics.go @@ -15,6 +15,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/hubserver" "github.com/ArdurAI/sith/internal/pep" ) @@ -35,16 +36,29 @@ type Config struct { // All label normalization occurs before a metric is created, so caller-controlled values cannot // increase cardinality or cross the privacy boundary. type Metrics struct { - gatherer prometheus.Gatherer - policyDecisions *prometheus.CounterVec - policyDuration *prometheus.HistogramVec - snapshotAttempts *prometheus.CounterVec - snapshotDuration *prometheus.HistogramVec + gatherer prometheus.Gatherer + policyDecisions *prometheus.CounterVec + policyDuration *prometheus.HistogramVec + policyAuditAttempts *prometheus.CounterVec + policyAuditDuration *prometheus.HistogramVec + snapshotAttempts *prometheus.CounterVec + snapshotDuration *prometheus.HistogramVec + fleetReadResults *prometheus.CounterVec + fleetReadFreshness *prometheus.CounterVec + readinessChecks *prometheus.CounterVec + readinessDuration *prometheus.HistogramVec + authAttempts *prometheus.CounterVec + authRefusals prometheus.Counter + authDeliveryDrops prometheus.Counter } var ( - _ pep.DecisionObserver = (*Metrics)(nil) - _ hubfleet.SnapshotObserver = (*Metrics)(nil) + _ pep.DecisionObserver = (*Metrics)(nil) + _ pep.AuditObserver = (*Metrics)(nil) + _ hubfleet.SnapshotObserver = (*Metrics)(nil) + _ hubfleet.FleetReadObserver = (*Metrics)(nil) + _ hubserver.ReadinessObserver = (*Metrics)(nil) + _ hubserver.AuthObserver = (*Metrics)(nil) ) // New constructs metrics against a caller-owned or fresh isolated registry. Registration errors @@ -68,6 +82,14 @@ func New(config Config) (*Metrics, error) { Namespace: "sith", Subsystem: "policy", Name: "decision_duration_seconds", Help: "Duration of completed Sith policy-read decisions by closed verb and outcome.", }, []string{"verb", "outcome"}), + policyAuditAttempts: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "policy", Name: "audit_attempts_total", + Help: "Total completed Sith policy-audit sink attempts by closed sink and outcome.", + }, []string{"sink", "outcome"}), + policyAuditDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "sith", Subsystem: "policy", Name: "audit_duration_seconds", + Help: "Duration of completed Sith policy-audit sink attempts by closed sink and outcome.", + }, []string{"sink", "outcome"}), snapshotAttempts: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "sith", Subsystem: "federation", Name: "spoke_snapshot_attempts_total", Help: "Total completed Sith federated spoke snapshot attempts by closed outcome.", @@ -76,6 +98,34 @@ func New(config Config) (*Metrics, error) { Namespace: "sith", Subsystem: "federation", Name: "spoke_snapshot_duration_seconds", Help: "Duration of completed Sith federated spoke snapshot attempts by closed outcome.", }, []string{"outcome"}), + fleetReadResults: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "federation", Name: "fleet_read_results_total", + Help: "Total authorized Sith federated fleet reads by closed coverage outcome.", + }, []string{"outcome"}), + fleetReadFreshness: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "federation", Name: "fleet_read_freshness_total", + Help: "Total authorized Sith federated fleet reads by closed request-time freshness outcome.", + }, []string{"outcome"}), + readinessChecks: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "hub", Name: "readiness_checks_total", + Help: "Total completed Sith Hub database readiness checks by closed outcome.", + }, []string{"outcome"}), + readinessDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "sith", Subsystem: "hub", Name: "readiness_check_duration_seconds", + Help: "Duration of completed Sith Hub database readiness checks by closed outcome.", + }, []string{"outcome"}), + authAttempts: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "auth", Name: "attempts_total", + Help: "Total completed Sith authentication verifier decisions by closed outcome.", + }, []string{"outcome"}), + authRefusals: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "auth", Name: "refusals_total", + Help: "Total authentication requests refused by Sith's sanitized middleware boundary.", + }), + authDeliveryDrops: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "sith", Subsystem: "auth", Name: "refusal_delivery_drops_total", + Help: "Total dropped bounded local authentication-refusal delivery records.", + }), } buildInfo := prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "sith", Name: "build_info", Help: "Sith build metadata with safe release identifiers only.", @@ -86,7 +136,7 @@ func New(config Config) (*Metrics, error) { }) buildInfo.Set(1) - registered := make([]prometheus.Collector, 0, 7) + registered := make([]prometheus.Collector, 0, 16) for _, collector := range []struct { name string collector prometheus.Collector @@ -96,8 +146,17 @@ func New(config Config) (*Metrics, error) { {name: "build info", collector: buildInfo}, {name: "policy decisions", collector: metrics.policyDecisions}, {name: "policy duration", collector: metrics.policyDuration}, + {name: "policy audit attempts", collector: metrics.policyAuditAttempts}, + {name: "policy audit duration", collector: metrics.policyAuditDuration}, {name: "snapshot attempts", collector: metrics.snapshotAttempts}, {name: "snapshot duration", collector: metrics.snapshotDuration}, + {name: "fleet read results", collector: metrics.fleetReadResults}, + {name: "fleet read freshness", collector: metrics.fleetReadFreshness}, + {name: "Hub readiness checks", collector: metrics.readinessChecks}, + {name: "Hub readiness duration", collector: metrics.readinessDuration}, + {name: "authentication attempts", collector: metrics.authAttempts}, + {name: "authentication refusals", collector: metrics.authRefusals}, + {name: "authentication-refusal delivery drops", collector: metrics.authDeliveryDrops}, } { if err := registry.Register(collector.collector); err != nil { for index := len(registered) - 1; index >= 0; index-- { @@ -107,12 +166,48 @@ func New(config Config) (*Metrics, error) { } registered = append(registered, collector.collector) } + for _, sink := range []pep.AuditSink{pep.AuditSinkDurable, pep.AuditSinkProcess} { + for _, outcome := range []pep.AuditOutcome{pep.AuditOutcomeSuccess, pep.AuditOutcomeError} { + metrics.policyAuditAttempts.WithLabelValues(string(sink), string(outcome)) + metrics.policyAuditDuration.WithLabelValues(string(sink), string(outcome)) + } + } + for _, outcome := range []hubfleet.FleetReadOutcome{ + hubfleet.FleetReadOutcomeComplete, + hubfleet.FleetReadOutcomeDegraded, + hubfleet.FleetReadOutcomeEmpty, + hubfleet.FleetReadOutcomeError, + } { + metrics.fleetReadResults.WithLabelValues(string(outcome)) + } + for _, outcome := range []hubfleet.FleetFreshnessOutcome{ + hubfleet.FleetFreshnessOutcomeFresh, + hubfleet.FleetFreshnessOutcomeStale, + hubfleet.FleetFreshnessOutcomeUnknown, + hubfleet.FleetFreshnessOutcomeEmpty, + hubfleet.FleetFreshnessOutcomeError, + } { + metrics.fleetReadFreshness.WithLabelValues(string(outcome)) + } + for _, outcome := range []hubserver.ReadinessOutcome{ + hubserver.ReadinessOutcomeReady, + hubserver.ReadinessOutcomeUnavailable, + } { + metrics.readinessChecks.WithLabelValues(string(outcome)) + metrics.readinessDuration.WithLabelValues(string(outcome)) + } + for _, outcome := range []hubserver.AuthOutcome{ + hubserver.AuthOutcomeAccepted, + hubserver.AuthOutcomeRefused, + } { + metrics.authAttempts.WithLabelValues(string(outcome)) + } return metrics, nil } // Handler returns an embeddable Prometheus exposition handler. It does not bind a port or make -// outbound calls; a future hub composition root owns listener, TLS, and scrape authorization. +// outbound calls; a composition root owns any listener and access boundary. func (metrics *Metrics) Handler() http.Handler { if metrics == nil || metrics.gatherer == nil { return http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { @@ -133,6 +228,18 @@ func (metrics *Metrics) ObserveDecision(verb pep.Verb, outcome pep.DecisionOutco metrics.policyDuration.WithLabelValues(verbLabel, outcomeLabel).Observe(normalizedDuration(duration)) } +// ObservePolicyAudit records one completed policy-audit sink attempt using only closed sink and +// outcome vocabularies. Invalid values are discarded instead of creating a caller-controlled +// series. +func (metrics *Metrics) ObservePolicyAudit(sink pep.AuditSink, outcome pep.AuditOutcome, duration time.Duration) { + if metrics == nil || metrics.policyAuditAttempts == nil || metrics.policyAuditDuration == nil || + !sink.Valid() || !outcome.Valid() { + return + } + metrics.policyAuditAttempts.WithLabelValues(string(sink), string(outcome)).Inc() + metrics.policyAuditDuration.WithLabelValues(string(sink), string(outcome)).Observe(normalizedDuration(duration)) +} + // ObserveSpokeSnapshot records one completed federated snapshot attempt using only a fixed outcome // vocabulary. It intentionally omits all tenant and spoke identifiers. func (metrics *Metrics) ObserveSpokeSnapshot(outcome hubfleet.SnapshotOutcome, duration time.Duration) { @@ -144,6 +251,49 @@ func (metrics *Metrics) ObserveSpokeSnapshot(outcome hubfleet.SnapshotOutcome, d metrics.snapshotDuration.WithLabelValues(outcomeLabel).Observe(normalizedDuration(duration)) } +// ObserveFleetRead records one authorized fleet read using only closed aggregate coverage and +// request-time freshness vocabularies. It omits every tenant-proportional or caller-controlled +// dimension. Invalid pairs are discarded rather than fabricating an observation. +func (metrics *Metrics) ObserveFleetRead(observation hubfleet.FleetReadObservation) { + if metrics == nil || metrics.fleetReadResults == nil || metrics.fleetReadFreshness == nil || !observation.Valid() { + return + } + metrics.fleetReadResults.WithLabelValues(string(observation.Outcome)).Inc() + metrics.fleetReadFreshness.WithLabelValues(string(observation.Freshness)).Inc() +} + +// ObserveReadiness records one completed application-database readiness check. Invalid values are +// discarded instead of creating a caller-controlled series or fabricating dependency failure. +func (metrics *Metrics) ObserveReadiness(outcome hubserver.ReadinessOutcome, duration time.Duration) { + if metrics == nil || metrics.readinessChecks == nil || metrics.readinessDuration == nil || !outcome.Valid() { + return + } + metrics.readinessChecks.WithLabelValues(string(outcome)).Inc() + metrics.readinessDuration.WithLabelValues(string(outcome)).Observe(normalizedDuration(duration)) +} + +// ObserveAuth records one already-sanitized verifier decision with a closed outcome. Refusals also +// increment the legacy unlabeled counter. No principal, workspace, correlation, request, credential +// mode, or failure reason is exposed. Invalid events are discarded rather than creating a series. +func (metrics *Metrics) ObserveAuth(event hubserver.AuthEvent) { + if metrics == nil || metrics.authAttempts == nil || event.Validate() != nil { + return + } + metrics.authAttempts.WithLabelValues(string(event.Outcome)).Inc() + if event.Outcome == hubserver.AuthOutcomeRefused && metrics.authRefusals != nil { + metrics.authRefusals.Inc() + } +} + +// ObserveAuthRefusalDeliveryDrop records one bounded process-local delivery drop. It carries no +// labels because authentication happens before any principal, workspace, or trusted correlation. +func (metrics *Metrics) ObserveAuthRefusalDeliveryDrop() { + if metrics == nil || metrics.authDeliveryDrops == nil { + return + } + metrics.authDeliveryDrops.Inc() +} + func normalizedVersion(value string) string { if value == "dev" || value == "unknown" || versionLabelPattern.MatchString(value) { return value diff --git a/internal/observability/metrics_test.go b/internal/observability/metrics_test.go index 7f5b860..97c7632 100644 --- a/internal/observability/metrics_test.go +++ b/internal/observability/metrics_test.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/ArdurAI/sith/internal/hubfleet" + "github.com/ArdurAI/sith/internal/hubserver" "github.com/ArdurAI/sith/internal/pep" ) @@ -23,8 +24,44 @@ func TestMetricsExposeOnlyBoundedSelfObservability(t *testing.T) { } metrics.ObserveDecision(pep.VerbFleetRead, pep.DecisionOutcomeAllow, 125*time.Millisecond) metrics.ObserveDecision(pep.Verb("workspace-a/token=secret"), pep.DecisionOutcome("untrusted"), -time.Second) + metrics.ObservePolicyAudit(pep.AuditSinkDurable, pep.AuditOutcomeSuccess, 15*time.Millisecond) + metrics.ObservePolicyAudit(pep.AuditSinkProcess, pep.AuditOutcomeError, -time.Second) + metrics.ObservePolicyAudit(pep.AuditSink("workspace-a/token=secret"), pep.AuditOutcome("untrusted"), time.Second) metrics.ObserveSpokeSnapshot(hubfleet.SnapshotOutcomeSuccess, 25*time.Millisecond) metrics.ObserveSpokeSnapshot(hubfleet.SnapshotOutcome("spoke-a/token=secret"), -time.Second) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeComplete, Freshness: hubfleet.FleetFreshnessOutcomeFresh, + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeDegraded, Freshness: hubfleet.FleetFreshnessOutcomeStale, + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeDegraded, Freshness: hubfleet.FleetFreshnessOutcomeUnknown, + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeEmpty, Freshness: hubfleet.FleetFreshnessOutcomeEmpty, + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeError, Freshness: hubfleet.FleetFreshnessOutcomeError, + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcome("workspace-a/token=secret"), + Freshness: hubfleet.FleetFreshnessOutcome("spoke-a/token=secret"), + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeComplete, Freshness: hubfleet.FleetFreshnessOutcomeStale, + }) + metrics.ObserveFleetRead(hubfleet.FleetReadObservation{ + Outcome: hubfleet.FleetReadOutcomeComplete, + Freshness: hubfleet.FleetFreshnessOutcome("spoke-a/token=secret"), + }) + metrics.ObserveReadiness(hubserver.ReadinessOutcomeReady, 10*time.Millisecond) + metrics.ObserveReadiness(hubserver.ReadinessOutcomeUnavailable, -time.Second) + metrics.ObserveReadiness(hubserver.ReadinessOutcome("database endpoint secret"), time.Second) + metrics.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeAccepted}) + metrics.ObserveAuth(hubserver.AuthEvent{Outcome: hubserver.AuthOutcomeRefused}) + metrics.ObserveAuth(hubserver.AuthEvent{Outcome: "token=secret"}) + metrics.ObserveAuthRefusalDeliveryDrop() response := httptest.NewRecorder() metrics.Handler().ServeHTTP(response, httptest.NewRequest(http.MethodGet, "http://metrics.invalid/metrics", nil)) @@ -36,20 +73,45 @@ func TestMetricsExposeOnlyBoundedSelfObservability(t *testing.T) { "sith_build_info", "sith_policy_decisions_total", "sith_policy_decision_duration_seconds", + "sith_policy_audit_attempts_total", + "sith_policy_audit_duration_seconds", + `sith_policy_audit_attempts_total{outcome="success",sink="durable"} 1`, + `sith_policy_audit_attempts_total{outcome="error",sink="durable"} 0`, + `sith_policy_audit_attempts_total{outcome="success",sink="process"} 0`, + `sith_policy_audit_attempts_total{outcome="error",sink="process"} 1`, "sith_federation_spoke_snapshot_attempts_total", "sith_federation_spoke_snapshot_duration_seconds", + `sith_federation_fleet_read_results_total{outcome="complete"} 1`, + `sith_federation_fleet_read_results_total{outcome="degraded"} 2`, + `sith_federation_fleet_read_results_total{outcome="empty"} 1`, + `sith_federation_fleet_read_results_total{outcome="error"} 1`, + `sith_federation_fleet_read_freshness_total{outcome="fresh"} 1`, + `sith_federation_fleet_read_freshness_total{outcome="stale"} 1`, + `sith_federation_fleet_read_freshness_total{outcome="unknown"} 1`, + `sith_federation_fleet_read_freshness_total{outcome="empty"} 1`, + `sith_federation_fleet_read_freshness_total{outcome="error"} 1`, + `sith_hub_readiness_checks_total{outcome="ready"} 1`, + `sith_hub_readiness_checks_total{outcome="unavailable"} 1`, + `sith_hub_readiness_check_duration_seconds_count{outcome="ready"} 1`, + `sith_hub_readiness_check_duration_seconds_count{outcome="unavailable"} 1`, + `sith_auth_attempts_total{outcome="accepted"} 1`, + `sith_auth_attempts_total{outcome="refused"} 1`, + "sith_auth_refusals_total 1", + "sith_auth_refusal_delivery_drops_total 1", `verb="fleet.read"`, `verb="invalid"`, `outcome="allow"`, `outcome="error"`, `outcome="success"`, `outcome="store-error"`, + `sink="durable"`, + `sink="process"`, } { if !strings.Contains(body, metric) { t.Fatalf("metrics output missing %q: %s", metric, body) } } - for _, forbidden := range []string{"workspace-a", "spoke-a", "token=secret", "untrusted"} { + for _, forbidden := range []string{"workspace-a", "spoke-a", "token=secret", "untrusted", "database endpoint secret"} { if strings.Contains(body, forbidden) { t.Fatalf("metrics output leaked %q: %s", forbidden, body) } @@ -80,14 +142,32 @@ func TestMetricsUseIndependentRegistriesAndNormalizeBuildLabels(t *testing.T) { if !strings.Contains(body, `sith_build_info{commit="unknown",version="unknown"} 1`) { t.Fatalf("unsafe build metadata was not normalized: %s", body) } + for _, preinitialized := range []string{ + `sith_federation_fleet_read_freshness_total{outcome="fresh"} 0`, + `sith_federation_fleet_read_freshness_total{outcome="stale"} 0`, + `sith_federation_fleet_read_freshness_total{outcome="unknown"} 0`, + `sith_federation_fleet_read_freshness_total{outcome="empty"} 0`, + `sith_federation_fleet_read_freshness_total{outcome="error"} 0`, + `sith_hub_readiness_checks_total{outcome="ready"} 0`, + `sith_hub_readiness_checks_total{outcome="unavailable"} 0`, + `sith_hub_readiness_check_duration_seconds_count{outcome="ready"} 0`, + `sith_hub_readiness_check_duration_seconds_count{outcome="unavailable"} 0`, + `sith_auth_attempts_total{outcome="accepted"} 0`, + `sith_auth_attempts_total{outcome="refused"} 0`, + `sith_auth_refusals_total 0`, + } { + if !strings.Contains(body, preinitialized) { + t.Fatalf("metrics output missing preinitialized readiness series %q: %s", preinitialized, body) + } + } } func TestMetricsRejectDuplicateRegistrations(t *testing.T) { registry := prometheus.NewPedanticRegistry() conflicting := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: "sith", Subsystem: "policy", Name: "decisions_total", - Help: "Total completed Sith policy-read decisions by closed verb and outcome.", - }, []string{"verb", "outcome"}) + Namespace: "sith", Subsystem: "hub", Name: "readiness_checks_total", + Help: "Total completed Sith Hub database readiness checks by closed outcome.", + }, []string{"outcome"}) if err := registry.Register(conflicting); err != nil { t.Fatal(err) } @@ -122,8 +202,17 @@ func assertSithMetricLabels(t *testing.T, metrics *Metrics) { "sith_build_info": {"commit": true, "version": true}, "sith_policy_decisions_total": {"outcome": true, "verb": true}, "sith_policy_decision_duration_seconds": {"outcome": true, "verb": true}, + "sith_policy_audit_attempts_total": {"outcome": true, "sink": true}, + "sith_policy_audit_duration_seconds": {"outcome": true, "sink": true}, "sith_federation_spoke_snapshot_attempts_total": {"outcome": true}, "sith_federation_spoke_snapshot_duration_seconds": {"outcome": true}, + "sith_federation_fleet_read_results_total": {"outcome": true}, + "sith_federation_fleet_read_freshness_total": {"outcome": true}, + "sith_hub_readiness_checks_total": {"outcome": true}, + "sith_hub_readiness_check_duration_seconds": {"outcome": true}, + "sith_auth_attempts_total": {"outcome": true}, + "sith_auth_refusals_total": {}, + "sith_auth_refusal_delivery_drops_total": {}, } for _, family := range families { labels, sithMetric := allowed[family.GetName()] diff --git a/internal/pep/approval_test.go b/internal/pep/approval_test.go new file mode 100644 index 0000000..14e4429 --- /dev/null +++ b/internal/pep/approval_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "testing" + + "github.com/ArdurAI/sith/internal/intent" +) + +func TestProposalApprovalBindingIsExactAndPrivacyMinimized(t *testing.T) { + t.Parallel() + + input, err := NewProposalInput( + "intent-250", "workspace-a", "user:operator", intent.VerbDeploymentRestart, + testProposalTarget(), digestFor("validated-arguments"), + ) + if err != nil { + t.Fatal(err) + } + binding, err := input.ApprovalBinding() + if err != nil { + t.Fatalf("ApprovalBinding() error = %v", err) + } + if err := binding.Validate(); err != nil { + t.Fatalf("ApprovalBinding.Validate() error = %v", err) + } + if binding.IntentID() != "intent-250" || binding.WorkspaceID() != "workspace-a" || + binding.Proposer() != "user:operator" || binding.ResolvedDigest() != input.resolvedDigest { + t.Fatalf("approval binding = %#v", binding) + } + if binding.ResolvedDigest() == input.argumentsDigest { + t.Fatal("approval binding exposed the argument-only digest instead of the resolved envelope") + } +} + +func TestProposalApprovalBindingRejectsInvalidOrMutatedProposal(t *testing.T) { + t.Parallel() + + if _, err := (ProposalInput{}).ApprovalBinding(); err == nil { + t.Fatal("zero proposal produced an approval binding") + } + if err := (ApprovalBinding{}).Validate(); err == nil { + t.Fatal("zero approval binding validated") + } + + input, err := NewProposalInput( + "intent-250", "workspace-a", "user:operator", intent.VerbDeploymentRestart, + testProposalTarget(), digestFor("validated-arguments"), + ) + if err != nil { + t.Fatal(err) + } + input.resolvedDigest = digestFor("attacker-replacement") + if _, err := input.ApprovalBinding(); err == nil { + t.Fatal("mutated proposal produced an approval binding") + } +} diff --git a/internal/pep/audit.go b/internal/pep/audit.go index 5a23a6f..39b1147 100644 --- a/internal/pep/audit.go +++ b/internal/pep/audit.go @@ -25,8 +25,8 @@ type slogAuditor struct { } func (auditor slogAuditor) Record(ctx context.Context, event AuditEvent) error { - if auditor.logger == nil { - return fmt.Errorf("record policy audit: logger is required") + if auditor.logger == nil || ctx == nil { + return fmt.Errorf("record policy audit: logger and context are required") } if err := event.Validate(); err != nil { return fmt.Errorf("record policy audit: %w", err) @@ -44,11 +44,19 @@ func (auditor slogAuditor) Record(ctx context.Context, event AuditEvent) error { "verdict", event.Verdict, "reason_code", event.ReasonCode, } + level := slog.LevelWarn if event.Verdict == VerdictAllow { - auditor.logger.InfoContext(ctx, "policy decision", attributes...) - return nil + level = slog.LevelInfo + } + handler := auditor.logger.Handler() + if !handler.Enabled(ctx, level) { + return fmt.Errorf("record policy audit: structured event level is disabled") + } + record := slog.NewRecord(time.Now(), level, "policy decision", 0) + record.Add(attributes...) + if err := handler.Handle(ctx, record); err != nil { + return fmt.Errorf("record policy audit: emit structured event: %w", err) } - auditor.logger.WarnContext(ctx, "policy decision", attributes...) return nil } @@ -66,7 +74,17 @@ func (event AuditEvent) Validate() error { if err := validateSafeText("policy audit actor", event.Actor, maxActorBytes); err != nil { return err } - if !event.Role.Valid() || event.Action != tenancy.ActionRead || !event.Verb.Valid() { + if !event.Role.Valid() || (event.Action != tenancy.ActionRead && event.Action != tenancy.ActionExportAudit && event.Action != tenancy.ActionProposeIntent) { + return fmt.Errorf("policy audit has unsupported role, action, or verb") + } + if !event.Role.Allows(event.Action) && (event.Verdict != VerdictDeny || (event.ReasonCode != "role-denied" && event.ReasonCode != "invalid-request")) { + return fmt.Errorf("policy audit has impossible role and action outcome") + } + if event.Verb == invalidVerb { + if event.Verdict != VerdictDeny || event.ReasonCode != "invalid-request" { + return fmt.Errorf("policy audit has unsafe invalid-request sentinel") + } + } else if !event.Verb.validForAction(event.Action) { return fmt.Errorf("policy audit has unsupported role, action, or verb") } return (Decision{Verdict: event.Verdict, ReasonCode: event.ReasonCode}).Validate() diff --git a/internal/pep/audit_metrics.go b/internal/pep/audit_metrics.go new file mode 100644 index 0000000..08d3d4b --- /dev/null +++ b/internal/pep/audit_metrics.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "fmt" + "time" +) + +// AuditSink is the closed production destination for one policy-audit attempt. +type AuditSink string + +// Closed production policy-audit sinks. +const ( + AuditSinkDurable AuditSink = "durable" + AuditSinkProcess AuditSink = "process" +) + +// Valid reports whether sink belongs to the closed policy-audit vocabulary. +func (sink AuditSink) Valid() bool { + return sink == AuditSinkDurable || sink == AuditSinkProcess +} + +// AuditOutcome is the bounded result of one policy-audit sink attempt. +type AuditOutcome string + +// Closed policy-audit attempt outcomes. +const ( + AuditOutcomeSuccess AuditOutcome = "success" + AuditOutcomeError AuditOutcome = "error" +) + +// Valid reports whether outcome belongs to the closed policy-audit vocabulary. +func (outcome AuditOutcome) Valid() bool { + return outcome == AuditOutcomeSuccess || outcome == AuditOutcomeError +} + +// AuditObserver receives passive measurements after a policy-audit sink attempt. Implementations +// must not block or mutate the audit path. The observed auditor isolates observer panics. +type AuditObserver interface { + ObservePolicyAudit(AuditSink, AuditOutcome, time.Duration) +} + +type observedAuditor struct { + sink AuditSink + auditor Auditor + observer AuditObserver +} + +// NewObservedAuditor decorates one closed policy-audit sink with passive measurements. The +// underlying auditor result remains authoritative: observer panics cannot suppress or create an +// audit failure. +func NewObservedAuditor(sink AuditSink, auditor Auditor, observer AuditObserver) (Auditor, error) { + if !sink.Valid() || auditor == nil || observer == nil { + return nil, fmt.Errorf("construct observed policy auditor: valid sink, auditor, and observer are required") + } + return observedAuditor{sink: sink, auditor: auditor, observer: observer}, nil +} + +func (auditor observedAuditor) Record(ctx context.Context, event AuditEvent) error { + if !auditor.sink.Valid() || auditor.auditor == nil || auditor.observer == nil { + return fmt.Errorf("record observed policy audit: valid sink, auditor, and observer are required") + } + startedAt := time.Now() + err := auditor.auditor.Record(ctx, event) + outcome := AuditOutcomeSuccess + if err != nil { + outcome = AuditOutcomeError + } + auditor.observe(outcome, time.Since(startedAt)) + return err +} + +func (auditor observedAuditor) observe(outcome AuditOutcome, duration time.Duration) { + defer func() { + _ = recover() + }() + auditor.observer.ObservePolicyAudit(auditor.sink, outcome, duration) +} diff --git a/internal/pep/audit_metrics_test.go b/internal/pep/audit_metrics_test.go new file mode 100644 index 0000000..0d8b793 --- /dev/null +++ b/internal/pep/audit_metrics_test.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "errors" + "testing" + "time" +) + +type auditObservation struct { + sink AuditSink + outcome AuditOutcome + duration time.Duration +} + +type recordingAuditObserver struct { + observations []auditObservation + panic bool +} + +func (observer *recordingAuditObserver) ObservePolicyAudit(sink AuditSink, outcome AuditOutcome, duration time.Duration) { + if observer.panic { + panic("observer failure") + } + observer.observations = append(observer.observations, auditObservation{sink: sink, outcome: outcome, duration: duration}) +} + +func TestObservedAuditorReportsSuccessAndPreservesCall(t *testing.T) { + t.Parallel() + + observer := &recordingAuditObserver{} + ctx := context.Background() + event := AuditEvent{} + var receivedContext context.Context + var receivedEvent AuditEvent + auditor, err := NewObservedAuditor(AuditSinkDurable, AuditFunc(func(callContext context.Context, callEvent AuditEvent) error { + receivedContext = callContext + receivedEvent = callEvent + return nil + }), observer) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(ctx, event); err != nil { + t.Fatal(err) + } + if receivedContext != ctx || receivedEvent != event { + t.Fatal("observed auditor changed the underlying call") + } + if len(observer.observations) != 1 || observer.observations[0].sink != AuditSinkDurable || + observer.observations[0].outcome != AuditOutcomeSuccess || observer.observations[0].duration < 0 { + t.Fatalf("observations = %#v", observer.observations) + } +} + +func TestObservedAuditorReportsAndPreservesFailure(t *testing.T) { + t.Parallel() + + want := errors.New("sink unavailable") + observer := &recordingAuditObserver{} + auditor, err := NewObservedAuditor(AuditSinkProcess, AuditFunc(func(context.Context, AuditEvent) error { + return want + }), observer) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), AuditEvent{}); !errors.Is(err, want) { + t.Fatalf("Record() error = %v, want original failure", err) + } + if len(observer.observations) != 1 || observer.observations[0].sink != AuditSinkProcess || + observer.observations[0].outcome != AuditOutcomeError { + t.Fatalf("observations = %#v", observer.observations) + } +} + +func TestObservedAuditorIsolatesObserverPanic(t *testing.T) { + t.Parallel() + + want := errors.New("authoritative audit failure") + observer := &recordingAuditObserver{panic: true} + auditor, err := NewObservedAuditor(AuditSinkDurable, AuditFunc(func(context.Context, AuditEvent) error { + return want + }), observer) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), AuditEvent{}); !errors.Is(err, want) { + t.Fatalf("Record() error = %v, want original failure", err) + } +} + +func TestObservedAuditorRejectsUnsafeConfiguration(t *testing.T) { + t.Parallel() + + observer := &recordingAuditObserver{} + sink := AuditFunc(func(context.Context, AuditEvent) error { return nil }) + for _, test := range []struct { + name string + sink AuditSink + auditor Auditor + observer AuditObserver + }{ + {name: "invalid sink", sink: AuditSink("workspace-a/token=secret"), auditor: sink, observer: observer}, + {name: "missing auditor", sink: AuditSinkDurable, observer: observer}, + {name: "missing observer", sink: AuditSinkDurable, auditor: sink}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if _, err := NewObservedAuditor(test.sink, test.auditor, test.observer); err == nil { + t.Fatal("unsafe observed-auditor configuration accepted") + } + }) + } + if err := (observedAuditor{}).Record(context.Background(), AuditEvent{}); err == nil { + t.Fatal("zero-value observed auditor accepted an event") + } +} diff --git a/internal/pep/audit_test.go b/internal/pep/audit_test.go index ff5ad0d..c7e5c87 100644 --- a/internal/pep/audit_test.go +++ b/internal/pep/audit_test.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "log/slog" "strings" "testing" @@ -74,6 +75,31 @@ func TestSlogAuditorRejectsUnsafeEventsWithoutEmission(t *testing.T) { } } +func TestSlogAuditorReportsHandlerFailure(t *testing.T) { + t.Parallel() + + want := errors.New("structured audit sink unavailable") + auditor, err := NewSlogAuditor(slog.New(failingAuditHandler{err: want})) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), policyAuditEvent(time.Now().UTC(), VerdictAllow, "phase-1-read")); !errors.Is(err, want) { + t.Fatalf("Record() error = %v, want handler failure", err) + } +} + +func TestSlogAuditorFailsClosedWhenEventLevelIsDisabled(t *testing.T) { + t.Parallel() + + auditor, err := NewSlogAuditor(slog.New(slog.NewTextHandler(&bytes.Buffer{}, &slog.HandlerOptions{Level: slog.LevelWarn}))) + if err != nil { + t.Fatal(err) + } + if err := auditor.Record(context.Background(), policyAuditEvent(time.Now().UTC(), VerdictAllow, "phase-1-read")); err == nil || !strings.Contains(err.Error(), "level is disabled") { + t.Fatalf("Record() error = %v, want disabled-level refusal", err) + } +} + func TestSlogAuditorRejectsUnsafeTraceIdentifierWithoutEmission(t *testing.T) { var output bytes.Buffer auditor, err := NewSlogAuditor(slog.New(slog.NewTextHandler(&output, nil))) @@ -110,9 +136,115 @@ func TestSlogAuditorTextHandlerPreservesSafeFieldsAndSeverity(t *testing.T) { } } +func TestSlogAuditorAcceptsProposalAndConstrainedInvalidRequestEvents(t *testing.T) { + var output bytes.Buffer + auditor, err := NewSlogAuditor(slog.New(slog.NewJSONHandler(&output, nil))) + if err != nil { + t.Fatal(err) + } + proposal := policyAuditEvent(time.Now().UTC(), VerdictAllow, "proposal-allowed") + proposal.Actor = "user:operator" + proposal.Role = tenancy.RoleOperator + proposal.Action = tenancy.ActionProposeIntent + proposal.Verb = "deployment.restart" + invalid := proposal + invalid.Verb = invalidVerb + invalid.Verdict = VerdictDeny + invalid.ReasonCode = "invalid-request" + invalidRead := policyAuditEvent(time.Now().UTC(), VerdictDeny, "invalid-request") + invalidRead.Verb = invalidVerb + for _, event := range []AuditEvent{proposal, invalid, invalidRead} { + if err := auditor.Record(context.Background(), event); err != nil { + t.Fatalf("Record(%s) error = %v", event.Verb, err) + } + } + logged := output.String() + for _, field := range []string{`"action":"propose-intent"`, `"verb":"deployment.restart"`, `"verb":"invalid"`} { + if !strings.Contains(logged, field) { + t.Fatalf("proposal audit missing %s: %s", field, logged) + } + } + for _, forbidden := range []string{"arguments_digest", "resolved_digest", "payments", "token", "secret"} { + if strings.Contains(logged, forbidden) { + t.Fatalf("proposal audit leaked %q: %s", forbidden, logged) + } + } +} + +func TestAuditEventRejectsUnsafeInvalidRequestSentinel(t *testing.T) { + base := policyAuditEvent(time.Now().UTC(), VerdictDeny, "invalid-request") + base.Verb = invalidVerb + if err := base.Validate(); err != nil { + t.Fatalf("baseline invalid-request event should validate: %v", err) + } + for _, mutate := range []func(*AuditEvent){ + func(event *AuditEvent) { event.Verdict = VerdictAllow }, + func(event *AuditEvent) { event.Verdict = VerdictRequireApproval }, + func(event *AuditEvent) { event.ReasonCode = "policy-deny" }, + } { + event := base + mutate(&event) + if err := event.Validate(); err == nil { + t.Fatalf("AuditEvent.Validate() accepted unsafe invalid sentinel: %#v", event) + } + } +} + +func TestAuditEventRejectsMalformedUTF8Actor(t *testing.T) { + t.Parallel() + + event := policyAuditEvent(time.Now().UTC(), VerdictAllow, "phase-1-read") + event.Actor = string([]byte{'u', 's', 'e', 'r', ':', 0x80}) + if err := event.Validate(); err == nil { + t.Fatal("AuditEvent.Validate() accepted a malformed UTF-8 actor") + } +} + +func TestAuditEventRejectsImpossibleProposalRoleOutcome(t *testing.T) { + base := policyAuditEvent(time.Now().UTC(), VerdictDeny, "role-denied") + base.Action = tenancy.ActionProposeIntent + base.Verb = "deployment.restart" + if err := base.Validate(); err != nil { + t.Fatalf("role-denied proposal event should validate: %v", err) + } + + invalidRequest := base + invalidRequest.ReasonCode = "invalid-request" + if err := invalidRequest.Validate(); err != nil { + t.Fatalf("pre-role invalid-request event should validate: %v", err) + } + + for _, mutate := range []func(*AuditEvent){ + func(event *AuditEvent) { event.Verdict = VerdictAllow; event.ReasonCode = "proposal-allowed" }, + func(event *AuditEvent) { + event.Verdict = VerdictRequireApproval + event.ReasonCode = "approval-required" + }, + func(event *AuditEvent) { event.ReasonCode = "policy-deny" }, + } { + event := base + mutate(&event) + if err := event.Validate(); err == nil { + t.Fatalf("AuditEvent.Validate() accepted impossible role outcome: %#v", event) + } + } +} + func policyAuditEvent(at time.Time, verdict Verdict, reason string) AuditEvent { return AuditEvent{ At: at, TraceID: tracing.ID("0123456789abcdef0123456789abcdef"), WorkspaceID: "workspace-a", Actor: "user:reader", Role: tenancy.RoleReader, Action: tenancy.ActionRead, Verb: VerbFleetRead, Verdict: verdict, ReasonCode: reason, } } + +type failingAuditHandler struct { + err error +} + +func (handler failingAuditHandler) Enabled(context.Context, slog.Level) bool { return true } + +func (handler failingAuditHandler) Handle(context.Context, slog.Record) error { return handler.err } + +func (handler failingAuditHandler) WithAttrs([]slog.Attr) slog.Handler { return handler } + +func (handler failingAuditHandler) WithGroup(string) slog.Handler { return handler } diff --git a/internal/pep/boundary_test.go b/internal/pep/boundary_test.go index b62abbe..c596b39 100644 --- a/internal/pep/boundary_test.go +++ b/internal/pep/boundary_test.go @@ -40,8 +40,23 @@ func TestPEPHasNoNetworkOrDispatchImports(t *testing.T) { } } -func TestAuditEventOmitsArgumentDigest(t *testing.T) { - if _, exists := reflect.TypeFor[AuditEvent]().FieldByName("ArgumentsDigest"); exists { - t.Fatal("audit event must not retain the policy argument digest") +func TestPolicyBoundaryOmitsRawProposalAndDigestMaterial(t *testing.T) { + assertExactFields(t, reflect.TypeFor[AuditEvent](), []string{ + "At", "TraceID", "WorkspaceID", "Actor", "Role", "Action", "Verb", "Verdict", "ReasonCode", + }) + assertExactFields(t, reflect.TypeFor[ProposalInput](), []string{ + "intentID", "workspaceID", "actor", "verb", "target", "argumentsDigest", "resolvedDigest", + }) +} + +func assertExactFields(t *testing.T, value reflect.Type, expected []string) { + t.Helper() + if value.NumField() != len(expected) { + t.Fatalf("%s fields = %d, want exactly %d", value.Name(), value.NumField(), len(expected)) + } + for _, name := range expected { + if _, exists := value.FieldByName(name); !exists { + t.Fatalf("%s is missing approved field %s", value.Name(), name) + } } } diff --git a/internal/pep/metrics.go b/internal/pep/metrics.go index ca2a051..8083072 100644 --- a/internal/pep/metrics.go +++ b/internal/pep/metrics.go @@ -6,10 +6,11 @@ import ( "context" "time" + "github.com/ArdurAI/sith/internal/intent" "github.com/ArdurAI/sith/internal/tracing" ) -// DecisionOutcome is the bounded self-observability result of one policy read attempt. +// DecisionOutcome is the bounded self-observability result of one policy decision. // It intentionally carries no workspace, actor, selector, credential, or reason-code material. type DecisionOutcome string @@ -21,7 +22,7 @@ const ( DecisionOutcomeError DecisionOutcome = "error" ) -// DecisionObserver receives passive, bounded measurements for policy reads. Implementations must +// DecisionObserver receives passive, bounded measurements for policy decisions. Implementations must // not block or mutate the authorization path. The enforcer isolates observer panics defensively. type DecisionObserver interface { ObserveDecision(verb Verb, outcome DecisionOutcome, duration time.Duration) @@ -66,7 +67,7 @@ func traceOutcome(outcome DecisionOutcome) tracing.Outcome { } func normalizedObservedVerb(verb Verb) Verb { - if verb.Valid() { + if verb.Valid() || intent.Verb(verb).Valid() { return verb } return "invalid" diff --git a/internal/pep/metrics_test.go b/internal/pep/metrics_test.go index b282594..df44a7f 100644 --- a/internal/pep/metrics_test.go +++ b/internal/pep/metrics_test.go @@ -65,6 +65,24 @@ func TestEnforcerRecoversFromPanickingObserver(t *testing.T) { } } +func TestEnforcerObservesCanonicalProposalVerb(t *testing.T) { + observer := &recordingDecisionObserver{} + enforcer, err := NewEnforcer(Config{ + Hook: proposalDecision(VerdictAllow, "proposal-allowed"), + Auditor: AuditFunc(func(context.Context, AuditEvent) error { return nil }), + Observer: observer, + }) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeProposal(context.Background(), operatorScope(t), testProposalInput(t)); err != nil { + t.Fatalf("AuthorizeProposal() error = %v", err) + } + if len(observer.events) != 1 || observer.events[0].verb != "deployment.restart" || observer.events[0].outcome != DecisionOutcomeAllow { + t.Fatalf("observations = %#v", observer.events) + } +} + type decisionObservation struct { verb Verb outcome DecisionOutcome diff --git a/internal/pep/pep.go b/internal/pep/pep.go index 1ddec42..e56f709 100644 --- a/internal/pep/pep.go +++ b/internal/pep/pep.go @@ -6,18 +6,33 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "strings" "time" "unicode" + "unicode/utf8" + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" "github.com/ArdurAI/sith/internal/tenancy" "github.com/ArdurAI/sith/internal/tracing" ) const ( - maxActorBytes = 256 - maxReasonCodeBytes = 64 + maxActorBytes = 256 + maxIntentIDBytes = 253 + maxReasonCodeBytes = 64 + maxTargetComponentBytes = 256 + invalidVerb = Verb("invalid") + proposalDigestDomain = "sith-pep-proposal/v1" +) + +// Stable policy refusal classes let later approval and action orchestration distinguish a deny +// from a pending approval without parsing error text. Both remain fail-closed outcomes. +var ( + ErrDenied = errors.New("policy denied request") + ErrApprovalRequired = errors.New("policy requires approval") ) // Verb identifies one closed PEP operation. New verbs are an explicit policy-boundary change. @@ -27,35 +42,52 @@ type Verb string const ( VerbFleetRead Verb = "fleet.read" VerbFleetCorrelate Verb = "fleet.correlate" + VerbFleetInventorySearch Verb = "fleet.inventory.search" VerbFleetImageSearch Verb = "fleet.image.search" VerbFleetCVESearch Verb = "fleet.cve.search" VerbFleetCVEIdentifierSearch Verb = "fleet.cve.identifier.search" VerbSpokeSnapshotRefresh Verb = "fleet.snapshot.refresh" + VerbAuditExport Verb = "audit.export" ) // Valid reports whether a verb belongs to the currently supported closed vocabulary. func (verb Verb) Valid() bool { switch verb { - case VerbFleetRead, VerbFleetCorrelate, VerbFleetImageSearch, VerbFleetCVESearch, VerbFleetCVEIdentifierSearch, VerbSpokeSnapshotRefresh: + case VerbFleetRead, VerbFleetCorrelate, VerbFleetInventorySearch, VerbFleetImageSearch, VerbFleetCVESearch, VerbFleetCVEIdentifierSearch, VerbSpokeSnapshotRefresh, VerbAuditExport: return true default: return false } } +func (verb Verb) validForAction(action tenancy.Action) bool { + switch action { + case tenancy.ActionRead: + return verb.Valid() && verb != VerbAuditExport + case tenancy.ActionExportAudit: + return verb == VerbAuditExport + case tenancy.ActionProposeIntent: + return intent.Verb(verb).Valid() + default: + return false + } +} + // Verdict is the PDP-compatible outcome returned at the policy hook. type Verdict string -// Supported policy outcomes. A non-allow outcome never reaches the read dependency. +// Supported policy outcomes. A non-allow outcome never reaches the downstream operation. const ( VerdictAllow Verdict = "allow" VerdictDeny Verdict = "deny" VerdictRequireApproval Verdict = "require-approval" ) -// Request is the normalized, post-authentication input to the policy hook. Scope identity comes -// only from a signed tenancy scope; raw credentials, headers, selectors, and result data are never -// carried into the audit event. +// Request is the normalized, post-authentication input to the policy hook. For reads, +// ArgumentsDigest binds canonical validated arguments. For proposals, it binds the complete +// resolved proposal envelope. Scope identity comes only from a signed tenancy scope; raw +// credentials, headers, selectors, targets, arguments, and result data are never carried into the +// hook or audit event. type Request struct { WorkspaceID tenancy.WorkspaceID Actor string @@ -72,6 +104,29 @@ type ReadInput struct { ArgumentsDigest string } +// ProposalInput is an immutable, privacy-minimizing binding for a validated and resolved typed +// proposal. Callers construct it only after handler-owned argument validation and target +// resolution. It retains digests and normalized identifiers, never raw arguments. +type ProposalInput struct { + intentID string + workspaceID tenancy.WorkspaceID + actor string + verb intent.Verb + target fleet.ResourceRef + argumentsDigest string + resolvedDigest string +} + +// ApprovalBinding is the privacy-minimizing, unforgeable view of one validated proposal needed by +// the approval store. Its fields remain private so code outside this package cannot assemble an +// approval from caller-controlled identifiers or a digest that did not come from NewProposalInput. +type ApprovalBinding struct { + intentID string + workspaceID tenancy.WorkspaceID + proposer string + resolvedDigest string +} + // NewReadInput hashes canonical typed arguments for the policy hook. Callers must validate their // concrete argument schema before constructing this input. func NewReadInput(verb Verb, canonicalArguments []byte) ReadInput { @@ -79,6 +134,68 @@ func NewReadInput(verb Verb, canonicalArguments []byte) ReadInput { return ReadInput{Verb: verb, ArgumentsDigest: "sha256:" + hex.EncodeToString(digest[:])} } +// NewProposalInput binds one exact resolved proposal. target must be the normalized target +// returned by planning and argumentsDigest must come from the already schema-validated argument +// document. The resulting digest changes if any bound identity, verb, target, or argument digest +// changes. +func NewProposalInput( + intentID string, + workspaceID tenancy.WorkspaceID, + actor string, + verb intent.Verb, + target fleet.ResourceRef, + argumentsDigest string, +) (ProposalInput, error) { + input := ProposalInput{ + intentID: intentID, workspaceID: workspaceID, actor: actor, verb: verb, + target: target, argumentsDigest: argumentsDigest, + } + if err := input.validateFields(); err != nil { + return ProposalInput{}, fmt.Errorf("construct policy proposal input: %w", err) + } + // An empty non-nil attributes map is semantically valid but remains caller-mutable. Discard it + // so the retained proposal binding cannot be changed or raced after construction. + input.target.Attributes = nil + input.resolvedDigest = input.digest() + return input, nil +} + +// ApprovalBinding returns the exact immutable identity and resolved digest that an approval may +// authorize. Raw arguments and target data never cross this boundary. +func (input ProposalInput) ApprovalBinding() (ApprovalBinding, error) { + if err := input.validate(); err != nil { + return ApprovalBinding{}, fmt.Errorf("derive proposal approval binding: invalid proposal") + } + return ApprovalBinding{ + intentID: input.intentID, workspaceID: input.workspaceID, + proposer: input.actor, resolvedDigest: input.resolvedDigest, + }, nil +} + +// Validate rejects an incomplete approval binding. Only ProposalInput.ApprovalBinding can create a +// non-zero binding outside this package because every field is private. +func (binding ApprovalBinding) Validate() error { + if validateSafeText("approval intent identifier", binding.intentID, maxIntentIDBytes) != nil || + tenancy.ValidateWorkspaceID(binding.workspaceID) != nil || + validateSafeText("approval proposer", binding.proposer, maxActorBytes) != nil || + !validDigest(binding.resolvedDigest) { + return fmt.Errorf("approval binding is invalid") + } + return nil +} + +// IntentID returns the immutable intent identifier bound to the approval. +func (binding ApprovalBinding) IntentID() string { return binding.intentID } + +// WorkspaceID returns the only workspace in which the approval may be used. +func (binding ApprovalBinding) WorkspaceID() tenancy.WorkspaceID { return binding.workspaceID } + +// Proposer returns the authenticated actor that created the bound proposal. +func (binding ApprovalBinding) Proposer() string { return binding.proposer } + +// ResolvedDigest returns the SHA-256 binding over the exact proposal envelope. +func (binding ApprovalBinding) ResolvedDigest() string { return binding.resolvedDigest } + // Decision records one PDP-compatible policy result using a safe reason code rather than free text. type Decision struct { Verdict Verdict @@ -113,7 +230,7 @@ type AuditEvent struct { ReasonCode string } -// Auditor durably records one PEP decision. An audit failure fails closed before a read runs. +// Auditor durably records one PEP decision. An audit failure fails closed before an operation runs. type Auditor interface { Record(context.Context, AuditEvent) error } @@ -135,7 +252,7 @@ type Config struct { Now func() time.Time } -// Enforcer applies the fixed Phase-1 read pipeline and creates one audit record for every decision. +// Enforcer applies the fixed policy pipeline and creates one audit record for every decision. type Enforcer struct { hook PolicyHook auditor Auditor @@ -161,15 +278,20 @@ func NewEnforcer(config Config) (*Enforcer, error) { return &Enforcer{hook: config.Hook, auditor: config.Auditor, observer: config.Observer, tracer: config.TraceObserver, now: config.Now}, nil } -// AllowReadHook is the temporary Phase-1 policy implementation. It allows only the closed read -// vocabulary; it is intentionally unsuitable for future write verbs. +// AllowReadHook is the temporary Phase-1 policy implementation. It allows only closed read-only +// operations, including the separately role-gated audit export; it is intentionally unsuitable for +// future write verbs. type AllowReadHook struct{} // Decide returns allow only for an already-validated read request. func (AllowReadHook) Decide(_ context.Context, request Request) (Decision, error) { - if request.Action != tenancy.ActionRead || !request.Verb.Valid() { + if (request.Action != tenancy.ActionRead && request.Action != tenancy.ActionExportAudit) || + !request.Verb.validForAction(request.Action) { return Decision{Verdict: VerdictDeny, ReasonCode: "unsupported-read"}, nil } + if request.Action == tenancy.ActionExportAudit { + return Decision{Verdict: VerdictAllow, ReasonCode: "phase-1-audit-export"}, nil + } return Decision{Verdict: VerdictAllow, ReasonCode: "phase-1-read"}, nil } @@ -177,40 +299,73 @@ func (AllowReadHook) Decide(_ context.Context, request Request) (Decision, error // hook → audit. A deny, approval requirement, malformed decision, hook error, or audit error blocks // the downstream reader. func (enforcer *Enforcer) AuthorizeRead(ctx context.Context, scope tenancy.Scope, input ReadInput) error { + request := Request{ + WorkspaceID: scope.WorkspaceID(), Actor: scope.Subject(), Role: scope.Role(), Action: tenancy.ActionRead, + Verb: input.Verb, ArgumentsDigest: input.ArgumentsDigest, + } + return enforcer.authorize(ctx, scope, request, true, "read") +} + +// AuthorizeAuditExport applies a dedicated admin-only policy boundary to disclosure of the retained +// workspace audit chain. The exact operation has no caller-controlled arguments. +func (enforcer *Enforcer) AuthorizeAuditExport(ctx context.Context, scope tenancy.Scope) error { + input := NewReadInput(VerbAuditExport, nil) + request := Request{ + WorkspaceID: scope.WorkspaceID(), Actor: scope.Subject(), Role: scope.Role(), Action: tenancy.ActionExportAudit, + Verb: input.Verb, ArgumentsDigest: input.ArgumentsDigest, + } + return enforcer.authorize(ctx, scope, request, true, "audit export") +} + +// AuthorizeProposal runs a resolved typed proposal through the same policy hook and mandatory +// audit boundary as reads. It grants no execution capability: AllowReadHook denies this action, and +// both deny and require-approval return typed fail-closed errors. +func (enforcer *Enforcer) AuthorizeProposal(ctx context.Context, scope tenancy.Scope, input ProposalInput) error { + request := Request{ + WorkspaceID: scope.WorkspaceID(), Actor: scope.Subject(), Role: scope.Role(), Action: tenancy.ActionProposeIntent, + Verb: Verb(input.verb), ArgumentsDigest: input.resolvedDigest, + } + inputValid := input.validate() == nil && input.workspaceID == scope.WorkspaceID() && input.actor == scope.Subject() + return enforcer.authorize(ctx, scope, request, inputValid, "proposal") +} + +func (enforcer *Enforcer) authorize( + ctx context.Context, + scope tenancy.Scope, + request Request, + inputValid bool, + operation string, +) error { if enforcer == nil || enforcer.hook == nil || enforcer.auditor == nil || ctx == nil { - return fmt.Errorf("authorize read: enforcer and context are required") + return fmt.Errorf("authorize %s: enforcer and context are required", operation) } traceContext, _, err := tracing.Ensure(ctx) if err != nil { - return fmt.Errorf("authorize read: establish trace context: %w", err) + return fmt.Errorf("authorize %s: establish trace context: %w", operation, err) } ctx = traceContext startedAt := time.Now() outcome := DecisionOutcomeError defer func() { - enforcer.observeDecision(input.Verb, outcome, time.Since(startedAt)) + enforcer.observeDecision(request.Verb, outcome, time.Since(startedAt)) enforcer.observeTrace(ctx, outcome, time.Since(startedAt)) }() - request := Request{ - WorkspaceID: scope.WorkspaceID(), Actor: scope.Subject(), Role: scope.Role(), Action: tenancy.ActionRead, - Verb: input.Verb, ArgumentsDigest: input.ArgumentsDigest, - } - if err := request.Validate(); err != nil { + if !inputValid || request.Validate() != nil { outcome = DecisionOutcomeDeny - return enforcer.refuse(ctx, request, VerdictDeny, "invalid-request", "authorize read: invalid policy request") + return enforcer.refuse(ctx, request, VerdictDeny, "invalid-request", fmt.Sprintf("authorize %s: invalid policy request", operation), ErrDenied) } - if err := scope.Authorize(tenancy.ActionRead); err != nil { + if err := scope.Authorize(request.Action); err != nil { outcome = DecisionOutcomeDeny - return enforcer.refuse(ctx, request, VerdictDeny, "role-denied", "authorize read: role does not permit read") + return enforcer.refuse(ctx, request, VerdictDeny, "role-denied", fmt.Sprintf("authorize %s: role does not permit %s", operation, request.Action), ErrDenied) } decision, err := enforcer.hook.Decide(ctx, request) if err != nil { outcome = DecisionOutcomeError - return enforcer.refuse(ctx, request, VerdictDeny, "hook-error", "authorize read: policy hook failed") + return enforcer.refuse(ctx, request, VerdictDeny, "hook-error", fmt.Sprintf("authorize %s: policy hook failed", operation), nil) } if err := decision.Validate(); err != nil { outcome = DecisionOutcomeError - return enforcer.refuse(ctx, request, VerdictDeny, "invalid-decision", "authorize read: policy hook returned an invalid decision") + return enforcer.refuse(ctx, request, VerdictDeny, "invalid-decision", fmt.Sprintf("authorize %s: policy hook returned an invalid decision", operation), nil) } if decision.Verdict != VerdictAllow { if decision.Verdict == VerdictRequireApproval { @@ -220,16 +375,16 @@ func (enforcer *Enforcer) AuthorizeRead(ctx context.Context, scope tenancy.Scope } if err := enforcer.record(ctx, request, decision); err != nil { outcome = DecisionOutcomeError - return fmt.Errorf("authorize read: audit policy refusal: %w", err) + return fmt.Errorf("authorize %s: audit policy refusal: %w", operation, err) } if decision.Verdict == VerdictRequireApproval { - return fmt.Errorf("authorize read: policy requires approval") + return fmt.Errorf("authorize %s: %w", operation, ErrApprovalRequired) } - return fmt.Errorf("authorize read: policy denied request") + return fmt.Errorf("authorize %s: %w", operation, ErrDenied) } if err := enforcer.record(ctx, request, decision); err != nil { outcome = DecisionOutcomeError - return fmt.Errorf("authorize read: audit policy decision: %w", err) + return fmt.Errorf("authorize %s: audit policy decision: %w", operation, err) } outcome = DecisionOutcomeAllow return nil @@ -243,13 +398,13 @@ func (request Request) Validate() error { if err := validateSafeText("policy actor", request.Actor, maxActorBytes); err != nil { return err } - if !request.Role.Valid() || request.Action != tenancy.ActionRead || !request.Verb.Valid() || !validDigest(request.ArgumentsDigest) { + if !request.Role.Valid() || !request.Verb.validForAction(request.Action) || !validDigest(request.ArgumentsDigest) { return fmt.Errorf("policy request uses an unsupported role, action, or verb") } return nil } -// Validate rejects incomplete, unknown, or unsafe PDP responses before they affect a read. +// Validate rejects incomplete, unknown, or unsafe PDP responses before they affect an operation. func (decision Decision) Validate() error { switch decision.Verdict { case VerdictAllow, VerdictDeny, VerdictRequireApproval: @@ -259,13 +414,40 @@ func (decision Decision) Validate() error { return validateReasonCode(decision.ReasonCode) } -func (enforcer *Enforcer) refuse(ctx context.Context, request Request, verdict Verdict, reasonCode, message string) error { +func (enforcer *Enforcer) refuse(ctx context.Context, request Request, verdict Verdict, reasonCode, message string, classification error) error { + request, auditable := normalizedAuditRequest(request) + if !auditable { + if classification != nil { + return fmt.Errorf("%s: %w", message, classification) + } + return fmt.Errorf("%s", message) + } if err := enforcer.record(ctx, request, Decision{Verdict: verdict, ReasonCode: reasonCode}); err != nil { + if classification != nil { + return fmt.Errorf("%s: %w: audit refusal: %w", message, classification, err) + } return fmt.Errorf("%s: audit refusal: %w", message, err) } + if classification != nil { + return fmt.Errorf("%s: %w", message, classification) + } return fmt.Errorf("%s", message) } +func normalizedAuditRequest(request Request) (Request, bool) { + if tenancy.ValidateWorkspaceID(request.WorkspaceID) != nil || validateSafeText("policy actor", request.Actor, maxActorBytes) != nil || + !request.Role.Valid() || (request.Action != tenancy.ActionRead && request.Action != tenancy.ActionExportAudit && request.Action != tenancy.ActionProposeIntent) { + return Request{}, false + } + if !request.Verb.validForAction(request.Action) { + request.Verb = invalidVerb + } + // AuditEvent deliberately has no binding-digest field. Clear it here as defense in depth so a + // malformed or caller-supplied value cannot survive refusal normalization into future sinks. + request.ArgumentsDigest = "" + return request, true +} + func (enforcer *Enforcer) record(ctx context.Context, request Request, decision Decision) error { traceID, ok := tracing.FromContext(ctx) if !ok { @@ -278,7 +460,7 @@ func (enforcer *Enforcer) record(ctx context.Context, request Request, decision } func validateSafeText(name, value string, maximum int) error { - if value == "" || strings.TrimSpace(value) != value || len(value) > maximum { + if value == "" || !utf8.ValidString(value) || strings.TrimSpace(value) != value || len(value) > maximum { return fmt.Errorf("%s is invalid", name) } for _, character := range value { @@ -313,3 +495,47 @@ func validDigest(value string) bool { } return true } + +func (input ProposalInput) validate() error { + if err := input.validateFields(); err != nil { + return err + } + if !validDigest(input.resolvedDigest) || input.resolvedDigest != input.digest() { + return fmt.Errorf("proposal binding digest is invalid") + } + return nil +} + +func (input ProposalInput) validateFields() error { + if validateSafeText("proposal intent identifier", input.intentID, maxIntentIDBytes) != nil || + tenancy.ValidateWorkspaceID(input.workspaceID) != nil || + validateSafeText("proposal actor", input.actor, maxActorBytes) != nil || !input.verb.Valid() || + validateProposalTarget(input.target) != nil || !validDigest(input.argumentsDigest) { + return fmt.Errorf("proposal fields are invalid") + } + return nil +} + +func (input ProposalInput) digest() string { + values := []string{ + proposalDigestDomain, input.intentID, string(input.workspaceID), input.actor, string(input.verb), + input.target.SourceKind, input.target.Scope, input.target.Kind, input.target.Namespace, input.target.Name, + input.argumentsDigest, + } + digest := sha256.Sum256([]byte(strings.Join(values, "\x00"))) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +func validateProposalTarget(target fleet.ResourceRef) error { + if len(target.Attributes) != 0 || + validateSafeText("proposal target source", target.SourceKind, maxTargetComponentBytes) != nil || + validateSafeText("proposal target scope", target.Scope, maxTargetComponentBytes) != nil || + validateSafeText("proposal target kind", target.Kind, maxTargetComponentBytes) != nil || + validateSafeText("proposal target name", target.Name, maxTargetComponentBytes) != nil { + return fmt.Errorf("proposal target is invalid") + } + if target.Namespace != "" && validateSafeText("proposal target namespace", target.Namespace, maxTargetComponentBytes) != nil { + return fmt.Errorf("proposal target is invalid") + } + return nil +} diff --git a/internal/pep/pep_test.go b/internal/pep/pep_test.go index 9d30edd..c309aaf 100644 --- a/internal/pep/pep_test.go +++ b/internal/pep/pep_test.go @@ -35,6 +35,43 @@ func TestEnforcerAllowsClosedReadAndAudits(t *testing.T) { } } +func TestEnforcerAllowsOnlyAdminAuditExportAndAuditsDedicatedAction(t *testing.T) { + for _, role := range []tenancy.Role{tenancy.RoleReader, tenancy.RoleOperator, tenancy.RoleApprover, tenancy.RoleAdmin} { + t.Run(string(role), func(t *testing.T) { + auditor := &recordingAuditor{} + enforcer, err := NewEnforcer(Config{Hook: AllowReadHook{}, Auditor: auditor}) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeAuditExport(context.Background(), testRoleScope(t, role)) + if role == tenancy.RoleAdmin { + if err != nil { + t.Fatalf("AuthorizeAuditExport() error = %v", err) + } + if len(auditor.events) != 1 || auditor.events[0].Action != tenancy.ActionExportAudit || + auditor.events[0].Verb != VerbAuditExport || auditor.events[0].Verdict != VerdictAllow || + auditor.events[0].ReasonCode != "phase-1-audit-export" { + t.Fatalf("admin audit event = %#v", auditor.events) + } + return + } + if !errors.Is(err, ErrDenied) || len(auditor.events) != 1 || + auditor.events[0].Verdict != VerdictDeny || auditor.events[0].ReasonCode != "role-denied" { + t.Fatalf("role %q error/events = %v/%#v", role, err, auditor.events) + } + }) + } + + auditor := &recordingAuditor{} + enforcer, err := NewEnforcer(Config{Hook: AllowReadHook{}, Auditor: auditor}) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeRead(context.Background(), testRoleScope(t, tenancy.RoleReader), NewReadInput(VerbAuditExport, nil)); !errors.Is(err, ErrDenied) || len(auditor.events) != 1 || auditor.events[0].Verb != invalidVerb { + t.Fatalf("ordinary read reused audit export verb: error/events = %v/%#v", err, auditor.events) + } +} + func TestEnforcerFailsClosedForUnsafePolicyOutcomes(t *testing.T) { tests := []struct { name string @@ -127,6 +164,20 @@ func TestReadInputBindsCanonicalArgumentsAndRejectsAlteredDigest(t *testing.T) { } } +func TestNormalizedAuditRequestErasesPolicyBindingDigest(t *testing.T) { + request := Request{ + WorkspaceID: "workspace-a", Actor: "user:reader", Role: tenancy.RoleReader, + Action: tenancy.ActionRead, Verb: VerbFleetRead, ArgumentsDigest: "token=caller-secret", + } + normalized, ok := normalizedAuditRequest(request) + if !ok { + t.Fatal("normalizedAuditRequest() rejected safe audit identity") + } + if normalized.ArgumentsDigest != "" { + t.Fatalf("normalized audit request retained binding digest %q", normalized.ArgumentsDigest) + } +} + type recordingAuditor struct { events []AuditEvent } @@ -148,3 +199,16 @@ func testScope(t *testing.T) tenancy.Scope { } return scope } + +func testRoleScope(t *testing.T, role tenancy.Role) tenancy.Scope { + t.Helper() + principal, err := tenancy.NewPrincipal("user:test", map[tenancy.WorkspaceID]tenancy.Role{"workspace-a": role}) + if err != nil { + t.Fatal(err) + } + scope, err := principal.Scope("workspace-a") + if err != nil { + t.Fatal(err) + } + return scope +} diff --git a/internal/pep/proposal_test.go b/internal/pep/proposal_test.go new file mode 100644 index 0000000..b91cfb9 --- /dev/null +++ b/internal/pep/proposal_test.go @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pep + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/tenancy" +) + +func TestEnforcerAuthorizesBoundTypedProposalAndAudits(t *testing.T) { + input := testProposalInput(t) + var captured Request + auditor := &recordingAuditor{} + enforcer, err := NewEnforcer(Config{ + Hook: HookFunc(func(_ context.Context, request Request) (Decision, error) { + captured = request + return Decision{Verdict: VerdictAllow, ReasonCode: "proposal-allowed"}, nil + }), + Auditor: auditor, + }) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeProposal(context.Background(), operatorScope(t), input); err != nil { + t.Fatalf("AuthorizeProposal() error = %v", err) + } + if captured.WorkspaceID != "workspace-a" || captured.Actor != "user:operator" || captured.Role != tenancy.RoleOperator || + captured.Action != tenancy.ActionProposeIntent || captured.Verb != Verb(intent.VerbDeploymentRestart) || + captured.ArgumentsDigest != input.resolvedDigest || !validDigest(captured.ArgumentsDigest) { + t.Fatalf("policy request = %#v", captured) + } + if len(auditor.events) != 1 { + t.Fatalf("audit events = %#v, want one", auditor.events) + } + event := auditor.events[0] + if event.WorkspaceID != "workspace-a" || event.Actor != "user:operator" || event.Role != tenancy.RoleOperator || + event.Action != tenancy.ActionProposeIntent || event.Verb != Verb(intent.VerbDeploymentRestart) || + event.Verdict != VerdictAllow || event.ReasonCode != "proposal-allowed" { + t.Fatalf("audit event = %#v", event) + } +} + +func TestAllowReadHookDeniesProposalByDefault(t *testing.T) { + enforcer, err := NewEnforcer(Config{Hook: AllowReadHook{}, Auditor: &recordingAuditor{}}) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeProposal(context.Background(), operatorScope(t), testProposalInput(t)) + if !errors.Is(err, ErrDenied) { + t.Fatalf("AuthorizeProposal() error = %v, want ErrDenied", err) + } +} + +func TestEnforcerRejectsNilProposalContext(t *testing.T) { + var hookCalls atomic.Int64 + var auditCalls atomic.Int64 + enforcer, err := NewEnforcer(Config{ + Hook: HookFunc(func(context.Context, Request) (Decision, error) { + hookCalls.Add(1) + return Decision{Verdict: VerdictAllow, ReasonCode: "unexpected"}, nil + }), + Auditor: AuditFunc(func(context.Context, AuditEvent) error { + auditCalls.Add(1) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + var nilContext context.Context + if err := enforcer.AuthorizeProposal(nilContext, operatorScope(t), testProposalInput(t)); err == nil || !strings.Contains(err.Error(), "context are required") { + t.Fatalf("AuthorizeProposal(nil) error = %v", err) + } + if hookCalls.Load() != 0 || auditCalls.Load() != 0 { + t.Fatalf("nil-context hook/audit calls = %d/%d, want zero", hookCalls.Load(), auditCalls.Load()) + } +} + +func TestEnforcerFailsClosedForUnsafeProposalPolicyOutcomes(t *testing.T) { + tests := []struct { + name string + hook PolicyHook + want Verdict + reason string + wantError error + contains string + auditFails bool + }{ + { + name: "deny", hook: proposalDecision(VerdictDeny, "policy-deny"), + want: VerdictDeny, reason: "policy-deny", wantError: ErrDenied, + }, + { + name: "approval", hook: proposalDecision(VerdictRequireApproval, "approval-required"), + want: VerdictRequireApproval, reason: "approval-required", wantError: ErrApprovalRequired, + }, + { + name: "hook error", hook: HookFunc(func(context.Context, Request) (Decision, error) { + return Decision{}, errors.New("pdp token=secret") + }), + want: VerdictDeny, reason: "hook-error", contains: "policy hook failed", + }, + { + name: "invalid decision", hook: proposalDecision("maybe", "invalid"), + want: VerdictDeny, reason: "invalid-decision", contains: "invalid decision", + }, + { + name: "audit failure", hook: proposalDecision(VerdictAllow, "proposal-allowed"), + contains: "audit policy decision", auditFails: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + auditor := &recordingAuditor{} + var sink Auditor = auditor + if test.auditFails { + sink = AuditFunc(func(context.Context, AuditEvent) error { return errors.New("audit unavailable") }) + } + enforcer, err := NewEnforcer(Config{Hook: test.hook, Auditor: sink}) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeProposal(context.Background(), operatorScope(t), testProposalInput(t)) + if err == nil { + t.Fatal("AuthorizeProposal() returned nil") + } + if test.wantError != nil && !errors.Is(err, test.wantError) { + t.Fatalf("AuthorizeProposal() error = %v, want %v", err, test.wantError) + } + if test.contains != "" && !strings.Contains(err.Error(), test.contains) { + t.Fatalf("AuthorizeProposal() error = %v, want %q", err, test.contains) + } + if strings.Contains(err.Error(), "token=secret") { + t.Fatalf("AuthorizeProposal() leaked hook error: %v", err) + } + if !test.auditFails && strings.Contains(fmt.Sprintf("%#v", auditor.events), "token=secret") { + t.Fatalf("AuthorizeProposal() leaked hook error into audit: %#v", auditor.events) + } + if !test.auditFails && (len(auditor.events) != 1 || auditor.events[0].Verdict != test.want || auditor.events[0].ReasonCode != test.reason) { + t.Fatalf("audit events = %#v, want %s/%s", auditor.events, test.want, test.reason) + } + }) + } +} + +func TestEnforcerProposalRoleMatrixFailsClosed(t *testing.T) { + for _, role := range []tenancy.Role{tenancy.RoleReader, tenancy.RoleApprover, tenancy.RoleAdmin} { + t.Run(string(role), func(t *testing.T) { + var hookCalls atomic.Int64 + auditor := &recordingAuditor{} + enforcer, err := NewEnforcer(Config{ + Hook: HookFunc(func(context.Context, Request) (Decision, error) { + hookCalls.Add(1) + return Decision{Verdict: VerdictAllow, ReasonCode: "unexpected"}, nil + }), + Auditor: auditor, + }) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeProposal(context.Background(), scopeForRole(t, role), testProposalInputFor(t, "user:"+string(role), "workspace-a")) + if err == nil || !errors.Is(err, ErrDenied) || !strings.Contains(err.Error(), "role does not permit") { + t.Fatalf("AuthorizeProposal() error = %v, want role denial", err) + } + if hookCalls.Load() != 0 { + t.Fatalf("policy hook calls = %d, want zero", hookCalls.Load()) + } + if len(auditor.events) != 1 || auditor.events[0].Verdict != VerdictDeny || auditor.events[0].ReasonCode != "role-denied" { + t.Fatalf("audit events = %#v", auditor.events) + } + }) + } +} + +func TestEnforcerRejectsTamperedProposalBindingBeforePolicyHook(t *testing.T) { + tests := []struct { + name string + mutate func(*ProposalInput) + }{ + {name: "workspace", mutate: func(input *ProposalInput) { input.workspaceID = "workspace-b" }}, + {name: "actor", mutate: func(input *ProposalInput) { input.actor = "user:other" }}, + {name: "verb", mutate: func(input *ProposalInput) { input.verb = "deployment.delete" }}, + {name: "target", mutate: func(input *ProposalInput) { input.target.Name = "payments-canary" }}, + {name: "arguments digest", mutate: func(input *ProposalInput) { input.argumentsDigest = digestFor("different") }}, + {name: "resolved digest", mutate: func(input *ProposalInput) { input.resolvedDigest = digestFor("forged") }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := testProposalInput(t) + test.mutate(&input) + var hookCalls atomic.Int64 + auditor := &recordingAuditor{} + enforcer, err := NewEnforcer(Config{ + Hook: HookFunc(func(context.Context, Request) (Decision, error) { + hookCalls.Add(1) + return Decision{Verdict: VerdictAllow, ReasonCode: "unexpected"}, nil + }), + Auditor: auditor, + }) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeProposal(context.Background(), operatorScope(t), input) + if err == nil || !errors.Is(err, ErrDenied) || !strings.Contains(err.Error(), "invalid policy request") { + t.Fatalf("AuthorizeProposal() error = %v, want invalid request", err) + } + if hookCalls.Load() != 0 { + t.Fatalf("policy hook calls = %d, want zero", hookCalls.Load()) + } + if len(auditor.events) != 1 || auditor.events[0].Verdict != VerdictDeny || auditor.events[0].ReasonCode != "invalid-request" { + t.Fatalf("audit events = %#v", auditor.events) + } + }) + } +} + +func TestProposalDigestBindsEveryResolvedEnvelopeField(t *testing.T) { + target := testProposalTarget() + base := proposalCase{ + intentID: "intent-230", workspaceID: "workspace-a", actor: "user:operator", + verb: intent.VerbDeploymentRestart, target: target, argumentsDigest: digestFor("validated-arguments"), + } + variants := []proposalCase{ + base, + base.with(func(value *proposalCase) { value.intentID = "intent-231" }), + base.with(func(value *proposalCase) { value.workspaceID = "workspace-b" }), + base.with(func(value *proposalCase) { value.actor = "user:operator-2" }), + base.with(func(value *proposalCase) { value.verb = intent.VerbDeploymentScale }), + base.with(func(value *proposalCase) { value.target.SourceKind = "kubernetes" }), + base.with(func(value *proposalCase) { value.target.Scope = "cluster-b" }), + base.with(func(value *proposalCase) { value.target.Kind = "statefulset" }), + base.with(func(value *proposalCase) { value.target.Namespace = "operations" }), + base.with(func(value *proposalCase) { value.target.Name = "payments-canary" }), + base.with(func(value *proposalCase) { value.argumentsDigest = digestFor("other-arguments") }), + } + seen := make(map[string]struct{}, len(variants)) + for index, variant := range variants { + input, err := NewProposalInput(variant.intentID, variant.workspaceID, variant.actor, variant.verb, variant.target, variant.argumentsDigest) + if err != nil { + t.Fatalf("NewProposalInput(variant %d) error = %v", index, err) + } + if !validDigest(input.resolvedDigest) { + t.Fatalf("variant %d digest = %q", index, input.resolvedDigest) + } + if _, exists := seen[input.resolvedDigest]; exists { + t.Fatalf("variant %d reused resolved digest %q", index, input.resolvedDigest) + } + seen[input.resolvedDigest] = struct{}{} + } +} + +func TestNewProposalInputRejectsMalformedEnvelopeFields(t *testing.T) { + base := proposalCase{ + intentID: "intent-230", workspaceID: "workspace-a", actor: "user:operator", + verb: intent.VerbDeploymentRestart, target: testProposalTarget(), argumentsDigest: digestFor("validated-arguments"), + } + tests := []struct { + name string + mutate func(*proposalCase) + }{ + {name: "empty intent ID", mutate: func(value *proposalCase) { value.intentID = "" }}, + {name: "control intent ID", mutate: func(value *proposalCase) { value.intentID = "intent\n230" }}, + {name: "empty workspace", mutate: func(value *proposalCase) { value.workspaceID = "" }}, + {name: "foreign whitespace actor", mutate: func(value *proposalCase) { value.actor = " user:operator" }}, + {name: "unknown verb", mutate: func(value *proposalCase) { value.verb = "deployment.delete" }}, + {name: "empty target source", mutate: func(value *proposalCase) { value.target.SourceKind = "" }}, + {name: "control target name", mutate: func(value *proposalCase) { value.target.Name = "payments\nsecret" }}, + {name: "target attributes", mutate: func(value *proposalCase) { value.target.Attributes = map[string]string{"token": "secret"} }}, + {name: "malformed arguments digest", mutate: func(value *proposalCase) { value.argumentsDigest = "sha256:ABC" }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + value := base + test.mutate(&value) + if _, err := NewProposalInput(value.intentID, value.workspaceID, value.actor, value.verb, value.target, value.argumentsDigest); err == nil { + t.Fatal("NewProposalInput() accepted malformed envelope") + } + }) + } +} + +func TestNewProposalInputSeversEmptyTargetAttributeAlias(t *testing.T) { + attributes := map[string]string{} + target := testProposalTarget() + target.Attributes = attributes + input, err := NewProposalInput( + "intent-230", "workspace-a", "user:operator", intent.VerbDeploymentRestart, + target, digestFor("validated-arguments"), + ) + if err != nil { + t.Fatalf("NewProposalInput() error = %v", err) + } + attributes["token"] = "caller-secret" + if input.target.Attributes != nil { + t.Fatalf("proposal retained caller-owned target attributes: %#v", input.target.Attributes) + } + enforcer, err := NewEnforcer(Config{ + Hook: proposalDecision(VerdictAllow, "proposal-allowed"), + Auditor: AuditFunc(func(context.Context, AuditEvent) error { return nil }), + }) + if err != nil { + t.Fatal(err) + } + if err := enforcer.AuthorizeProposal(context.Background(), operatorScope(t), input); err != nil { + t.Fatalf("AuthorizeProposal() changed after caller map mutation: %v", err) + } +} + +func TestEnforcerAuthorizesBoundProposalConcurrently(t *testing.T) { + const workers = 64 + var hookCalls atomic.Int64 + var auditCalls atomic.Int64 + enforcer, err := NewEnforcer(Config{ + Hook: HookFunc(func(context.Context, Request) (Decision, error) { + hookCalls.Add(1) + return Decision{Verdict: VerdictAllow, ReasonCode: "proposal-allowed"}, nil + }), + Auditor: AuditFunc(func(context.Context, AuditEvent) error { + auditCalls.Add(1) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + input := testProposalInput(t) + scope := operatorScope(t) + var group sync.WaitGroup + errorsByWorker := make(chan error, workers) + for worker := 0; worker < workers; worker++ { + group.Add(1) + go func() { + defer group.Done() + errorsByWorker <- enforcer.AuthorizeProposal(context.Background(), scope, input) + }() + } + group.Wait() + close(errorsByWorker) + for err := range errorsByWorker { + if err != nil { + t.Fatalf("AuthorizeProposal() concurrent error = %v", err) + } + } + if hookCalls.Load() != workers || auditCalls.Load() != workers { + t.Fatalf("hook/audit calls = %d/%d, want %d/%d", hookCalls.Load(), auditCalls.Load(), workers, workers) + } +} + +func FuzzProposalInputRejectsTamperedBinding(f *testing.F) { + f.Add("target-a", uint8(0)) + f.Add("target-b", uint8(5)) + f.Fuzz(func(t *testing.T, replacement string, field uint8) { + input := testProposalInput(t) + marker := fmt.Sprintf("caller-%x", sha256.Sum256([]byte(replacement))) + switch field % 6 { + case 0: + input.intentID = marker + case 1: + input.actor = marker + case 2: + input.verb = intent.Verb(marker) + case 3: + input.target.Name = marker + case 4: + input.argumentsDigest = marker + case 5: + input.resolvedDigest = marker + } + var hookCalls atomic.Int64 + var audits []AuditEvent + enforcer, err := NewEnforcer(Config{ + Hook: HookFunc(func(context.Context, Request) (Decision, error) { + hookCalls.Add(1) + return Decision{Verdict: VerdictAllow, ReasonCode: "unexpected"}, nil + }), + Auditor: AuditFunc(func(_ context.Context, event AuditEvent) error { + audits = append(audits, event) + return nil + }), + }) + if err != nil { + t.Fatal(err) + } + err = enforcer.AuthorizeProposal(context.Background(), operatorScope(t), input) + if err == nil { + t.Fatal("AuthorizeProposal() accepted a tampered proposal") + } + if strings.Contains(err.Error(), marker) || strings.Contains(fmt.Sprintf("%#v", audits), marker) { + t.Fatalf("AuthorizeProposal() leaked caller material") + } + if hookCalls.Load() != 0 { + t.Fatalf("policy hook calls = %d, want zero", hookCalls.Load()) + } + if len(audits) != 1 || audits[0].Action != tenancy.ActionProposeIntent || audits[0].Verdict != VerdictDeny || audits[0].ReasonCode != "invalid-request" { + t.Fatalf("tampered proposal audits = %#v, want one sanitized invalid-request denial", audits) + } + }) +} + +func proposalDecision(verdict Verdict, reason string) PolicyHook { + return HookFunc(func(context.Context, Request) (Decision, error) { + return Decision{Verdict: verdict, ReasonCode: reason}, nil + }) +} + +func testProposalInput(t testing.TB) ProposalInput { + t.Helper() + return testProposalInputFor(t, "user:operator", "workspace-a") +} + +func testProposalInputFor(t testing.TB, actor string, workspaceID tenancy.WorkspaceID) ProposalInput { + t.Helper() + input, err := NewProposalInput( + "intent-230", workspaceID, actor, intent.VerbDeploymentRestart, + testProposalTarget(), digestFor("validated-arguments"), + ) + if err != nil { + t.Fatalf("NewProposalInput() error = %v", err) + } + return input +} + +func testProposalTarget() fleet.ResourceRef { + return fleet.ResourceRef{SourceKind: "argocd", Scope: "cluster-a", Kind: "deployment", Namespace: "payments", Name: "payments"} +} + +func digestFor(value string) string { + return NewReadInput(VerbFleetRead, []byte(value)).ArgumentsDigest +} + +func operatorScope(t testing.TB) tenancy.Scope { + t.Helper() + return scopeForRole(t, tenancy.RoleOperator) +} + +func scopeForRole(t testing.TB, role tenancy.Role) tenancy.Scope { + t.Helper() + subject := "user:" + string(role) + principal, err := tenancy.NewPrincipal(subject, map[tenancy.WorkspaceID]tenancy.Role{"workspace-a": role}) + if err != nil { + t.Fatal(err) + } + scope, err := principal.Scope("workspace-a") + if err != nil { + t.Fatal(err) + } + return scope +} + +type proposalCase struct { + intentID string + workspaceID tenancy.WorkspaceID + actor string + verb intent.Verb + target fleet.ResourceRef + argumentsDigest string +} + +func (value proposalCase) with(mutate func(*proposalCase)) proposalCase { + mutate(&value) + return value +} + +func (value proposalCase) String() string { + return fmt.Sprintf("%s/%s/%s", value.workspaceID, value.verb, value.intentID) +} diff --git a/internal/privacy/boundary_test.go b/internal/privacy/boundary_test.go index 056a95e..71c55d1 100644 --- a/internal/privacy/boundary_test.go +++ b/internal/privacy/boundary_test.go @@ -15,12 +15,27 @@ import ( ) var approvedNetworkImports = map[string]map[string]bool{ - "internal/cli/mcp.go": {"net": true, "net/http": true}, - "internal/cli/ui.go": {"net": true, "net/http": true}, + "internal/cli/mcp.go": {"net": true, "net/http": true}, + "internal/cli/ui.go": {"net": true, "net/http": true}, + // This pure projector uses net/url only to remove credentials, queries, and fragments from + // already-fetched Argo CD facts. It has no socket, HTTP, or Kubernetes client capability. + "internal/connector/argocd/project.go": {"net/url": true}, "internal/connector/kubeconfig/local_streams.go": {"net/http": true, "net/url": true}, "internal/hubserver/auth.go": {"net/http": true}, - "internal/hubserver/exchange.go": {"net": true, "net/http": true}, - "internal/hubserver/fleet.go": {"net/http": true, "net/url": true}, + // The bearer-only audit export adapter has one exact inbound route and no listener, outbound + // client, database implementation, connector, filesystem, process, or local-operation capability. + "internal/hubserver/audit_export.go": {"net/http": true, "net/url": true}, + // Browser OIDC is a Hub-only code+PKCE broker. It accepts no local-mode traffic, uses no + // caller-controlled endpoint, and keeps all proofs and session JWTs server-side. + "internal/hubserver/browser_oidc.go": {"net": true, "net/http": true, "net/url": true}, + // The Hub console is a fixed same-origin read adapter over the existing tenant-scoped source. + // It has no listener, outbound request, connector, local-operation, refresh, or write seam. + "internal/hubserver/console.go": {"net/http": true, "net/url": true}, + "internal/hubserver/exchange.go": {"net": true, "net/http": true}, + "internal/hubserver/fleet.go": {"net/http": true, "net/url": true}, + // Fixed inbound-only Hub probes return body-free status codes. They have no listener, dial, + // endpoint selection, tenant data, persistence, or outbound HTTP capability. + "internal/hubserver/probes.go": {"net/http": true}, // AWS STS egress is endpoint-pinned, SigV4-profiled, redirect-disabled, and never used by local mode. "internal/hubauth/aws_sts.go": {"net/http": true, "net/url": true}, "internal/hubauth/oidc.go": {"net": true, "net/http": true, "net/netip": true, "net/url": true}, @@ -34,12 +49,16 @@ var approvedNetworkImports = map[string]map[string]bool{ "k8s.io/client-go/dynamic": true, "k8s.io/client-go/kubernetes": true, "k8s.io/client-go/rest": true, "google.golang.org/grpc": true, "google.golang.org/grpc/credentials": true, }, - // The in-cluster hub composition root is the reviewed boundary for its fixed TLS listener and - // scoped Kubernetes client. It delegates every spoke credential read to the direct OCM adapter. + // The in-cluster hub composition root is the reviewed boundary for its fixed TLS listener, + // browser OIDC routes, and scoped Kubernetes client. It delegates every spoke credential read to + // the direct OCM adapter. "internal/hubruntime/config.go": { - "net": true, "k8s.io/client-go/kubernetes": true, "k8s.io/client-go/rest": true, + "net": true, "net/http": true, "k8s.io/client-go/kubernetes": true, "k8s.io/client-go/rest": true, }, - "internal/hubruntime/runtime.go": {"net": true, "net/http": true}, + "internal/hubruntime/runtime.go": {"net": true, "net/http": true}, + // The Hub-only operator metrics listener is opt-in, exact-loopback-only, and has one fixed + // read-only route. It is separately bound from tenant APIs and has no local-mode path. + "internal/hubruntime/metrics.go": {"net": true, "net/http": true}, "internal/mcpserver/server.go": {"net": true, "net/http": true, "net/url": true}, "internal/observability/metrics.go": {"net/http": true}, "internal/webui/api.go": {"net/http": true}, @@ -54,9 +73,12 @@ var approvedFilesystemWrites = map[string]map[string]bool{ } var approvedProcessImports = map[string]map[string]bool{ - "internal/cli/local.go": {"os/exec": true}, - "internal/cli/ui.go": {"os/exec": true}, - "internal/tui/local_actions.go": {"os/exec": true}, + // The Hub-only process audit sink starts the current trusted Sith executable with an inherited + // bounded Unix datagram FD. It passes no environment, config, request values, or credentials. + "internal/auditdelivery/process.go": {"os/exec": true}, + "internal/cli/local.go": {"os/exec": true}, + "internal/cli/ui.go": {"os/exec": true}, + "internal/tui/local_actions.go": {"os/exec": true}, } var forbiddenTelemetryPrefixes = []string{ diff --git a/internal/remediation/boundary_test.go b/internal/remediation/boundary_test.go new file mode 100644 index 0000000..eb356fa --- /dev/null +++ b/internal/remediation/boundary_test.go @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: Apache-2.0 + +package remediation + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +var allowedProductionImports = map[string]bool{ + "context": true, + "crypto/sha1": true, + "crypto/sha256": true, + "encoding/hex": true, + "encoding/json": true, + "fmt": true, + "path": true, + "reflect": true, + "slices": true, + "sort": true, + "strings": true, + "time": true, + "unicode": true, + "unicode/utf8": true, + "github.com/ArdurAI/sith/internal/brain": true, + "github.com/ArdurAI/sith/internal/connector": true, + "github.com/ArdurAI/sith/internal/fleet": true, + "github.com/ArdurAI/sith/internal/intent": true, + "github.com/ArdurAI/sith/internal/intentargs": true, + "github.com/ArdurAI/sith/internal/tenancy": true, +} + +func TestRemediationPackageTreeHasNoIOAuthorityOrPolicyImports(t *testing.T) { + t.Parallel() + err := filepath.WalkDir(".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".go" || strings.HasSuffix(path, "_test.go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + if err != nil { + return err + } + for _, imported := range file.Imports { + importPath, err := strconv.Unquote(imported.Path.Value) + if err != nil { + return err + } + if !allowedProductionImports[importPath] { + t.Errorf("remediation production file %s imports unreviewed package %q", path, importPath) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk remediation package tree: %v", err) + } +} + +func TestGitOpsBoundaryOmitsAuthorityAndKeepsBundleOpaque(t *testing.T) { + t.Parallel() + assertExactFields(t, reflect.TypeFor[GitOpsProvenanceInput](), []string{ + "Workspace", "Subject", "Sources", "ObservedAt", "ValidUntil", "Handler", "Repository", + "BaseRef", "BaseCommit", "FilePath", "ObservedBlobSHA", "DesiredContent", "Title", "Body", + "CommitMessage", "EvidenceRefs", + }) + assertExactFields(t, reflect.TypeFor[Resolution](), []string{ + "Status", "Target", "Arguments", "ArgumentsDigest", "EvidenceRefs", "Reasons", + }) + assertExactFields(t, reflect.TypeFor[GitOpsResolver](), []string{"handler", "now"}) + + bundle := reflect.TypeFor[GitOpsProvenanceBundle]() + if bundle.NumField() != 17 { + t.Fatalf("GitOpsProvenanceBundle fields = %d, want exact reviewed shape", bundle.NumField()) + } + for index := range bundle.NumField() { + field := bundle.Field(index) + if field.IsExported() { + t.Fatalf("GitOpsProvenanceBundle exposes mutable field %s", field.Name) + } + } + + for _, value := range []reflect.Type{ + reflect.TypeFor[GitOpsProvenanceInput](), reflect.TypeFor[GitOpsProvenanceBundle](), reflect.TypeFor[Resolution](), + } { + for index := range value.NumField() { + name := strings.ToLower(value.Field(index).Name) + for _, forbidden := range []string{"actor", "role", "intent", "approval", "policy", "credential", "token", "secret", "signature", "endpoint"} { + if strings.Contains(name, forbidden) { + t.Fatalf("%s exposes forbidden authority field %s", value.Name(), value.Field(index).Name) + } + } + } + } +} + +func TestGitSourceSnapshotBoundaryIsObservedOnlyAndOpaque(t *testing.T) { + t.Parallel() + assertExactFields(t, reflect.TypeFor[GitSourceSnapshotInput](), []string{ + "Workspace", "Subject", "Sources", "ObservedAt", "ValidUntil", "Repository", "BaseRef", + "BaseCommit", "FilePath", "ObservedBlobSHA", "CurrentContent", "EvidenceRefs", + }) + + snapshot := reflect.TypeFor[GitSourceSnapshot]() + if snapshot.NumField() != 13 { + t.Fatalf("GitSourceSnapshot fields = %d, want exact reviewed shape", snapshot.NumField()) + } + for index := range snapshot.NumField() { + if field := snapshot.Field(index); field.IsExported() { + t.Fatalf("GitSourceSnapshot exposes mutable field %s", field.Name) + } + } + if snapshot.NumMethod() != 2 || snapshot.Method(0).Name != "Freshness" || snapshot.Method(1).Name != "Version" { + t.Fatalf("GitSourceSnapshot methods = %#v, want only Freshness and Version", snapshot) + } + + for _, value := range []reflect.Type{reflect.TypeFor[GitSourceSnapshotInput](), snapshot} { + for index := range value.NumField() { + name := strings.ToLower(value.Field(index).Name) + for _, forbidden := range []string{ + "desired", "title", "body", "commitmessage", "handler", "actor", "role", "intent", + "approval", "policy", "credential", "token", "secret", "signature", "endpoint", "dispatch", + } { + if strings.Contains(name, forbidden) { + t.Fatalf("%s exposes forbidden change or authority field %s", value.Name(), value.Field(index).Name) + } + } + } + } +} + +func TestDesiredChangeBoundaryIsTransformerOwnedAndOpaque(t *testing.T) { + t.Parallel() + change := reflect.TypeFor[DesiredChange]() + if change.NumField() != 5 { + t.Fatalf("DesiredChange fields = %d, want exact reviewed shape", change.NumField()) + } + for index := range change.NumField() { + field := change.Field(index) + if field.IsExported() { + t.Fatalf("DesiredChange exposes mutable field %s", field.Name) + } + name := strings.ToLower(field.Name) + for _, forbidden := range []string{ + "title", "body", "commitmessage", "handler", "actor", "role", "intent", "approval", + "policy", "credential", "token", "secret", "signature", "endpoint", "dispatch", "execute", + } { + if strings.Contains(name, forbidden) { + t.Fatalf("DesiredChange exposes forbidden authority field %s", field.Name) + } + } + } + if change.NumMethod() != 1 || change.Method(0).Name != "Version" { + t.Fatalf("DesiredChange methods = %#v, want only Version", change) + } + + err := filepath.WalkDir(".", func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() || filepath.Ext(path) != ".go" || strings.HasSuffix(path, "_test.go") { + return nil + } + file, parseErr := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if parseErr != nil { + return parseErr + } + for _, rawDeclaration := range file.Decls { + declaration, ok := rawDeclaration.(*ast.FuncDecl) + if ok && declaration.Recv == nil && token.IsExported(declaration.Name.Name) && + returnsDesiredChange(declaration) { + t.Errorf("DesiredChange construction escaped the reviewed package boundary as %s in %s", declaration.Name.Name, path) + } + } + return nil + }) + if err != nil { + t.Fatalf("inspect DesiredChange package boundary: %v", err) + } +} + +func returnsDesiredChange(declaration *ast.FuncDecl) bool { + if declaration.Type.Results == nil { + return false + } + found := false + for _, result := range declaration.Type.Results.List { + ast.Inspect(result.Type, func(node ast.Node) bool { + identifier, ok := node.(*ast.Ident) + if ok && identifier.Name == "DesiredChange" { + found = true + return false + } + return !found + }) + } + return found +} + +func assertExactFields(t *testing.T, value reflect.Type, expected []string) { + t.Helper() + if value.NumField() != len(expected) { + t.Fatalf("%s fields = %d, want exactly %d", value.Name(), value.NumField(), len(expected)) + } + for index, name := range expected { + if value.Field(index).Name != name { + t.Fatalf("%s field %d = %s, want %s", value.Name(), index, value.Field(index).Name, name) + } + } +} diff --git a/internal/remediation/desired_change.go b/internal/remediation/desired_change.go new file mode 100644 index 0000000..3e517d4 --- /dev/null +++ b/internal/remediation/desired_change.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 + +package remediation + +import ( + "fmt" + "sort" + "strings" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const ( + // DesiredChangeVersion is the first immutable snapshot-to-output binding contract. + DesiredChangeVersion = "desired-change/v1" + + maxDesiredChangeEvidenceRefs = 32 +) + +// DesiredChange is immutable after construction. It binds exact proposed bytes to one exact +// GitSourceSnapshot, one reviewed transformer version, and cited evidence. Its fields remain +// private, and construction remains package-private until a concrete transformer policy is +// separately reviewed. +type DesiredChange struct { + version string + snapshot GitSourceSnapshot + transformerVersion string + desiredContent string + evidenceRefs []fleet.ResourceRef +} + +// Version reports the closed desired-change contract without exposing proposed bytes or source +// state. +func (change DesiredChange) Version() string { return change.version } + +// newDesiredChange is deliberately package-private. Only a separately reviewed deterministic +// transformer or declarative renderer in this package may turn output bytes into a DesiredChange; +// request and runtime callers cannot relabel arbitrary bytes as trusted transformer output. +func newDesiredChange( + snapshot GitSourceSnapshot, + transformerVersion string, + desiredContent string, + evidenceRefs []fleet.ResourceRef, +) (DesiredChange, error) { + change := DesiredChange{ + version: DesiredChangeVersion, + snapshot: cloneGitSourceSnapshot(snapshot), + transformerVersion: transformerVersion, + desiredContent: desiredContent, + evidenceRefs: cloneResourceRefs(evidenceRefs), + } + sort.Slice(change.evidenceRefs, func(left, right int) bool { + return resourceRefLess(change.evidenceRefs[left], change.evidenceRefs[right]) + }) + if err := change.validate(); err != nil { + return DesiredChange{}, fmt.Errorf("construct desired change: change is invalid") + } + return change, nil +} + +func (change DesiredChange) validate() error { + if change.version != DesiredChangeVersion || change.snapshot.validate() != nil || + !validTransformerVersion(change.transformerVersion) || + !validGitSourceContent(change.desiredContent) || + change.desiredContent == change.snapshot.currentContent || len(change.evidenceRefs) < 2 || + len(change.evidenceRefs) > maxDesiredChangeEvidenceRefs { + return fmt.Errorf("desired change is invalid") + } + + subjectAttached := false + blobAttached := false + for index, ref := range change.evidenceRefs { + if validateStableRef(ref) != nil || + (index > 0 && !resourceRefLess(change.evidenceRefs[index-1], ref)) { + return fmt.Errorf("desired change evidence is invalid") + } + subjectAttached = subjectAttached || sameResourceRef(ref, change.snapshot.subject) + blobAttached = blobAttached || gitSourceBlobRefMatches(ref, change.snapshot) + } + if !subjectAttached || !blobAttached { + return fmt.Errorf("desired change evidence is unattached") + } + return nil +} + +func cloneGitSourceSnapshot(snapshot GitSourceSnapshot) GitSourceSnapshot { + cloned := snapshot + cloned.subject = cloneResourceRef(snapshot.subject) + cloned.evidenceRefs = cloneResourceRefs(snapshot.evidenceRefs) + return cloned +} + +func validTransformerVersion(value string) bool { + if validateSafeText(value, maxIdentityBytes, false) != nil || value != strings.ToLower(value) { + return false + } + parts := strings.Split(value, "/") + if len(parts) != 2 { + return false + } + for _, part := range parts { + if part == "" || !isLowerAlphaNumeric(part[0]) || !isLowerAlphaNumeric(part[len(part)-1]) { + return false + } + for _, character := range part { + if (character < 'a' || character > 'z') && (character < '0' || character > '9') && + character != '-' && character != '_' && character != '.' { + return false + } + } + } + return true +} + +func isLowerAlphaNumeric(value byte) bool { + return (value >= 'a' && value <= 'z') || (value >= '0' && value <= '9') +} diff --git a/internal/remediation/desired_change_test.go b/internal/remediation/desired_change_test.go new file mode 100644 index 0000000..a956d49 --- /dev/null +++ b/internal/remediation/desired_change_test.go @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: Apache-2.0 + +package remediation + +import ( + "fmt" + "slices" + "strings" + "sync" + "testing" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const testTransformerVersion = "fixture-renderer/v1" + +func TestDesiredChangePreservesExactSnapshotAndOutput(t *testing.T) { + t.Parallel() + snapshot := mustGitSourceSnapshot(t) + evidence := validDesiredChangeEvidence(snapshot) + slices.Reverse(evidence) + desiredContent := "apiVersion: v1\r\nmetadata:\n name: café\t\nspec:\n replicas: 4\n" + + change, err := newDesiredChange(snapshot, testTransformerVersion, desiredContent, evidence) + if err != nil { + t.Fatalf("newDesiredChange() error = %v", err) + } + if change.Version() != DesiredChangeVersion || change.transformerVersion != testTransformerVersion || + change.desiredContent != desiredContent || !sameGitSourceSnapshot(change.snapshot, snapshot) { + t.Fatalf("change did not preserve the exact binding: %#v", change) + } + if len(change.evidenceRefs) != 2 || !sameResourceRef(change.evidenceRefs[0], testBlobRef()) || + !sameResourceRef(change.evidenceRefs[1], testSubjectRef()) { + t.Fatalf("evidence = %#v, want canonical blob and subject order", change.evidenceRefs) + } + + reordered, err := newDesiredChange( + snapshot, + testTransformerVersion, + desiredContent, + validDesiredChangeEvidence(snapshot), + ) + if err != nil { + t.Fatalf("newDesiredChange(reordered) error = %v", err) + } + if !slices.EqualFunc(change.evidenceRefs, reordered.evidenceRefs, sameResourceRef) { + t.Fatalf("evidence ordering changed with caller order: %#v != %#v", change.evidenceRefs, reordered.evidenceRefs) + } +} + +func TestDesiredChangeConstructionIsMutationIsolated(t *testing.T) { + t.Parallel() + snapshot := mustGitSourceSnapshot(t) + evidence := validDesiredChangeEvidence(snapshot) + desiredContent := "replicas: 4\n" + change, err := newDesiredChange(snapshot, testTransformerVersion, desiredContent, evidence) + if err != nil { + t.Fatal(err) + } + + snapshot.subject.Name = "mutated-subject" + snapshot.repository.Repository = "other" + snapshot.baseCommit = strings.Repeat("c", 40) + snapshot.currentContent = "mutated source" + snapshot.evidenceRefs[0].Name = "mutated-snapshot-evidence" + evidence[0].Name = "mutated-change-evidence" + + if err := change.validate(); err != nil { + t.Fatalf("change retained caller mutation: %v", err) + } + if change.snapshot.subject.Name != "payments" || change.snapshot.repository.Repository != "sith" || + change.snapshot.baseCommit != testBaseSHA || change.snapshot.currentContent == "mutated source" || + change.snapshot.evidenceRefs[0].Name == "mutated-snapshot-evidence" || + change.evidenceRefs[1].Name == "mutated-change-evidence" || change.desiredContent != desiredContent { + t.Fatalf("change retained caller-owned state: %#v", change) + } +} + +func TestNewDesiredChangeRejectsInvalidClaims(t *testing.T) { + tests := []struct { + name string + mutate func(*desiredChangeFixture) + }{ + {"zero snapshot", func(input *desiredChangeFixture) { input.snapshot = GitSourceSnapshot{} }}, + {"forged snapshot", func(input *desiredChangeFixture) { input.snapshot.version = "forged/v9" }}, + {"forged snapshot subject", func(input *desiredChangeFixture) { input.snapshot.subject.Name = "other" }}, + {"empty transformer", func(input *desiredChangeFixture) { input.transformerVersion = "" }}, + {"uppercase transformer", func(input *desiredChangeFixture) { input.transformerVersion = "Fixture/v1" }}, + {"unversioned transformer", func(input *desiredChangeFixture) { input.transformerVersion = "fixture-renderer" }}, + {"ambiguous transformer", func(input *desiredChangeFixture) { input.transformerVersion = "fixture/renderer/v1" }}, + {"option-shaped transformer", func(input *desiredChangeFixture) { input.transformerVersion = "-fixture/v1" }}, + {"spaced transformer", func(input *desiredChangeFixture) { input.transformerVersion = "fixture renderer/v1" }}, + {"invalid UTF-8 output", func(input *desiredChangeFixture) { input.desiredContent = string([]byte{0xff}) }}, + {"NUL output", func(input *desiredChangeFixture) { input.desiredContent = "secret\x00value" }}, + {"oversized output", func(input *desiredChangeFixture) { + input.desiredContent = strings.Repeat("x", maxGitSourceSnapshotContentBytes+1) + }}, + {"no-op output", func(input *desiredChangeFixture) { + input.desiredContent = input.snapshot.currentContent + }}, + {"no evidence", func(input *desiredChangeFixture) { input.evidenceRefs = nil }}, + {"subject-only evidence", func(input *desiredChangeFixture) { + input.evidenceRefs = []fleet.ResourceRef{input.snapshot.subject} + }}, + {"blob-only evidence", func(input *desiredChangeFixture) { + input.evidenceRefs = []fleet.ResourceRef{testBlobRef()} + }}, + {"foreign subject evidence", func(input *desiredChangeFixture) { input.evidenceRefs[0].Name = "other" }}, + {"foreign blob evidence", func(input *desiredChangeFixture) { + input.evidenceRefs[1].Name = strings.Repeat("e", 40) + }}, + {"duplicate evidence", func(input *desiredChangeFixture) { + input.evidenceRefs = append(input.evidenceRefs, input.evidenceRefs[0]) + }}, + {"unsafe evidence", func(input *desiredChangeFixture) { input.evidenceRefs[0].Name = "secret\nforged" }}, + {"evidence attributes", func(input *desiredChangeFixture) { + input.evidenceRefs[0].Attributes = map[string]string{"uid": "private"} + }}, + {"too much evidence", func(input *desiredChangeFixture) { + for index := len(input.evidenceRefs); index <= maxDesiredChangeEvidenceRefs; index++ { + input.evidenceRefs = append(input.evidenceRefs, fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "Observation", + Namespace: "ArdurAI/sith", Name: fmt.Sprintf("evidence-%02d", index), + }) + } + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := validDesiredChangeFixture(t) + test.mutate(&input) + change, err := newDesiredChange( + input.snapshot, + input.transformerVersion, + input.desiredContent, + input.evidenceRefs, + ) + if err == nil || change.Version() != "" { + t.Fatalf("newDesiredChange() = %#v, %v, want rejection", change, err) + } + if strings.Contains(err.Error(), "secret") || len(err.Error()) > 160 { + t.Fatalf("constructor leaked or returned unbounded error: %q", err) + } + }) + } +} + +func TestDesiredChangeRejectsForgedState(t *testing.T) { + t.Parallel() + fixture := validDesiredChangeFixture(t) + change, err := newDesiredChange( + fixture.snapshot, + fixture.transformerVersion, + fixture.desiredContent, + fixture.evidenceRefs, + ) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + mutate func(*DesiredChange) + }{ + {"version", func(value *DesiredChange) { value.version = "forged/v9" }}, + {"snapshot", func(value *DesiredChange) { value.snapshot.observedBlobSHA = strings.Repeat("e", 40) }}, + {"transformer", func(value *DesiredChange) { value.transformerVersion = "forged" }}, + {"no-op output", func(value *DesiredChange) { value.desiredContent = value.snapshot.currentContent }}, + {"evidence order", func(value *DesiredChange) { slices.Reverse(value.evidenceRefs) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + forged := change + forged.snapshot = cloneGitSourceSnapshot(change.snapshot) + forged.evidenceRefs = cloneResourceRefs(change.evidenceRefs) + test.mutate(&forged) + if err := forged.validate(); err == nil { + t.Fatalf("forged change validated: %#v", forged) + } + }) + } +} + +func TestDesiredChangeValidationIsConcurrentAndReadOnly(t *testing.T) { + t.Parallel() + fixture := validDesiredChangeFixture(t) + change, err := newDesiredChange( + fixture.snapshot, + fixture.transformerVersion, + fixture.desiredContent, + fixture.evidenceRefs, + ) + if err != nil { + t.Fatal(err) + } + before := change + before.snapshot = cloneGitSourceSnapshot(change.snapshot) + before.evidenceRefs = cloneResourceRefs(change.evidenceRefs) + + const readers = 64 + errors := make(chan error, readers) + var group sync.WaitGroup + for range readers { + group.Add(1) + go func() { + defer group.Done() + if validateErr := change.validate(); validateErr != nil { + errors <- validateErr + return + } + if change.Version() != DesiredChangeVersion { + errors <- fmt.Errorf("version = %q", change.Version()) + } + }() + } + group.Wait() + if !sameDesiredChange(change, before) { + t.Fatalf("concurrent validation mutated change: %#v != %#v", change, before) + } + close(errors) + for err := range errors { + t.Error(err) + } +} + +func FuzzDesiredChangeConstructor(f *testing.F) { + for _, seed := range []struct{ transformer, content string }{ + {testTransformerVersion, "replicas: 4\n"}, + {"invalid", "secret\x00value"}, + {"renderer/2026-07-22", "name: café\r\n"}, + } { + f.Add(seed.transformer, seed.content) + } + f.Fuzz(func(t *testing.T, transformer, content string) { + snapshot := mustGitSourceSnapshot(t) + change, err := newDesiredChange(snapshot, transformer, content, validDesiredChangeEvidence(snapshot)) + if err != nil { + return + } + if change.transformerVersion != transformer || change.desiredContent != content || + !sameGitSourceSnapshot(change.snapshot, snapshot) { + t.Fatalf("accepted change altered its exact binding: %#v", change) + } + if validateErr := change.validate(); validateErr != nil { + t.Fatalf("accepted change did not revalidate: %v", validateErr) + } + }) +} + +type desiredChangeFixture struct { + snapshot GitSourceSnapshot + transformerVersion string + desiredContent string + evidenceRefs []fleet.ResourceRef +} + +func validDesiredChangeFixture(t testing.TB) desiredChangeFixture { + t.Helper() + snapshot := mustGitSourceSnapshot(t) + return desiredChangeFixture{ + snapshot: snapshot, + transformerVersion: testTransformerVersion, + desiredContent: "replicas: 4\n", + evidenceRefs: validDesiredChangeEvidence(snapshot), + } +} + +func mustGitSourceSnapshot(t testing.TB) GitSourceSnapshot { + t.Helper() + snapshot, err := NewGitSourceSnapshot(validGitSourceSnapshotInput()) + if err != nil { + t.Fatalf("NewGitSourceSnapshot() error = %v", err) + } + return snapshot +} + +func validDesiredChangeEvidence(snapshot GitSourceSnapshot) []fleet.ResourceRef { + return []fleet.ResourceRef{cloneResourceRef(snapshot.subject), testBlobRef()} +} + +func sameGitSourceSnapshot(left, right GitSourceSnapshot) bool { + return left.version == right.version && left.workspace == right.workspace && + sameResourceRef(left.subject, right.subject) && left.source == right.source && + left.observedAt.Equal(right.observedAt) && left.validUntil.Equal(right.validUntil) && + left.repository == right.repository && left.baseRef == right.baseRef && + left.baseCommit == right.baseCommit && left.filePath == right.filePath && + left.observedBlobSHA == right.observedBlobSHA && left.currentContent == right.currentContent && + slices.EqualFunc(left.evidenceRefs, right.evidenceRefs, sameResourceRef) +} + +func sameDesiredChange(left, right DesiredChange) bool { + return left.version == right.version && sameGitSourceSnapshot(left.snapshot, right.snapshot) && + left.transformerVersion == right.transformerVersion && left.desiredContent == right.desiredContent && + slices.EqualFunc(left.evidenceRefs, right.evidenceRefs, sameResourceRef) +} diff --git a/internal/remediation/git_snapshot.go b/internal/remediation/git_snapshot.go new file mode 100644 index 0000000..545db96 --- /dev/null +++ b/internal/remediation/git_snapshot.go @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: Apache-2.0 + +package remediation + +import ( + "crypto/sha1" //nolint:gosec // GitHub Git object identity is SHA-1; this is not a security digest. + "crypto/sha256" + "encoding/hex" + "fmt" + "path" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + // GitSourceSnapshotVersion is the first immutable observed-Git-state contract. + GitSourceSnapshotVersion = "git-source-snapshot/v1" + // GitSourceSnapshotAdapterVersion pins the canonical GitHub Git-object observation contract. + GitSourceSnapshotAdapterVersion = "github-git-source-snapshot/2026-03-10" + + maxGitSourceSnapshotContentBytes = 64 << 10 + maxGitSourceSnapshotValidity = 5 * time.Minute + maxGitSourceSnapshotEvidenceRefs = 32 +) + +// GitSourceFreshness classifies an immutable observation against a caller-supplied trusted time. +type GitSourceFreshness string + +// Closed freshness states. The interval is fresh exactly when ObservedAt <= now < ValidUntil. +const ( + GitSourceFresh GitSourceFreshness = "fresh" + GitSourceFuture GitSourceFreshness = "future" + GitSourceStale GitSourceFreshness = "stale" +) + +// GitSourceSnapshotInput is accepted only from a canonical source adapter boundary. It contains +// observed Git state, never desired bytes, PR metadata, authority, credentials, or execution state. +type GitSourceSnapshotInput struct { + Workspace tenancy.WorkspaceID + Subject fleet.ResourceRef + Sources []SourceIdentity + ObservedAt time.Time + ValidUntil time.Time + Repository RepositoryIdentity + BaseRef string + BaseCommit string + FilePath string + ObservedBlobSHA string + CurrentContent string + EvidenceRefs []fleet.ResourceRef +} + +// GitSourceSnapshot is immutable after construction. Fields remain private so later composition +// cannot rewrite an observed identity, Git precondition, or byte sequence. +type GitSourceSnapshot struct { + version string + workspace tenancy.WorkspaceID + subject fleet.ResourceRef + source SourceIdentity + observedAt time.Time + validUntil time.Time + repository RepositoryIdentity + baseRef string + baseCommit string + filePath string + observedBlobSHA string + currentContent string + evidenceRefs []fleet.ResourceRef +} + +// Version reports the closed snapshot contract without exposing mutable observation fields. +func (snapshot GitSourceSnapshot) Version() string { return snapshot.version } + +// NewGitSourceSnapshot validates and defensively copies one canonical Git observation. It performs +// no I/O and does not infer or render a desired change. +func NewGitSourceSnapshot(input GitSourceSnapshotInput) (GitSourceSnapshot, error) { + if len(input.Sources) != 1 { + return GitSourceSnapshot{}, fmt.Errorf("construct Git source snapshot: exactly one source is required") + } + snapshot := GitSourceSnapshot{ + version: GitSourceSnapshotVersion, + workspace: input.Workspace, + subject: cloneResourceRef(input.Subject), + source: input.Sources[0], + observedAt: input.ObservedAt.UTC(), + validUntil: input.ValidUntil.UTC(), + repository: input.Repository, + baseRef: input.BaseRef, + baseCommit: input.BaseCommit, + filePath: input.FilePath, + observedBlobSHA: input.ObservedBlobSHA, + currentContent: input.CurrentContent, + evidenceRefs: cloneResourceRefs(input.EvidenceRefs), + } + sort.Slice(snapshot.evidenceRefs, func(left, right int) bool { + return resourceRefLess(snapshot.evidenceRefs[left], snapshot.evidenceRefs[right]) + }) + if err := snapshot.validate(); err != nil { + return GitSourceSnapshot{}, fmt.Errorf("construct Git source snapshot: snapshot is invalid") + } + return snapshot, nil +} + +// Freshness classifies the snapshot against a trusted clock value. Construction deliberately does +// not read a clock so identical source input always produces the same immutable value. +func (snapshot GitSourceSnapshot) Freshness(now time.Time) (GitSourceFreshness, error) { + if err := snapshot.validate(); err != nil { + return "", fmt.Errorf("inspect Git source snapshot: snapshot is invalid") + } + now = now.UTC() + if now.IsZero() { + return "", fmt.Errorf("inspect Git source snapshot: trusted time is required") + } + if now.Before(snapshot.observedAt) { + return GitSourceFuture, nil + } + if !now.Before(snapshot.validUntil) { + return GitSourceStale, nil + } + return GitSourceFresh, nil +} + +func (snapshot GitSourceSnapshot) validate() error { + if snapshot.version != GitSourceSnapshotVersion || tenancy.ValidateWorkspaceID(snapshot.workspace) != nil || + validateStableRef(snapshot.subject) != nil || validateGitSourceSnapshotSource(snapshot.source) != nil || + validateRepository(snapshot.repository) != nil || snapshot.source.NativeID != snapshot.repository.nativeID() || + snapshot.observedAt.IsZero() || snapshot.validUntil.IsZero() || + !snapshot.observedAt.Before(snapshot.validUntil) || + snapshot.validUntil.Sub(snapshot.observedAt) > maxGitSourceSnapshotValidity || + !validGitSourceBaseRef(snapshot.baseRef) || !validObjectID(snapshot.baseCommit) || + !validGitSourcePath(snapshot.filePath) || !validGitSourceContent(snapshot.currentContent) || + len(snapshot.baseCommit) != len(snapshot.observedBlobSHA) || + !gitSourceBlobMatchesContent(snapshot.observedBlobSHA, snapshot.currentContent) || len(snapshot.evidenceRefs) < 2 || + len(snapshot.evidenceRefs) > maxGitSourceSnapshotEvidenceRefs { + return fmt.Errorf("git source snapshot is invalid") + } + + subjectAttached := false + blobAttached := false + for index, ref := range snapshot.evidenceRefs { + if validateStableRef(ref) != nil || + (index > 0 && !resourceRefLess(snapshot.evidenceRefs[index-1], ref)) { + return fmt.Errorf("git source snapshot evidence is invalid") + } + subjectAttached = subjectAttached || sameResourceRef(ref, snapshot.subject) + blobAttached = blobAttached || gitSourceBlobRefMatches(ref, snapshot) + } + if !subjectAttached || !blobAttached { + return fmt.Errorf("git source snapshot evidence is unattached") + } + return nil +} + +func validateGitSourceSnapshotSource(source SourceIdentity) error { + if source.Kind != gitHubSourceKind || source.AdapterVersion != GitSourceSnapshotAdapterVersion || + validateSafeText(source.NativeID, maxIdentityBytes, false) != nil { + return fmt.Errorf("git source snapshot source is invalid") + } + return nil +} + +func validGitSourceBaseRef(value string) bool { + if validateSafeText(value, maxIdentityBytes, false) != nil || strings.HasPrefix(value, ".") || + strings.HasPrefix(value, "-") || strings.HasPrefix(value, "/") || + strings.HasPrefix(strings.ToLower(value), "refs/") || strings.EqualFold(value, "HEAD") || + value == "@" || validObjectID(value) || strings.HasSuffix(value, ".") || strings.HasSuffix(value, "/") || + strings.Contains(value, "..") || strings.Contains(value, "//") || strings.Contains(value, "@{") || + strings.HasSuffix(strings.ToLower(value), ".lock") { + return false + } + for _, character := range value { + if unicode.IsSpace(character) || strings.ContainsRune("~^:?*[]\\", character) { + return false + } + } + for _, component := range strings.Split(value, "/") { + if component == "" || strings.HasPrefix(component, ".") || strings.HasSuffix(strings.ToLower(component), ".lock") { + return false + } + } + return true +} + +func validGitSourcePath(value string) bool { + if validateSafeText(value, maxIdentityBytes, false) != nil || strings.HasPrefix(value, "/") || + strings.HasSuffix(value, "/") || strings.Contains(value, "\\") || path.Clean(value) != value { + return false + } + for _, component := range strings.Split(value, "/") { + if component == "" || component == "." || component == ".." || strings.EqualFold(component, ".git") { + return false + } + } + return true +} + +func validGitSourceContent(value string) bool { + return len(value) <= maxGitSourceSnapshotContentBytes && utf8.ValidString(value) && + !strings.ContainsRune(value, '\x00') +} + +func gitSourceBlobMatchesContent(objectID, content string) bool { + if !validObjectID(objectID) { + return false + } + payload := make([]byte, 0, len(content)+64) + payload = fmt.Appendf(payload, "blob %d\x00", len(content)) + payload = append(payload, content...) + switch len(objectID) { + case sha1.Size * 2: + digest := sha1.Sum(payload) //nolint:gosec // Git object identity, not a security digest. + return objectID == hex.EncodeToString(digest[:]) + case sha256.Size * 2: + digest := sha256.Sum256(payload) + return objectID == hex.EncodeToString(digest[:]) + default: + return false + } +} + +func gitSourceBlobRefMatches(ref fleet.ResourceRef, snapshot GitSourceSnapshot) bool { + return ref.SourceKind == snapshot.source.Kind && ref.Scope == snapshot.repository.Host && + ref.Kind == "Blob" && ref.Namespace == snapshot.repository.Owner+"/"+snapshot.repository.Repository && + ref.Name == snapshot.observedBlobSHA && len(ref.Attributes) == 0 +} diff --git a/internal/remediation/git_snapshot_test.go b/internal/remediation/git_snapshot_test.go new file mode 100644 index 0000000..3cb7a8c --- /dev/null +++ b/internal/remediation/git_snapshot_test.go @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: Apache-2.0 + +package remediation + +import ( + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/fleet" +) + +const testSnapshotBlobSHA = "c63745ccdd30a4492aed8a39e04b7f482ace3612" + +const testSnapshotSHA256Blob = "c611609efc93463233f57218a49edeea21922ab692dbb3b1fe535a52afe4545a" + +const emptyGitBlobSHA = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + +func TestGitSourceSnapshotPreservesExactObservedState(t *testing.T) { + t.Parallel() + input := validGitSourceSnapshotInput() + slices.Reverse(input.EvidenceRefs) + snapshot, err := NewGitSourceSnapshot(input) + if err != nil { + t.Fatalf("NewGitSourceSnapshot() error = %v", err) + } + if snapshot.Version() != GitSourceSnapshotVersion || snapshot.workspace != testWorkspace || + !sameResourceRef(snapshot.subject, testSubjectRef()) || snapshot.source != input.Sources[0] || + snapshot.repository != input.Repository || snapshot.baseRef != "dev" || snapshot.baseCommit != testBaseSHA || + snapshot.filePath != "deploy/payments.yaml" || snapshot.observedBlobSHA != testSnapshotBlobSHA || + snapshot.currentContent != input.CurrentContent || !snapshot.observedAt.Equal(input.ObservedAt) || + !snapshot.validUntil.Equal(input.ValidUntil) { + t.Fatalf("snapshot did not preserve exact observed state: %#v", snapshot) + } + if len(snapshot.evidenceRefs) != 2 || !sameResourceRef(snapshot.evidenceRefs[0], testBlobRef()) || + !sameResourceRef(snapshot.evidenceRefs[1], testSubjectRef()) { + t.Fatalf("evidence = %#v, want canonical blob and subject order", snapshot.evidenceRefs) + } + got, err := snapshot.Freshness(testNow) + if err != nil || got != GitSourceFresh { + t.Fatalf("Freshness() = %q, %v, want %q", got, err, GitSourceFresh) + } + + reordered := validGitSourceSnapshotInput() + reorderedSnapshot, err := NewGitSourceSnapshot(reordered) + if err != nil { + t.Fatalf("NewGitSourceSnapshot(reordered) error = %v", err) + } + if !slices.EqualFunc(snapshot.evidenceRefs, reorderedSnapshot.evidenceRefs, sameResourceRef) { + t.Fatalf("evidence ordering changed with caller order: %#v != %#v", snapshot.evidenceRefs, reorderedSnapshot.evidenceRefs) + } +} + +func TestGitSourceSnapshotConstructionIsMutationIsolated(t *testing.T) { + t.Parallel() + input := validGitSourceSnapshotInput() + snapshot, err := NewGitSourceSnapshot(input) + if err != nil { + t.Fatal(err) + } + + input.Workspace = "workspace-b" + input.Subject.Name = "mutated-subject" + input.Sources[0].NativeID = "github.com/other/repository" + input.Repository.Repository = "other" + input.BaseRef = "main" + input.BaseCommit = strings.Repeat("c", 40) + input.FilePath = "other/file.yaml" + input.ObservedBlobSHA = strings.Repeat("d", 40) + input.CurrentContent = "mutated" + input.EvidenceRefs[0].Name = "mutated-evidence" + + if err := snapshot.validate(); err != nil { + t.Fatalf("snapshot retained caller mutation: %v", err) + } + if snapshot.workspace != testWorkspace || snapshot.subject.Name != "payments" || + snapshot.source.NativeID != "github.com/ArdurAI/sith" || snapshot.repository.Repository != "sith" || + snapshot.baseRef != "dev" || snapshot.baseCommit != testBaseSHA || snapshot.filePath != "deploy/payments.yaml" || + snapshot.observedBlobSHA != testSnapshotBlobSHA || snapshot.currentContent == "mutated" || + snapshot.evidenceRefs[1].Name == "mutated-evidence" { + t.Fatalf("snapshot retained caller-owned state: %#v", snapshot) + } +} + +func TestGitSourceSnapshotAcceptsSHA256ObjectIdentity(t *testing.T) { + t.Parallel() + input := validGitSourceSnapshotInput() + input.BaseCommit = strings.Repeat("a", 64) + input.ObservedBlobSHA = testSnapshotSHA256Blob + input.EvidenceRefs[1].Name = testSnapshotSHA256Blob + snapshot, err := NewGitSourceSnapshot(input) + if err != nil { + t.Fatalf("NewGitSourceSnapshot() error = %v", err) + } + if snapshot.baseCommit != input.BaseCommit || snapshot.observedBlobSHA != testSnapshotSHA256Blob { + t.Fatalf("SHA-256 identities were not preserved: %#v", snapshot) + } +} + +func TestGitSourceSnapshotAcceptsExactEmptyFile(t *testing.T) { + t.Parallel() + input := validGitSourceSnapshotInput() + input.CurrentContent = "" + input.ObservedBlobSHA = emptyGitBlobSHA + input.EvidenceRefs[1].Name = emptyGitBlobSHA + snapshot, err := NewGitSourceSnapshot(input) + if err != nil { + t.Fatalf("NewGitSourceSnapshot() error = %v", err) + } + if snapshot.currentContent != "" || snapshot.observedBlobSHA != emptyGitBlobSHA { + t.Fatalf("empty file observation was not preserved: %#v", snapshot) + } +} + +func TestNewGitSourceSnapshotRejectsInvalidClaims(t *testing.T) { + tests := []struct { + name string + mutate func(*GitSourceSnapshotInput) + }{ + {"no source", func(input *GitSourceSnapshotInput) { input.Sources = nil }}, + {"multiple sources", func(input *GitSourceSnapshotInput) { input.Sources = append(input.Sources, input.Sources[0]) }}, + {"invalid workspace", func(input *GitSourceSnapshotInput) { input.Workspace = " workspace-a" }}, + {"subject attributes", func(input *GitSourceSnapshotInput) { input.Subject.Attributes = map[string]string{"uid": "private"} }}, + {"source kind", func(input *GitSourceSnapshotInput) { input.Sources[0].Kind = "gitlab" }}, + {"source adapter", func(input *GitSourceSnapshotInput) { input.Sources[0].AdapterVersion = "future/v2" }}, + {"source repository mismatch", func(input *GitSourceSnapshotInput) { input.Sources[0].NativeID = "github.com/ArdurAI/other" }}, + {"repository URL", func(input *GitSourceSnapshotInput) { input.Repository.Host = "https://github.com" }}, + {"repository suffix", func(input *GitSourceSnapshotInput) { input.Repository.Repository = "sith.git" }}, + {"zero observation", func(input *GitSourceSnapshotInput) { input.ObservedAt = time.Time{} }}, + {"zero validity", func(input *GitSourceSnapshotInput) { input.ValidUntil = time.Time{} }}, + {"reversed validity", func(input *GitSourceSnapshotInput) { input.ValidUntil = input.ObservedAt }}, + {"unbounded validity", func(input *GitSourceSnapshotInput) { + input.ValidUntil = input.ObservedAt.Add(maxGitSourceSnapshotValidity + time.Nanosecond) + }}, + {"symbolic base", func(input *GitSourceSnapshotInput) { input.BaseRef = "HEAD" }}, + {"full base ref", func(input *GitSourceSnapshotInput) { input.BaseRef = "refs/heads/dev" }}, + {"option-shaped base", func(input *GitSourceSnapshotInput) { input.BaseRef = "--upload-pack=evil" }}, + {"commit-shaped base", func(input *GitSourceSnapshotInput) { input.BaseRef = testBaseSHA }}, + {"reflog base", func(input *GitSourceSnapshotInput) { input.BaseRef = "dev@{1}" }}, + {"single at base", func(input *GitSourceSnapshotInput) { input.BaseRef = "@" }}, + {"ambiguous base", func(input *GitSourceSnapshotInput) { input.BaseRef = "release..next" }}, + {"locked base", func(input *GitSourceSnapshotInput) { input.BaseRef = "dev.lock" }}, + {"invalid base commit", func(input *GitSourceSnapshotInput) { input.BaseCommit = strings.ToUpper(testBaseSHA) }}, + {"short base commit", func(input *GitSourceSnapshotInput) { input.BaseCommit = testBaseSHA[:12] }}, + {"mixed object algorithms", func(input *GitSourceSnapshotInput) { input.BaseCommit = strings.Repeat("a", 64) }}, + {"invalid blob", func(input *GitSourceSnapshotInput) { input.ObservedBlobSHA = "not-a-blob" }}, + {"blob content mismatch", func(input *GitSourceSnapshotInput) { input.ObservedBlobSHA = testBlobSHA }}, + {"unsafe relative path", func(input *GitSourceSnapshotInput) { input.FilePath = "../secret.yaml" }}, + {"absolute path", func(input *GitSourceSnapshotInput) { input.FilePath = "/deploy/payments.yaml" }}, + {"unclean path", func(input *GitSourceSnapshotInput) { input.FilePath = "deploy//payments.yaml" }}, + {"backslash path", func(input *GitSourceSnapshotInput) { input.FilePath = "deploy\\payments.yaml" }}, + {"Git metadata path", func(input *GitSourceSnapshotInput) { input.FilePath = ".git/config" }}, + {"invalid UTF-8 content", func(input *GitSourceSnapshotInput) { input.CurrentContent = string([]byte{0xff}) }}, + {"NUL content", func(input *GitSourceSnapshotInput) { input.CurrentContent = "secret\x00value" }}, + {"unbounded content", func(input *GitSourceSnapshotInput) { + input.CurrentContent = strings.Repeat("x", maxGitSourceSnapshotContentBytes+1) + }}, + {"no evidence", func(input *GitSourceSnapshotInput) { input.EvidenceRefs = nil }}, + {"subject-only evidence", func(input *GitSourceSnapshotInput) { input.EvidenceRefs = []fleet.ResourceRef{input.Subject} }}, + {"blob-only evidence", func(input *GitSourceSnapshotInput) { input.EvidenceRefs = []fleet.ResourceRef{testBlobRef()} }}, + {"foreign blob evidence", func(input *GitSourceSnapshotInput) { input.EvidenceRefs[1].Name = strings.Repeat("e", 40) }}, + {"duplicate evidence", func(input *GitSourceSnapshotInput) { + input.EvidenceRefs = append(input.EvidenceRefs, input.EvidenceRefs[0]) + }}, + {"unsafe evidence", func(input *GitSourceSnapshotInput) { input.EvidenceRefs[0].Name = "commit\nforged" }}, + {"too much evidence", func(input *GitSourceSnapshotInput) { + for index := len(input.EvidenceRefs); index <= maxGitSourceSnapshotEvidenceRefs; index++ { + input.EvidenceRefs = append(input.EvidenceRefs, fleet.ResourceRef{ + SourceKind: "github", Scope: "github.com", Kind: "Observation", + Namespace: "ArdurAI/sith", Name: fmt.Sprintf("evidence-%02d", index), + }) + } + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := validGitSourceSnapshotInput() + test.mutate(&input) + snapshot, err := NewGitSourceSnapshot(input) + if err == nil || snapshot.Version() != "" { + t.Fatalf("NewGitSourceSnapshot() = %#v, %v, want rejection", snapshot, err) + } + if strings.Contains(err.Error(), "secret") || len(err.Error()) > 160 { + t.Fatalf("constructor leaked or returned unbounded error: %q", err) + } + }) + } +} + +func TestGitSourceSnapshotFreshnessUsesTrustedTime(t *testing.T) { + t.Parallel() + snapshot, err := NewGitSourceSnapshot(validGitSourceSnapshotInput()) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + now time.Time + want GitSourceFreshness + }{ + {"future", snapshot.observedAt.Add(-time.Nanosecond), GitSourceFuture}, + {"observed boundary", snapshot.observedAt, GitSourceFresh}, + {"fresh", testNow, GitSourceFresh}, + {"expiry boundary", snapshot.validUntil, GitSourceStale}, + {"stale", snapshot.validUntil.Add(time.Nanosecond), GitSourceStale}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := snapshot.Freshness(test.now) + if err != nil || got != test.want { + t.Fatalf("Freshness(%s) = %q, %v, want %q", test.now, got, err, test.want) + } + }) + } + if got, err := snapshot.Freshness(time.Time{}); err == nil || got != "" { + t.Fatalf("Freshness(zero) = %q, %v, want closed error", got, err) + } + forged := snapshot + forged.version = "forged/v9" + if got, err := forged.Freshness(testNow); err == nil || got != "" { + t.Fatalf("Freshness(forged) = %q, %v, want closed error", got, err) + } +} + +func TestGitSourceSnapshotFreshnessIsConcurrentAndReadOnly(t *testing.T) { + t.Parallel() + snapshot, err := NewGitSourceSnapshot(validGitSourceSnapshotInput()) + if err != nil { + t.Fatal(err) + } + const readers = 64 + errors := make(chan error, readers) + var group sync.WaitGroup + for range readers { + group.Add(1) + go func() { + defer group.Done() + got, freshnessErr := snapshot.Freshness(testNow) + if freshnessErr != nil { + errors <- freshnessErr + return + } + if got != GitSourceFresh { + errors <- fmt.Errorf("freshness = %q", got) + } + }() + } + group.Wait() + close(errors) + for err := range errors { + t.Error(err) + } +} + +func FuzzGitSourceSnapshotConstructor(f *testing.F) { + for _, seed := range []struct{ baseRef, filePath, content string }{ + {"dev", "deploy/payments.yaml", "replicas: 3\n"}, + {"HEAD", "../secret", "\x00"}, + {"release/v1", "manifests/café.yaml", "name: café\r\n"}, + } { + f.Add(seed.baseRef, seed.filePath, seed.content) + } + f.Fuzz(func(t *testing.T, baseRef, filePath, content string) { + input := validGitSourceSnapshotInput() + input.BaseRef = baseRef + input.FilePath = filePath + input.CurrentContent = content + snapshot, err := NewGitSourceSnapshot(input) + if err != nil { + return + } + if snapshot.baseRef != baseRef || snapshot.filePath != filePath || snapshot.currentContent != content { + t.Fatalf("accepted snapshot changed exact source bytes: %#v", snapshot) + } + if got, freshnessErr := snapshot.Freshness(testNow); freshnessErr != nil || got != GitSourceFresh { + t.Fatalf("accepted snapshot Freshness() = %q, %v", got, freshnessErr) + } + }) +} + +func validGitSourceSnapshotInput() GitSourceSnapshotInput { + subject := testSubjectRef() + return GitSourceSnapshotInput{ + Workspace: testWorkspace, + Subject: subject, + Sources: []SourceIdentity{{ + Kind: gitHubSourceKind, AdapterVersion: GitSourceSnapshotAdapterVersion, + NativeID: "github.com/ArdurAI/sith", + }}, + ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), + Repository: RepositoryIdentity{Host: "github.com", Owner: "ArdurAI", Repository: "sith"}, + BaseRef: "dev", BaseCommit: testBaseSHA, FilePath: "deploy/payments.yaml", + ObservedBlobSHA: testSnapshotBlobSHA, + CurrentContent: "apiVersion: v1\r\nmetadata:\n name: café\t\n", + EvidenceRefs: []fleet.ResourceRef{subject, testBlobRef()}, + } +} + +func testBlobRef() fleet.ResourceRef { + return fleet.ResourceRef{ + SourceKind: gitHubSourceKind, Scope: "github.com", Kind: "Blob", + Namespace: "ArdurAI/sith", Name: testSnapshotBlobSHA, + } +} diff --git a/internal/remediation/gitops.go b/internal/remediation/gitops.go new file mode 100644 index 0000000..c39123b --- /dev/null +++ b/internal/remediation/gitops.go @@ -0,0 +1,611 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package remediation resolves inert Brain candidates against source-owned provenance without +// authorizing, persisting, dispatching, or executing an action. +package remediation + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "slices" + "sort" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/ArdurAI/sith/internal/brain" + "github.com/ArdurAI/sith/internal/connector" + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/intentargs" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const ( + // GitOpsProvenanceVersion is the first immutable source-to-resolver contract. + GitOpsProvenanceVersion = "gitops-provenance/v1" + // GitOpsSourceAdapterVersion pins the canonical GitHub Git-object observation contract. + GitOpsSourceAdapterVersion = "github-gitops-provenance/2026-03-10" + maxIdentityBytes = 512 + maxBundleContentBytes = 64 << 10 + maxBundleValidity = 5 * time.Minute + gitHubSourceKind = "github" + openPRTargetKind = "Repository" +) + +// ResolutionStatus distinguishes a provenance-complete result from a fail-closed abstention. +type ResolutionStatus string + +// Closed resolution states. +const ( + ResolutionReady ResolutionStatus = "ready" + ResolutionAbstained ResolutionStatus = "abstained" +) + +// AbstentionReason is a bounded, non-sensitive explanation for refusing to resolve a candidate. +type AbstentionReason string + +// Closed abstention reasons. Callers must branch on these values rather than parse error text. +const ( + ReasonCandidateMissing AbstentionReason = "candidate-missing" + ReasonCandidateInvalid AbstentionReason = "candidate-invalid" + ReasonCandidateUnsupported AbstentionReason = "candidate-unsupported" + ReasonVerdictInvalid AbstentionReason = "verdict-invalid" + ReasonVerdictUnconfirmed AbstentionReason = "verdict-unconfirmed" + ReasonFleetAmbiguous AbstentionReason = "fleet-ambiguous" + ReasonProvenanceMissing AbstentionReason = "provenance-missing" + ReasonProvenanceAmbiguous AbstentionReason = "provenance-ambiguous" + ReasonProvenanceInvalid AbstentionReason = "provenance-invalid" + ReasonProvenanceFuture AbstentionReason = "provenance-future" + ReasonProvenanceStale AbstentionReason = "provenance-stale" + ReasonWorkspaceMismatch AbstentionReason = "workspace-mismatch" + ReasonSubjectMismatch AbstentionReason = "subject-mismatch" + ReasonHandlerContractDrift AbstentionReason = "handler-contract-drift" + ReasonHandlerRejected AbstentionReason = "handler-rejected" + ReasonHandlerTargetMismatch AbstentionReason = "handler-target-mismatch" + ReasonHandlerOutputMismatch AbstentionReason = "handler-output-mismatch" +) + +// SourceIdentity identifies the one canonical adapter observation that owns a provenance bundle. +type SourceIdentity struct { + Kind string + AdapterVersion string + NativeID string +} + +// HandlerContract pins provenance to the exact typed-action adapter and argument schema it was +// assembled for. A handler upgrade invalidates an older bundle instead of silently reinterpreting +// it. +type HandlerContract struct { + Kind string + AdapterVersion string + SchemaDigest string +} + +// RepositoryIdentity is one configured Git repository, without a credential or endpoint URL. +type RepositoryIdentity struct { + Host string + Owner string + Repository string +} + +// GitOpsProvenanceInput is accepted only from a canonical source adapter boundary. Actor, role, +// intent ID, approval state, and policy decisions intentionally have no field here. +type GitOpsProvenanceInput struct { + Workspace tenancy.WorkspaceID + Subject fleet.ResourceRef + Sources []SourceIdentity + ObservedAt time.Time + ValidUntil time.Time + Handler HandlerContract + Repository RepositoryIdentity + BaseRef string + BaseCommit string + FilePath string + ObservedBlobSHA string + DesiredContent string + Title string + Body string + CommitMessage string + EvidenceRefs []fleet.ResourceRef +} + +// GitOpsProvenanceBundle is immutable after construction. Its fields stay private so downstream +// request code cannot rewrite source identity or Git preconditions after validation. +type GitOpsProvenanceBundle struct { + version string + workspace tenancy.WorkspaceID + subject fleet.ResourceRef + source SourceIdentity + observedAt time.Time + validUntil time.Time + handler HandlerContract + repository RepositoryIdentity + baseRef string + baseCommit string + filePath string + observedBlobSHA string + desiredContent string + title string + body string + commitMessage string + evidenceRefs []fleet.ResourceRef +} + +// Version reports the closed bundle contract without exposing mutable provenance fields. +func (bundle GitOpsProvenanceBundle) Version() string { return bundle.version } + +// NewGitOpsProvenanceBundle validates and defensively copies one source-owned bundle. Exact +// GitHub path, SHA, size, and repository policy remain handler-owned and are revalidated during +// resolution. +func NewGitOpsProvenanceBundle(input GitOpsProvenanceInput) (GitOpsProvenanceBundle, error) { + if len(input.Sources) != 1 { + return GitOpsProvenanceBundle{}, fmt.Errorf("construct GitOps provenance: exactly one source is required") + } + bundle := GitOpsProvenanceBundle{ + version: GitOpsProvenanceVersion, + workspace: input.Workspace, + subject: cloneResourceRef(input.Subject), + source: input.Sources[0], + observedAt: input.ObservedAt.UTC(), + validUntil: input.ValidUntil.UTC(), + handler: input.Handler, + repository: input.Repository, + baseRef: input.BaseRef, + baseCommit: input.BaseCommit, + filePath: input.FilePath, + observedBlobSHA: input.ObservedBlobSHA, + desiredContent: input.DesiredContent, + title: input.Title, + body: input.Body, + commitMessage: input.CommitMessage, + evidenceRefs: cloneResourceRefs(input.EvidenceRefs), + } + if err := bundle.validate(); err != nil { + return GitOpsProvenanceBundle{}, fmt.Errorf("construct GitOps provenance: bundle is invalid") + } + sort.Slice(bundle.evidenceRefs, func(left, right int) bool { + return resourceRefLess(bundle.evidenceRefs[left], bundle.evidenceRefs[right]) + }) + for index := 1; index < len(bundle.evidenceRefs); index++ { + if sameResourceRef(bundle.evidenceRefs[index-1], bundle.evidenceRefs[index]) { + return GitOpsProvenanceBundle{}, fmt.Errorf("construct GitOps provenance: evidence references are not unique") + } + } + return bundle, nil +} + +// GitOpsHandler is the pure validation seam implemented by the planning-only GitHub adapter. +// CanonicalizeOpenPRArgs performs no network request or mutation. +type GitOpsHandler interface { + Descriptor() connector.Descriptor + CanonicalizeOpenPRArgs(json.RawMessage) (fleet.ResourceRef, json.RawMessage, error) +} + +// GitOpsResolver resolves one candidate using a trusted server clock and the live handler +// contract. It retains no bundle or result state. +type GitOpsResolver struct { + handler GitOpsHandler + now func() time.Time +} + +// NewGitOpsResolver validates the injected pure handler and server clock before accepting work. +func NewGitOpsResolver(handler GitOpsHandler, now func() time.Time) (*GitOpsResolver, error) { + if isNilHandler(handler) || now == nil { + return nil, fmt.Errorf("construct GitOps resolver: handler and clock are required") + } + if _, _, err := inspectHandler(handler.Descriptor()); err != nil { + return nil, fmt.Errorf("construct GitOps resolver: handler contract is invalid") + } + return &GitOpsResolver{handler: handler, now: now}, nil +} + +// HandlerContractFor returns the immutable adapter/schema binding a canonical source uses when it +// constructs provenance. It does not validate provenance or grant authority. +func HandlerContractFor(handler GitOpsHandler) (HandlerContract, error) { + if isNilHandler(handler) { + return HandlerContract{}, fmt.Errorf("inspect GitOps handler: handler is required") + } + contract, _, err := inspectHandler(handler.Descriptor()) + if err != nil { + return HandlerContract{}, fmt.Errorf("inspect GitOps handler: contract is invalid") + } + return contract, nil +} + +// Resolution contains either a provenance-complete, handler-validated argument document or one +// or more closed abstention reasons. Ready output is still not a proposal or authorization. +type Resolution struct { + Status ResolutionStatus + Target fleet.ResourceRef + Arguments json.RawMessage + ArgumentsDigest string + EvidenceRefs []fleet.ResourceRef + Reasons []AbstentionReason +} + +// Resolve fails closed unless one confirmed R2/R4 candidate and one exact source-owned bundle +// satisfy the live handler contract. workspace must later come from authenticated server scope; +// it is deliberately separate from candidate and provenance data. +func (resolver *GitOpsResolver) Resolve( + ctx context.Context, + workspace tenancy.WorkspaceID, + verdict brain.Verdict, + bundles []GitOpsProvenanceBundle, +) (Resolution, error) { + if resolver == nil || isNilHandler(resolver.handler) || resolver.now == nil || ctx == nil { + return Resolution{}, fmt.Errorf("resolve GitOps provenance: resolver and context are required") + } + if err := ctx.Err(); err != nil { + return Resolution{}, fmt.Errorf("resolve GitOps provenance: %w", err) + } + if err := tenancy.ValidateWorkspaceID(workspace); err != nil { + return Resolution{}, fmt.Errorf("resolve GitOps provenance: workspace is invalid") + } + if verdict.RemediationCandidate == nil { + return abstain(ReasonCandidateMissing), nil + } + if err := verdict.RemediationCandidate.Validate(); err != nil { + return abstain(ReasonCandidateInvalid), nil + } + if !validBrainVerdictRef(verdict) { + return abstain(ReasonVerdictInvalid), nil + } + if verdict.RemediationCandidate.Verb != intent.VerbGitOpsOpenPR || + (verdict.Rule != brain.RuleOOMKilled && verdict.Rule != brain.RuleConfigDrift) { + return abstain(ReasonCandidateUnsupported), nil + } + if verdict.FleetWide { + return abstain(ReasonFleetAmbiguous), nil + } + if verdict.Status != brain.StatusConfirmed { + return abstain(ReasonVerdictUnconfirmed), nil + } + if len(bundles) == 0 { + return abstain(ReasonProvenanceMissing), nil + } + if len(bundles) != 1 { + return abstain(ReasonProvenanceAmbiguous), nil + } + + bundle := bundles[0] + if err := bundle.validate(); err != nil { + return abstain(ReasonProvenanceInvalid), nil + } + reason, err := resolver.freshness(bundle) + if err != nil { + return Resolution{}, err + } + if reason != "" { + return abstain(reason), nil + } + if bundle.workspace != workspace { + return abstain(ReasonWorkspaceMismatch), nil + } + if !sameResourceRef(bundle.subject, verdict.Ref) { + return abstain(ReasonSubjectMismatch), nil + } + + contract, schema, err := inspectHandler(resolver.handler.Descriptor()) + if err != nil || contract != bundle.handler || bundle.source.Kind != contract.Kind { + return abstain(ReasonHandlerContractDrift), nil + } + arguments, err := bundle.arguments() + if err != nil { + return abstain(ReasonProvenanceInvalid), nil + } + target, canonical, err := resolver.handler.CanonicalizeOpenPRArgs(arguments) + if err != nil { + return abstain(ReasonHandlerRejected), nil + } + if err := ctx.Err(); err != nil { + return Resolution{}, fmt.Errorf("resolve GitOps provenance: %w", err) + } + if !targetMatchesRepository(target, contract.Kind, bundle.repository) { + return abstain(ReasonHandlerTargetMismatch), nil + } + if err := schema.Validate(canonical); err != nil { + return abstain(ReasonHandlerContractDrift), nil + } + if !canonicalMatchesBundle(canonical, bundle) { + return abstain(ReasonHandlerOutputMismatch), nil + } + postContract, _, err := inspectHandler(resolver.handler.Descriptor()) + if err != nil || postContract != contract { + return abstain(ReasonHandlerContractDrift), nil + } + if err := ctx.Err(); err != nil { + return Resolution{}, fmt.Errorf("resolve GitOps provenance: %w", err) + } + reason, err = resolver.freshness(bundle) + if err != nil { + return Resolution{}, err + } + if reason != "" { + return abstain(reason), nil + } + digest := sha256.Sum256(canonical) + return Resolution{ + Status: ResolutionReady, + Target: cloneResourceRef(target), + Arguments: append(json.RawMessage(nil), canonical...), + ArgumentsDigest: "sha256:" + hex.EncodeToString(digest[:]), + EvidenceRefs: cloneResourceRefs(bundle.evidenceRefs), + }, nil +} + +func (resolver *GitOpsResolver) freshness(bundle GitOpsProvenanceBundle) (AbstentionReason, error) { + now := resolver.now().UTC() + if now.IsZero() { + return "", fmt.Errorf("resolve GitOps provenance: clock returned an invalid time") + } + if now.Before(bundle.observedAt) { + return ReasonProvenanceFuture, nil + } + if !now.Before(bundle.validUntil) { + return ReasonProvenanceStale, nil + } + return "", nil +} + +type openPRArguments struct { + BaseRef string `json:"base_ref"` + ExpectedBaseSHA string `json:"expected_base_sha"` + Title string `json:"title"` + Body string `json:"body,omitempty"` + CommitMessage string `json:"commit_message"` + Changes []fileChange `json:"changes"` +} + +type fileChange struct { + Operation string `json:"operation"` + Path string `json:"path"` + Content *string `json:"content"` + ExpectedBlobSHA *string `json:"expected_blob_sha"` +} + +func (bundle GitOpsProvenanceBundle) arguments() (json.RawMessage, error) { + content := bundle.desiredContent + observed := bundle.observedBlobSHA + encoded, err := json.Marshal(openPRArguments{ + BaseRef: bundle.baseRef, ExpectedBaseSHA: bundle.baseCommit, + Title: bundle.title, Body: bundle.body, CommitMessage: bundle.commitMessage, + Changes: []fileChange{{ + Operation: "update", Path: bundle.filePath, + Content: &content, ExpectedBlobSHA: &observed, + }}, + }) + if err != nil { + return nil, fmt.Errorf("encode GitOps provenance arguments") + } + return json.RawMessage(encoded), nil +} + +func (bundle GitOpsProvenanceBundle) validate() error { + if bundle.version != GitOpsProvenanceVersion || tenancy.ValidateWorkspaceID(bundle.workspace) != nil || + validateStableRef(bundle.subject) != nil || bundle.observedAt.IsZero() || bundle.validUntil.IsZero() || + !bundle.observedAt.Before(bundle.validUntil) || bundle.validUntil.Sub(bundle.observedAt) > maxBundleValidity || + validateSource(bundle.source) != nil || + validateHandlerContract(bundle.handler) != nil || bundle.source.Kind != bundle.handler.Kind || + validateRepository(bundle.repository) != nil || bundle.source.NativeID != bundle.repository.nativeID() || + !validProvenanceBaseRef(bundle.baseRef) || !validObjectID(bundle.baseCommit) || + validateSafeText(bundle.filePath, maxIdentityBytes, false) != nil || + !validObjectID(bundle.observedBlobSHA) || + validateSafeText(bundle.title, maxIdentityBytes, false) != nil || + validateMultilineText(bundle.body, 16<<10, true) != nil || + validateSafeText(bundle.commitMessage, maxIdentityBytes, false) != nil || + !validBundleContent(bundle.desiredContent) || len(bundle.evidenceRefs) == 0 { + return fmt.Errorf("GitOps provenance bundle is invalid") + } + for _, ref := range bundle.evidenceRefs { + if validateStableRef(ref) != nil { + return fmt.Errorf("GitOps provenance evidence is invalid") + } + } + return nil +} + +func inspectHandler(descriptor connector.Descriptor) (HandlerContract, *intentargs.Schema, error) { + if descriptor.Kind != gitHubSourceKind || + descriptor.ConnKind != connector.KindTypedAction || descriptor.Owner != "sith" || + validateSafeText(descriptor.AdapterVersion, maxIdentityBytes, false) != nil || + !slices.Equal(descriptor.WireVersions, []connector.WireVersion{connector.CurrentWireVersion()}) || + !slices.Equal(descriptor.Capabilities, []connector.Capability{connector.CapPlan}) || + !slices.Equal(descriptor.Verbs, []intent.Verb{intent.VerbGitOpsOpenPR}) || + len(descriptor.ArgSchemas) != 1 { + return HandlerContract{}, nil, fmt.Errorf("handler descriptor is not the exact GitOps planning contract") + } + raw, present := descriptor.ArgSchemas[intent.VerbGitOpsOpenPR] + if !present || len(raw) == 0 { + return HandlerContract{}, nil, fmt.Errorf("handler schema is missing") + } + schema, err := intentargs.Compile(append(json.RawMessage(nil), raw...)) + if err != nil { + return HandlerContract{}, nil, fmt.Errorf("handler schema is invalid") + } + digest := sha256.Sum256(raw) + return HandlerContract{ + Kind: descriptor.Kind, AdapterVersion: descriptor.AdapterVersion, + SchemaDigest: "sha256:" + hex.EncodeToString(digest[:]), + }, schema, nil +} + +func validateHandlerContract(contract HandlerContract) error { + if validateSafeText(contract.Kind, maxIdentityBytes, false) != nil || + validateSafeText(contract.AdapterVersion, maxIdentityBytes, false) != nil || !validDigest(contract.SchemaDigest) { + return fmt.Errorf("handler contract is invalid") + } + return nil +} + +func validateSource(source SourceIdentity) error { + if source.Kind != gitHubSourceKind || source.AdapterVersion != GitOpsSourceAdapterVersion || + validateSafeText(source.NativeID, maxIdentityBytes, false) != nil { + return fmt.Errorf("source identity is invalid") + } + return nil +} + +func validateRepository(repository RepositoryIdentity) error { + if validateSafeText(repository.Host, maxIdentityBytes, false) != nil || + validateSafeText(repository.Owner, maxIdentityBytes, false) != nil || + validateSafeText(repository.Repository, maxIdentityBytes, false) != nil || + repository.Host != strings.ToLower(repository.Host) || strings.Contains(repository.Host, "://") || + strings.ContainsAny(repository.Host+repository.Owner+repository.Repository, "/\\") || + strings.HasSuffix(strings.ToLower(repository.Repository), ".git") { + return fmt.Errorf("repository identity is invalid") + } + return nil +} + +func (repository RepositoryIdentity) nativeID() string { + return repository.Host + "/" + repository.Owner + "/" + repository.Repository +} + +func validBrainVerdictRef(verdict brain.Verdict) bool { + if validateStableRef(verdict.Ref) != nil || len(verdict.Citations) == 0 { + return false + } + attached := false + for _, citation := range verdict.Citations { + if citation.Stale || validateStableRef(citation.Ref) != nil { + return false + } + attached = attached || sameResourceRef(citation.Ref, verdict.Ref) + } + return attached +} + +func targetMatchesRepository(target fleet.ResourceRef, sourceKind string, repository RepositoryIdentity) bool { + return len(target.Attributes) == 0 && target.SourceKind == sourceKind && target.Scope == repository.Host && + target.Kind == openPRTargetKind && target.Namespace == repository.Owner && target.Name == repository.Repository +} + +func validateStableRef(ref fleet.ResourceRef) error { + if len(ref.Attributes) != 0 || validateSafeText(ref.SourceKind, maxIdentityBytes, false) != nil || + validateSafeText(ref.Scope, maxIdentityBytes, false) != nil || + validateSafeText(ref.Kind, maxIdentityBytes, false) != nil || + validateSafeText(ref.Namespace, maxIdentityBytes, true) != nil || + validateSafeText(ref.Name, maxIdentityBytes, false) != nil { + return fmt.Errorf("resource reference is invalid") + } + return nil +} + +func validateSafeText(value string, maximum int, allowEmpty bool) error { + if (!allowEmpty && value == "") || len(value) > maximum || !utf8.ValidString(value) || + strings.TrimSpace(value) != value || strings.ContainsRune(value, '\x00') { + return fmt.Errorf("text is invalid") + } + for _, character := range value { + if unicode.IsControl(character) { + return fmt.Errorf("text is invalid") + } + } + return nil +} + +func validateMultilineText(value string, maximum int, allowEmpty bool) error { + if (!allowEmpty && value == "") || len(value) > maximum || !utf8.ValidString(value) || + strings.TrimSpace(value) != value || strings.ContainsRune(value, '\x00') { + return fmt.Errorf("text is invalid") + } + for _, character := range value { + if unicode.IsControl(character) && character != '\n' && character != '\t' { + return fmt.Errorf("text is invalid") + } + } + return nil +} + +func validBundleContent(value string) bool { + return len(value) <= maxBundleContentBytes && utf8.ValidString(value) && !strings.ContainsRune(value, '\x00') +} + +func validDigest(value string) bool { + if !strings.HasPrefix(value, "sha256:") || len(value) != len("sha256:")+sha256.Size*2 { + return false + } + hexValue := strings.TrimPrefix(value, "sha256:") + _, err := hex.DecodeString(hexValue) + return err == nil && hexValue == strings.ToLower(hexValue) +} + +func validObjectID(value string) bool { + if len(value) != 40 && len(value) != 64 { + return false + } + _, err := hex.DecodeString(value) + return err == nil && value == strings.ToLower(value) +} + +func validProvenanceBaseRef(value string) bool { + return validateSafeText(value, maxIdentityBytes, false) == nil && !strings.EqualFold(value, "HEAD") && + !strings.HasPrefix(value, "-") && !strings.HasPrefix(strings.ToLower(value), "refs/") && !validObjectID(value) +} + +func canonicalMatchesBundle(canonical json.RawMessage, bundle GitOpsProvenanceBundle) bool { + var decoded openPRArguments + if err := json.Unmarshal(canonical, &decoded); err != nil || len(decoded.Changes) != 1 { + return false + } + change := decoded.Changes[0] + return decoded.BaseRef == bundle.baseRef && decoded.ExpectedBaseSHA == bundle.baseCommit && + decoded.Title == bundle.title && decoded.Body == bundle.body && decoded.CommitMessage == bundle.commitMessage && + change.Operation == "update" && change.Path == bundle.filePath && change.Content != nil && + *change.Content == bundle.desiredContent && change.ExpectedBlobSHA != nil && + *change.ExpectedBlobSHA == bundle.observedBlobSHA +} + +func abstain(reasons ...AbstentionReason) Resolution { + return Resolution{Status: ResolutionAbstained, Reasons: append([]AbstentionReason(nil), reasons...)} +} + +func isNilHandler(handler GitOpsHandler) bool { + if handler == nil { + return true + } + value := reflect.ValueOf(handler) + return (value.Kind() == reflect.Chan || value.Kind() == reflect.Func || value.Kind() == reflect.Interface || + value.Kind() == reflect.Map || value.Kind() == reflect.Pointer || value.Kind() == reflect.Slice) && value.IsNil() +} + +func cloneResourceRefs(refs []fleet.ResourceRef) []fleet.ResourceRef { + cloned := make([]fleet.ResourceRef, len(refs)) + for index, ref := range refs { + cloned[index] = cloneResourceRef(ref) + } + return cloned +} + +func cloneResourceRef(ref fleet.ResourceRef) fleet.ResourceRef { + cloned := ref + if ref.Attributes != nil { + cloned.Attributes = make(map[string]string, len(ref.Attributes)) + for key, value := range ref.Attributes { + cloned.Attributes[key] = value + } + } + return cloned +} + +func sameResourceRef(left, right fleet.ResourceRef) bool { + return left.SourceKind == right.SourceKind && left.Scope == right.Scope && left.Kind == right.Kind && + left.Namespace == right.Namespace && left.Name == right.Name && len(left.Attributes) == 0 && len(right.Attributes) == 0 +} + +func resourceRefLess(left, right fleet.ResourceRef) bool { + leftParts := [...]string{left.SourceKind, left.Scope, left.Kind, left.Namespace, left.Name} + rightParts := [...]string{right.SourceKind, right.Scope, right.Kind, right.Namespace, right.Name} + for index := range leftParts { + if leftParts[index] != rightParts[index] { + return leftParts[index] < rightParts[index] + } + } + return false +} diff --git a/internal/remediation/gitops_test.go b/internal/remediation/gitops_test.go new file mode 100644 index 0000000..31d3ea7 --- /dev/null +++ b/internal/remediation/gitops_test.go @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: Apache-2.0 + +package remediation + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/ArdurAI/sith/internal/brain" + "github.com/ArdurAI/sith/internal/connector" + githubconnector "github.com/ArdurAI/sith/internal/connector/github" + "github.com/ArdurAI/sith/internal/fleet" + "github.com/ArdurAI/sith/internal/intent" + "github.com/ArdurAI/sith/internal/tenancy" +) + +const testWorkspace tenancy.WorkspaceID = "workspace-a" + +const ( + testBaseSHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + testBlobSHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +) + +var testNow = time.Date(2026, 7, 22, 18, 0, 0, 0, time.UTC) + +func TestGitOpsResolverProducesDeterministicR2AndR4Arguments(t *testing.T) { + t.Parallel() + for _, rule := range []brain.RuleID{brain.RuleOOMKilled, brain.RuleConfigDrift} { + rule := rule + t.Run(string(rule), func(t *testing.T) { + t.Parallel() + fixture := newGitOpsFixture(t, rule) + first, err := fixture.resolver.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + assertReadyResolution(t, first) + + secondInput := validGitOpsInput(fixture.bundle.handler) + slices.Reverse(secondInput.EvidenceRefs) + secondBundle, err := NewGitOpsProvenanceBundle(secondInput) + if err != nil { + t.Fatalf("construct reordered bundle: %v", err) + } + second, err := fixture.resolver.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{secondBundle}) + if err != nil { + t.Fatalf("second Resolve() error = %v", err) + } + if !slices.Equal(first.Arguments, second.Arguments) || first.ArgumentsDigest != second.ArgumentsDigest || + !slices.EqualFunc(first.EvidenceRefs, second.EvidenceRefs, sameResourceRef) { + t.Fatalf("resolution changed with evidence ordering:\nfirst=%#v\nsecond=%#v", first, second) + } + }) + } +} + +func TestGitOpsProvenanceAndResolutionAreMutationIsolated(t *testing.T) { + t.Parallel() + planner := newGitHubPlanner(t) + contract, err := HandlerContractFor(planner) + if err != nil { + t.Fatal(err) + } + input := validGitOpsInput(contract) + bundle, err := NewGitOpsProvenanceBundle(input) + if err != nil { + t.Fatal(err) + } + + input.Subject.Name = "mutated-subject" + input.Sources[0].NativeID = "github.com/other/repository" + input.EvidenceRefs[0].Name = "mutated-evidence" + resolver, err := NewGitOpsResolver(planner, func() time.Time { return testNow }) + if err != nil { + t.Fatal(err) + } + verdict := validGitOpsVerdict(brain.RuleConfigDrift) + first, err := resolver.Resolve(context.Background(), testWorkspace, verdict, []GitOpsProvenanceBundle{bundle}) + if err != nil { + t.Fatal(err) + } + assertReadyResolution(t, first) + + first.Arguments[0] = '!' + first.Target.Name = "mutated-target" + first.Target.Attributes = map[string]string{"token": "forbidden"} + first.EvidenceRefs[0].Name = "mutated-output" + first.Reasons = append(first.Reasons, ReasonCandidateInvalid) + + second, err := resolver.Resolve(context.Background(), testWorkspace, verdict, []GitOpsProvenanceBundle{bundle}) + if err != nil { + t.Fatal(err) + } + assertReadyResolution(t, second) + if len(second.Arguments) == 0 || second.Arguments[0] != '{' || second.Target.Name != "sith" || + len(second.Target.Attributes) != 0 || second.EvidenceRefs[0].Name == "mutated-output" || len(second.Reasons) != 0 { + t.Fatalf("second resolution retained caller mutation: %#v", second) + } +} + +func TestGitOpsResolverAbstainsClosed(t *testing.T) { + fixture := newGitOpsFixture(t, brain.RuleConfigDrift) + tests := []struct { + name string + want AbstentionReason + mutate func(*brain.Verdict, *[]GitOpsProvenanceBundle) + }{ + {"candidate missing", ReasonCandidateMissing, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { verdict.RemediationCandidate = nil }}, + {"candidate mutated", ReasonCandidateInvalid, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { + verdict.RemediationCandidate.RequiredProvenance[0] = brain.ProvenanceArgoRevision + }}, + {"verdict ref attributes", ReasonVerdictInvalid, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { + verdict.Ref.Attributes = map[string]string{"native": "untrusted"} + }}, + {"stale citation", ReasonVerdictInvalid, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { verdict.Citations[0].Stale = true }}, + {"unattached citation", ReasonVerdictInvalid, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { verdict.Citations[0].Ref.Name = "other" }}, + {"unsupported R1", ReasonCandidateUnsupported, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { + verdict.Rule = brain.RuleBadDeploy + verdict.RemediationCandidate = &brain.RemediationCandidate{Verb: intent.VerbArgoCDRollback, RequiredProvenance: []brain.ProvenanceRequirement{ + brain.ProvenanceArgoApplicationTarget, brain.ProvenanceArgoRevision, + }} + }}, + {"fleet verdict", ReasonFleetAmbiguous, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { verdict.FleetWide = true }}, + {"unconfirmed verdict", ReasonVerdictUnconfirmed, func(verdict *brain.Verdict, _ *[]GitOpsProvenanceBundle) { verdict.Status = brain.StatusUnconfirmed }}, + {"missing provenance", ReasonProvenanceMissing, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { *bundles = nil }}, + {"ambiguous provenance", ReasonProvenanceAmbiguous, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { *bundles = append(*bundles, (*bundles)[0]) }}, + {"invalid provenance", ReasonProvenanceInvalid, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { (*bundles)[0].version = "forged/v9" }}, + {"future provenance", ReasonProvenanceFuture, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { + (*bundles)[0].observedAt = testNow.Add(time.Second) + (*bundles)[0].validUntil = testNow.Add(2 * time.Minute) + }}, + {"stale provenance", ReasonProvenanceStale, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { (*bundles)[0].validUntil = testNow }}, + {"foreign workspace", ReasonWorkspaceMismatch, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { (*bundles)[0].workspace = "workspace-b" }}, + {"unattached subject", ReasonSubjectMismatch, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { (*bundles)[0].subject.Name = "other" }}, + {"source contract forged", ReasonProvenanceInvalid, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { + (*bundles)[0].source.AdapterVersion = "forged/v1" + }}, + {"handler adapter drift", ReasonHandlerContractDrift, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { + (*bundles)[0].handler.AdapterVersion = "gitops-open-pr/2099-01-01" + }}, + {"handler schema drift", ReasonHandlerContractDrift, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { + (*bundles)[0].handler.SchemaDigest = "sha256:" + strings.Repeat("c", 64) + }}, + {"unsafe path rejected by handler", ReasonHandlerRejected, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { (*bundles)[0].filePath = "../secret.yaml" }}, + {"configured base mismatch rejected by handler", ReasonHandlerRejected, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { (*bundles)[0].baseRef = "main" }}, + {"oversized content rejected by handler", ReasonHandlerRejected, func(_ *brain.Verdict, bundles *[]GitOpsProvenanceBundle) { + (*bundles)[0].desiredContent = strings.Repeat("x", (16<<10)+1) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + verdict := cloneVerdict(fixture.verdict) + bundles := []GitOpsProvenanceBundle{cloneBundle(fixture.bundle)} + test.mutate(&verdict, &bundles) + got, err := fixture.resolver.Resolve(context.Background(), testWorkspace, verdict, bundles) + if err != nil { + t.Fatalf("Resolve() error = %v", err) + } + assertAbstention(t, got, test.want) + }) + } +} + +func TestNewGitOpsProvenanceBundleRejectsInvalidClaims(t *testing.T) { + planner := newGitHubPlanner(t) + contract, err := HandlerContractFor(planner) + if err != nil { + t.Fatal(err) + } + tests := []struct { + name string + mutate func(*GitOpsProvenanceInput) + }{ + {"no source", func(input *GitOpsProvenanceInput) { input.Sources = nil }}, + {"multiple sources", func(input *GitOpsProvenanceInput) { input.Sources = append(input.Sources, input.Sources[0]) }}, + {"invalid workspace", func(input *GitOpsProvenanceInput) { input.Workspace = " workspace-a" }}, + {"subject attributes", func(input *GitOpsProvenanceInput) { input.Subject.Attributes = map[string]string{"uid": "private"} }}, + {"source kind", func(input *GitOpsProvenanceInput) { input.Sources[0].Kind = "gitlab" }}, + {"source adapter", func(input *GitOpsProvenanceInput) { input.Sources[0].AdapterVersion = "future/v2" }}, + {"source repository mismatch", func(input *GitOpsProvenanceInput) { input.Sources[0].NativeID = "github.com/ArdurAI/other" }}, + {"invalid repository", func(input *GitOpsProvenanceInput) { input.Repository.Repository = "sith.git" }}, + {"zero observation", func(input *GitOpsProvenanceInput) { input.ObservedAt = time.Time{} }}, + {"reversed validity", func(input *GitOpsProvenanceInput) { input.ValidUntil = input.ObservedAt }}, + {"unbounded validity", func(input *GitOpsProvenanceInput) { + input.ValidUntil = input.ObservedAt.Add(maxBundleValidity + time.Nanosecond) + }}, + {"symbolic base", func(input *GitOpsProvenanceInput) { input.BaseRef = "HEAD" }}, + {"full base ref", func(input *GitOpsProvenanceInput) { input.BaseRef = "refs/heads/dev" }}, + {"commit-shaped base", func(input *GitOpsProvenanceInput) { input.BaseRef = testBaseSHA }}, + {"invalid base commit", func(input *GitOpsProvenanceInput) { input.BaseCommit = strings.ToUpper(testBaseSHA) }}, + {"invalid blob", func(input *GitOpsProvenanceInput) { input.ObservedBlobSHA = "not-a-blob" }}, + {"empty title", func(input *GitOpsProvenanceInput) { input.Title = "" }}, + {"NUL content", func(input *GitOpsProvenanceInput) { input.DesiredContent = "secret\x00value" }}, + {"unbounded content", func(input *GitOpsProvenanceInput) { + input.DesiredContent = strings.Repeat("x", maxBundleContentBytes+1) + }}, + {"no evidence", func(input *GitOpsProvenanceInput) { input.EvidenceRefs = nil }}, + {"duplicate evidence", func(input *GitOpsProvenanceInput) { + input.EvidenceRefs = append(input.EvidenceRefs, input.EvidenceRefs[0]) + }}, + {"unsafe evidence", func(input *GitOpsProvenanceInput) { input.EvidenceRefs[0].Name = "commit\nforged" }}, + {"noncanonical digest", func(input *GitOpsProvenanceInput) { input.Handler.SchemaDigest = "sha256:" + strings.Repeat("A", 64) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + input := validGitOpsInput(contract) + test.mutate(&input) + bundle, err := NewGitOpsProvenanceBundle(input) + if err == nil || bundle.Version() != "" { + t.Fatalf("NewGitOpsProvenanceBundle() = %#v, %v, want rejection", bundle, err) + } + if strings.Contains(err.Error(), "secret") || len(err.Error()) > 160 { + t.Fatalf("constructor leaked or returned unbounded error: %q", err) + } + }) + } +} + +func TestGitOpsResolverRejectsHandlerMismatchAndDrift(t *testing.T) { + tests := []struct { + name string + want AbstentionReason + proxy func(GitOpsHandler) GitOpsHandler + }{ + {"target mismatch", ReasonHandlerTargetMismatch, func(base GitOpsHandler) GitOpsHandler { + return &handlerProxy{base: base, canonicalize: func(target fleet.ResourceRef, document json.RawMessage) (fleet.ResourceRef, json.RawMessage) { + target.Name = "other" + return target, document + }} + }}, + {"output commit mismatch", ReasonHandlerOutputMismatch, func(base GitOpsHandler) GitOpsHandler { + return &handlerProxy{base: base, canonicalize: func(target fleet.ResourceRef, document json.RawMessage) (fleet.ResourceRef, json.RawMessage) { + var args openPRArguments + if err := json.Unmarshal(document, &args); err != nil { + panic(err) + } + args.ExpectedBaseSHA = strings.Repeat("c", 40) + encoded, err := json.Marshal(args) + if err != nil { + panic(err) + } + return target, encoded + }} + }}, + {"output blob mismatch", ReasonHandlerOutputMismatch, func(base GitOpsHandler) GitOpsHandler { + return &handlerProxy{base: base, canonicalize: func(target fleet.ResourceRef, document json.RawMessage) (fleet.ResourceRef, json.RawMessage) { + var args openPRArguments + if err := json.Unmarshal(document, &args); err != nil { + panic(err) + } + blob := strings.Repeat("d", 40) + args.Changes[0].ExpectedBlobSHA = &blob + encoded, err := json.Marshal(args) + if err != nil { + panic(err) + } + return target, encoded + }} + }}, + {"repository mismatch", ReasonHandlerTargetMismatch, func(base GitOpsHandler) GitOpsHandler { + return &handlerProxy{base: base, canonicalize: func(target fleet.ResourceRef, document json.RawMessage) (fleet.ResourceRef, json.RawMessage) { + target.Namespace = "Other" + return target, document + }} + }}, + {"post-canonicalization descriptor drift", ReasonHandlerContractDrift, func(base GitOpsHandler) GitOpsHandler { + return &handlerProxy{base: base, mutateDescriptor: func(descriptor connector.Descriptor, call int) connector.Descriptor { + if call >= 3 { + descriptor.AdapterVersion += "/drifted" + } + return descriptor + }} + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + base := newGitHubPlanner(t) + contract, err := HandlerContractFor(base) + if err != nil { + t.Fatal(err) + } + bundle, err := NewGitOpsProvenanceBundle(validGitOpsInput(contract)) + if err != nil { + t.Fatal(err) + } + resolver, err := NewGitOpsResolver(test.proxy(base), func() time.Time { return testNow }) + if err != nil { + t.Fatal(err) + } + got, err := resolver.Resolve(context.Background(), testWorkspace, validGitOpsVerdict(brain.RuleConfigDrift), []GitOpsProvenanceBundle{bundle}) + if err != nil { + t.Fatal(err) + } + assertAbstention(t, got, test.want) + }) + } +} + +func TestGitOpsResolverRejectsInvalidDependenciesAndCancellation(t *testing.T) { + t.Parallel() + var typedNil *handlerProxy + if resolver, err := NewGitOpsResolver(typedNil, func() time.Time { return testNow }); err == nil || resolver != nil { + t.Fatalf("NewGitOpsResolver(typed nil) = %#v, %v", resolver, err) + } + if resolver, err := NewGitOpsResolver(newGitHubPlanner(t), nil); err == nil || resolver != nil { + t.Fatalf("NewGitOpsResolver(nil clock) = %#v, %v", resolver, err) + } + if _, err := HandlerContractFor(typedNil); err == nil { + t.Fatal("HandlerContractFor() accepted typed nil") + } + + fixture := newGitOpsFixture(t, brain.RuleConfigDrift) + var nilResolver *GitOpsResolver + if _, err := nilResolver.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}); err == nil { + t.Fatal("Resolve() accepted a nil resolver") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := fixture.resolver.Resolve(ctx, testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}); err == nil || !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("Resolve(canceled) error = %v", err) + } + var missingContext context.Context + if _, err := fixture.resolver.Resolve(missingContext, testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}); err == nil { + t.Fatal("Resolve(nil context) succeeded") + } + if _, err := fixture.resolver.Resolve(context.Background(), " invalid", fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}); err == nil { + t.Fatal("Resolve(invalid workspace) succeeded") + } + zeroClock, err := NewGitOpsResolver(newGitHubPlanner(t), func() time.Time { return time.Time{} }) + if err != nil { + t.Fatal(err) + } + if _, err := zeroClock.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}); err == nil { + t.Fatal("Resolve(zero clock) succeeded") + } +} + +func TestNewGitOpsResolverRejectsNoncanonicalHandlerDescriptors(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(connector.Descriptor) connector.Descriptor + }{ + {"wrong owner", func(descriptor connector.Descriptor) connector.Descriptor { + descriptor.Owner = "caller" + return descriptor + }}, + {"wire version missing", func(descriptor connector.Descriptor) connector.Descriptor { + descriptor.WireVersions = nil + return descriptor + }}, + {"execute capability", func(descriptor connector.Descriptor) connector.Descriptor { + descriptor.Capabilities = append(descriptor.Capabilities, connector.CapExecute) + return descriptor + }}, + {"extra verb", func(descriptor connector.Descriptor) connector.Descriptor { + descriptor.Verbs = append(descriptor.Verbs, intent.VerbDeploymentRestart) + return descriptor + }}, + {"invalid schema", func(descriptor connector.Descriptor) connector.Descriptor { + descriptor.ArgSchemas[intent.VerbGitOpsOpenPR] = json.RawMessage(`{"type":"not-a-type"}`) + return descriptor + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + proxy := &handlerProxy{base: newGitHubPlanner(t), mutateDescriptor: func(descriptor connector.Descriptor, _ int) connector.Descriptor { + return test.mutate(descriptor) + }} + if resolver, err := NewGitOpsResolver(proxy, func() time.Time { return testNow }); err == nil || resolver != nil { + t.Fatalf("NewGitOpsResolver() = %#v, %v, want descriptor rejection", resolver, err) + } + }) + } +} + +func TestGitOpsResolverHonorsCancellationAfterHandlerValidation(t *testing.T) { + t.Parallel() + base := newGitHubPlanner(t) + contract, err := HandlerContractFor(base) + if err != nil { + t.Fatal(err) + } + bundle, err := NewGitOpsProvenanceBundle(validGitOpsInput(contract)) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + proxy := &handlerProxy{base: base, canonicalize: func(target fleet.ResourceRef, document json.RawMessage) (fleet.ResourceRef, json.RawMessage) { + cancel() + return target, document + }} + resolver, err := NewGitOpsResolver(proxy, func() time.Time { return testNow }) + if err != nil { + t.Fatal(err) + } + if _, err := resolver.Resolve(ctx, testWorkspace, validGitOpsVerdict(brain.RuleConfigDrift), []GitOpsProvenanceBundle{bundle}); err == nil || !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("Resolve() error = %v, want cancellation", err) + } +} + +func TestGitOpsResolverHonorsCancellationAfterFinalContractCheck(t *testing.T) { + t.Parallel() + base := newGitHubPlanner(t) + contract, err := HandlerContractFor(base) + if err != nil { + t.Fatal(err) + } + bundle, err := NewGitOpsProvenanceBundle(validGitOpsInput(contract)) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + proxy := &handlerProxy{base: base, mutateDescriptor: func(descriptor connector.Descriptor, call int) connector.Descriptor { + if call >= 3 { + cancel() + } + return descriptor + }} + resolver, err := NewGitOpsResolver(proxy, func() time.Time { return testNow }) + if err != nil { + t.Fatal(err) + } + if _, err := resolver.Resolve(ctx, testWorkspace, validGitOpsVerdict(brain.RuleConfigDrift), []GitOpsProvenanceBundle{bundle}); err == nil || !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("Resolve() error = %v, want final-check cancellation", err) + } +} + +func TestGitOpsResolverRechecksFreshnessBeforeReturningReady(t *testing.T) { + t.Parallel() + planner := newGitHubPlanner(t) + contract, err := HandlerContractFor(planner) + if err != nil { + t.Fatal(err) + } + bundle, err := NewGitOpsProvenanceBundle(validGitOpsInput(contract)) + if err != nil { + t.Fatal(err) + } + clockCalls := 0 + resolver, err := NewGitOpsResolver(planner, func() time.Time { + clockCalls++ + if clockCalls == 1 { + return testNow + } + return testNow.Add(2 * time.Minute) + }) + if err != nil { + t.Fatal(err) + } + got, err := resolver.Resolve(context.Background(), testWorkspace, validGitOpsVerdict(brain.RuleConfigDrift), []GitOpsProvenanceBundle{bundle}) + if err != nil { + t.Fatal(err) + } + assertAbstention(t, got, ReasonProvenanceStale) +} + +func TestGitOpsResolverIsConcurrentAndDeterministic(t *testing.T) { + t.Parallel() + fixture := newGitOpsFixture(t, brain.RuleConfigDrift) + want, err := fixture.resolver.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}) + if err != nil { + t.Fatal(err) + } + wantJSON, err := json.Marshal(want) + if err != nil { + t.Fatal(err) + } + + const workers = 64 + errors := make(chan error, workers) + var wait sync.WaitGroup + for range workers { + wait.Add(1) + go func() { + defer wait.Done() + got, resolveErr := fixture.resolver.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{fixture.bundle}) + if resolveErr != nil { + errors <- resolveErr + return + } + encoded, marshalErr := json.Marshal(got) + if marshalErr != nil { + errors <- marshalErr + return + } + if !slices.Equal(encoded, wantJSON) { + errors <- fmt.Errorf("nondeterministic resolution") + } + }() + } + wait.Wait() + close(errors) + for err := range errors { + t.Error(err) + } +} + +func FuzzGitOpsResolverNeverPanics(f *testing.F) { + fixture := newGitOpsFixture(f, brain.RuleConfigDrift) + f.Add("deploy/payments.yaml", "replicas: 4\n", "dev") + f.Add("../secret", "marker\x00value", "HEAD") + f.Fuzz(func(t *testing.T, path, content, baseRef string) { + bundle := cloneBundle(fixture.bundle) + bundle.filePath = path + bundle.desiredContent = content + bundle.baseRef = baseRef + got, err := fixture.resolver.Resolve(context.Background(), testWorkspace, fixture.verdict, []GitOpsProvenanceBundle{bundle}) + if err != nil { + if len(err.Error()) > 160 { + t.Fatalf("unbounded error length %d", len(err.Error())) + } + return + } + encoded, marshalErr := json.Marshal(got) + if marshalErr != nil { + t.Fatal(marshalErr) + } + if len(encoded) > maxBundleContentBytes+(16<<10) { + t.Fatalf("unbounded resolution length %d", len(encoded)) + } + }) +} + +type gitOpsFixture struct { + resolver *GitOpsResolver + verdict brain.Verdict + bundle GitOpsProvenanceBundle +} + +func newGitOpsFixture(t testing.TB, rule brain.RuleID) gitOpsFixture { + t.Helper() + planner := newGitHubPlanner(t) + contract, err := HandlerContractFor(planner) + if err != nil { + t.Fatalf("HandlerContractFor() error = %v", err) + } + bundle, err := NewGitOpsProvenanceBundle(validGitOpsInput(contract)) + if err != nil { + t.Fatalf("NewGitOpsProvenanceBundle() error = %v", err) + } + resolver, err := NewGitOpsResolver(planner, func() time.Time { return testNow }) + if err != nil { + t.Fatalf("NewGitOpsResolver() error = %v", err) + } + return gitOpsFixture{resolver: resolver, verdict: validGitOpsVerdict(rule), bundle: bundle} +} + +func newGitHubPlanner(t testing.TB) *githubconnector.OpenPRPlanner { + t.Helper() + planner, err := githubconnector.NewOpenPRPlanner(githubconnector.OpenPRPlannerConfig{ + Host: "github.com", Owner: "ArdurAI", Repository: "sith", BaseRef: "dev", + }) + if err != nil { + t.Fatalf("NewOpenPRPlanner() error = %v", err) + } + return planner +} + +func validGitOpsInput(contract HandlerContract) GitOpsProvenanceInput { + subject := testSubjectRef() + return GitOpsProvenanceInput{ + Workspace: testWorkspace, + Subject: subject, + Sources: []SourceIdentity{{ + Kind: gitHubSourceKind, AdapterVersion: GitOpsSourceAdapterVersion, NativeID: "github.com/ArdurAI/sith", + }}, + ObservedAt: testNow.Add(-time.Minute), ValidUntil: testNow.Add(time.Minute), Handler: contract, + Repository: RepositoryIdentity{Host: "github.com", Owner: "ArdurAI", Repository: "sith"}, + BaseRef: "dev", BaseCommit: testBaseSHA, FilePath: "deploy/payments.yaml", ObservedBlobSHA: testBlobSHA, + DesiredContent: "replicas: 4\n", Title: "Reconcile payments resources", Body: "Source-owned drift remediation\n\nEvidence is attached.", + CommitMessage: "Reconcile payments resources", + EvidenceRefs: []fleet.ResourceRef{ + subject, + {SourceKind: gitHubSourceKind, Scope: "github.com", Kind: "Blob", Namespace: "ArdurAI/sith", Name: testBlobSHA}, + }, + } +} + +func validGitOpsVerdict(rule brain.RuleID) brain.Verdict { + ref := testSubjectRef() + lens, predicate, observed := fleet.LensDesired, "desired.drift", "OutOfSync" + if rule == brain.RuleOOMKilled { + lens, predicate, observed = fleet.LensLive, "pod.reason", "OOMKilled" + } + return brain.Verdict{ + Rule: rule, Status: brain.StatusConfirmed, Ref: ref, + Citations: []brain.Citation{{ + Ref: ref, Lens: lens, Predicate: predicate, Observed: observed, + Weight: 60, ObservedAt: testNow.Add(-2 * time.Minute), Source: "fixture", + }}, + RemediationCandidate: &brain.RemediationCandidate{ + Verb: intent.VerbGitOpsOpenPR, + RequiredProvenance: []brain.ProvenanceRequirement{ + brain.ProvenanceGitRepository, brain.ProvenanceGitBaseRef, brain.ProvenanceGitBaseCommit, + brain.ProvenanceGitFilePath, brain.ProvenanceGitObservedBlob, brain.ProvenanceGitDesiredContent, + }, + }, + } +} + +func testSubjectRef() fleet.ResourceRef { + return fleet.ResourceRef{SourceKind: "kubeconfig", Scope: "alpha", Kind: "Deployment", Namespace: "prod", Name: "payments"} +} + +func assertReadyResolution(t testing.TB, got Resolution) { + t.Helper() + if got.Status != ResolutionReady || len(got.Reasons) != 0 || got.ArgumentsDigest == "" || len(got.EvidenceRefs) != 2 { + t.Fatalf("resolution = %#v, want ready", got) + } + wantTarget := fleet.ResourceRef{SourceKind: gitHubSourceKind, Scope: "github.com", Kind: openPRTargetKind, Namespace: "ArdurAI", Name: "sith"} + if !sameResourceRef(got.Target, wantTarget) { + t.Fatalf("target = %#v, want %#v", got.Target, wantTarget) + } + var args openPRArguments + if err := json.Unmarshal(got.Arguments, &args); err != nil { + t.Fatalf("decode arguments: %v", err) + } + if args.BaseRef != "dev" || args.ExpectedBaseSHA != testBaseSHA || len(args.Changes) != 1 || + args.Changes[0].Path != "deploy/payments.yaml" || args.Changes[0].Content == nil || + *args.Changes[0].Content != "replicas: 4\n" || args.Changes[0].ExpectedBlobSHA == nil || + *args.Changes[0].ExpectedBlobSHA != testBlobSHA { + t.Fatalf("arguments = %#v, want exact source-owned values", args) + } + digest := sha256.Sum256(got.Arguments) + wantDigest := "sha256:" + hex.EncodeToString(digest[:]) + if got.ArgumentsDigest != wantDigest { + t.Fatalf("digest = %q, want %q", got.ArgumentsDigest, wantDigest) + } + if got.EvidenceRefs[0].String() > got.EvidenceRefs[1].String() { + t.Fatalf("evidence refs are not canonical: %#v", got.EvidenceRefs) + } +} + +func assertAbstention(t testing.TB, got Resolution, want AbstentionReason) { + t.Helper() + if got.Status != ResolutionAbstained || !slices.Equal(got.Reasons, []AbstentionReason{want}) || + !zeroResourceRef(got.Target) || len(got.Arguments) != 0 || got.ArgumentsDigest != "" || len(got.EvidenceRefs) != 0 { + t.Fatalf("resolution = %#v, want closed abstention %q", got, want) + } +} + +func zeroResourceRef(ref fleet.ResourceRef) bool { + return ref.SourceKind == "" && ref.Scope == "" && ref.Kind == "" && ref.Namespace == "" && ref.Name == "" && len(ref.Attributes) == 0 +} + +func cloneVerdict(verdict brain.Verdict) brain.Verdict { + cloned := verdict + cloned.Ref = cloneResourceRef(verdict.Ref) + cloned.Citations = slices.Clone(verdict.Citations) + for index := range cloned.Citations { + cloned.Citations[index].Ref = cloneResourceRef(cloned.Citations[index].Ref) + } + if verdict.RemediationCandidate != nil { + candidate := *verdict.RemediationCandidate + candidate.RequiredProvenance = slices.Clone(verdict.RemediationCandidate.RequiredProvenance) + cloned.RemediationCandidate = &candidate + } + return cloned +} + +func cloneBundle(bundle GitOpsProvenanceBundle) GitOpsProvenanceBundle { + cloned := bundle + cloned.subject = cloneResourceRef(bundle.subject) + cloned.evidenceRefs = cloneResourceRefs(bundle.evidenceRefs) + return cloned +} + +type handlerProxy struct { + base GitOpsHandler + descriptorCalls int + mutateDescriptor func(connector.Descriptor, int) connector.Descriptor + canonicalize func(fleet.ResourceRef, json.RawMessage) (fleet.ResourceRef, json.RawMessage) +} + +func (proxy *handlerProxy) Descriptor() connector.Descriptor { + proxy.descriptorCalls++ + descriptor := cloneDescriptor(proxy.base.Descriptor()) + if proxy.mutateDescriptor != nil { + return proxy.mutateDescriptor(descriptor, proxy.descriptorCalls) + } + return descriptor +} + +func (proxy *handlerProxy) CanonicalizeOpenPRArgs(arguments json.RawMessage) (fleet.ResourceRef, json.RawMessage, error) { + target, document, err := proxy.base.CanonicalizeOpenPRArgs(arguments) + if err != nil || proxy.canonicalize == nil { + return target, document, err + } + target, document = proxy.canonicalize(target, append(json.RawMessage(nil), document...)) + return target, document, nil +} + +func cloneDescriptor(descriptor connector.Descriptor) connector.Descriptor { + cloned := descriptor + cloned.WireVersions = slices.Clone(descriptor.WireVersions) + cloned.Capabilities = slices.Clone(descriptor.Capabilities) + cloned.Verbs = slices.Clone(descriptor.Verbs) + cloned.ArgSchemas = make(map[intent.Verb]json.RawMessage, len(descriptor.ArgSchemas)) + for verb, schema := range descriptor.ArgSchemas { + cloned.ArgSchemas[verb] = append(json.RawMessage(nil), schema...) + } + return cloned +} diff --git a/internal/tenancy/model.go b/internal/tenancy/model.go index 9d9e226..2d6e0f3 100644 --- a/internal/tenancy/model.go +++ b/internal/tenancy/model.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "unicode" + "unicode/utf8" ) const maxIdentityLength = 256 @@ -35,6 +36,7 @@ type Action string // Supported authorization classes. const ( ActionRead Action = "read" + ActionExportAudit Action = "export-audit" ActionProposeIntent Action = "propose-intent" ActionApproveIntent Action = "approve-intent" ActionManageWorkspace Action = "manage-workspace" @@ -102,14 +104,14 @@ func (role Role) Allows(action Action) bool { case RoleApprover: return action == ActionRead || action == ActionApproveIntent case RoleAdmin: - return action == ActionRead || action == ActionManageWorkspace + return action == ActionRead || action == ActionExportAudit || action == ActionManageWorkspace default: return false } } func validateIdentity(name, value string) error { - if value == "" || strings.TrimSpace(value) != value { + if value == "" || !utf8.ValidString(value) || strings.TrimSpace(value) != value { return fmt.Errorf("%s must be a non-empty, trimmed value", name) } if len(value) > maxIdentityLength { @@ -124,7 +126,7 @@ func validateIdentity(name, value string) error { } func validateDisplayName(name, value string) error { - if value == "" || strings.TrimSpace(value) != value { + if value == "" || !utf8.ValidString(value) || strings.TrimSpace(value) != value { return fmt.Errorf("%s must be a non-empty, trimmed value", name) } if len(value) > maxIdentityLength { diff --git a/internal/tenancy/model_test.go b/internal/tenancy/model_test.go index 8d64a2e..19c0847 100644 --- a/internal/tenancy/model_test.go +++ b/internal/tenancy/model_test.go @@ -7,7 +7,7 @@ import "testing" func TestRoleAllowsExactActionClasses(t *testing.T) { t.Parallel() - actions := []Action{ActionRead, ActionProposeIntent, ActionApproveIntent, ActionManageWorkspace, "unknown"} + actions := []Action{ActionRead, ActionExportAudit, ActionProposeIntent, ActionApproveIntent, ActionManageWorkspace, "unknown"} tests := []struct { role Role allowed map[Action]bool @@ -15,7 +15,7 @@ func TestRoleAllowsExactActionClasses(t *testing.T) { {role: RoleReader, allowed: map[Action]bool{ActionRead: true}}, {role: RoleOperator, allowed: map[Action]bool{ActionRead: true, ActionProposeIntent: true}}, {role: RoleApprover, allowed: map[Action]bool{ActionRead: true, ActionApproveIntent: true}}, - {role: RoleAdmin, allowed: map[Action]bool{ActionRead: true, ActionManageWorkspace: true}}, + {role: RoleAdmin, allowed: map[Action]bool{ActionRead: true, ActionExportAudit: true, ActionManageWorkspace: true}}, {role: "owner", allowed: map[Action]bool{}}, } for _, test := range tests { @@ -86,6 +86,15 @@ func TestTenancyModelsRejectAmbiguousIdentity(t *testing.T) { {name: "padded tenant key", run: func() error { return (Workspace{ID: "a", Name: "A", TenantKey: " tenant-a"}).Validate() }}, {name: "unknown role", run: func() error { return (Membership{WorkspaceID: "a", Subject: "alice", Role: "owner"}).Validate() }}, {name: "control character", run: func() error { return (Membership{WorkspaceID: "a\n", Subject: "alice", Role: RoleReader}).Validate() }}, + {name: "malformed UTF-8 workspace ID", run: func() error { + return ValidateWorkspaceID(WorkspaceID(string([]byte{'a', 0x80}))) + }}, + {name: "malformed UTF-8 display name", run: func() error { + return (Workspace{ID: "a", Name: string([]byte{'A', 0x80}), TenantKey: "tenant-a"}).Validate() + }}, + {name: "malformed UTF-8 subject", run: func() error { + return (Membership{WorkspaceID: "a", Subject: string([]byte{'a', 0x80}), Role: RoleReader}).Validate() + }}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/internal/tui/model.go b/internal/tui/model.go index 2aec24f..b9d716a 100644 --- a/internal/tui/model.go +++ b/internal/tui/model.go @@ -418,9 +418,14 @@ func (model *Model) applyCommand() (tea.Model, tea.Cmd) { model.closeLocalSessions() return model, tea.Quit case "pause": - model.store.SetPaused(true) + if err := model.store.SetPaused(fleet.LocalWorkspace, true); err != nil { + model.lastError = err.Error() + } case "resume": - model.store.SetPaused(false) + if err := model.store.SetPaused(fleet.LocalWorkspace, false); err != nil { + model.lastError = err.Error() + return model, nil + } return model, model.syncCommand() case "refresh": return model, model.syncCommand() @@ -510,7 +515,7 @@ func (model *Model) syncLensCommand() tea.Cmd { func (model *Model) waitCommand(after uint64) tea.Cmd { return func() tea.Msg { - version, err := model.store.WaitForChange(model.ctx, after) + version, err := model.store.WaitForChange(model.ctx, fleet.LocalWorkspace, after) return cacheChangedMsg{version: version, err: err} } } diff --git a/internal/tui/model_test.go b/internal/tui/model_test.go index f08ba3b..031fdec 100644 --- a/internal/tui/model_test.go +++ b/internal/tui/model_test.go @@ -91,8 +91,8 @@ func TestModelNavigationFilterPauseAndCoverage(t *testing.T) { _, _ = model.Update(keyMessage(":")) _, _ = model.Update(keyMessage("pause")) _, _ = model.Update(specialKey(tea.KeyEnter)) - if !store.Paused() || !strings.Contains(strings.ToLower(model.View().Content), "paused") { - t.Fatalf("paused state/view = %v/%q", store.Paused(), model.View().Content) + if !store.Paused(fleet.LocalWorkspace) || !strings.Contains(strings.ToLower(model.View().Content), "paused") { + t.Fatalf("paused state/view = %v/%q", store.Paused(fleet.LocalWorkspace), model.View().Content) } _, command := model.Update(keyMessage("ctrl+r")) if command == nil { @@ -314,7 +314,7 @@ func populatedStore(t *testing.T, pods int) *fleetcache.Store { }, }, now)) } - if err := store.Replace("Pod", fleet.QueryResult{Facts: podFacts, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}}); err != nil { + if err := store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{Facts: podFacts, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}}); err != nil { t.Fatalf("Replace(Pod) error = %v", err) } deployFacts := []fleet.Fact{ @@ -325,7 +325,7 @@ func populatedStore(t *testing.T, pods int) *fleetcache.Store { "spec": map[string]any{"replicas": 3}, "status": map[string]any{"availableReplicas": 0, "updatedReplicas": 0}, }, now), } - if err := store.Replace("Deployment", fleet.QueryResult{Facts: deployFacts, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}}); err != nil { + if err := store.Replace(fleet.LocalWorkspace, "Deployment", fleet.QueryResult{Facts: deployFacts, Coverage: fleet.Coverage{Requested: 2, Reachable: 2}}); err != nil { t.Fatalf("Replace(Deployment) error = %v", err) } return store diff --git a/internal/webui/server_test.go b/internal/webui/server_test.go index 77517ce..e5d6dac 100644 --- a/internal/webui/server_test.go +++ b/internal/webui/server_test.go @@ -394,7 +394,7 @@ func populatedWebStore(t *testing.T) *fleetcache.Store { store := fleetcache.New() store.SetDiscovery(fleet.LocalWorkspace, connector.Discovery{Scopes: []connector.Scope{{Name: "alpha", Reachable: true, ObservedAt: now}}}) observed := json.RawMessage(`{"apiVersion":"v1","kind":"Pod","metadata":{"name":"api","namespace":"apps"},"status":{"phase":"Running"}}`) - err := store.Replace("Pod", fleet.QueryResult{ + err := store.Replace(fleet.LocalWorkspace, "Pod", fleet.QueryResult{ Facts: []fleet.Fact{{Evidence: fleet.Evidence{ Ref: fleet.ResourceRef{SourceKind: "test", Scope: "alpha", Kind: "Pod", Namespace: "apps", Name: "api"}, Kind: fleet.FactInventory, Observed: observed, ObservedAt: now, diff --git a/monitoring/sith-hub.rules.test.yml b/monitoring/sith-hub.rules.test.yml new file mode 100644 index 0000000..e5aa356 --- /dev/null +++ b/monitoring/sith-hub.rules.test.yml @@ -0,0 +1,884 @@ +# SPDX-License-Identifier: Apache-2.0 + +rule_files: + - sith-hub.rules.yml + +evaluation_interval: 1m + +tests: + - name: policy audit failure aggregates hostile source labels and honors the hold window + interval: 1m + input_series: + - series: 'sith_policy_audit_attempts_total{sink="durable",outcome="error",job="hub-a",instance="10.0.0.1:9090",workspace="must-not-propagate"}' + values: '0 1+0x12' + - series: 'sith_policy_audit_attempts_total{sink="process",outcome="error",job="hub-b",instance="10.0.0.2:9090",trace="must-not-propagate"}' + values: '0+0x13' + alert_rule_test: + - eval_time: 2m + alertname: SithHubPolicyAuditFailure + exp_alerts: [] + - eval_time: 3m + alertname: SithHubPolicyAuditFailure + exp_alerts: + - exp_labels: + component: sith-hub + severity: critical + exp_annotations: + summary: Sith hub is refusing governed reads because policy-audit delivery failed + description: At least one durable or process policy-audit sink failed during the last five minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubpolicyauditfailure + + - name: policy audit counter reset does not fabricate a failure + interval: 1m + input_series: + - series: 'sith_policy_audit_attempts_total{sink="durable",outcome="error"}' + values: '100+0x5 0+0x10' + alert_rule_test: + - eval_time: 9m + alertname: SithHubPolicyAuditFailure + exp_alerts: [] + + - name: missing policy audit series stays quiet + alert_rule_test: + - eval_time: 10m + alertname: SithHubPolicyAuditFailure + exp_alerts: [] + + - name: sustained policy decision errors aggregate hostile labels then resolve + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow",job="hub-a",instance="10.0.0.1:9090"}' + values: '0+2x45' + - series: 'sith_policy_decisions_total{verb="deployment.restart",outcome="error",workspace="must-not-propagate",trace="must-not-propagate"}' + values: '0+0.2x20 4+0x25' + alert_rule_test: + - eval_time: 19m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + - eval_time: 20m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub policy decisions are returning sustained errors + description: More than five percent of at least twenty aggregate eligible policy decisions ended in error over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubpolicydecisionerrorratiohigh + - eval_time: 36m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: missing policy decision series stays quiet + alert_rule_test: + - eval_time: 30m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: policy decision errors stay quiet below minimum volume + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow"}' + values: '0+0.4x35' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="error"}' + values: '0+0.1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: policy decision error ratio exactly at five percent stays quiet + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow"}' + values: '0+19x35' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="error"}' + values: '0+1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: policy decision volume exactly at twenty satisfies the inclusive guard + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow"}' + values: '0+0x14 18+0x20' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="error"}' + values: '0+0x14 2+0x20' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="not-counted"}' + values: '0+0x14 1000+0x20' + promql_expr_test: + - expr: sum(increase(sith_policy_decisions_total{outcome=~"allow|deny|require-approval|error"}[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + alert_rule_test: + - eval_time: 24m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + - eval_time: 25m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub policy decisions are returning sustained errors + description: More than five percent of at least twenty aggregate eligible policy decisions ended in error over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubpolicydecisionerrorratiohigh + + - name: ordinary policy denies and approval requirements stay quiet + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="deny"}' + values: '0+20x35' + - series: 'sith_policy_decisions_total{verb="deployment.restart",outcome="require-approval"}' + values: '0+20x35' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow"}' + values: '0+9x35' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="error"}' + values: '0+1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: transient policy decision errors recover before the hold window + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow"}' + values: '0+2x35' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="error"}' + values: '0+0.4x4 1.6+0x31' + alert_rule_test: + - eval_time: 14m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + - eval_time: 30m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: policy decision error counter reset does not fabricate errors + interval: 1m + input_series: + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="allow"}' + values: '0+2x30' + - series: 'sith_policy_decisions_total{verb="fleet.read",outcome="error"}' + values: '100+0x5 0+0x25' + alert_rule_test: + - eval_time: 20m + alertname: SithHubPolicyDecisionErrorRatioHigh + exp_alerts: [] + + - name: authentication refusal delivery drop aggregates source labels and honors the hold window + interval: 1m + input_series: + - series: 'sith_auth_refusal_delivery_drops_total{job="hub-a",instance="10.0.0.1:9090",actor="must-not-propagate"}' + values: '0 1+0x20' + alert_rule_test: + - eval_time: 5m + alertname: SithHubAuthRefusalDeliveryDrop + exp_alerts: [] + - eval_time: 6m + alertname: SithHubAuthRefusalDeliveryDrop + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub dropped structured authentication-refusal evidence + description: At least one bounded authentication-refusal record could not be delivered during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubauthrefusaldeliverydrop + + - name: refusal-only authentication aggregates hostile labels and honors exact volume and hold guards + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused",job="hub-a",instance="10.0.0.1:9090",actor="must-not-propagate"}' + values: '0+0x14 12+0x20' + - series: 'sith_auth_attempts_total{outcome="refused",job="hub-b",instance="10.0.0.2:9090",workspace="must-not-propagate"}' + values: '0+0x14 8+0x20' + - series: 'sith_auth_attempts_total{outcome="accepted",job="hub-a",instance="10.0.0.1:9090",principal="must-not-propagate"}' + values: '0+0x25 1+0x10' + - series: 'sith_auth_attempts_total{outcome="accepted",job="hub-b",instance="10.0.0.2:9090"}' + values: '0+0x35' + promql_expr_test: + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 and sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0 + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + alert_rule_test: + - eval_time: 24m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + - eval_time: 25m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub authentication traffic is persistently refusal-only + description: At least twenty aggregate authentication attempts were refused and none were accepted over fifteen minutes; a recent scraped sample from the preinitialized accepted-outcome series was present during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubauthenticationrefusalonly + - eval_time: 26m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: missing authentication attempt series stay quiet + alert_rule_test: + - eval_time: 30m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: missing accepted authentication series stays quiet even with refusals + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '0+2x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: stale accepted authentication samples cannot satisfy the recent telemetry guard + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '0+0x1 20+0x20' + - series: 'sith_auth_attempts_total{outcome="accepted"}' + values: '0+0x1 _x20' + promql_expr_test: + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 + eval_time: 12m + exp_samples: + - labels: '{}' + value: 20 + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 and sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0 + eval_time: 12m + exp_samples: [] + alert_rule_test: + - eval_time: 12m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: refusal-only authentication stays quiet below minimum volume + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '0+0x14 19+0x20' + - series: 'sith_auth_attempts_total{outcome="accepted"}' + values: '0+0x35' + promql_expr_test: + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 19 + alert_rule_test: + - eval_time: 25m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: accepted-only authentication traffic stays quiet + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '0+0x35' + - series: 'sith_auth_attempts_total{outcome="accepted"}' + values: '0+2x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: one accepted authentication suppresses high refusal traffic + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '0+0x14 20+0x20' + - series: 'sith_auth_attempts_total{outcome="accepted"}' + values: '0+0x14 1+0x20' + promql_expr_test: + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 and sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0 + eval_time: 25m + exp_samples: [] + alert_rule_test: + - eval_time: 25m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: transient refusal-only authentication clears before the hold window + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '0+0x14 20+0x20' + - series: 'sith_auth_attempts_total{outcome="accepted"}' + values: '0+0x19 1+0x15' + promql_expr_test: + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 and sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0 + eval_time: 19m + exp_samples: + - labels: '{}' + value: 20 + - expr: sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 and sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 and sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0 + eval_time: 25m + exp_samples: [] + alert_rule_test: + - eval_time: 19m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + - eval_time: 25m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: authentication refusal counter reset does not fabricate traffic + interval: 1m + input_series: + - series: 'sith_auth_attempts_total{outcome="refused"}' + values: '100+0x5 0+0x30' + - series: 'sith_auth_attempts_total{outcome="accepted"}' + values: '0+0x35' + alert_rule_test: + - eval_time: 20m + alertname: SithHubAuthenticationRefusalOnly + exp_alerts: [] + + - name: sustained snapshot failures fire only after ratio volume and hold guards + interval: 1m + input_series: + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="success",job="hub-a",instance="10.0.0.1:9090",spoke="must-not-propagate"}' + values: '0+2x35' + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="transport",job="hub-b",instance="10.0.0.2:9090",endpoint="must-not-propagate"}' + values: '0+0.2x35' + alert_rule_test: + - eval_time: 19m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: [] + - eval_time: 20m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub federation snapshot failures are sustained + description: More than five percent of at least twenty aggregate spoke snapshot attempts failed over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfederationsnapshotfailureratiohigh + + - name: snapshot ratio above threshold stays quiet below minimum volume + interval: 1m + input_series: + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="success"}' + values: '0+0.4x35' + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="deadline"}' + values: '0+0.1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: [] + + - name: snapshot volume above minimum stays quiet below failure threshold + interval: 1m + input_series: + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="success"}' + values: '0+2x35' + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="store-error"}' + values: '0+0.05x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: [] + + - name: snapshot ratio exactly at five percent stays quiet + interval: 1m + input_series: + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="success"}' + values: '0+19x35' + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="invalid-snapshot"}' + values: '0+1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: [] + + - name: snapshot volume exactly at twenty satisfies the inclusive guard + interval: 1m + input_series: + - series: 'sith_federation_spoke_snapshot_attempts_total{outcome="canceled"}' + values: '0+0x14 20+0x20' + promql_expr_test: + - expr: sum(increase(sith_federation_spoke_snapshot_attempts_total[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + alert_rule_test: + - eval_time: 24m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: [] + - eval_time: 25m + alertname: SithHubFederationSnapshotFailureRatioHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub federation snapshot failures are sustained + description: More than five percent of at least twenty aggregate spoke snapshot attempts failed over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfederationsnapshotfailureratiohigh + + - name: sustained fleet read degradation aggregates hostile labels then resolves + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="complete",job="hub-a",instance="10.0.0.1:9090"}' + values: '0+2x45' + - series: 'sith_federation_fleet_read_results_total{outcome="degraded",workspace="must-not-propagate",spoke="must-not-propagate"}' + values: '0+0.1x20 2+0x25' + - series: 'sith_federation_fleet_read_results_total{outcome="error",trace="must-not-propagate",endpoint="must-not-propagate"}' + values: '0+0.1x20 2+0x25' + - series: 'sith_federation_fleet_read_results_total{outcome="empty",actor="must-not-propagate"}' + values: '0+100x45' + alert_rule_test: + - eval_time: 19m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + - eval_time: 20m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub fleet reads are returning sustained coverage degradation + description: More than five percent of at least twenty aggregate eligible fleet reads were degraded or failed over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfleetreadcoveragedegradationhigh + - eval_time: 36m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: legitimate empty fleet reads stay quiet and cannot satisfy volume + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="empty"}' + values: '0+100x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: missing fleet read series stays quiet + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: adverse fleet read ratio stays quiet below eligible volume despite empty traffic + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="complete"}' + values: '0+0.4x35' + - series: 'sith_federation_fleet_read_results_total{outcome="error"}' + values: '0+0.1x35' + - series: 'sith_federation_fleet_read_results_total{outcome="empty"}' + values: '0+100x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: fleet read adverse ratio exactly at five percent stays quiet + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="complete"}' + values: '0+19x35' + - series: 'sith_federation_fleet_read_results_total{outcome="degraded"}' + values: '0+1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: fleet read eligible volume exactly at twenty satisfies the inclusive guard + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="complete"}' + values: '0+0x14 18+0x20' + - series: 'sith_federation_fleet_read_results_total{outcome="error"}' + values: '0+0x14 2+0x20' + - series: 'sith_federation_fleet_read_results_total{outcome="empty"}' + values: '0+100x35' + promql_expr_test: + - expr: sum(increase(sith_federation_fleet_read_results_total{outcome=~"complete|degraded|error"}[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + alert_rule_test: + - eval_time: 24m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + - eval_time: 25m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub fleet reads are returning sustained coverage degradation + description: More than five percent of at least twenty aggregate eligible fleet reads were degraded or failed over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfleetreadcoveragedegradationhigh + + - name: transient fleet read degradation recovers before the hold window + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="complete"}' + values: '0+2x35' + - series: 'sith_federation_fleet_read_results_total{outcome="degraded"}' + values: '0+0.4x3 1.2+0x32' + alert_rule_test: + - eval_time: 14m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + - eval_time: 30m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: adverse fleet read counter reset does not fabricate degradation + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_results_total{outcome="complete"}' + values: '0+2x30' + - series: 'sith_federation_fleet_read_results_total{outcome="error"}' + values: '100+0x5 0+0x25' + alert_rule_test: + - eval_time: 20m + alertname: SithHubFleetReadCoverageDegradationHigh + exp_alerts: [] + + - name: sustained proven-stale fleet reads aggregate hostile labels then resolve + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="fresh",job="hub-a",instance="10.0.0.1:9090"}' + values: '0+2x45' + - series: 'sith_federation_fleet_read_freshness_total{outcome="stale",workspace="must-not-propagate",spoke="must-not-propagate"}' + values: '0+0.2x20 4+0x25' + - series: 'sith_federation_fleet_read_freshness_total{outcome="unknown",trace="must-not-propagate"}' + values: '0+100x45' + - series: 'sith_federation_fleet_read_freshness_total{outcome="error",endpoint="must-not-propagate"}' + values: '0+100x45' + - series: 'sith_federation_fleet_read_freshness_total{outcome="empty",actor="must-not-propagate"}' + values: '0+100x45' + alert_rule_test: + - eval_time: 19m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + - eval_time: 20m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub fleet reads are returning sustained proven-stale results + description: More than five percent of at least twenty aggregate freshness-eligible fleet reads were proven stale over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfleetreadstalenesshigh + - eval_time: 36m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: unknown error and empty freshness outcomes stay quiet and cannot satisfy volume + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="unknown"}' + values: '0+100x35' + - series: 'sith_federation_fleet_read_freshness_total{outcome="error"}' + values: '0+100x35' + - series: 'sith_federation_fleet_read_freshness_total{outcome="empty"}' + values: '0+100x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: missing fleet read freshness series stays quiet + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: stale fleet read ratio stays quiet below eligible volume despite excluded traffic + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="fresh"}' + values: '0+0.4x35' + - series: 'sith_federation_fleet_read_freshness_total{outcome="stale"}' + values: '0+0.1x35' + - series: 'sith_federation_fleet_read_freshness_total{outcome="unknown"}' + values: '0+100x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: stale fleet read ratio exactly at five percent stays quiet + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="fresh"}' + values: '0+19x35' + - series: 'sith_federation_fleet_read_freshness_total{outcome="stale"}' + values: '0+1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: fleet read freshness volume exactly at twenty satisfies the inclusive guard + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="fresh"}' + values: '0+0x14 18+0x20' + - series: 'sith_federation_fleet_read_freshness_total{outcome="stale"}' + values: '0+0x14 2+0x20' + - series: 'sith_federation_fleet_read_freshness_total{outcome="unknown"}' + values: '0+100x35' + promql_expr_test: + - expr: sum(increase(sith_federation_fleet_read_freshness_total{outcome=~"fresh|stale"}[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + alert_rule_test: + - eval_time: 24m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + - eval_time: 25m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub fleet reads are returning sustained proven-stale results + description: More than five percent of at least twenty aggregate freshness-eligible fleet reads were proven stale over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfleetreadstalenesshigh + + - name: transient proven-stale fleet reads recover before the hold window + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="fresh"}' + values: '0+2x35' + - series: 'sith_federation_fleet_read_freshness_total{outcome="stale"}' + values: '0+0.4x3 1.2+0x32' + alert_rule_test: + - eval_time: 14m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + - eval_time: 30m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: stale fleet read counter reset does not fabricate staleness + interval: 1m + input_series: + - series: 'sith_federation_fleet_read_freshness_total{outcome="fresh"}' + values: '0+2x30' + - series: 'sith_federation_fleet_read_freshness_total{outcome="stale"}' + values: '100+0x5 0+0x25' + alert_rule_test: + - eval_time: 20m + alertname: SithHubFleetReadStalenessHigh + exp_alerts: [] + + - name: sustained database readiness degradation aggregates hostile labels then resolves + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready",job="hub-a",instance="10.0.0.1:9090"}' + values: '0+2x45' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable",workspace="must-not-propagate",endpoint="must-not-propagate"}' + values: '0+0.2x20 4+0x25' + alert_rule_test: + - eval_time: 19m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + - eval_time: 20m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub database-backed readiness is persistently degraded + description: More than five percent of at least twenty aggregate database readiness checks were unavailable over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubdatabasereadinessdegradationhigh + - eval_time: 36m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: missing database readiness series stays quiet + alert_rule_test: + - eval_time: 30m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: all-ready database checks stay quiet + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready"}' + values: '0+2x35' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable"}' + values: '0+0x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: database readiness degradation stays quiet below minimum volume + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready"}' + values: '0+0.4x35' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable"}' + values: '0+0.1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: database readiness unavailable ratio exactly at five percent stays quiet + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready"}' + values: '0+19x35' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable"}' + values: '0+1x35' + alert_rule_test: + - eval_time: 30m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: database readiness volume exactly at twenty satisfies the inclusive guard + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready"}' + values: '0+0x14 18+0x20' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable"}' + values: '0+0x14 2+0x20' + promql_expr_test: + - expr: sum(increase(sith_hub_readiness_checks_total{outcome=~"ready|unavailable"}[15m])) + eval_time: 25m + exp_samples: + - labels: '{}' + value: 20 + alert_rule_test: + - eval_time: 24m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + - eval_time: 25m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub database-backed readiness is persistently degraded + description: More than five percent of at least twenty aggregate database readiness checks were unavailable over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubdatabasereadinessdegradationhigh + + - name: transient database readiness degradation recovers before the hold window + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready"}' + values: '0+2x35' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable"}' + values: '0+0.4x3 1.2+0x32' + alert_rule_test: + - eval_time: 14m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + - eval_time: 30m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: unavailable database readiness counter reset does not fabricate degradation + interval: 1m + input_series: + - series: 'sith_hub_readiness_checks_total{outcome="ready"}' + values: '0+2x30' + - series: 'sith_hub_readiness_checks_total{outcome="unavailable"}' + values: '100+0x5 0+0x25' + alert_rule_test: + - eval_time: 20m + alertname: SithHubDatabaseReadinessDegradationHigh + exp_alerts: [] + + - name: initially missing Hub telemetry honors the hold then fires + alert_rule_test: + - eval_time: 4m + alertname: SithHubTelemetryMissing + exp_alerts: [] + - eval_time: 5m + alertname: SithHubTelemetryMissing + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub telemetry is absent from the rule evaluator + description: No Sith build-info sample reached the rule evaluator during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubtelemetrymissing + + - name: current Hub build info stays quiet and arbitrary source labels do not matter + interval: 1m + input_series: + - series: 'sith_build_info{version="v1.2.3",commit="abcdef0",job="hub-a",instance="10.0.0.1:9090",workspace="must-not-propagate"}' + values: '1+0x20' + - series: 'sith_build_info{version="v1.2.4",commit="abcdef1",job="hub-b",trace="must-not-propagate"}' + values: '1+0x20' + alert_rule_test: + - eval_time: 20m + alertname: SithHubTelemetryMissing + exp_alerts: [] + + - name: bounded Hub telemetry gap stays quiet + interval: 1m + input_series: + - series: 'sith_build_info{version="v1.2.3",commit="abcdef0"}' + values: '1+0x5 _x9 1+0x10' + alert_rule_test: + - eval_time: 14m + alertname: SithHubTelemetryMissing + exp_alerts: [] + - eval_time: 20m + alertname: SithHubTelemetryMissing + exp_alerts: [] + + - name: sustained Hub telemetry disappearance fires then recovery resolves + interval: 1m + input_series: + - series: 'sith_build_info{version="v1.2.3",commit="abcdef0",endpoint="must-not-propagate"}' + values: '1+0x5 _x15 1+0x10' + alert_rule_test: + - eval_time: 19m + alertname: SithHubTelemetryMissing + exp_alerts: [] + - eval_time: 20m + alertname: SithHubTelemetryMissing + exp_alerts: + - exp_labels: + component: sith-hub + severity: warning + exp_annotations: + summary: Sith hub telemetry is absent from the rule evaluator + description: No Sith build-info sample reached the rule evaluator during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubtelemetrymissing + - eval_time: 21m + alertname: SithHubTelemetryMissing + exp_alerts: [] diff --git a/monitoring/sith-hub.rules.yml b/monitoring/sith-hub.rules.yml new file mode 100644 index 0000000..4247f8b --- /dev/null +++ b/monitoring/sith-hub.rules.yml @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 + +groups: + - name: sith-hub.failure-signals + interval: 1m + limit: 9 + rules: + - alert: SithHubPolicyAuditFailure + expr: sum(increase(sith_policy_audit_attempts_total{outcome="error"}[5m])) > 0 + for: 2m + labels: + component: sith-hub + severity: critical + annotations: + summary: Sith hub is refusing governed reads because policy-audit delivery failed + description: At least one durable or process policy-audit sink failed during the last five minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubpolicyauditfailure + + - alert: SithHubPolicyDecisionErrorRatioHigh + expr: | + ( + sum(increase(sith_policy_decisions_total{outcome="error"}[15m])) + / + clamp_min(sum(increase(sith_policy_decisions_total{outcome=~"allow|deny|require-approval|error"}[15m])), 1) + ) > 0.05 + and + sum(increase(sith_policy_decisions_total{outcome=~"allow|deny|require-approval|error"}[15m])) >= 20 + for: 10m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub policy decisions are returning sustained errors + description: More than five percent of at least twenty aggregate eligible policy decisions ended in error over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubpolicydecisionerrorratiohigh + + - alert: SithHubAuthRefusalDeliveryDrop + expr: sum(increase(sith_auth_refusal_delivery_drops_total[10m])) > 0 + for: 5m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub dropped structured authentication-refusal evidence + description: At least one bounded authentication-refusal record could not be delivered during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubauthrefusaldeliverydrop + + - alert: SithHubAuthenticationRefusalOnly + expr: | + sum(increase(sith_auth_attempts_total{outcome="refused"}[15m])) >= 20 + and + sum(increase(sith_auth_attempts_total{outcome="accepted"}[15m])) == 0 + and + sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0 + for: 10m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub authentication traffic is persistently refusal-only + description: At least twenty aggregate authentication attempts were refused and none were accepted over fifteen minutes; a recent scraped sample from the preinitialized accepted-outcome series was present during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubauthenticationrefusalonly + + - alert: SithHubFederationSnapshotFailureRatioHigh + expr: | + ( + sum(increase(sith_federation_spoke_snapshot_attempts_total{outcome!="success"}[15m])) + / + clamp_min(sum(increase(sith_federation_spoke_snapshot_attempts_total[15m])), 1) + ) > 0.05 + and + sum(increase(sith_federation_spoke_snapshot_attempts_total[15m])) >= 20 + for: 10m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub federation snapshot failures are sustained + description: More than five percent of at least twenty aggregate spoke snapshot attempts failed over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfederationsnapshotfailureratiohigh + + - alert: SithHubFleetReadCoverageDegradationHigh + expr: | + ( + sum(increase(sith_federation_fleet_read_results_total{outcome=~"degraded|error"}[15m])) + / + clamp_min(sum(increase(sith_federation_fleet_read_results_total{outcome=~"complete|degraded|error"}[15m])), 1) + ) > 0.05 + and + sum(increase(sith_federation_fleet_read_results_total{outcome=~"complete|degraded|error"}[15m])) >= 20 + for: 10m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub fleet reads are returning sustained coverage degradation + description: More than five percent of at least twenty aggregate eligible fleet reads were degraded or failed over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfleetreadcoveragedegradationhigh + + - alert: SithHubFleetReadStalenessHigh + expr: | + ( + sum(increase(sith_federation_fleet_read_freshness_total{outcome="stale"}[15m])) + / + clamp_min(sum(increase(sith_federation_fleet_read_freshness_total{outcome=~"fresh|stale"}[15m])), 1) + ) > 0.05 + and + sum(increase(sith_federation_fleet_read_freshness_total{outcome=~"fresh|stale"}[15m])) >= 20 + for: 10m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub fleet reads are returning sustained proven-stale results + description: More than five percent of at least twenty aggregate freshness-eligible fleet reads were proven stale over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubfleetreadstalenesshigh + + - alert: SithHubDatabaseReadinessDegradationHigh + expr: | + ( + sum(increase(sith_hub_readiness_checks_total{outcome="unavailable"}[15m])) + / + clamp_min(sum(increase(sith_hub_readiness_checks_total{outcome=~"ready|unavailable"}[15m])), 1) + ) > 0.05 + and + sum(increase(sith_hub_readiness_checks_total{outcome=~"ready|unavailable"}[15m])) >= 20 + for: 10m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub database-backed readiness is persistently degraded + description: More than five percent of at least twenty aggregate database readiness checks were unavailable over fifteen minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubdatabasereadinessdegradationhigh + + - alert: SithHubTelemetryMissing + expr: absent_over_time(sith_build_info[10m]) + for: 5m + labels: + component: sith-hub + severity: warning + annotations: + summary: Sith hub telemetry is absent from the rule evaluator + description: No Sith build-info sample reached the rule evaluator during the last ten minutes. + runbook_url: https://github.com/ArdurAI/sith/blob/dev/docs/runbooks/hub-alerts.md#sithhubtelemetrymissing diff --git a/sessions/2026-07-15-e10-loopback-metrics.md b/sessions/2026-07-15-e10-loopback-metrics.md new file mode 100644 index 0000000..ff6c035 --- /dev/null +++ b/sessions/2026-07-15-e10-loopback-metrics.md @@ -0,0 +1,17 @@ +# Session — 2026-07-15 — e10-loopback-metrics + +**Builder:** Gnani Rahul Nutakki · **Model/effort:** autonomous · **Branch:** gnanirahulnutakki/feat/e10-loopback-metrics +**Slice(s):** E10 F10.1b / #177 · **Status:** in-progress + +--- + +[G] Goal: compose the existing isolated Hub self-observability registry into an opt-in, loopback-only operator endpoint that unblocks the bounded audit-log delivery drop counter in #140. +[S] Scope: strict `SITH_HUB_METRICS_LISTEN_ADDR` validation, runtime-owned `GET /metrics` listener, Helm opt-in metadata, docs, and tests. Out: tenant routes, Services, ingress, remote telemetry, persistence, automatic collectors, and Kubernetes/request/credential labels. +[A] Action: verified the Go 1.26 `http.Server` shutdown/close semantics and Kubernetes same-Pod localhost model; selected a separate local HTTP listener to prevent process-wide counters from leaking through tenant fleet APIs. +[T] Test: `go mod verify`; `govulncheck ./...`; targeted and full `go test -race -count=1 ./...`; `make ci`; pinned-Helm chart contract; `make e2e-isolation` (including the fixed 50,000x cross-workspace selector fuzz campaign); `make release-check`; and `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` all passed. The final two-cluster Kind fleet/OCI contract run passed in 162.641s. +[R] Review: manual privacy/red-team review confirmed exact-IP loopback validation, a fixed read-only route, no tenant handler reuse, and no Service or ingress exposure. CodeRabbit reviewed the final uncommitted diff with 0 findings. GitHub Dependabot, code-scanning, and secret-scanning queues were 0/0/0. +[C] Checkpoint #1: pending signed DCO/GSTACK commit — Docker cleanup reclaimed 1.365 GB after the final Kind run; `kind get clusters` reports none. + +--- + +**Session close:** validated and ready for signed DCO/GSTACK commit, PR, and exact post-merge `dev` CI verification. **Open questions touched:** none; disabled-by-default, exact-loopback binding is the safe default. diff --git a/sessions/2026-07-15-e10-process-audit-sink.md b/sessions/2026-07-15-e10-process-audit-sink.md new file mode 100644 index 0000000..fa4edc5 --- /dev/null +++ b/sessions/2026-07-15-e10-process-audit-sink.md @@ -0,0 +1,18 @@ +# Session — 2026-07-15 — e10-process-audit-sink + +**Builder:** Gnani Rahul Nutakki · **Model/effort:** autonomous · **Branch:** gnanirahulnutakki/feat/e10-process-audit-sink +**Slice(s):** E10 F10.3b / #140 · **Status:** in-progress + +--- + +[G] Goal: ship bounded, nonblocking local delivery for the closed Hub authentication-refusal event now that #177 supplies the opt-in loopback-only self-observation seam. +[S] Scope: process-supervised Unix datagram delivery, fixed child record validation, one unlabeled drop counter, runtime lifecycle ownership, docs, and adversarial tests. Out: listeners, socket pathnames, Services, ingress, exporters, queues, persistence, remote telemetry, request metadata, credentials, raw payload retention, and generic event delivery. +[A] Action: reviewed the current synchronous `slog` limitation and the Go process/FD contracts; selected a same-binary child with only inherited FD 3 and stderr. The parent uses nonblocking datagram send and explicitly kills/reaps the child on Hub shutdown. +[T] Test: `go mod tidy`; `go mod verify`; full `go test -race -count=1 ./...`; `make ci`; `make e2e-isolation`; `make e2e-helm HELM=/Volumes/EXTENDED/MacData/tools/bin/helm`; `make release-check`; and `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` all pass after the final runtime correction. The final real two-cluster Kind gate completed in 164.910s and left no Kind clusters; Docker cleanup reclaimed 1.365GB. PR CI initially exposed a Linux-only test flaw: closing a Unix datagram descriptor does not portably wake a concurrent receive. The test now validates the closed wire shape directly, while the subprocess tests cover the production force-kill/reap lifecycle; it passes `go test -race -count=100 ./internal/auditdelivery` and a fresh full local `make ci`. +[R] Review: manual red-team pass verified the fixed two-byte parent record, nonblocking send/drop accounting, child-only stderr blocking, forced reap, and no new listener/persistence/remote path. CodeRabbit corrected two documentation claims: the child warning is not audit evidence, and a same-container child is not a filesystem/network sandbox. It then found a real opt-in regression: unconditional registry construction made `metrics != nil` insufficient to guard listener startup. `newOptionalLoopbackMetricsServer` now guards the exact configured address, and its injected-factory regression test proves the disabled path performs no bind. The pre-intent/E6 documentation boundary was clarified. Final CodeRabbit pass: 0 findings. +[S] Security: live GitHub Dependabot, code-scanning, and secret-scanning queues: 0 / 0 / 0. `origin/dev` verified at `f5a2c00` before commit. +[C] Checkpoint #3: amend the signed DCO/GSTACK commit with the cross-platform test correction, force-update the feature branch, and require fresh PR CI. + +--- + +**Session close:** awaiting fresh PR CI after test correction · **Open questions touched:** none; the child is force-reaped rather than relying on Unix datagram peer-close behavior, which has no portable EOF wakeup. diff --git a/sessions/2026-07-15-e8-browser-oidc-session.md b/sessions/2026-07-15-e8-browser-oidc-session.md new file mode 100644 index 0000000..22d225e --- /dev/null +++ b/sessions/2026-07-15-e8-browser-oidc-session.md @@ -0,0 +1,21 @@ +# Session — 2026-07-15 — e8-browser-oidc-session + +**Builder:** gnanirahulnutakki · **Branch:** gnanirahulnutakki/feat/e8-browser-oidc-session +**Slice(s):** E8 F8.1a · #175 · **Status:** in-progress + +--- + +[G] Goal: implement #175's issuer-pinned OIDC authorization-code and PKCE browser-session boundary for the Hub without broadening the existing bearer-only fleet API. +[S] Scope: OIDC browser authorization/token exchange, bounded single-use transactions, secure cookie handoff, runtime/Helm configuration, tests, README, and release evidence. Out: console rendering, cookie authentication for fleet APIs, refresh tokens, client secrets, and write operations. +[A] Action: verified #172 remains an external package-visibility prerequisite and advanced to the independent E8 slice; confirmed current GitHub Dependabot, code-scanning, and secret-scanning queues are all empty; traced the existing strict OIDC verifier, forced-RLS binding, session issuer, and Hub runtime boundary. +[A] Action: added issuer-origin-pinned Authorization Code + PKCE S256 discovery/token handling, exact nonce-bound browser ID-token exchange, bounded single-use process-local login transactions, secure host-only cookie delivery, and a Hub runtime/Helm deployment contract. The bearer fleet API remains unchanged and cookie-only requests fail its authentication gate. +[A] Action: retained the stricter raw OIDC exchange profile while allowing the nonce-bound browser ID-token profile to omit optional `nbf`, as permitted by OIDC Core; browser tokens retain exact issuer, one exact client audience, type, algorithm, expiry, issue-time, nonce, and forced-RLS membership checks. Added the Hub-only imports to the explicit privacy boundary allowlist; local-mode packages remain outside it. +[A] Action: completed a CodeRabbit agent review pass and manual red-team review of transaction replay, callback/host binding, endpoint pinning, token leakage, cookie scope, middleware separation, RLS scope, and deployment secret handling. No actionable external review finding was returned; corrected CI static-analysis findings and the privacy-boundary allowlist gate. +[T] Test: `make ci` passed (gofmt/goimports, vet, golangci-lint, govulncheck, race suite, privacy boundary, scripts, performance, e2e, build); `make e2e-isolation` passed with PostgreSQL RLS controls and 50,003 selector fuzz executions; `make release-check` passed; `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` passed; `make e2e-helm HELM=/Volumes/EXTENDED/MacData/tools/bin/helm-v4.2.2` passed; `make e2e-oci` passed. Final module verification and govulncheck passed; final GitHub security queues were Dependabot/code-scanning/secret-scanning `0/0/0`; Docker prune reclaimed 2.23 GB across the final cleanup and no Kind clusters remain. +[C] Checkpoint #1: implementation, review, documentation, and local validation complete; next: create the signed DCO/GSTACK commit, push the small PR into `dev`, and verify exact post-merge CI before closing #175. + + + +--- + +**Session close:** ready for review · **Open questions touched:** Q13 — the browser session stays a Hub-only seam and does not alter local-mode upgrade behavior. diff --git a/sessions/2026-07-16-argocd-application-facts.md b/sessions/2026-07-16-argocd-application-facts.md new file mode 100644 index 0000000..30de799 --- /dev/null +++ b/sessions/2026-07-16-argocd-application-facts.md @@ -0,0 +1,96 @@ +# Argo CD Application graph facts + +- Builder: gnanirahulnutakki +- Effort: standard +- Branch: `gnanirahulnutakki/feat/argocd-application-facts` +- Issue: `#206` +- Status: landed on `dev` through #208; exact post-merge proof recorded below + +## Goal + +Establish the first hand-written Wave-1 Argo CD fact contract by projecting one already-authorized +`Application` CRD into bounded, deterministic LIVE, DESIRED, TIMELINE, and drift graph evidence. + +## Decision + +Build the pure projector before a network client or the E12 gRPC framework. The repository requires +the day-1 adapters to prove their behavior before generalization, and this boundary avoids inventing +new credential/configuration APIs while kubeconfig pagination work remains active in #190. + +The projector receives one already-fetched object plus explicit workspace, scope, and observation +time. It performs no network access, credential loading, shell execution, planning, verification, +sync, or rollback. A later reader can reuse this exact fact contract without changing graph or brain +semantics. + +## Safety and boundedness + +- Validate the Argo API group/kind and cluster/namespace-bounded identity before emitting facts. +- Serialize allowlisted fields only. Repository and destination URLs lose userinfo, query, and + fragment data; raw Helm/Kustomize/plugin parameters and status messages are never copied. +- Cap sources at 16, retained history at the newest 32 entries, total facts at 36, and every encoded + fact payload at 16 KiB. +- Mark the oldest retained history entry when earlier history was truncated. +- Missing status, sync, health, or history emits fewer facts instead of fabricated evidence. +- Keep a source-level test that rejects network, dynamic-client, shell, and mutation seams. + +## Progress + +[G] Add a production-safe Argo Application fact projector for #206. +[S] Limit this slice to pure normalization plus real API-server proof; discovery/query and mutations +remain explicit non-goals. +[A] Reconciled #46 and #30 against live code and specs. E14's local rule engine is shipped; its +governed renderer remains blocked on E4/E5. E12 explicitly requires hand-written adapters before +the subprocess framework. +[A] Opened #206 and created a dedicated EXTENDED worktree from the exact `origin/dev` merge head. +[A] Implemented deterministic desired, health, sync-drift, and bounded sync-history projections with +OTel-aligned cluster/namespace identity. +[T] Added malformed identity/status, URL credential stripping, raw Helm parameter non-retention, +payload budget, history truncation, abstention, determinism, and no-mutation boundary tests. +[T] Added a real kind regression that installs a minimal Application CRD, creates and reads one +Application through the API server, projects it, assembles the graph, and proves secret markers are +absent. +[R] Red-team review caught that valid Helm OCI repositories can omit a URL scheme. Updated the +sanitizer to accept documented scheme-less OCI and SCP-like SSH forms while stripping userinfo and +rejecting query/fragment credentials, local paths, and unsupported schemes. + +## Verification + +- Focused race tests and privacy-boundary tests pass; final projector coverage is 80.5%. +- `golangci-lint`, `go vet`, `govulncheck`, full repository race tests, policy scripts, tagged E2E, + and the production build pass under `make ci`. +- `make e2e-isolation` passes, including 100,024 final workspace-isolation fuzz executions. +- Focused real-kind projection passes in 32.783 seconds with pinned kind v0.32.0; the complete kind + fanout, OCI, and Argo suite passes in 190.748 seconds. +- The final full kind fanout, OCI, and Argo suite passes in 193.362 seconds. +- `make release-check` passes two reproducible GoReleaser builds, archive/SBOM verification, and + the multi-architecture distroless OCI layout contract. +- README was reviewed. No update is warranted because this slice adds no user-facing command, + network reader, configuration, credential flow, or supported runtime behavior yet. +- The final full-gate rerun after the red-team OCI compatibility correction passes. +- Notion decision: `3a02637e-db07-8194-8160-e78cb189cc86`. +- Notion session: `3a02637e-db07-8143-9bf5-f5629753a902`. + +## Landed closure + +- Issue #206 closed after PR #208 merged into `dev` on 2026-07-17. +- Exact signed/DCO feature commit: `73ea5cc970f49b2f1a67730e3a5c42489edb594d`. +- Exact merge commit: `311eb1c6c25f0c7dc76487fe0bedfa3a3dde6054`. +- Hosted PR CI, release, CodeQL, and CodeRabbit checks all passed. +- Exact post-merge `dev` CI: . +- Exact post-merge CodeQL: . +- Post-merge Dependabot, code-scanning, and secret-scanning queues were `0 / 0 / 0`. + +Primary compatibility references: + +- +- +- + +## Checkpoint + +- `2026-07-16/argocd-application-facts#1` + +## Open questions + +- Network discovery/query composition remains a later child slice. It must reuse existing + kubeconfig authorization and bounded pagination rather than add another credential store. diff --git a/sessions/2026-07-16-brain-image-evidence.md b/sessions/2026-07-16-brain-image-evidence.md new file mode 100644 index 0000000..270166c --- /dev/null +++ b/sessions/2026-07-16-brain-image-evidence.md @@ -0,0 +1,36 @@ +# Fleet image correlation evidence + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/brain-image-evidence` +- Issue: `#182` +- Status: complete + +## Goal + +Require fresh, cited, deterministic image repo-digest evidence before Sith emits a fleet-wide failure correlation. + +## Scope + +- Reject stale repo-digest observations from fleet correlation. +- Cite one canonical digest observation for every correlated entity. +- Deduplicate repeated observations independently of input order. +- Preserve canonical rule, abstention, and ranking behavior. + +## Tests + +- `go test -race -count=1 ./internal/brain` +- Stale digest rejection, cited correlation, duplicate selection, full input-order reversal, and canonical source-reference tie-break coverage. +- `make ci` +- `make e2e-isolation` +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` +- `GOPATH=/Volumes/EXTENDED/MacData/go make release-check GORELEASER=/Volumes/EXTENDED/MacData/tools/bin/goreleaser` + +## Review + +- Preserved base failure citations because the fleet verdict must prove both the failure and the shared image. +- Added a canonical full-reference tie-break after independent review found equal-time/source/value observations could remain input-order dependent. + +## Checkpoint + +- `2026-07-16/brain-image-evidence#1` diff --git a/sessions/2026-07-16-browser-oidc-rate-limit.md b/sessions/2026-07-16-browser-oidc-rate-limit.md new file mode 100644 index 0000000..4021c37 --- /dev/null +++ b/sessions/2026-07-16-browser-oidc-rate-limit.md @@ -0,0 +1,20 @@ +# Session — 2026-07-16 — browser-oidc-rate-limit + +**Builder:** gnanirahulnutakki · **Effort:** deep · **Branch:** gnanirahulnutakki/fix/browser-oidc-rate-limit +**Slice(s):** Phase 1 browser identity hardening · #180 · **Status:** ready for review + +--- + +[G] Goal: fix #180 so every browser OIDC login or callback request consumes exactly one rate-limit attempt without weakening transaction replay, PKCE, nonce, cookie, or token-disclosure controls. +[S] Scope: browser OIDC request accounting and focused regression coverage. Out: limiter algorithm changes, distributed rate limiting, bearer-route authentication, provider verification, and session lifetime policy. +[A] Action: removed the callback-only second limiter debit; the common request admission path remains the single fail-closed limiter boundary for both login and callback requests. +[A] Action: added a production-boundary regression proving ten full login flows consume exactly twenty requests and that the next request is rejected before a transaction or provider exchange is created. +[A] Action: addressed independent review feedback by asserting the guarded limiter count after every login and callback phase, preventing compensating debit errors from satisfying only the aggregate limit. +[T] Test: focused `go test -race -count=1 ./internal/hubserver`, `make ci`, `make e2e-isolation` with PostgreSQL RLS plus 50,000 fuzz executions, `make e2e-kind`, and `make release-check` all passed. A second independent review completed with no findings. +[C] Checkpoint #1: implementation, regression, red-team review, and required local gates complete; next: create the signed DCO/GSTACK commit and open the small PR into `dev`. + + + +--- + +**Session close:** ready for review · **Open questions touched:** none diff --git a/sessions/2026-07-16-fleetcache-workspace-mutations.md b/sessions/2026-07-16-fleetcache-workspace-mutations.md new file mode 100644 index 0000000..171f937 --- /dev/null +++ b/sessions/2026-07-16-fleetcache-workspace-mutations.md @@ -0,0 +1,46 @@ +# Fleet-cache workspace mutation boundaries + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/fleetcache-workspace-mutations` +- Issue: `#187` +- Status: ready for review + +## Goal + +Qualify every fleet-cache data mutation and its operational state by an explicit workspace boundary. + +## Scope + +- Key records, coverage, warm state, sync state, update time, and errors by workspace. +- Carry an explicit workspace on watch snapshots, upserts, deletes, and errors. +- Reject missing or mixed workspace mutations atomically. +- Preserve `QueryScoped` ownership revalidation and workspace-scope the local-UI pause control. +- Keep pagination issues #185 and #190, connector framework work, and external release blockers out of scope. + +## Progress + +[G] Prove and repair the cross-workspace mutation collisions tracked by #187. +[S] Limit changes to connector watch envelopes, the in-memory fleet cache, local hydration callers, and focused unit/integration coverage. +[A] Revalidated `origin/dev` at `a54ebe4`, confirmed #187 open, and preserved the isolated EXTENDED worktree across the #185 merge. +[T] Added race/fuzz regressions for identical resource identity collisions, cross-workspace watch-error contamination, and change-notification isolation; the real two-cluster kind suite now replays a live Pod identity through two independent workspaces. + +## Verification + +- `make ci` passed on the final tree: formatting, vet, lint, vulnerability scan, race tests, safety scripts, performance, subprocess e2e, and build. +- `make e2e-isolation` passed with PostgreSQL RLS and 50,000 executions for each scoped-read and foreign-mutation fuzzer. +- `make e2e-kind` passed against two real kind clusters in 153.102 seconds. +- `make release-check` passed two reproducible snapshot builds, archive/SBOM verification, formula generation, and multi-platform OCI layout verification. +- Red-team review confirmed workspace validation occurs before lock acquisition or mutation; record keys, kind aliases, per-kind watch errors, replace, snapshot, upsert, delete, pause, sync, version, and change-wait paths cannot alter or signal a foreign workspace. +- `README.md` was reviewed; its public workspace-scoped cache description remains accurate, so the implementation evidence belongs in `docs/ROADMAP.md` rather than duplicating internals in the README. +- Notion decision log: `https://app.notion.com/p/39f2637edb078163b98ef0ac07b992e2` +- Notion session checkpoint: `https://app.notion.com/p/39f2637edb0781cfb6a1fa1dd2f91d48` + +## Checkpoint + +- `2026-07-16/fleetcache-workspace-mutations#1` +- `2026-07-16/fleetcache-workspace-mutations#2` + +## Open questions + +- None. Missing or mixed workspace identity fails closed; this is a security invariant, not a configurable policy. diff --git a/sessions/2026-07-16-fleetcache-workspace-scope.md b/sessions/2026-07-16-fleetcache-workspace-scope.md new file mode 100644 index 0000000..cae613f --- /dev/null +++ b/sessions/2026-07-16-fleetcache-workspace-scope.md @@ -0,0 +1,35 @@ +# Fleet-cache workspace scope metadata + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/fleetcache-workspace-scope` +- Issue: `#183` +- Status: complete + +## Goal + +Keep discovery metadata for identically named cluster scopes isolated by workspace. + +## Scope + +- Key scope metadata by workspace and scope name. +- Preserve fail-closed behavior for guessed and ambiguous scopes. +- Ensure refreshing one workspace cannot mutate or remove another workspace's metadata. +- Keep record and coverage mutation-contract redesign in a separate issue. + +## Tests + +- `go test -race -count=1 ./internal/fleetcache` +- `PATH=/Users/nutakki/.local/bin:/Volumes/EXTENDED/MacData/tools/bin:/opt/homebrew/bin:/usr/bin:/bin GOPATH=/Volumes/EXTENDED/MacData/go make ci` +- `PATH=/Users/nutakki/.local/bin:/Volumes/EXTENDED/MacData/tools/bin:/opt/homebrew/bin:/usr/bin:/bin GOPATH=/Volumes/EXTENDED/MacData/go make e2e-isolation` +- `PATH=/Users/nutakki/.local/bin:/Volumes/EXTENDED/MacData/tools/bin:/opt/homebrew/bin:/usr/bin:/bin GOPATH=/Volumes/EXTENDED/MacData/go make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` +- `PATH=/Users/nutakki/.local/bin:/Volumes/EXTENDED/MacData/tools/bin:/opt/homebrew/bin:/usr/bin:/bin GOPATH=/Volumes/EXTENDED/MacData/go make release-check GORELEASER=/Volumes/EXTENDED/MacData/tools/bin/goreleaser` + +## Review + +- An independent review found ambiguous success still cleared global coverage failures despite leaving workspace metadata untouched. +- Fixed by rejecting non-unique scope membership before any reachability, coverage, or last-error mutation, with a prior-watch-failure regression. + +## Checkpoint + +- `2026-07-16/fleetcache-workspace-scope#1` diff --git a/sessions/2026-07-16-github-merged-pr-facts.md b/sessions/2026-07-16-github-merged-pr-facts.md new file mode 100644 index 0000000..22da180 --- /dev/null +++ b/sessions/2026-07-16-github-merged-pr-facts.md @@ -0,0 +1,92 @@ +# GitHub merged pull-request timeline facts + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/feat/github-merged-pr-facts` +- Issue: `#212` +- Status: locally verified; awaiting signed PR and exact post-merge proof + +## Goal + +Define Sith's first GitHub normalization contract by projecting one already-authorized, already- +fetched REST `Get a pull request` response into bounded TIMELINE evidence without adding an HTTP +client, token/configuration seam, persistence, repository-to-workload inference, or mutation. + +## Scope + +- Pin the source shape to GitHub REST API version `2026-03-10`. +- Take workspace, host, owner, repository, pull number, and collection time only from trusted caller + input. Response URLs and nested repository objects never define identity or tenancy. +- Emit one merge event only from internally consistent `merged=true` evidence. Valid unmerged pull + requests abstain, including when GitHub supplies a pre-merge test-merge SHA. +- Retain only pull number, event kind, head/base/merge commit SHAs, and merge time. +- Keep every event unattached until a separately governed repository-to-workload relation exists. +- Bound source bytes, JSON depth, caller identity, fact count, encoded payload, and clock skew. + +## Decision + +Use an exact-key, allowlist-only JSON decoder around the documented GitHub response. Go's default +struct decoder accepts case-insensitive key aliases; exact-key extraction prevents casing variants +from redefining a required field while still allowing GitHub to add unrelated response fields. +Duplicate keys, trailing JSON, excessive nesting, invalid UTF-8, and type mismatches fail closed. + +The source merge timestamp orders the TIMELINE fact, while caller collection time bounds it with a +five-minute clock-skew allowance. Strict zero-skew comparison would create avoidable outages on a +slightly skewed node; an event beyond the fixed allowance remains invalid. Both SHA-1 and SHA-256 +hex commit identifiers are accepted so the fact contract does not hard-code one repository object +format. + +## Security, operability, and cost + +- Titles, bodies, users, labels, URLs, nested repository data, clone tokens, branch labels, and all + unknown fields are discarded before graph construction and never echoed in validation errors. +- The source-level boundary test rejects network, filesystem, process, database, gRPC, client-go, + credential, persistence, and mutation seams. +- This pure in-memory projector creates no infrastructure and no runtime cloud cost. A later live + reader must use least-privilege GitHub read permission and account for API rate limits and egress. + +## Progress and verification + +[G] Normalize merged GitHub pull requests into bounded TIMELINE facts for #212. +[S] Pure response projection only; live auth/fetching, desired manifests, governed PR writes, and +repository-to-workload relations remain out of scope. +[A] Verified the E12 Wave-1 contract against repository specs and the current official GitHub REST +API versioning and pull-request endpoint documentation, then opened #212 from exact `origin/dev`. +[A] Implemented exact-key parsing, caller-trusted identity, honest unmerged abstention, bounded +clock skew, allowlisted payload construction, and permanently unattached graph facts for this slice. +[T] Race-enabled focused tests pass with 94.9% statement coverage. Adversarial cases cover secret +non-retention, exact-key aliases, duplicate/trailing/deep JSON, malformed types/times/SHAs, every +input budget, graph validity, determinism, abstention, and the no-capability boundary. +[T] A ten-second fuzz campaign completed more than 550,000 executions without a panic, invalid +fact, capability escape, excess fact count, or oversized payload. +[T] `make ci` passes formatting, vet, pinned lint with zero findings, vulnerability scanning with no +findings, the full race suite, policy tests, warm-view performance, subprocess E2E, and the build. +[A] Hosted review found two valid hardening gaps: the initial capability test blacklisted known APIs +instead of failing closed on any new import or declaration, and wrapped JSON errors could echo an +attacker-sized numeric literal. The final boundary exact-allowlists every production import, type, +value, function, and method, rejects injected interfaces, and returns fixed field-level decode +errors with an explicit overlong-number regression. +[T] After review hardening, `make ci` and `make e2e-isolation` pass again. The final +`make e2e-kind` passes the pinned Kubernetes fanout, OCI, and Argo suite in 237.406 seconds. +`make release-check` passes module verification, two reproducible release builds, +archive/SPDX SBOM inspection, Homebrew formula generation, and the multi-platform distroless OCI +layout. The release check used an isolated temporary `GOPATH` because this machine's configured +GOPATH root contains another `go.mod`. +[T] `README.md` was reviewed in full. No update is warranted because this slice adds no user-facing +command, configuration, authentication flow, endpoint, runtime connector, or supported behavior; +the roadmap and this engineering checkpoint are the correct documentation surfaces. + +Primary compatibility references: + +- +- + +## Checkpoint + +- `2026-07-16/github-merged-pr-facts#1` +- `2026-07-16/github-merged-pr-facts#2` + +## Open questions + +- Live GitHub authentication and fetching remain a later connector child. That slice should use + the documented least-privilege `Pull requests: read` permission and reuse this projector. diff --git a/sessions/2026-07-16-helm-pin-alignment.md b/sessions/2026-07-16-helm-pin-alignment.md new file mode 100644 index 0000000..16620f4 --- /dev/null +++ b/sessions/2026-07-16-helm-pin-alignment.md @@ -0,0 +1,79 @@ +# Helm tooling pin alignment + +- Builder: gnanirahulnutakki +- Effort: standard +- Branch: `gnanirahulnutakki/fix/helm-pin-alignment` +- Issue: `#197` +- Status: ready for review + +## Goal + +Align every live Helm tool boundary on the official v4.2.3 patch release and reject version +lookalikes that could bypass a prefix comparison. + +## Scope + +- Pin CI, the hub chart contract, the M0 OCM runner, and current experiment documentation to + `v4.2.3`. +- Verify CI's Linux amd64 archive against the official SHA-256 + `e9b88b4ee95b18c706839c28d3a0220e5bc470e9cd9262410c90793c45ff8b7c`. +- Accept only the exact semantic release or Helm's real `+g` build metadata. +- Add policy coverage that fails if executable and documented pins diverge. +- Leave historical session records unchanged; this session is the dated evidence for the new pin. + +## Progress + +[G] Repair the stale and divergent Helm pins tracked by #197. +[S] Limit the slice to tool-version admission, checksum verification, policy coverage, current +experiment/roadmap documentation, and this session checkpoint. +[A] Revalidated `origin/dev`, confirmed there is no active #197 owner or PR, and created a separate +EXTENDED worktree. +[A] Queried the official Helm GitHub release API: v4.2.3 is a non-draft, non-prerelease release +published 2026-07-09. The official Linux amd64 checksum endpoint returns the pinned digest above. +[A] Downloaded the official Darwin arm64 archive over TLS, verified its published SHA-256, and +observed the real short-version output `v4.2.3+g43e8b7f`. +[A] Aligned CI, the M0 runner, the real Helm chart contract, and current experiment evidence on +v4.2.3. Both shell and Go validators now accept only the exact release or lowercase +`+g<7-40 hex>` commit metadata. +[T] Added a cross-file policy gate plus positive and adversarial version cases covering patch +lookalikes, prereleases, vendor suffixes, short hashes, uppercase hashes, whitespace, and extra +output. +[A] Rebasing before commit incorporated #194's disjoint worker-pool merge (`231c89e`) so the final +signed slice starts from the current live `dev` integration head. + +## Verification + +- `bash -n`, focused ShellCheck (excluding the runner's pre-existing SC2174), and + `git diff --check` pass. +- `tests/scripts/helm_tooling_policy_test.sh` passes all 7 cross-file pin and checksum assertions. +- `tests/scripts/m0_ocm_falsification_safety_test.sh` passes all 29 assertions, including the + exact-version and five lookalike rejection cases. +- `make e2e-helm HELM=/Volumes/EXTENDED/MacData/tools/bin/helm-v4.2.3` passes the real chart + contract in 2.374 seconds on the final rebased tree with the checksum-verified upstream binary. +- Final `make ci` on the rebased live-`dev` tree passes with zero lint findings, no known + vulnerabilities, the full race suite, all shell policy suites, performance, subprocess e2e, + and build. +- `make e2e-isolation` passes PostgreSQL/RLS controls and both 50,000-execution cross-workspace + fuzz campaigns. +- `make e2e-kind` passes the two-cluster fan-out and OCI contract in 153.604 seconds. +- `make release-check` passes two reproducible snapshots, SPDX SBOM verification, Homebrew formula + generation, and the multi-platform release OCI layout. The first invocation inherited the + machine's stale `/Users/nutakki/go` global GOPATH and stopped before artifact creation; the + verified rerun used the canonical EXTENDED GOPATH. +- `make e2e-ocm` passes the real hub-plus-two-spoke lab with the verified Helm v4.2.3 binary: + all four add-ons converged, M0 passed in 220 seconds, both direct ClusterProxy tests passed, and + cleanup removed all three clusters. +- Red-team review confirmed there is no prefix fallback: malformed pins fail the cross-file gate, + shell command substitution admits no extra output, and the Go path removes only the one expected + trailing newline before applying the anchored release pattern. +- The README was reviewed before commit. No user command changes; the experiment and roadmap docs + are the authoritative surfaces for this tool-policy correction. + +## Checkpoint + +- `2026-07-16/helm-pin-alignment#1` + +## Open questions + +- None. Custom vendor suffixes are deliberately rejected because they are not the verified + upstream release artifact. diff --git a/sessions/2026-07-16-hub-refresh-coalescing.md b/sessions/2026-07-16-hub-refresh-coalescing.md new file mode 100644 index 0000000..902b8ee --- /dev/null +++ b/sessions/2026-07-16-hub-refresh-coalescing.md @@ -0,0 +1,50 @@ +# Hub refresh coalescing + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/hub-refresh-coalescing` +- Issue: `#195` +- Status: ready for review + +## Goal + +Collapse duplicate concurrent hub refreshes without crossing workspace, authorization, cancellation, request-context, or trace boundaries. + +## Scope + +- Authorize every caller through the existing role and PEP pipeline before coordination. +- Keep at most one active refresh flight per validated workspace while allowing different workspaces to progress independently. +- Run shared work on a fresh internal context that contains no caller cancellation, credential, request value, or request trace. +- Return defensive copies of the closed coverage result and one closed error result. +- Remove completed, failed, and panicking flights without retaining per-request state. +- Keep spoke worker-pool behavior (#194) and database snapshot isolation (#193) out of this slice. + +## Progress + +[G] Repair duplicate same-workspace refresh work and last-writer timing races tracked by #195. +[S] Limit the change to collector coordination, adversarial race tests, and operator/roadmap documentation. +[A] Revalidated `origin/dev` at `566fc96`, confirmed no open PR collision, and created an isolated EXTENDED worktree. +[T] Added deterministic authorization, same/cross-workspace, leader/waiter cancellation, shared failure, panic cleanup, result-copy, and trace/context isolation regressions. + +## Verification + +- Focused `go test -race -count=100 ./internal/hubfleet` passed. +- Focused hubfleet, hubserver, and hubruntime race suites passed. +- `make ci` passed formatting, vet, lint with zero findings, vulnerability scanning with no findings, the full race suite, safety scripts, performance, subprocess e2e, and build. +- `make e2e-isolation` passed PostgreSQL RLS packages and 50,000 executions for each fleet-cache workspace-isolation fuzzer. +- `make e2e-kind` passed the real two-cluster and OCI image contracts in 157.014 seconds with the pinned kind v0.32.0 toolchain. +- `make release-check` passed module verification, two reproducible GoReleaser snapshots, SPDX SBOM validation, Homebrew formula generation, and the multi-platform OCI layout. +- Red-team review confirmed denied callers never join, caller cancellation/values/traces cannot reach shared work, coverage slices do not alias, and completion/failure/panic paths remove coordinator state. +- `README.md` documents the operator-visible authorization, workspace, cancellation, and request-context boundaries. +- Hosted PR checks and exact post-merge `dev` gates remain pending. +- Notion decision log: `https://app.notion.com/p/3a02637edb0781f3933ef6d6f52268b6` +- Notion session checkpoint: `https://app.notion.com/p/3a02637edb0781fb82ddd9dc2d20408e` + +## Checkpoint + +- `2026-07-16/hub-refresh-coalescing#1` +- `2026-07-16/hub-refresh-coalescing#2` + +## Open questions + +- None. Coordination is intentionally per workspace and occurs only after each caller's own policy decision. diff --git a/sessions/2026-07-16-hub-spoke-worker-pool.md b/sessions/2026-07-16-hub-spoke-worker-pool.md new file mode 100644 index 0000000..5090558 --- /dev/null +++ b/sessions/2026-07-16-hub-spoke-worker-pool.md @@ -0,0 +1,50 @@ +# Hub spoke worker pool + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/hub-spoke-worker-pool` +- Issue: `#194` +- Status: ready for review + +## Goal + +Bound and parallelize one workspace refresh so unavailable spokes do not delay healthy peers serially or create unbounded proxy/API pressure. + +## Scope + +- Add a validated `MaxConcurrentSpokes` collector setting with a default of four and a hard maximum of 64. +- Parallelize transport and snapshot validation only; keep store writes, coverage mutation, metrics, and trace observations serialized. +- Preserve independent per-spoke deadlines, failure categories, retained-stale accounting, and sorted coverage. +- Stop admission on parent cancellation or the first store failure, cancel in-flight work, and wait for every worker before returning. +- Keep refresh coalescing (#195), repeatable-read database snapshots (#193), and OCM transport authorization (#104) out of this slice. + +## Progress + +[G] Repair serial N-times-timeout hub refresh latency tracked by #194. +[S] Limit changes to collector fan-out, adversarial race tests, and operator/roadmap evidence. +[A] Revalidated the canonical remote and live issue/PR inventory at `origin/dev` `ca45dc3`, then created an isolated EXTENDED worktree. +[T] Added deterministic worker-cap, timeout-wave, healthy-peer, cancellation, store-failure, coverage-order, and closed-observability regressions. + +## Verification + +- Focused `go test -race ./internal/hubfleet` passed. +- Synchronization-heavy cancellation and admission tests passed 100 repeated race-detector runs. +- The four-spoke timeout regression completes in a single one-second parallel wave. Its one-worker negative control failed as intended at 4.004 seconds. +- Red-team review found and repaired a worker-goroutine panic escape and panic-path worker leak; transport and store panic regressions now prove closed errors, peer cancellation/join, and clean later refreshes. +- Independent CodeRabbit review completed twice; the exact final tree returned zero findings across all six changed files. +- `make ci` passed formatting, vet, lint with zero findings, vulnerability scanning with no findings, the full race suite, safety scripts, and build. +- `make e2e-isolation` passed PostgreSQL RLS packages and both 50,000-execution workspace-isolation fuzzers. +- `make e2e-kind` passed the real two-cluster and immutable OCI image contracts in 159.378 seconds with kind v0.32.0. +- `make release-check` passed module verification, two reproducible GoReleaser snapshots, SPDX SBOM validation, Homebrew formula generation, and the multi-platform OCI layout. +- Notion decision log: `https://app.notion.com/p/3a02637edb078131aea9d5452a51caca` +- Notion session checkpoint: `https://app.notion.com/p/3a02637edb0781d4b7c2c0d6f50ee8a6` +- Hosted PR and exact post-merge `dev` gates remain pending. + +## Checkpoint + +- `2026-07-16/hub-spoke-worker-pool#1` +- `2026-07-16/hub-spoke-worker-pool#2` + +## Open questions + +- None. The packaged runtime keeps the conservative default; embedded construction may choose only a validated finite limit. diff --git a/sessions/2026-07-16-hubdb-repeatable-read.md b/sessions/2026-07-16-hubdb-repeatable-read.md new file mode 100644 index 0000000..604c13a --- /dev/null +++ b/sessions/2026-07-16-hubdb-repeatable-read.md @@ -0,0 +1,50 @@ +# Hub fleet repeatable-read snapshot + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/hubdb-repeatable-read` +- Issue: `#193` +- Status: ready for review + +## Goal + +Return persisted hub facts, coverage, and staleness from one workspace-scoped PostgreSQL snapshot. + +## Scope + +- Keep existing workspace write transactions at `READ COMMITTED, READ WRITE`. +- Read cluster state and facts inside one `REPEATABLE READ, READ ONLY` transaction. +- Set and verify the workspace RLS scope inside that same transaction. +- Fail closed if any returned fact lacks a registered spoke in the captured cluster-state snapshot. +- Add a deterministic PostgreSQL regression that commits `ReplaceSnapshot` between the two reads. +- Keep hub collection concurrency, refresh coalescing, and OCM lifecycle findings out of this slice. + +## Progress + +[G] Repair mixed persisted fact and coverage snapshots tracked by #193. +[S] Limit changes to the hub database transaction boundary, focused PostgreSQL proof, and roadmap evidence. +[A] Revalidated `origin/dev` at `566fc96` and created an isolated EXTENDED worktree. +[T] Added a two-connection deterministic interleaving that verifies repeatable-read, read-only mode, transaction-local RLS, and coherent old-old or new-new results. + +## Verification + +- Focused package race tests pass. +- The PostgreSQL RLS integration regression passes on the restored repeatable-read tree. +- Red-team negative control: downgrading the query to `READ COMMITTED` returned generation 2 facts with generation 1 `Stale=true` and `StaleFor=2h0m0s`; the regression failed as intended. +- Independent CodeRabbit review completed with zero findings. +- `make ci` passed formatting, vet, lint, vulnerability scanning, the full race suite, safety scripts, performance, subprocess e2e, and build. +- `make e2e-isolation` passed PostgreSQL RLS coverage and both 50,000-execution fleet-cache isolation fuzzers. +- `make e2e-kind` passed against two real kind clusters in 163.456 seconds. +- `make release-check` passed reproducible archives, SPDX SBOM validation, Homebrew formula generation, and the multi-platform OCI layout. +- Notion decision log: `https://app.notion.com/p/3a02637edb0781909fc0d9629635d08c` +- Notion session checkpoint: `https://app.notion.com/p/3a02637edb078148baefddf316ea1469` +- PR and exact post-merge `dev` evidence remain pending. + +## Checkpoint + +- `2026-07-16/hubdb-repeatable-read#1` +- `2026-07-16/hubdb-repeatable-read#2` + +## Open questions + +- None. The stronger transaction mode is private to read snapshots and does not alter write-path semantics. diff --git a/sessions/2026-07-16-kubeconfig-directory-race.md b/sessions/2026-07-16-kubeconfig-directory-race.md new file mode 100644 index 0000000..723a7f7 --- /dev/null +++ b/sessions/2026-07-16-kubeconfig-directory-race.md @@ -0,0 +1,50 @@ +# Kubeconfig directory race-safe import + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/kubeconfig-directory-race` +- Issue: `#196` +- Status: ready for review + +## Goal + +Keep directory import bound to the originally selected directory and prevent validation-to-open races from reading replacement or external files. + +## Scope + +- Open one descriptor-backed `os.Root` after validating the selected directory identity. +- Traverse and open entries only through that root. +- Verify the opened root and regular files match the identities observed before parsing. +- Reject deferred local credential files and exec commands that name an absolute or relative + executable path after the root closes. +- Reject deferred CA, client certificate/key, token-file, and executable-path reads; embedded data + and bare exec commands resolved through `PATH` remain supported. +- Preserve entry-count, depth, and byte bounds plus relative, content-free diagnostics. +- Keep generic table pagination, watch bootstrap bounds, and hub concurrency findings out of this slice. + +## Progress + +[G] Repair the path-based validation-to-open race tracked by #196. +[S] Limit changes to kubeconfig directory traversal, focused adversarial tests, and roadmap evidence. +[A] Revalidated `origin/dev` at `f6fe9f4` and created an isolated EXTENDED worktree. +[T] Added deterministic pre-open root replacement, post-open pathname replacement, regular-file replacement, external-symlink swap, and replaced-ancestor regressions; the focused race suite passes. + +## Verification + +- `make ci` passed formatting, vet, lint, vulnerability scan, the full race suite, safety scripts, performance, subprocess e2e, and build. +- `make e2e-isolation` passed PostgreSQL RLS and 50,000 executions for each fleet-cache isolation fuzzer. +- `make e2e-kind` passed against two real kind clusters in 153.347 seconds. +- `make release-check` passed reproducible snapshot archives, SPDX SBOM validation, Homebrew formula generation, and multi-platform OCI layout verification. +- Red-team review added a live post-open non-symlink identity check and deterministic nested-ancestor replacement coverage. +- `README.md` documents the embedded credential-data and PATH-resolved exec-command constraints that prevent deferred reads from reopening the race. +- Notion decision log: `https://app.notion.com/p/39f2637edb078117a356d22f0fa569cb` +- Notion session checkpoint: `https://app.notion.com/p/39f2637edb07815c8f66eb0f714a41cf` + +## Checkpoint + +- `2026-07-16/kubeconfig-directory-race#1` +- `2026-07-16/kubeconfig-directory-race#2` + +## Open questions + +- None. Root or file identity ambiguity fails closed before parsing. diff --git a/sessions/2026-07-16-kubeconfig-query-pagination.md b/sessions/2026-07-16-kubeconfig-query-pagination.md new file mode 100644 index 0000000..66cb155 --- /dev/null +++ b/sessions/2026-07-16-kubeconfig-query-pagination.md @@ -0,0 +1,41 @@ +# Kubeconfig query pagination + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/kubeconfig-query-pagination` +- Issue: `#185` +- Status: complete + +## Goal + +Bound Kubernetes resource-list response size and materialization across kubeconfig contexts without changing deterministic fleet-wide query limits. + +## Scope + +- Request Kubernetes lists in pages of at most 250 objects. +- Divide a fixed 10,000-object scan budget deterministically across sorted contexts. +- Stop pathological continuation chains after 128 pages per context. +- Discard partial scope facts when a continuation request fails or times out. +- Surface truncated contexts through coverage, cache state, text rendering, and brain abstention. +- Preserve last-known cache rows outside a truncated prefix and clear truncation only after a complete watch snapshot. + +## Tests + +- `go test -race -count=1 ./internal/fleet ./internal/connector ./internal/connector/kubeconfig ./internal/fleetcache ./internal/fleetrender ./internal/brain` +- Pagination, opaque continuation, deterministic multi-context budgets, cancellation, and fail-closed coverage unit regressions. +- Real two-cluster kind pagination and fleet-wide limit regression. +- `make ci` +- `make e2e-isolation` including 50,012 scoped fuzz executions. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` +- `GOPATH=/Volumes/EXTENDED/MacData/go make release-check GORELEASER=/Volumes/EXTENDED/MacData/tools/bin/goreleaser` + +## Review + +- Kept the scan budget independent of the caller's output limit so name/prefix filters and global sorting retain their prior semantics whenever coverage is complete. +- Marked truncated cache snapshots degraded and limited recovery to full watch snapshots rather than ordinary liveness events. +- Independent CodeRabbit review reported zero findings across all changed implementation and regression files. +- Generic server-side Table responses remain a separate bounded-materialization issue under `#190`. + +## Checkpoint + +- `2026-07-16/kubeconfig-query-pagination#1` diff --git a/sessions/2026-07-16-kubeconfig-timeout-bound.md b/sessions/2026-07-16-kubeconfig-timeout-bound.md new file mode 100644 index 0000000..686b190 --- /dev/null +++ b/sessions/2026-07-16-kubeconfig-timeout-bound.md @@ -0,0 +1,44 @@ +# Kubeconfig timeout containment + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/kubeconfig-timeout-bound` +- Issue: `#181` +- Status: complete + +## Goal + +Bound kubeconfig operations whose underlying client-go credential helper ignores context cancellation while preserving progress for healthy contexts. + +## Scope + +- Add adapter-owned admission control for timed operations. +- Retain per-context capacity until the underlying operation exits. +- Limit retained operations per kubeconfig context without a globally exhaustible gate. +- Cover ignored cancellation with unit and real subprocess regressions. + +## Actions + +- Reproduced unbounded goroutine growth with a probe that waits after its caller times out. +- Confirmed client-go's exec authenticator starts credential helpers without `CommandContext`. + +## Tests + +- `go test -race -count=1 ./internal/connector/kubeconfig` +- Synthetic ignored-cancellation containment and healthy-peer regression. +- Real client-go exec credential subprocess containment, recovery, and secret-exclusion regression. +- `make ci` +- `make e2e-isolation` including 50,000 fuzz executions. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` +- `make release-check` + +## Review + +- Added explicit reachability validation after independent review. +- Replaced the global limit with per-context quarantine after a two-wedged-context red-team case showed global capacity could deny healthy peers. +- Closed the simultaneous admission-release and deadline race with cancellation checks under and after gate acquisition. +- Kept pre-existing query pagination and result-budget work in a separate issue. + +## Checkpoint + +- `2026-07-16/kubeconfig-timeout-bound#1` diff --git a/sessions/2026-07-16-ocm-addon-wait-lifecycle.md b/sessions/2026-07-16-ocm-addon-wait-lifecycle.md new file mode 100644 index 0000000..74dd63c --- /dev/null +++ b/sessions/2026-07-16-ocm-addon-wait-lifecycle.md @@ -0,0 +1,71 @@ +# OCM add-on wait lifecycle + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/ocm-addon-wait-lifecycle` +- Issue: `#198` +- Status: ready for review + +## Goal + +Make the M0 OCM add-on convergence wait tolerate asynchronous creation and delete/recreate +transitions without granting separate creation and availability timeout windows. + +## Scope + +- Use one absolute deadline for each `ManagedClusterAddOn` from first creation check through + confirmed `Available=True`. +- Query only machine-formatted UID and `Available` condition fields from the current object. +- Treat NotFound, deletion, recreation, missing conditions, `False`, and `Unknown` as bounded + progress states. +- Fail closed on authorization or API errors, malformed identity, malformed or duplicate + `Available` conditions, and deadline exhaustion. +- Keep cluster registration waits, chart versions, OCM transport behavior, and Helm pin alignment + outside this slice. + +## Progress + +[G] Repair the lifecycle race and doubled timeout window tracked by #198. +[S] Limit the implementation to the M0 experiment runner, deterministic safety harness, experiment +and roadmap evidence, and this session checkpoint. +[A] Revalidated current `dev`, confirmed #194 is owned by another active task, reserved a separate +EXTENDED worktree, and checked the official kubectl get/wait machine-output contracts. +[A] Replaced the two-phase get/wait sequence with a shared-deadline polling loop. Each get is bounded +by the remaining budget, suppresses only NotFound, validates closed UID/condition output, and +requires two consecutive ready reads for the same UID. +[T] The deterministic safety harness now covers delayed creation, old-object deletion plus new UID +recreation, terminal read failures without response-body leakage, duplicate/malformed conditions, +and shared-deadline exhaustion. + +## Verification + +- `bash -n` passes for the runner and its safety harness. +- `bash tests/scripts/m0_ocm_falsification_safety_test.sh` passes all 23 assertions. +- `git diff --check` passes. +- The repository README was reviewed; no user-facing command or product behavior changed, so the + experiment and roadmap are the authoritative documentation for this harness-only correction. +- Focused safety passed five consecutive runs; the final post-review safety run also passed all 23 + assertions with `bash -n`, focused ShellCheck, and `git diff --check` green. +- `make ci` passed twice; the final run includes zero lint findings, no known vulnerabilities, the + complete race suite, all shell policy suites, warm-cache performance, subprocess e2e, and build. +- `make e2e-isolation` passed the PostgreSQL/RLS suites and both 50,000-execution cross-workspace + fuzzers. +- `make e2e-kind` passed the real two-cluster fan-out and OCI image contract in 159.546 seconds. +- `make e2e-ocm` passed the affected real hub-plus-two-spoke lab in 163 seconds: all four add-ons + converged through the lifecycle-safe wait, topology/credential/RBAC/outbound-only controls passed, + both direct ClusterProxy tests passed, and cleanup removed all clusters. +- `make release-check` passed two reproducible GoReleaser snapshots, SPDX SBOM verification, + Homebrew formula generation, and the multi-platform release OCI layout. +- Red-team review added an explicit post-request absolute-deadline check and confirmed that only UID + plus the closed Available status are parsed; kubectl stderr and object bodies remain suppressed. +- Hosted PR and exact post-merge `dev` evidence remain pending. + +## Checkpoint + +- `2026-07-16/ocm-addon-wait-lifecycle#1` +- `2026-07-16/ocm-addon-wait-lifecycle#2` + +## Open questions + +- None. A transient absent/missing condition remains retryable; malformed or duplicate condition + values remain terminal. diff --git a/sessions/2026-07-16-prometheus-alert-facts.md b/sessions/2026-07-16-prometheus-alert-facts.md new file mode 100644 index 0000000..b9b9bd1 --- /dev/null +++ b/sessions/2026-07-16-prometheus-alert-facts.md @@ -0,0 +1,86 @@ +# Prometheus alert facts + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/feat/prometheus-alert-facts` +- Issue: `#209` +- Status: ready for review + +## Goal + +Define Sith's first Prometheus normalization contract by projecting an already-authorized +`GET /api/v1/alerts` success response into bounded TELEMETRY facts without adding an HTTP client, +credential seam, telemetry store, or mutation path. + +## Scope + +- Accept only the documented success envelope and active-alert fields. +- Treat caller-provided workspace, cluster scope, and observation time as the trust boundary. +- Retain only `alertname` plus a small correlation allowlist; discard annotations, unknown labels, + and any response-provided cluster identity. +- Attach only one unambiguous Pod, Node, Deployment, StatefulSet, or DaemonSet identity. Preserve + every other valid alert as unattached evidence instead of guessing. +- Bound raw bytes, JSON depth, alerts, labels, label bytes, facts, and encoded payloads before + integration code. +- Keep live fetching, authentication, endpoint discovery, persistence, and the E12 framework out of + this contract-defining slice. + +## Decision + +Prometheus alert labels are untrusted evidence, not tenancy. The projector hashes only the +allowlisted, normalized alert identity into provenance, derives a safe graph resource name from the +same digest, rejects collisions within one response, and never permits a `cluster` label to replace +the trusted scope. Annotations are decoded only to validate the documented response shape and are +never projected or fingerprinted. + +The alert `state` and finite value remain observations, while `activeAt` plus the allowlisted label +identity define one active alert instance. Input order therefore cannot change output order or +identity. Missing Kubernetes identity is valid abstention; multiple identity candidates are invalid +ambiguity and fail the entire projection. + +## Trade-offs + +- The Prometheus alerts endpoint is documented under API v1 but explicitly has weaker stability + guarantees than the overarching v1 API. `ProtocolVersion = alerts/v1` isolates that source shape + so future additive handling does not silently redefine normalized facts. +- Dropping unknown labels protects the privacy boundary but can collapse two source alerts that + differ only by discarded labels. A same-response collision fails closed instead of coalescing; + widening the allowlist requires a deliberate contract review. +- This pure projector creates no infrastructure and no runtime cloud cost. A future live adapter + will add request volume, Prometheus query load, authentication, availability, and egress concerns. + +## Verification + +- Focused unit and graph tests pass with 94.0% statement coverage. +- The race-enabled package test passes. +- Adversarial coverage includes duplicate/trailing/malformed JSON, invalid UTF-8 and control data, + excessive nesting, non-finite values, unsupported states, ambiguous identities, secret + non-retention, deterministic ordering, empty-result abstention, and every declared size/count + budget. +- The source boundary test rejects network, gRPC, client-go, process execution, plugin/syscall, and + mutation seams. +- `README.md` was reviewed. No user-facing command, configuration, endpoint, or runtime behavior is + introduced, so the roadmap and this checkpoint are the appropriate documentation surfaces. +- `make ci` passes formatting, vet, pinned lint with zero findings, the current vulnerability scan + with no findings, the complete race suite, all shell policy suites, warm-cache performance, + subprocess E2E, and the production build. +- `make e2e-isolation` passes PostgreSQL/RLS and both 50,000-execution workspace-isolation fuzzers. +- `make e2e-kind` passes in 242.829 seconds with repository-pinned kind v0.32.0 and Kubernetes + v1.36.1. The machine's installed kind v0.30.0 failed first because it cannot load images into the + pinned node's containerd config version 4; no source correction was required. +- `make release-check` passes two reproducible snapshots, archive and SPDX SBOM verification, + Homebrew formula generation, and the multi-platform distroless OCI layout. An isolated temporary + `GOPATH` was required because this machine's configured GOPATH root contains another `go.mod`, + which caused Go 1.26.5 `go mod verify` to lose the worktree module context. +- Notion decision log: `https://app.notion.com/p/3a02637edb07814e89d7c416173ee653`. +- Notion session checkpoint: `https://app.notion.com/p/3a02637edb0781d3b512eb8fc68d9460`. +- Hosted PR and exact post-merge evidence remain pending. + +## Checkpoint + +- `2026-07-16/prometheus-alert-facts#1` +- `2026-07-16/prometheus-alert-facts#2` + +## Open questions + +- None for this slice. Live endpoint discovery and authentication belong to a later connector issue. diff --git a/sessions/2026-07-16-table-materialization.md b/sessions/2026-07-16-table-materialization.md new file mode 100644 index 0000000..3203137 --- /dev/null +++ b/sessions/2026-07-16-table-materialization.md @@ -0,0 +1,47 @@ +# Bounded generic Table materialization + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/table-materialization` +- Issue: `#190` +- Status: ready for review + +## Goal + +Bound the second server-side Kubernetes Table query used to render generic resources so a remote API server cannot bypass the fleet query's materialization budget. + +## Scope + +- Paginate Table lists with opaque continuation tokens and the existing 250-row query page size. +- Cap each decoded Table page at 4 MiB and the complete Table request at 16 MiB. +- Retain display fields only for objects that survive query selection inside the per-scope item budget. +- Reject ignored row limits, continuation cycles, expired or invalid continuation responses, and page/byte-budget exhaustion without restarting. +- Keep the unbounded dynamic watch bootstrap tracked by #192 out of this slice. + +## Progress + +[G] Repair generic server-table over-materialization tracked by #190. +[S] Limit production changes to the kubeconfig Table transport, query retention contract, call sites, and related evidence. +[A] Added bounded streaming decode, opaque pagination, row/byte budgets, retain-key filtering, and a Table-only error-body guard at the existing reviewed kubeconfig HTTP boundary. +[A] Preserved HTTP status classification while replacing untrusted non-2xx bodies with constant-size synthetic Kubernetes Status responses. +[T] Added adversarial unit cases for opaque multi-page tokens, selected-row retention, ignored limits, continuation cycles, expired and invalid continuations, body/token non-disclosure, and oversized pages. +[T] Expanded the real two-cluster kind gate with 260 ConfigMaps and proved a second-page object retains the API server's Data column. + +## Verification + +- Focused kubeconfig and privacy race suites passed; targeted golangci-lint returned zero findings. +- `make ci` passed formatting, vet, lint with zero findings, vulnerability scanning with no findings, the full race suite, privacy/safety contracts, performance, binary E2E, and build. +- `make e2e-isolation` passed digest-pinned PostgreSQL RLS and both 50,000-execution workspace-isolation fuzzers. +- `make e2e-kind` passed the real two-cluster pagination and immutable OCI image contracts in 210.245 seconds on the final tree. +- `make release-check` passed module verification, reproducible GoReleaser snapshots, SPDX SBOM validation, Homebrew formula generation, and the multi-platform OCI layout. +- Secret-pattern preflight passed. +- CodeRabbit identified two valid test/status-semantic findings; both were repaired, and the exact final tree returned zero findings across all nine files then in scope. +- Hosted PR and exact post-merge `dev` gates remain pending. + +## Checkpoint + +- `2026-07-16/table-materialization#1` + +## Open questions + +- None. The watch bootstrap's dynamic List remains separately tracked by #192. diff --git a/sessions/2026-07-16-watch-bootstrap.md b/sessions/2026-07-16-watch-bootstrap.md new file mode 100644 index 0000000..3f652d1 --- /dev/null +++ b/sessions/2026-07-16-watch-bootstrap.md @@ -0,0 +1,67 @@ +# Bounded watch bootstrap snapshots + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/fix/watch-bootstrap` +- Issue: `#192` +- Status: ready for review + +## Goal + +Prevent kubeconfig list-watch hydration from materializing an unbounded collection before opening +each per-scope, per-kind watch stream. + +## Scope + +- Page the dynamic List at 250 objects under one absolute request deadline. +- Accept only complete snapshots within 10,000 objects and 128 pages per scope and kind. +- Preserve opaque continuation tokens and require one stable, nonempty resource version across pages. +- Emit `WatchError` and open no stream when limits, continuation, cancellation, or snapshot + consistency checks fail. +- Keep the already-landed generic Kubernetes Table transport bound from #190 unchanged. + +## Progress + +[G] Repair unbounded watch bootstrap materialization tracked by #192. +[S] Limit production changes to kubeconfig watch bootstrap loading plus operator and roadmap evidence. +[A] Added complete, consistent paginated bootstrap loading and opened each watch from the verified +snapshot resource version only after all pages succeed. +[A] Rejected empty or changed resource versions, nil responses, ignored limits, continuation cycles, +expired/failed continuations, cancellation, and object/page budget exhaustion without retaining +partial objects. +[A] Sanitized continuation failures so opaque tokens and remote response text never enter emitted +errors. +[T] Added direct pagination, budget, consistency, cancellation, non-disclosure, and fail-closed +adapter regressions under the race detector. +[T] Extended the real two-cluster kind gate with a direct ConfigMap watch and proved +`sith-table-page-259` appears in the completed snapshot before the stream opens. +[R] Red-team review found two client-go fake constraints: root fake list actions discard supplied +pagination options and the object tracker accepts numeric watch resource versions only. Tests now +record options before that fake boundary and use the tracker's real numeric RV while pure unit tests +retain opaque token and RV coverage. + +## Verification + +- The complete kubeconfig race suite passed, plus 100 repeated race runs of the pagination, + cancellation, non-disclosure, and fail-closed watch tests. +- Secret-pattern preflight found no credential literals or private-key material in the changed tree. +- CodeRabbit returned no actionable findings on the implementation diff; strict manual red-team + review added nil-response and generic continuation non-disclosure coverage. +- `make ci` passed formatting, vet, lint with zero findings, `govulncheck` with no vulnerabilities, + full repository race/coverage tests, policy scripts, tagged E2E, performance, and build on the + rebased final tree. +- `make e2e-isolation` passed digest-pinned PostgreSQL RLS and 100,020 final workspace-isolation + fuzz executions. +- `make e2e-kind` passed the combined real-cluster fanout/watch pagination, immutable OCI, and Argo + projection contracts in 236.712 seconds on the rebased final tree. +- `make release-check` passed reproducible multi-platform archives, SPDX SBOM verification, + Homebrew generation, and the multi-architecture distroless OCI layout. +- Hosted PR and exact post-merge `dev` gates remain pending. + +## Checkpoint + +- `2026-07-16/watch-bootstrap#1` + +## Open questions + +- None. diff --git a/sessions/2026-07-17-e10-database-readiness.md b/sessions/2026-07-17-e10-database-readiness.md new file mode 100644 index 0000000..5868456 --- /dev/null +++ b/sessions/2026-07-17-e10-database-readiness.md @@ -0,0 +1,73 @@ +# Session — 2026-07-17 — E10 database-aware Hub readiness + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-database-readiness` +**Slice:** E10 F10.5a / [#244](https://github.com/ArdurAI/sith/issues/244) · **Status:** ready for review +**Decision record:** [Notion](https://app.notion.com/p/3a12637edb0781cea875d0e5aba58aa2) + +## [G] Goal + +Replace the Hub chart's TCP-only readiness claim with a bounded application-level decision that +detects PostgreSQL loss without turning a dependency outage into a restart storm. + +## [D] Design + +- `GET /healthz` is dependency-free process liveness on the existing TLS listener. +- `GET /readyz` calls only the existing least-privilege application pool's `Ping` under a + server-owned one-second deadline. +- Both routes are unauthenticated because kubelet probes have no Sith identity, but they accept + only an exact GET with no query or encoded-path variation and emit only fixed empty responses. +- Database errors, endpoints, credentials, tenant state, and panics collapse to one empty `503`. +- OCM and spoke reachability are excluded: partial reachability belongs in fleet coverage and must + not make the whole Hub unavailable. +- The chart uses HTTPS `httpGet` probes on the named `https` port with explicit timeout, success, + and failure thresholds. It adds no listener, Service, secret, metric, or cloud resource. + +## [R] Research and trade-offs + +- Kubernetes distinguishes readiness, which removes a Pod from Service traffic, from liveness, + which restarts it. The current primary guide also documents HTTPS probes and minimal dedicated + endpoints: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +- pgxpool `Ping` acquires a connection and executes an empty SQL statement, which directly tests + the existing pool without crossing a tenant query boundary: + https://pkg.go.dev/github.com/jackc/pgx/v5/pgxpool#Pool.Ping +- Coupling PostgreSQL to liveness was rejected because a shared database outage could restart every + Hub replica. A separate plaintext probe listener was rejected because Kubernetes supports HTTPS + probes and the Hub already owns the required TLS listener. + +## [S] Scope boundary + +No OCM call, spoke call, tenant query, RLS scope, error body, dependency label, listener, Service, +Ingress, exporter, credential, database topology change, or cloud resource is added. + +## [T] Verification plan + +- Focused race tests for `internal/hubserver`, `internal/hubruntime`, and `internal/hubdb`. +- Helm v4.2.3 lint/render contract for both profiles and optional browser/metrics combinations. +- Full `make ci`, forced PostgreSQL/RLS isolation, release checks, standalone OCI/Helm gates, and + the digest-pinned Kubernetes v1.36.1 two-cluster Kind test. +- Independent secret/diff review and CodeRabbit review before merge; exact post-merge `dev` CI, + CodeQL, and GitHub security queues before issue closure. + +## [V] Local verification + +- Focused race tests pass for `internal/hubserver`, `internal/hubruntime`, `internal/hubdb`, and + the production privacy boundary. Probe fixtures cover ready, dependency error, timeout, caller + cancellation, panic, wrong method, query, trailing path, encoded path, nil construction, and + authenticated fleet fallback isolation. +- Official Helm v4.2.3 lint/render validation passes for light/heavy, browser OIDC, loopback + metrics, and combined browser-plus-metrics modes. The rendered contract requires HTTPS + `/healthz` and `/readyz`, the named `https` port, explicit one-second timeouts, and no TCP probe. +- Full `make ci` passes with zero lint findings, no reachable vulnerabilities, all race tests, + privacy boundaries, shell policies, Prometheus rules, warm-cache performance, binary smoke, + normal e2e, and build. +- Forced PostgreSQL/RLS isolation passes with 75.1% hub-db coverage; both cross-workspace fuzz + campaigns complete 50,000 executions with no new failure. +- The digest-pinned Kubernetes v1.36.1 two-cluster gate passes in 240.5 seconds. Standalone + multi-architecture OCI validation also passes. +- The aggregate `make release-check` wrapper reproduces the known machine-local `go mod verify` + module-discovery anomaly. Every substantive stage passes in isolated invocations: GoReleaser + config, two clean four-platform snapshots, archive/SPDX SBOM validation, Homebrew formula, + release-derived linux/amd64+linux/arm64 OCI layout, and exact digest equality. +- CodeRabbit's complete independent review reports no findings across all thirteen changed files. + +Hosted PR checks, review, and exact post-merge `dev` proof remain pending. diff --git a/sessions/2026-07-17-e10-fleet-read-coverage-alert.md b/sessions/2026-07-17-e10-fleet-read-coverage-alert.md new file mode 100644 index 0000000..988d96f --- /dev/null +++ b/sessions/2026-07-17-e10-fleet-read-coverage-alert.md @@ -0,0 +1,91 @@ +# Session — 2026-07-17 — E10 fleet-read coverage alert + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-fleet-read-alert` +**Slice:** E10 F10.4b / [#242](https://github.com/ArdurAI/sith/issues/242) · **Status:** ready for review + +## [G] Goal + +Turn the bounded F10.1d fleet-read outcome counter into one actionable aggregate warning without +claiming snapshot freshness, selecting an SLO target, or adding tenant-proportional observability. + +## [D] Design + +- The numerator is `degraded|error`; the eligible denominator is `complete|degraded|error`. +- `empty` is excluded from both sides because a consistent zero-scope read is legitimate and must + neither fire nor dilute the alert. +- The warning requires more than 5% adverse outcomes among at least 20 eligible + `complete|degraded|error` reads over 15 minutes, sustained for 10 minutes. +- A top-level aggregate removes the closed outcome and arbitrary scrape labels before alert + creation. Output labels and annotations are fixed strings. +- The existing operator-owned loopback scrape, rule evaluator, Alertmanager, receiver, and + metamonitoring boundaries remain unchanged. + +## [R] Research and trade-offs + +- Prometheus recommends a small number of symptom-oriented, actionable alerts with slack for small + blips: https://prometheus.io/docs/practices/alerting/ +- Google SRE treats an SLO target as a stakeholder/product decision and calls out low-traffic error + budget alerting as a special case: https://sre.google/workbook/alerting-on-slos/ +- A warning symptom alert with a minimum-volume guard was selected instead of a page or formal burn + alert. There is no negotiated target, traffic baseline, or response owner for a defensible SLO. +- Treating `empty` as an error was rejected because an internally consistent zero-scope workspace + is valid. Including it only in the denominator was rejected because it could hide degradation. + +## [S] Scope boundary + +No workspace, tenant, spoke, resource, selector, principal, trace, endpoint, age, or raw-error label +is added. There is no Service, ServiceMonitor, PrometheusRule CRD, exporter, remote-write path, +receiver, dashboard, freshness guarantee, dispatch/PDP signal, SLO target, or error budget. + +## [T] Verification plan + +- Promtool fixtures for sustained mixed degradation/error, legitimate empty reads, absent series, + low traffic, exact 5%, exact minimum volume, transient recovery, resolution, counter reset, and + hostile source labels. +- Repository contract tests for rule count/limit, canonical expression, fixed output labels, static + annotations, runbook anchor, and forbidden dynamic/CRD patterns. +- Full CI/race, vulnerability, forced RLS isolation, release/SBOM reproducibility, Helm/OCI, real + two-cluster Kind, independent review, hosted CI/CodeQL, security queues, and exact post-merge + `dev` proof before closeout. + +## [O] Security, operability, and cost + +The blast radius is one additional aggregate warning. Four existing counter series feed one short +expression per minute and at most one new alert instance. There is no cloud resource, spoke egress, +listener, storage, or tenant-proportional cost. + +## [K] Knowledge checkpoint + +Notion decision record: +https://app.notion.com/p/3a12637edb0781f2b206fd00d891ba31 + +## [V] Local verification + +Local verification is complete on base `be442ff44630ac4ba80a3ebc7869a6ec591d5ffa`; hosted PR checks +and exact post-merge `dev` proof remain pending until the reviewed commit is pushed. + +- `make test-alert-rules PROMTOOL=/tmp/sith-prometheus-3.13.1/promtool` validates four rules and all + fixtures. The new fixtures cover + mixed degradation/error, hostile input labels, high empty traffic, empty-only and absent series, + low eligible volume, exact 5%, exact 20, transient recovery, resolution, and counter reset. +- `make ci` passes with pinned golangci-lint v2.12.2, govulncheck v1.6.0, and promtool v3.13.1: + formatting, vet, zero lint findings, no reachable vulnerabilities, race/coverage, shell policies, + performance, binary e2e, OCI instruction contract, and build are green. Observability coverage is + 92.5%. +- `make e2e-isolation` passes with 75.2% `hubdb` coverage. Both cross-tenant fuzz campaigns complete + exactly 50,000 executions; the mutation campaign found two additional interesting corpus inputs + and remained green. +- The live official Helm v4.2.3 darwin-arm64 archive matches SHA-256 + `048ecf5ad3160f83d918f9fe945238d2132b079640f7b106175331c25f242c64`; the fail-closed Helm + contract passes through `make e2e-helm`. `make e2e-oci` passes standalone multi-architecture OCI + validation. +- `make e2e-kind KIND=/tmp/sith-tools/kind` passes with Kind v0.32.0 against the digest-pinned + Kubernetes v1.36.1 node image, covering the fleet, OCI, and Argo two-cluster gate in 240.563 + seconds. +- The aggregate release target reaches the known machine-local `go mod verify` discovery anomaly. + Direct GoReleaser and releasecheck commands pass the substantive gates independently: config, + two clean four-platform snapshots, archive and SPDX SBOM validation, Homebrew formula, + release-derived multi-architecture OCI layout, and exact first/second artifact digest equality. +- CodeRabbit's first complete review identified two documentation issues: shared ratio-alert volume + terminology and an ambiguous local-versus-hosted verification status. Both were corrected; the + second complete review reports zero findings across all eight changed files. diff --git a/sessions/2026-07-17-e10-fleet-read-outcomes.md b/sessions/2026-07-17-e10-fleet-read-outcomes.md new file mode 100644 index 0000000..3f9ff15 --- /dev/null +++ b/sessions/2026-07-17-e10-fleet-read-outcomes.md @@ -0,0 +1,85 @@ +# Session — 2026-07-17 — E10 bounded fleet-read outcomes + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-fleet-read-outcomes` +**Slice:** E10 F10.1d / [#240](https://github.com/ArdurAI/sith/issues/240) · **Status:** ready for review + +## [G] Goal + +Create the smallest production signal needed to classify authorized Hub fleet-read coverage +outcomes, without retaining tenant, spoke, resource, selector, or raw-error dimensions and without +claiming freshness or a complete F10.4 SLO. + +## [D] Design + +- `hubfleet.Source` emits one result only after PEP authorization and the tenant-scoped reader call. +- The closed outcomes are `complete`, `degraded`, `empty`, and `error`. Coverage assessment and the + result/count consistency check run before the empty check, so malformed zero-request coverage + fails closed as `degraded`. +- A reader error remains wrapped with its original cause and emits only `error`. +- The optional observer is panic-isolated; metrics cannot change authorization, database behavior, + response data, or availability. +- `observability.Metrics` preinitializes exactly four aggregate counter series and the production + runtime injects the same observer into API and console fleet reads. + +## [S] Scope boundary + +No workspace, spoke, resource, selector, route, principal, trace, snapshot age, or raw error becomes +a label. There is no new listener, Service, ServiceMonitor, PrometheusRule, exporter, remote-write +path, receiver, SLO target, error budget, or alert. Dispatch and PDP signals remain future work. + +## [T] Verification plan + +- Unit fixtures for complete, stale/incomplete, malformed, empty, and reader-error results. +- Explicit tests that policy refusal emits nothing and observer panic cannot affect success or the + original reader error. +- Prometheus exposition tests for exact preinitialized series, invalid-value normalization, label + allowlisting, independent registries, and rollback after a late duplicate registration. +- API, console, and real two-cluster runtime wiring tests. +- Full CI/race, vulnerability, forced PostgreSQL RLS/isolation, release/SBOM reproducibility, + Helm/OCI, and real two-cluster gates before review and merge. + +## [O] Security, operability, and cost + +The blast radius is a process-local counter increment after an authorized read. Cardinality is +constant at four series, so scrape and storage cost do not grow with tenants or spokes. The signal +cannot authorize, route, dispatch, or identify a tenant, and the existing loopback-only scrape +boundary remains unchanged. + +## [R] Research and trade-offs + +- Prometheus metric naming guidance requires one logical quantity per metric and a `_total` suffix + for accumulating counts: https://prometheus.io/docs/practices/naming/ +- Prometheus instrumentation guidance warns that every labelset consumes RAM, CPU, disk, and network + and recommends keeping most metrics unlabeled or below ten series: + https://prometheus.io/docs/practices/instrumentation/ +- One closed `outcome` label was selected instead of per-gap, tenant, or spoke dimensions. It keeps + the series count at four while preserving a meaningful aggregate coverage-outcome ratio. +- `complete` means the existing coverage contract reported no gaps; it is not a snapshot-age + guarantee. Snapshot ages were rejected for this slice because a histogram would require a reviewed + bucket and sampling contract. The existing tenant-scoped response remains the diagnostic source + for exact stale and unreachable scopes. + +## [V] Local verification + +- Focused and repository-wide race suites pass. Full `make ci` passes with pinned golangci-lint + v2.12.2, govulncheck v1.6.0, and Prometheus promtool v3.13.1: formatting, vet, zero lint findings, + no reachable vulnerabilities, coverage, shell policies, portable alert tests, performance, + binary e2e, and build are green. Observability coverage is 92.5%. +- Forced PostgreSQL RLS/isolation passes with 75.2% `hubdb` coverage. Both cross-tenant fuzz campaigns + complete 50,000 executions; the mutation campaign found one additional interesting corpus input + and remained green. +- The official Helm v4.2.3 darwin-arm64 archive checksum was verified before the Helm contract ran. + Standalone multi-architecture OCI validation passes. +- Kind v0.32.0 against the digest-pinned Kubernetes v1.36.1 node image passes the full fleet, OCI, + and Argo two-cluster gate in 237.454 seconds. +- The aggregate release target reaches the known machine-local `go mod verify` discovery anomaly. + Its substantive gates pass independently: GoReleaser configuration, two reproducible four-platform + snapshots, archive and SPDX SBOM validation, Homebrew formula, multi-architecture release OCI + layout, and identical first/second artifact digests. +- Independent CodeRabbit review first reported zero code findings, then identified one documentation + ambiguity that conflated complete coverage with freshness. The wording now explicitly rejects that + guarantee; final review reports zero findings across all 14 changed files. Manual red-team review + added a result/count consistency guard so corrupt cross-field coverage cannot inflate the complete + or empty outcomes. +- Notion decision and validation checkpoint: + https://app.notion.com/p/3a12637edb0781c7b611da38e74f6e30 diff --git a/sessions/2026-07-17-e10-policy-audit-metrics.md b/sessions/2026-07-17-e10-policy-audit-metrics.md new file mode 100644 index 0000000..ed765ef --- /dev/null +++ b/sessions/2026-07-17-e10-policy-audit-metrics.md @@ -0,0 +1,72 @@ +# Session — 2026-07-17 — E10 policy-audit metrics + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/policy-audit-metrics` +**Slice:** E10 F10.1c / [#234](https://github.com/ArdurAI/sith/issues/234) · **Status:** ready for review + +## [G] Goal + +Expose the fail-closed policy-audit sinks through Sith's existing isolated, loopback-only +Prometheus registry so operators can distinguish database and process-log failures and latency +without identity-bearing or caller-controlled labels. + +## [D] Design + +- `pep` owns a typed passive audit-observer contract and an auditor decorator. The decorator calls + the authoritative sink first, reports its exact success/error result afterward, and isolates an + observer panic without changing the returned audit error. +- The runtime decorates the durable database and structured process-log sinks separately before + passing them to the existing database-first ordered auditor. A durable failure therefore emits no + false process attempt. +- `observability.Metrics` implements the contract with two fixed metric families and only the + closed `durable|process` and `success|error` vocabularies. Invalid values are discarded. +- The registry remains non-global and pull-only. No listener, Service, ingress, exporter, remote + write, persistence, tenant label, or external-system telemetry is added. + +## [R] Research and trade-offs + +- Prometheus recommends application-prefixed names, base-unit suffixes, and avoiding high-cardinality + labels: https://prometheus.io/docs/practices/naming/ +- Prometheus instrumentation guidance treats cardinality as a primary operating cost: + https://prometheus.io/docs/practices/instrumentation/ +- Histograms preserve aggregatable latency distributions; Sith retains the repository's existing + classic-histogram convention: https://prometheus.io/docs/practices/histograms/ +- The bounded cost is four counter label combinations plus four fixed histogram combinations. No + tenant-proportional series or new infrastructure cost is introduced. + +## [T] Planned evidence + +- Unit and race tests for success, error preservation, invalid configuration, and observer panic. +- Ordered integration tests proving durable failure suppresses the process attempt and process + failure follows durable success. +- A real scrape and label allowlist proving hostile values do not create series or leak data. +- Full CI, forced PostgreSQL RLS isolation, vulnerability scan, reproducible release/SBOM, Helm, + and real two-cluster fan-out gates before review and merge. + +## [V] Local validation + +- Focused race tests and `go vet` pass for `internal/pep`, `internal/observability`, and + `internal/hubruntime`. +- Full `make ci` passes with the repository-pinned golangci-lint v2.12.2: zero lint issues, no + vulnerabilities, the complete race suite, script policy checks, performance budget, binary e2e, + and build. +- `make e2e-isolation` passes the PostgreSQL forced-RLS suite at 75.2% `hubdb` coverage and both + fixed 50,000-case cross-workspace fuzz campaigns. +- The canonical local release target reaches the known machine-wide `go mod verify` discovery + anomaly even though `go env GOMOD` resolves this checkout. Its substantive steps pass + independently: two four-platform GoReleaser snapshots, distribution and SPDX SBOM verification, + Homebrew formula, multi-architecture OCI layout, and identical artifact digests. Clean hosted CI + remains the authoritative module-verification gate. +- Helm v4.2.3 was downloaded from `get.helm.sh` and verified against its official checksum; the + pinned Helm chart contract and cross-platform OCI contract pass. The real two-cluster Kind fleet, + OCI, and Argo projection suite passes in 238.583 seconds. + +## [C] Review checkpoint + +- Manual red-team review confirmed that the observer receives no event contents, error value, + tenant identity, or request metadata; invalid typed values create no series; and a panic cannot + replace the authoritative sink result. +- The runtime wraps each sink before the existing ordered composition, so durable failure still + prevents process delivery and its metric, while process failure remains distinguishable after a + successful durable append. +- CodeRabbit reviewed all eight implementation and validation files in the uncommitted diff and + reported zero findings. diff --git a/sessions/2026-07-17-e10-portable-alert-rules.md b/sessions/2026-07-17-e10-portable-alert-rules.md new file mode 100644 index 0000000..1b62f01 --- /dev/null +++ b/sessions/2026-07-17-e10-portable-alert-rules.md @@ -0,0 +1,72 @@ +# Session — 2026-07-17 — E10 portable hub alert rules + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-portable-alert-rules` +**Slice:** E10 F10.4a / [#238](https://github.com/ArdurAI/sith/issues/238) · **Status:** ready for review + +## [G] Goal + +Turn the Hub's existing fixed-cardinality self-observability metrics into tested, actionable alerts +without widening the loopback scrape boundary or pretending that missing dispatch/PDP/freshness +signals already satisfy the full F10.4 SLO contract. + +## [D] Design + +- A native Prometheus rule file remains independent of Helm and the Prometheus Operator. Sith adds + no Service, ServiceMonitor, PrometheusRule CRD, exporter, remote write, or notification receiver. +- Every expression uses a top-level aggregate, so arbitrary scrape/remote-write labels cannot fan + out or leak into the resulting alert. Alert labels and annotations are fixed strings only. +- Policy-audit failures are critical because they fail governed reads closed. Authentication log + drops and sustained aggregate snapshot failures are warnings with explicit hold and volume guards. +- Missing series remain an external metamonitoring concern rather than being misreported as healthy + or unhealthy by a rule that cannot distinguish disabled metrics from a broken scrape. + +## [R] Research and trade-offs + +- Prometheus recommends few, symptom-oriented, actionable alerts with slack for small blips: + https://prometheus.io/docs/practices/alerting/ +- Prometheus documents native rule validation and deterministic unit fixtures through `promtool`: + https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/ and + https://prometheus.io/docs/prometheus/latest/configuration/unit_testing_rules/ +- The latest verified upstream release on 2026-07-17 is Prometheus v3.13.1, published 2026-07-10; + CI pins its official Linux amd64 archive checksum. +- A chart-rendered PrometheusRule was rejected because Sith intentionally has no remotely scrapeable + Service or ServiceMonitor. Rendering a rule CR without a data path would be a misleading partial + integration and would add an optional-CRD admission dependency. + +## [T] Validation plan + +- `promtool check rules` plus fixture-driven alert tests for thresholds, hold windows, minimum + volume, missing series, counter resets, and hostile input labels. +- A Go contract test locks the rule/metric allowlist, fixed labels, static annotations, aggregation, + group cardinality limit, and runbook links. +- Full CI/race, vulnerability, forced PostgreSQL isolation, release/SBOM reproducibility, Helm/OCI, + and real two-cluster Kind gates before review and merge. + +## [O] Security, operability, and cost + +The evaluator runs three short-window expressions once per minute and can emit at most three alert +instances. No existing metric series, runtime listener, network path, or storage contract changes. +Prometheus/Alertmanager capacity, external labels, notification routing, and metamonitoring remain +operator-owned. + +## [V] Local verification + +- Official Prometheus v3.13.1 `promtool check rules` and fixture tests pass. Fixtures cover exact + 5% and exact 20-attempt boundaries, hold windows, minimum volume, missing series, counter reset, + and hostile source labels that must not reach alerts. +- Full `make ci` passes with the pinned golangci-lint v2.12.2 and govulncheck v1.6.0: formatting, + vet, zero lint findings, no reachable vulnerabilities, full race/coverage, shell policies, + performance, binary e2e, and build are green. Observability coverage is 93.1%. +- Forced PostgreSQL RLS/isolation passes with 75.2% `hubdb` coverage; both cross-tenant fuzz + campaigns complete 50,000+ executions. +- Helm v4.2.3 and the standalone multi-architecture OCI contract pass. Kind v0.32.0 against the + digest-pinned Kubernetes v1.36.1 node passes the complete fleet/OCI/Argo suite in 236.537 seconds. +- The aggregate release target reaches the known machine-local `go mod verify` discovery anomaly. + Its substantive gates pass independently: two reproducible four-platform snapshots, archive and + SPDX SBOM validation, Homebrew formula, multi-architecture OCI layout, and identical digests. +- CodeRabbit's first complete review produced four applicable test-contract findings: exact + comparator fixtures, canonical expression locking, complete output-label allowlisting, and + relative `PROMTOOL` path handling. All were fixed; the second complete review reports zero + findings. The GitHub-side linter then identified Bash `! grep`/`errexit` behavior in two negative + policy checks; both now use explicit fail-closed branches and pass ShellCheck. +- Notion checkpoint: https://app.notion.com/p/3a02637edb078115a74ac6cd5deb61f9 diff --git a/sessions/2026-07-17-e10-readiness-alert.md b/sessions/2026-07-17-e10-readiness-alert.md new file mode 100644 index 0000000..35923b2 --- /dev/null +++ b/sessions/2026-07-17-e10-readiness-alert.md @@ -0,0 +1,54 @@ +# Session — 2026-07-17 — E10 Hub database-readiness alert + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-readiness-alert` +**Slice:** E10 F10.4c / [#248](https://github.com/ArdurAI/sith/issues/248) · **Status:** ready for commit +**Decision record:** [Notion](https://app.notion.com/p/3a12637edb0781349e9fd9a3487e61c2) + +## [G] Goal + +Turn sustained database-backed Hub readiness degradation into one actionable portable warning without +inventing a freshness SLO, creating monitoring infrastructure, or widening metric cardinality. + +## [D] Design + +- Aggregate `unavailable` over all completed `ready|unavailable` checks in a 15-minute window. +- Require more than 5% failures and at least 20 completed checks, continuously for 10 minutes. +- Aggregate away every source label and emit only static `component` and `severity` labels. +- Keep notification routing and the scrape/rule pipeline operator-owned; this is a warning symptom, + not an error-budget page or metamonitoring substitute. + +## [S] Security, privacy, and cost boundary + +No tenant, workspace, spoke, actor, request, endpoint, credential, error, panic, or dependency label +is emitted. The slice adds one aggregate alert vector over existing counters and no listener, Service, +ServiceMonitor, PrometheusRule, Alertmanager, receiver, exporter, remote write, persistence, cloud +resource, or spoke egress. + +## [T] Verification plan + +- `promtool check rules` plus fixtures for missing, all-ready, low-volume, exact-threshold, inclusive + volume, transient, sustained, recovery, hostile-label aggregation, and counter-reset behavior. +- Static Go/shell contracts for exact expression, labels, annotations, group bound, and pinned tool. +- Full CI, forced PostgreSQL/RLS isolation, reproducible release/SBOM, standalone OCI and pinned Helm, + real two-cluster Kind, CodeRabbit, CodeQL, security queues, and exact post-merge `dev` proof. + +## [R] Primary references + +- [Prometheus alerting best practices](https://prometheus.io/docs/practices/alerting/) +- [Prometheus alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) + +## [V] Current evidence + +- Pinned Prometheus 3.13.1 parses all five rules; fixtures prove missing/all-ready/low-volume data, + the exact threshold, inclusive volume, transient failure, sustained firing after the hold, recovery, + hostile-label aggregation, and counter-reset behavior. +- The focused race suite and full `make ci` pass formatting, vet, zero-finding lint, vulnerability + scanning with no reachable vulnerabilities, all race tests, policy scripts, Prometheus fixtures, + performance, binary smoke, and build. `internal/observability` coverage remains 93.1%. +- Forced PostgreSQL/RLS isolation passes with `hubdb` coverage at 75.1%; both 50,000-case + cross-workspace fuzz campaigns pass. +- Reproducible release verification passes twice for four platforms, including SPDX SBOMs, + distribution verification, Homebrew formula, and the release-derived multi-architecture OCI layout. +- Standalone OCI and pinned Helm v4.2.3 contracts pass. +- The digest-pinned Kubernetes v1.36.1 real two-cluster Kind gate passes in 236.012 seconds. +- CodeRabbit CLI v0.6.5 reviews all nine changed files with no findings. diff --git a/sessions/2026-07-17-e10-readiness-metrics.md b/sessions/2026-07-17-e10-readiness-metrics.md new file mode 100644 index 0000000..d7d0d10 --- /dev/null +++ b/sessions/2026-07-17-e10-readiness-metrics.md @@ -0,0 +1,64 @@ +# Session — 2026-07-17 — E10 Hub database readiness metrics + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-readiness-metrics` +**Slice:** E10 F10.1e / [#246](https://github.com/ArdurAI/sith/issues/246) · **Status:** ready for commit +**Decision record:** [Notion](https://app.notion.com/p/3a12637edb0781baa05dcddc674bf9bb) + +## [G] Goal + +Expose bounded attempts, failures, and latency for the Hub's existing database-aware readiness +check without changing the probe response, widening the scrape boundary, or creating +tenant-proportional metrics. + +## [D] Design + +- A probe-owned observer receives exactly one completed valid readiness check as `ready` or + `unavailable` plus duration. +- Database error, server deadline, caller cancellation, and recovered checker panic all collapse to + `unavailable`. Rejected request variants never reach the checker or observer. +- The existing isolated registry preinitializes two counter series and two matching histogram + outcomes. Invalid observation values are discarded. +- Observer panic is recovered at the instrumentation seam and cannot change the HTTP status, empty + body, database timeout, or checker call count. +- Production injects the already-owned registry; no new metrics listener or dependency is created. + +## [S] Security, privacy, and cost boundary + +No tenant, workspace, spoke, actor, request, endpoint, credential, error, panic, or dependency-class +label is emitted. The slice adds no Service, ServiceMonitor, PrometheusRule, exporter, queue, +persistence, remote write, cloud resource, or spoke egress. Cost is limited to two fixed counter +series and histogram buckets for two outcomes per Hub process plus bounded local metric work on the +existing readiness Ping. + +## [T] Verification plan + +- Focused race tests for probe completion, error, timeout, cancellation, checker panic, observer + panic, invalid requests, exposition, label allowlists, independent registries, and registration + rollback. +- Full `make ci`, forced PostgreSQL/RLS isolation, release/SBOM reproducibility, standalone Helm and + OCI contracts, and the digest-pinned real two-cluster Kind gate. +- Independent diff/secret review and CodeRabbit review before signed DCO/GSTACK commit; exact hosted + CI, CodeQL, security queues, and post-merge `dev` proof before issue closure. + +## [V] Current evidence + +- Focused race tests pass for `internal/hubserver`, `internal/observability`, `internal/hubruntime`, + and the production privacy boundary. Fixtures cover success, dependency error, timeout, caller + cancellation, checker panic, observer panic, invalid requests, exact call/observation counts, + preinitialized series, invalid-value suppression, label allowlists, independent registries, and + registration rollback. +- Full `make ci` passes formatting, vet, zero-finding lint, vulnerability scanning with no reachable + vulnerabilities, the complete race suite, shell policies, Prometheus rule fixtures, performance, + binary e2e, and build. `internal/observability` coverage is 93.1%. +- Forced PostgreSQL/RLS isolation passes with `hubdb` coverage at 75.1%; both 50,000-case + cross-workspace fuzz campaigns pass. +- `make release-check` passes after command-scoping `GOPATH` to the configured module-cache root: + module integrity, GoReleaser configuration, two reproducible four-platform snapshots, SPDX SBOM, + distribution verification, Homebrew formula, and release-derived multi-arch OCI layout are green. + No global Go setting was changed. +- Standalone OCI passes. The Helm contract correctly rejects machine-default Helm v4.1.4 and passes + with the repository-pinned v4.2.3 binary. +- The digest-pinned Kubernetes v1.36.1 two-cluster Kind gate passes in 235.949 seconds. +- CodeRabbit CLI v0.6.5 completes a 10-file uncommitted-diff review with no findings. +- Primary design guidance: [Prometheus instrumentation](https://prometheus.io/docs/practices/instrumentation/) + and [metric/label naming](https://prometheus.io/docs/practices/naming/). diff --git a/sessions/2026-07-17-e5-single-use-approvals.md b/sessions/2026-07-17-e5-single-use-approvals.md new file mode 100644 index 0000000..f3169b6 --- /dev/null +++ b/sessions/2026-07-17-e5-single-use-approvals.md @@ -0,0 +1,63 @@ +# E5 F5.9a — exact single-use approval grants + +[G] Goal: implement GitHub issue #250 as the durable, execution-free core beneath F5.9 approval +elicitation. Keep all repository, worktree, test-cache, and durable knowledge artifacts on +`/Volumes/EXTENDED`; checkpoint the decision and evidence in both Notion and Obsidian. + +[T] Research: MCP 2025-06-18 Elicitation defines `elicitation/create`, structured flat primitive +schemas, accept/decline/cancel outcomes, server/client validation, and the prohibition on requesting +sensitive information. PostgreSQL 18 documents that conditional `UPDATE ... RETURNING` can make +the one-row consumption result authoritative, FORCE RLS applies the workspace policy to the table +owner, and `FOR SHARE` blocks concurrent `UPDATE`/`DELETE` of current membership rows while a grant +or consume transaction validates them. `FOR KEY SHARE` was deliberately rejected because it does +not block a role-only update. + +[D] Decision: expose a private-field `pep.ApprovalBinding` only from a validated immutable +`ProposalInput`. The binding carries the intent id, workspace, proposer, and complete resolved +proposal digest; it never carries raw arguments, targets, justification, credentials, tokens, or +elicitation content. The database stores that minimal binding with an authenticated distinct +approver and timestamps. + +[D] Decision: preserve the approval row as durable evidence. The least-privilege application role +has `SELECT`, `INSERT`, and column-level `UPDATE (consumed_at)` only. It cannot change workspace, +intent, proposer, approver, digest, or approval time and cannot delete a row. This is distinct from +the append-only E6 policy-audit hash chain because an authorization grant must transition exactly +once while audit history never transitions. + +[A] Action: added migration `0010_approval_grants.sql`, automatic migration privilege repair and +catalog validation, a current-membership and role-checked creation path, and a conditional exact +consume path. The catalog audit requires exactly one complete workspace policy per table; this +rejects additional permissive policies that PostgreSQL would otherwise OR with the intended RLS +predicate. Unknown id, foreign workspace, wrong intent, wrong digest, pre-approval time, stale +membership, and replay all return the same stable `ErrApprovalGrantUnavailable` classification. + +[T] Test: pure Go package tests cover binding provenance, mutation rejection, opaque identifier +generation and entropy failure, schema privacy/RLS contracts, and fuzz the identifier vocabulary. +The digest-pinned PostgreSQL 18.4 integration test covers current role lookup, role refusal, +self-approval refusal, RLS foreign read/write/consume denial, altered and additional permissive +policy negative controls, immutable-column/delete privilege denial, correct consume, +approve-then-swap refusal, replay refusal, and exactly one success from two concurrent consumers. + +[S] Security boundary: no MCP transport, Ardur PDP, expiry policy, multi-approver rule, credential, +connector, signed dispatch, shell/filesystem write, generic execute/apply surface, or production +mutation is added. A future dispatcher must still re-run the full PEP/PDP and exact binding check. + +[C] Cost: one small indexed row and one short scoped transaction per approval. No service, queue, +polling, egress, or cloud resource is introduced. Retention and expiry policy remain explicit later +work rather than an invented default. + +[V] Current evidence: focused pure Go and race tests pass; proposal-binding and approval-id fuzz +campaigns each execute 50,000 inputs; the real PostgreSQL 18.4 isolation suite passes at 75.6% +`hubdb` coverage with two additional 50,000-input repository fuzz campaigns. Full `make ci` passes +with zero lint findings and no reachable vulnerabilities. Reproducible release archives, SPDX +SBOMs, Homebrew formula, two-platform OCI layout, Helm 4.2.3 contract, OCI runtime contract, and a +fresh Kind v0.32.0 / Kubernetes v1.36.1 cluster pass. CodeRabbit's first pass found the additional +permissive-policy audit gap and two minor documentation/test issues; all were corrected, and its +second pass reports no findings. Hosted PR and exact post-merge `dev` evidence remain pending. + +[R] Primary references: +- https://modelcontextprotocol.io/specification/2025-06-18/client/elicitation +- https://www.postgresql.org/docs/current/sql-update.html +- https://www.postgresql.org/docs/current/ddl-rowsecurity.html +- https://www.postgresql.org/docs/current/sql-select.html#SQL-FOR-UPDATE-SHARE +- https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS diff --git a/sessions/2026-07-17-elasticsearch-log-causes.md b/sessions/2026-07-17-elasticsearch-log-causes.md new file mode 100644 index 0000000..7f84745 --- /dev/null +++ b/sessions/2026-07-17-elasticsearch-log-causes.md @@ -0,0 +1,128 @@ +# Elasticsearch bounded log-cause facts + +- Builder: gnanirahulnutakki +- Effort: deep +- Branch: `gnanirahulnutakki/feat/elasticsearch-log-causes` +- Issue: `#214` +- Status: locally verified; awaiting signed PR and exact post-merge proof + +## Goal + +Define Sith's first Elasticsearch normalization contract by projecting one already-authorized, +already-fetched Search API response into bounded TELEMETRY cause facts for R3 without adding an +HTTP client, endpoint/index configuration, credential custody, persistence, or mutation. + +## Scope + +- Accept only the current ECS/Filebeat field profile: `@timestamp`, `message`, + `orchestrator.cluster.name`, `kubernetes.namespace`, `kubernetes.pod.name`, and optional + `kubernetes.container.name`. +- Take workspace, cluster scope, namespace, Pod, optional container, query window, and collection + time only from trusted caller input. When container is supplied, every hit must carry that exact + value. When omitted, the scope is deliberately Pod-wide: hits may carry any container or omit the + field, and the aggregate fact carries no container identity. +- Require every hit timestamp to fall in the inclusive trusted `[WindowStart, WindowEnd]` interval. + The interval duration is capped at fifteen minutes; this is a query-width cap, not a freshness + claim. `ObservedAt` only prevents `WindowEnd` from exceeding collection time by more than the + five-minute clock-skew allowance. A future live reader must query the identical bounds. +- Classify a closed cause taxonomy: `panic`, `missing-config`, or `dependency-failure`. +- Emit at most one aggregate fact per cause with count and first/last event time. +- Discard raw messages, index/document IDs, source documents, unknown fields, labels, URLs, query + text, and user data before graph construction. + +## Decision + +Prove the pure evidence boundary before any live Elasticsearch adapter. The response must be +complete, untimed-out, not early-terminated, and have zero failed shards. `_source`, ignored raw +values, highlights, inner hits, unknown `fields` members, ambiguous arrays, absent cluster identity, +and identity mismatches fail closed. Successful empty or unclassified results return zero facts and +never claim that logs or the wider fleet are clean. + +The ECS `orchestrator.cluster.name` field is mandatory even though Elastic documents that some +Kubernetes deployments do not populate it. In a multi-cluster hub, namespace and Pod names alone +are not a safe join key. Operators must populate the cluster field or Sith abstains; it never guesses. + +## Security, operability, and cost + +- Raw logs can contain credentials, personal data, and internal addresses. Classification happens + in bounded memory, errors never echo field contents, and only the closed derived answer survives. +- The source-level boundary test exact-allowlists every production import and declaration and + rejects injected interfaces plus network, filesystem, process, database, credential, persistence, + gRPC, client-go, and mutation seams. +- This pure projector creates no infrastructure and no runtime cloud cost. A future live reader must + use TLS, an allowlisted index/data-stream target, finite timeout/window/size budgets, + `allow_partial_search_results=false`, and index `read` only. Wide wildcards and deep pagination + would add data-exposure and Elasticsearch CPU/memory cost and remain out of scope. + +## Progress and verification + +[G] Normalize one bounded Elasticsearch log search into R3 TELEMETRY facts for #214. +[S] Pure ECS response projection only; live querying, auth, index discovery, pagination/PIT/scroll, +raw-log retention, negative evidence, and out-of-process framework work remain out of scope. +[A] Verified the E12/R3 contract against repository specs and current official Elastic Search API, +selected-fields, ECS, Kubernetes-field, and privilege documentation; no duplicate issue or code +existed. Opened #214 and based an isolated worktree on exact `origin/dev` merge `4c1e194`. +[A] Implemented bounded input/response parsing, exact cluster/namespace/Pod/container matching, +closed conservative classification, deterministic aggregation, and sanitized entity-attached facts. +[T] Focused race tests pass with 95.7% statement coverage. Adversarial cases cover secret +non-retention, classifier specificity and false positives, partial/failed shards, `_source`, ignored +and expanded content, unknown fields, malformed types, duplicate/trailing/deep JSON, attacker-sized +numbers, size/count/time budgets, identity confusion, determinism, abstention, and AST boundaries. +[T] Native fuzzing completed 1,329,594 executions with no panic, invalid fact, non-atomic error, +capability escape, or excess fact count. +[T] `make ci` passes formatting, vet, lint with zero findings, vulnerability scanning with no +findings, the full race suite, policy tests, performance, subprocess E2E, and build. +[T] `make e2e-isolation` passes real PostgreSQL RLS plus both 50,000-execution workspace-isolation +fuzzers. `make e2e-kind` passes pinned two-cluster fan-out, OCI, and Argo tests in 236.347 seconds. +[T] `make release-check` passes module verification, two reproducible release builds, archive/SPDX +SBOM validation, Homebrew formula generation, and the multi-platform distroless OCI layout. +[T] CodeRabbit's committed-diff review found two valid fail-closed gaps: present JSON `null` values +could decode as absent values, and the declaration boundary keyed methods only by their bare name. +The projector now rejects every present `null` response field, while the boundary allowlist keys +methods by receiver type and has a regression proving identically named methods remain distinct. +[T] Post-review focused race tests still pass at 95.7% statement coverage, and native fuzzing +completed 4,185,770 executions without a failure. The complete post-review matrix is green: +`make ci`, `make e2e-isolation`, `make e2e-kind` (236.131 seconds), and `make release-check`. +[T] The follow-up committed-diff review found that allowing the standard `io` package for its EOF +sentinel also left `io.Reader` available to future fields or parameters. The boundary now rejects +every `io` selector except `io.EOF`, forbids import aliases, pins the exact public projector +signature, and includes adversarial regressions for reader parameters, reader results, interface +inputs, receiver changes, and reader fields. Focused race tests and the full `make ci` gate pass +after this test-only hardening; production behavior is unchanged. +[T] A second follow-up review showed that a one-way declaration-name allowlist could still miss +deletions or structural changes such as a callback added to `Projection`. The boundary is now +bidirectional: the exact production file and declaration sets must exist, the complete +comment-independent production AST must match its reviewed SHA-256 fingerprint, and Projection's +nine value-only fields are independently shape-checked for readable failures. Regressions reject +callback, reader, and extra-response fields. Focused race tests and `make ci` pass after this +test-only change. +[T] The final contract-clarity review asked for exact optional-container and time-window semantics. +`TestProjectLogCausesAcceptsInclusiveWindowAndOptionalContainer` already proves both endpoint +inclusion and Pod-wide matching without a container field; the roadmap and this decision record now +state those rules explicitly for the future live reader. +[T] `README.md` was reviewed in full. No update is warranted because this slice adds no user-facing +command, configuration, authentication flow, endpoint, runtime connector, or supported behavior; +the roadmap and this checkpoint are the correct documentation surfaces. + +Primary compatibility references: + +- +- +- +- +- +- + +## Checkpoint + +- `2026-07-17/elasticsearch-log-causes#1` +- `2026-07-17/elasticsearch-log-causes#2` +- `2026-07-17/elasticsearch-log-causes#3` +- `2026-07-17/elasticsearch-log-causes#4` +- `2026-07-17/elasticsearch-log-causes#5` + +## Open questions + +- Live endpoint/index configuration, authorization, mapping discovery, and query execution remain a + later connector child. That slice must reuse this projector and enforce the documented request + contract rather than introduce another normalization path. diff --git a/sessions/2026-07-17-hub-cve-evidence-console.md b/sessions/2026-07-17-hub-cve-evidence-console.md new file mode 100644 index 0000000..8283d19 --- /dev/null +++ b/sessions/2026-07-17-hub-cve-evidence-console.md @@ -0,0 +1,65 @@ +# Session — 2026-07-17 — E8 runtime CVE evidence console + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/console-cve-evidence` +**Slice:** E8 F8.1e / [#236](https://github.com/ArdurAI/sith/issues/236) · **Status:** ready for review + +## [G] Goal + +Let an authenticated Hub operator ask where one exact CVE was observed in immutable runtime images +without exposing bearer credentials, raw scanner payloads, a refresh path, or a second query engine. + +## [D] Design + +- The cookie-authenticated route reuses `hubfleet.CVESearcher.SearchByIdentifier`, including its + tenant scope, dedicated PEP verb, freshness calculation, and persisted database query. +- A fourth purpose-specific HMAC proof is bound to the browser session, workspace, expiry, and CVE + route. Cross-site Fetch Metadata, duplicate proofs/cookies, bearer fallback, foreign workspaces, + and non-canonical queries fail before the PEP or database. +- The route reads 257 facts as a sentinel, renders at most 256, and fails closed on malformed, + duplicate, selector-mismatched, mutable-image, or provenance-bearing stored evidence. +- The browser receives only cluster scope, immutable digest, exact identifier, canonical severity, + observation time, staleness, and named coverage. DOM rendering uses text-only nodes under the + existing no-store, same-origin CSP boundary. + +## [R] Research and trade-offs + +- OWASP recommends server-generated CSRF tokens and Fetch Metadata as complementary defenses: + https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html +- MDN documents `connect-src 'self'` as the browser network boundary for same-origin fetches: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Security-Policy/connect-src +- WCAG requires identified input errors and descriptive labels; the explicit CVE form includes both: + https://www.w3.org/WAI/WCAG22/Understanding/error-identification and + https://www.w3.org/WAI/WCAG22/Techniques/general/G131.html +- Calling the bearer API from JavaScript was rejected because it would put a bearer token inside + the browser surface. Direct database access was rejected because it would duplicate PEP/query + semantics. Automatic scanner refresh was rejected because the console is a persisted-read client. + +## [T] Local validation + +- Focused race tests, `go vet`, JavaScript syntax validation, diff checks, and golangci-lint pass + for `internal/hubserver` and `internal/hubruntime`. +- Adversarial tests cover purpose separation, expiry/replay, cross-site and foreign-workspace + refusal, strict canonical query parsing, generic error responses, projection allowlisting, + duplicate JSON members/facts, selector and digest mismatch, provenance/display rejection, + 257-row overflow, inconsistent coverage, and fuzzed stored CVE observations. +- Full `make ci` passes with zero lint issues, no reachable vulnerabilities, the complete race + suite, shell-policy tests, performance budget, binary e2e, and build. +- PostgreSQL forced RLS passes at 75.2% `hubdb` coverage; both isolation fuzz campaigns pass 50,000 + executions. +- The local release target reaches the known machine-wide `go mod verify` discovery anomaly. Its + substantive steps pass independently: two reproducible four-platform snapshots, SPDX SBOM and + distribution verification, Homebrew formula, multi-architecture OCI layout, and matching digests. +- Helm v4.2.3, standalone multi-architecture OCI, and the real two-cluster Kind fleet/OCI/Argo suite + pass; Kind completed in 236.309 seconds. +- Desktop (1440 px) and mobile (390 px) browser inspection pass. A browser harness also proves a + malformed CVE DTO fails closed and a complete valid DTO renders. +- CodeRabbit's first pass produced four applicable hardening suggestions and one inapplicable + nonexistent-field suggestion. After the fixes, its second full uncommitted-diff review reported + zero findings. + +## [O] Operability and cost + +Each explicit lookup adds one bounded Hub database read. It performs no scanner refresh, spoke +request, connector call, write, or cloud API request, so there is no new infrastructure or egress +cost. Incomplete coverage remains visible and prevents the UI from turning an empty partial result +into a CVE-free claim. diff --git a/sessions/2026-07-17-hub-fleet-console.md b/sessions/2026-07-17-hub-fleet-console.md new file mode 100644 index 0000000..9a1359c --- /dev/null +++ b/sessions/2026-07-17-hub-fleet-console.md @@ -0,0 +1,60 @@ +# Session — 2026-07-17 — hub-fleet-console + +**Builder:** gnanirahulnutakki · **Effort:** deep · **Branch:** +`gnanirahulnutakki/feat/hub-fleet-console` +**Slice(s):** E8 / F8.1b · #218 · **Status:** ready for review + +--- + +[G] Goal: render one tenant-scoped, coverage-honest Hub fleet console for #218 without exposing a +privileged browser path. +[S] Scope: the existing browser OIDC success redirect, a separate cookie/session adapter over the +persisted `hubfleet.Source`, fixed embedded HTML/CSS/JavaScript, production-boundary tests, and +operator documentation. Out: collector refresh, connector calls, local operations, inventory +records, correlation, service selection, proposals, approvals, writes, and generic cookie auth. +[A] Action: mounted exact GET-only page, fleet, and asset routes only when browser OIDC is fully +configured. The callback redirects to the transaction-bound workspace path and accepts no return +URL. The bearer fleet API remains bearer-only. +[A] Action: bound the fleet read to one exact `__Host-sith-session` cookie, signed membership, the +existing PEP, same-origin Fetch Metadata, and a five-minute process-key HMAC proof scoped to the +session, workspace, fleet-read purpose, and expiry. Restart, duplicate credentials, foreign scope, +missing or expired proof, methods, queries, and reader errors fail closed with generic responses. +[A] Action: built the responsive coverage rail and cluster ledger with current, stale, partial, +unreachable, inconsistent, and unaccounted evidence. Named gaps are visible text as well as rail +segments, invalid reads clear the prior timestamp, empty scope makes no health claim, and the +renderer uses no browser storage, inline/external assets, automatic polling, or DOM HTML injection. +[A] Action: pinned the production file, import set, assets, routes, and forbidden mutation +capabilities structurally. Added a real `ServeMux` test plus adversarial session, CSRF, tenant, +reader-error, asset, and configuration coverage. +[T] Test: focused race tests pass with 86.2% statement coverage for `internal/hubserver`; Hub runtime +and privacy boundary tests pass, JavaScript syntax is valid, and source formatting is clean. +[T] Test: full `make ci` passes formatting, vet, lint with zero findings, vulnerability scanning +with no findings, the complete race suite, operator policy tests, performance budget, subprocess +E2E, and build. +[T] Test: `make e2e-isolation` passes forced-RLS PostgreSQL coverage and two 50,000-execution +cross-workspace fuzzers. `make e2e-kind` passes the pinned real two-cluster fan-out, OCI, and Argo +tests in 237.786 seconds. +[T] Test: isolated-GOPATH `make release-check` passes module verification, two reproducible release +builds, archive and SPDX SBOM verification, Homebrew formula generation, and the multi-platform +distroless OCI layout after the final asset changes. +[T] Test: rendered desktop and 390-pixel mobile views preserve coverage hierarchy, named-gap +legibility, keyboard focus, and readable cluster status. Reduced-motion behavior is present. +[T] Test: CodeRabbit first identified inaccessible hover-only gap names and a stale timestamp after +failed reads. Both were corrected; the full CI gate passed again and the follow-up review completed +with no findings. +[T] Test: `README.md` was reviewed and updated because this slice changes the supported browser +OIDC completion path and adds a user-visible Hub endpoint and security boundary. +[C] Checkpoint #1: implementation, red-team review, responsive visual inspection, independent +review, and all required local gates complete; next: create the signed DCO/GSTACK commit and open a +small PR into `dev`. + +Primary compatibility references: + +- +- +- +- + +--- + +**Session close:** ready for review · **Open questions touched:** none diff --git a/sessions/2026-07-17-hub-health-correlation.md b/sessions/2026-07-17-hub-health-correlation.md new file mode 100644 index 0000000..c0d67be --- /dev/null +++ b/sessions/2026-07-17-hub-health-correlation.md @@ -0,0 +1,55 @@ +# Session — 2026-07-17 — hub-health-correlation + +**Builder:** gnanirahulnutakki · **Effort:** deep · **Branch:** +`gnanirahulnutakki/feat/hub-health-correlation` +**Slice(s):** E8 / F8.1c · #220 · **Status:** ready for review + +--- + +[G] Goal: render one tenant-scoped, coverage-honest answer to “where is this exact resource not +Healthy?” without adding a privileged browser path. +[S] Scope: compose the existing PEP-governed `hubfleet.Correlator` into the browser-OIDC Hub +runtime, add one GET-only console adapter, purpose-separated request proof, minimal response +projection, explicit-submit renderer, adversarial tests, and operator documentation. Out: polling, +collector refresh, connector calls, local operations, persistence, writes, selectors, Secret +resources, arbitrary health predicates, and service/workspace pickers. +[A] Action: added strict canonical query parsing before the correlator, fixed `health_not=Healthy`, +a 257-row sentinel for a 256-match response bound, and generic fail-closed errors for unsafe input, +PEP/storage failures, over-bound results, or unexpected stored fact shapes. +[A] Action: separated the correlation HMAC purpose from the fleet-snapshot proof while retaining +the exact signed session/workspace/expiry binding, same-origin Fetch Metadata check, no-store +headers, restrictive CSP, and bearer-auth refusal. +[A] Action: projected only cluster scope, exact resource identity, normalized health, observation +time, stale state, and bounded coverage. Raw observations, attributes, workspace fields, +provenance, native IDs, deep links, and source payloads never enter the browser response. +[A] Action: added a responsive, keyboard-accessible explicit-submit form and one fleet-wide answer +with named stale, unreachable, truncated, unaccounted, and inconsistent gaps. Rendering uses only +`textContent`/DOM construction and clears prior answers on failure. +[T] Test: focused race coverage passes at 85.9% for `internal/hubserver`; correlator and Hub runtime +race suites pass. Unsafe request and hostile stored-shape tests prove fail-closed behavior before a +query or browser projection respectively. +[T] Test: full `make ci` passes twice, including formatting, vet, lint with zero findings, +vulnerability scanning with no findings, the complete race suite, policy checks, performance +budget, subprocess E2E, and build. The second run includes the independent-review correction. +[T] Test: `make e2e-isolation` passes forced-RLS PostgreSQL integration plus both 50,000-execution +cross-workspace fuzzers. The existing database-backed two-spoke correlator coverage verifies the +same tenant-scoped read seam composed by this slice. +[T] Test: pinned real two-cluster kind fan-out, OCI, and Argo projection checks pass in 237.836 +seconds. Isolated-GOPATH `make release-check` passes module verification, two reproducible release +builds, archive and SPDX SBOM checks, Homebrew generation, and multi-platform distroless OCI layout. +[T] Test: desktop and 390-pixel mobile rendering preserve the exact-query form, explicit-submit +behavior, adjacent named gaps, health/stale badges, and readable resource identity. JavaScript +syntax and reduced-motion behavior pass. +[T] Test: CodeRabbit found that the completed result hid the existing assistive-technology status +region. The renderer now keeps a visually hidden live completion announcement and avoids duplicate +live regions; the follow-up full-diff review reports zero findings. Its request to mark the session +merged was correctly deferred because no merge evidence existed yet. +[T] Test: `README.md` was reviewed and updated because this slice adds a supported Hub endpoint, +browser-visible behavior, security boundary, and per-submit database cost. +[C] Checkpoint #1: implementation, red-team tests, responsive inspection, independent review, and +all required local gates complete; next: create the signed DCO/GSTACK commit and open a small PR +into `dev`. + +--- + +**Session close:** ready for review · **Open questions touched:** none diff --git a/sessions/2026-07-18-approval-lifecycle-audit.md b/sessions/2026-07-18-approval-lifecycle-audit.md new file mode 100644 index 0000000..497823b --- /dev/null +++ b/sessions/2026-07-18-approval-lifecycle-audit.md @@ -0,0 +1,48 @@ +# Session — 2026-07-18 — approval-lifecycle-audit + +**Builder:** gnanirahulnutakki · **Effort:** deep · **Branch:** +`gnanirahulnutakki/feat/approval-lifecycle-audit` +**Slice(s):** E6 / F6.1b · #252 · **Status:** ready for review + +--- + +[G] Goal: append privacy-minimized approval creation and consumption evidence to the tenant hash +chain atomically with each successful single-use grant mutation. +[S] Scope: evolve the retained PostgreSQL chain format, reuse one transaction-local append +primitive, bind lifecycle pairs to the exact immutable grant, preserve format-1 verification, +add hostile rollback and mixed-chain tests, and document rollout ordering. Out: endpoints, MCP, +PDP policy, multi-approver policy, expiry, credentials, signing, dispatch, connectors, filesystem +or shell access, generic execution, and production mutation. +[A] Action: added migration 0011 with closed format-2 lifecycle kinds and shapes, format-1 writer +defaults for schema rollout, forced-RLS-compatible columns, and immutable application-role +privileges. +[A] Action: refactored the chain append into a transaction-local primitive. Approval creation and +consumption append `approval-created` and `approval-consumed` after their row mutation but before +transaction commit, so an append failure rolls the mutation back. +[A] Action: used a domain-separated SHA-256 digest over the opaque 128-bit grant ID, workspace, +intent ID, proposer, approver, resolved proposal digest, and approval timestamp. The lifecycle +entry contains no raw target, arguments, justification, credentials, tokens, elicitation content, +or free-form reason. +[A] Action: retained policy decisions on exact format 1 and introduced format 2 only for approval +lifecycle entries. README documents that verifiers must upgrade before lifecycle traffic is +enabled during a rolling deployment. +[T] Test: package and race suites pass. Two 50,000-execution fuzz campaigns cover format-1 chain +field framing and approval evidence framing. +[T] Test: the PostgreSQL 18.4 integration suite proves tenant RLS, immutable privileges, +mixed-format verification, concurrent one-winner consumption, stable refused paths, retained-row +and hash/head tamper detection, creation rollback on audit failure, and consumption rollback on +audit failure. +[T] Test: full `make ci` passes with formatting, vet, lint, reachable-vulnerability scanning, +complete race tests, policy checks, alert validation, performance budget, subprocess E2E, and +build. `make release-check` passes reproducible archives, SPDX SBOMs, Homebrew generation, and the +multi-platform distroless OCI layout. +[T] Test: Helm and OCI E2E pass. The pinned two-cluster Kind suite passes in 239.070 seconds. +[T] Test: CodeRabbit CLI 0.6.5 reviewed all seven changed files against `origin/dev` and returned +zero findings. `README.md` was reviewed and updated for the new security and rolling-upgrade +boundary. +[C] Checkpoint #1: implementation, rollback falsification, mixed-chain compatibility, full gates, +documentation, and independent review complete; next: open the signed DCO/GSTACK PR into `dev`. + +--- + +**Session close:** ready for review · **Open questions touched:** none diff --git a/sessions/2026-07-18-e10-auth-outcome-counter.md b/sessions/2026-07-18-e10-auth-outcome-counter.md new file mode 100644 index 0000000..7f48a72 --- /dev/null +++ b/sessions/2026-07-18-e10-auth-outcome-counter.md @@ -0,0 +1,73 @@ +# E10 F10.1h bounded authentication outcomes + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-auth-outcome-counter` + +**Slice:** E10 F10.1h / [#272](https://github.com/ArdurAI/sith/issues/272) · **Status:** local gates clean; landing pending + +**Base:** `origin/dev` at `89cffa5cb6f4e51c8c5f4ef9410f323ee044f493` + +## [G] Goal + +Add the smallest trustworthy authentication-attempt denominator for Sith's existing local bearer +and browser-session verifier boundaries without expanding the audit or privacy surface. + +## [S] Scope and decision + +- `sith_auth_attempts_total{outcome="accepted|refused"}` has exactly two preinitialized series. +- `accepted` is emitted immediately after local verifier success, before workspace authorization or + the protected handler. A valid credential forbidden from a workspace is still accepted authn. +- `refused` is emitted exactly once on each existing uniform HTTP 401 path and also increments the + legacy unlabeled `sith_auth_refusals_total` counter exactly once. +- The process-supervised audit observer and slog adapter remain refusal-only. Accepted events cannot + log, write a datagram, increment a delivery-drop counter, or start child work. +- The event and metric contain no credential, reason, tenant, workspace, identity, token, IP, path, + method, request, error, trace, correlation, authorization, or handler-result dimension. + +## [A] Analysis and nonclaims + +- Authentication success and workspace authorization are separate decisions. Emitting accepted + before `Principal.Scope` preserves that boundary and avoids a misleading denominator. +- A single bounded `outcome` label follows Prometheus guidance to expose one logical counter family + whose known series are initialized at startup. +- This does not cover provider exchange/callback failures, authorization denials, handler outcomes, + or every future authentication mode. +- This slice publishes counters only. It does not define a ratio, threshold, brute-force detector, + alert, SLO, error budget, page, listener, Service, exporter, persistence, remote write, retention, + or cloud resource. +- Runtime cost is one fixed counter increment per completed verifier decision. Existing scraper and + time-series retention costs remain operator-owned. + +## [T] Verification plan + +- Unit tests: closed event outcomes, accepted/refused increments, zero-at-start series, invalid-event + silence, legacy refusal compatibility, and forbidden-label inspection. +- Boundary tests: bearer success, browser success before authorization, all existing refusal paths, + and observer panic isolation preserve governed HTTP behavior. +- Refusal-sink tests: accepted events produce no log, datagram, or delivery-drop increment. +- Gates: focused and race suites, full CI, forced-RLS/isolation fuzz, release/Helm/OCI/two-cluster + Kind validation, complete-diff CodeRabbit review, signed DCO/GSTACK commit, exact-head hosted CI + and CodeQL, empty review/security queues, merge, and exact post-merge `dev` proof. + +## Sources + +- [Prometheus instrumentation](https://prometheus.io/docs/practices/instrumentation/) +- [Prometheus metric naming](https://prometheus.io/docs/practices/naming/) + +## [C] Local verification checkpoint + +- Focused package tests and focused race tests pass for `internal/hubserver`, + `internal/observability`, `internal/auditdelivery`, and `internal/hubruntime`. +- `make ci` passes: formatting, golangci-lint with zero issues, vet, `govulncheck` with no + reachable vulnerabilities, repository-wide race/coverage, safety policies, eight portable alert + rules, latency guard, tagged binary e2e, and build. `internal/observability` coverage is 94.7%. +- `make e2e-isolation` passes PostgreSQL 18.4 forced-RLS suites at 76.2% `hubdb` coverage plus both + 50,000-execution cross-workspace fuzz campaigns. +- `make release-check` passes two reproducible Darwin/Linux amd64/arm64 builds, SPDX SBOMs, + checksums, Homebrew metadata, and the release-derived two-platform OCI layout. +- Pinned Helm 4.2.3 and standalone two-platform OCI contract gates pass. +- The Kubernetes v1.36.1 real two-cluster Kind gate passes in 238.997 seconds. Independent cleanup + checks find no Kind clusters and no Sith/Kind test containers afterward. +- README review is complete and documents the exact verifier-before-authorization boundary, legacy + counter compatibility, refusal-only sinks, privacy exclusions, and nonclaims. +- Repeated complete 16-file CodeRabbit reviews have zero findings. The final checkpointed + secret-signature scan found zero candidates. diff --git a/sessions/2026-07-18-e10-auth-refusal-metric.md b/sessions/2026-07-18-e10-auth-refusal-metric.md new file mode 100644 index 0000000..57b93ee --- /dev/null +++ b/sessions/2026-07-18-e10-auth-refusal-metric.md @@ -0,0 +1,76 @@ +# E10 F10.1g bounded authentication-refusal metric + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-auth-refusal-metric` + +**Slice:** E10 F10.1g / [#266](https://github.com/ArdurAI/sith/issues/266) · **Status:** in progress + +**Base:** `origin/dev` at `15656aaf47331775e5619fc11d431bfd3fdf5dcf` + +## [G] Goal + +Add the smallest truthful E10 security-observability substrate: one process-wide count of requests +refused by the existing sanitized bearer/session middleware boundary. + +## [S] Scope and decision + +- `sith_auth_refusals_total` is a preinitialized Prometheus counter with zero labels. +- One valid `hubserver.AuthEvent{Outcome: refused}` produces one increment. +- Runtime fanout delivers independently to the existing process-local audit observer and the metric + observer, with panic isolation per destination. +- The existing uniform HTTP 401 response and structured local refusal delivery remain unchanged. +- No credential mode, failure reason, tenant, workspace, actor, principal, token, IP, path, request, + trace, or correlation data enters the metric. + +## [A] Analysis and nonclaims + +- Authentication refusal precedes a trusted principal and workspace, so adding identity or request + labels would create both a sensitive-data boundary and attacker-controlled cardinality. +- A refusal ratio needs a trustworthy success or total-attempt denominator. The current observer + deliberately emits only a sanitized refusal, so this slice does not invent a ratio or threshold. +- This does not cover successful authentication, OIDC provider exchange/callback failures, + authorization denials, or every future authentication mode. +- This is not a brute-force detector, alert, SLO, error budget, page, or complete security-monitoring + control. It adds no listener, Service, exporter, persistence, remote write, cloud resource, or + retention of other systems' telemetry. +- Increment and fanout overhead is constant. Operators retain responsibility for existing scrape + collector and storage cost. + +## [T] Verification plan + +- Unit tests: closed event validation, defensive fanout copy, configuration rejection, per-observer + panic isolation, zero-at-start, exact increment, invalid-event silence, and forbidden-label scan. +- Handler tests: bearer/API and browser-session/console refusals remain uniform; valid auth is silent. +- Runtime review: the same composite observer is passed to fleet, audit-export, and console handlers; + only the process observer owns shutdown. +- Gates: focused and race suites, full CI, vulnerability analysis, release/Helm/OCI/e2e gates, + CodeRabbit complete-diff review, signed DCO/GSTACK commit, exact-head CI/CodeQL, empty review and + security queues, merge, and exact post-merge `dev` CI/CodeQL. + +## Sources + +- [Prometheus instrumentation](https://prometheus.io/docs/practices/instrumentation/) +- [Prometheus metric naming](https://prometheus.io/docs/practices/naming/) +- [OWASP Logging Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html) + +## [C] Local verification checkpoint + +- Focused package tests and focused race tests pass for `internal/hubserver`, + `internal/observability`, and `internal/hubruntime`. +- `make ci` passes with the pinned EXTENDED-hosted Prometheus 3.13.1 binary: formatting, + golangci-lint with zero issues, vet, `govulncheck` with no reachable vulnerabilities, repository + race/coverage, all safety policies, eight portable alert rules, latency guard, tagged binary e2e, + and build. `internal/observability` coverage is 94.4%. +- The first `make ci` attempt reached the alert-rule gate after code and security checks, then + stopped because `promtool` was absent from that shell's PATH. The corrected run used the verified + EXTENDED binary and passed without a source change. +- `make e2e-isolation` passes PostgreSQL 18.4 forced-RLS suites plus both 50,000-execution workspace + fuzz campaigns. +- `make release-check` passes reproducible Darwin/Linux amd64/arm64 archives, SPDX SBOMs, checksums, + Homebrew metadata, and the release-derived multi-platform OCI layout. +- Pinned Helm 4.2.3 and cross-platform OCI contract gates pass. +- The Kubernetes v1.36.1 two-cluster Kind gate passes in 241.386 seconds; `kind get clusters` is + empty afterward. +- Two complete CodeRabbit passes found only overview wording ambiguity. Both findings were fixed by + naming the current signal a sanitized authentication-refusal count and keeping future derived + rates conditional on trustworthy denominators. The final complete ten-file review has no + findings. diff --git a/sessions/2026-07-18-e10-auth-refusal-only-alert.md b/sessions/2026-07-18-e10-auth-refusal-only-alert.md new file mode 100644 index 0000000..607b855 --- /dev/null +++ b/sessions/2026-07-18-e10-auth-refusal-only-alert.md @@ -0,0 +1,143 @@ +# Session — 2026-07-18 — E10 authentication refusal-only warning + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-auth-refusal-only-alert` +**Slice:** [#274](https://github.com/ArdurAI/sith/issues/274), E10 [#28](https://github.com/ArdurAI/sith/issues/28) · **Status:** hosted review correction in progress + +**Base:** `origin/dev` at `4096320d42b512c943615b3da8ef5d1a3ce80839` + +--- + +## [G] Goal + +Add one portable, aggregate warning for sustained refusal-only Hub authentication traffic without +inventing a workload-independent ratio, attributing an actor, or claiming attack detection. + +## [A] Decision and implementation + +- Alert when at least 20 aggregate `refused` attempts and zero `accepted` attempts occur over + 15 minutes, with a continuous 10-minute hold. +- Aggregate away the closed outcome and every scrape/source label; emit only fixed component and + severity labels plus static annotations. +- Stay quiet when the accepted series is missing. Partial telemetry cannot prove refusal-only + traffic; the separate missing-telemetry rule remains the metamonitoring signal. +- Require at least one recent scraped sample from the preinitialized accepted-outcome series during + the most recent 10 minutes. This proves series visibility, not an accepted event, and prevents old + samples in the 15-minute range from satisfying the freshness guard after accepted telemetry stops. +- Treat any accepted verifier decision as suppression, even if later workspace authorization + denies the request. + +## [A] Rejected alternative + +A generic five-percent refusal ratio was rejected. OWASP says authentication successes and +failures should be monitored but explicitly rejects one-size-fits-all monitoring and alerting. +Prometheus recommends a small set of simple symptom alerts with slack. Sith has no negotiated +authentication objective or traffic baseline that makes five percent meaningful. + +## [S] Security, operability, and cost boundary + +No tenant, workspace, actor, identity, intent, trace, request, credential, endpoint, verifier +error, or scrape/source label survives aggregation. The warning does not claim brute force, +credential stuffing, account compromise, an SLO, an error budget, a page, OIDC-provider coverage, +authorization-denial coverage, or monitoring-path health. It adds one expression evaluated once +per minute over existing fixed-cardinality series and at most one warning instance, with no runtime +path, listener, Service, exporter, storage, remote write, receiver, credential, network request, or +cloud resource. + +## [T] Verification plan + +- Go and promtool contracts pin the exact expression, 15-minute window, inclusive 20-refusal + guard, 10-minute hold, static annotations, fixed labels, and ninth-rule limit. +- Fixtures prove sustained firing/resolution, hostile-label aggregation, the exact boundary, and + silence for missing/stale/partial data, low volume, accepted traffic, transient bursts, and + resets. +- Remaining gates: repeated complete-diff review, signed DCO/GSTACK, exact-head hosted gates, merge, + empty review/security queues, and exact post-merge `dev` proof. + +## Primary sources + +- [OWASP Authentication Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Authentication_Cheat_Sheet.html) +- [OWASP Logging Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html) +- [Prometheus alerting practices](https://prometheus.io/docs/practices/alerting/) + +## [C] Checkpoint + +Notion and the curated Obsidian decision are synchronized. The rule, contracts, fixtures, and +operator documentation are implemented locally. + +- Pinned Prometheus 3.13.1 validates all nine rules and every fixture. +- The focused observability package passes with the race detector. +- The first wider shell-policy run correctly failed because its bounded alert count remained at + eight. Updating that explicit contract to nine fixed the only failure; the complete shell-policy + suite now passes. +- Fixtures directly prove the combined expression, exact 20-refusal boundary, 10-minute pending + state, accepted-event suppression, hostile-label aggregation, partial-data silence, low volume, + transient recovery, and counter-reset safety. + +## [T] Full local proof + +- `make ci` passes formatting, lint with zero findings, vet, reachable-vulnerability analysis with + no findings, repository-wide race/coverage, shell policies, all nine rules, performance, tagged + binary end-to-end, and build. `internal/observability` coverage is 94.7%. +- PostgreSQL 18.4 forced-RLS tests pass at 76.2% `hubdb` coverage, followed by both 50,000-case + cross-workspace isolation fuzz campaigns. +- Two reproducible GoReleaser builds, SPDX SBOMs, checksums, Homebrew metadata, and the + release-derived two-platform OCI layout pass. +- Pinned Helm 4.2.3 and the standalone Linux amd64/arm64 OCI contract pass. +- The digest-pinned Kubernetes v1.36.1 two-cluster fan-out, OCI, and Argo projection gate passes in + 241.828 seconds. Independent cleanup finds no Kind clusters or Kind containers. +- One unrelated auto-remove `sith-local-ops` container created at `2026-07-18T01:07:24Z` was already + running and was left untouched; it is not evidence from or residue of this cluster run. + +README review is complete. The first 25,756-byte diff/secret inspection found zero signature +candidates. The first complete eight-file CodeRabbit review found one valid minor stale-status list +in this journal; this correction removes gates that the later evidence already proves passed. +Repeated final secret inspection/review, signing, hosted gates, merge, and exact post-merge proof +remain pending. + +The corrected 25,907-byte candidate has zero secret-signature candidates, and the second complete +eight-file CodeRabbit review reports zero findings. This journal-only evidence update is included +in one final scan and complete review before the source tree is frozen for signing. + +## [A] Hosted review correction + +Exact-head CI `29656534080`, CodeQL `29656533683`, and hosted CodeRabbit completed on signed head +`fe38d4929cc4cb81ab6e88b194b6c109986b6e11`. The hosted review found one valid correctness gap: +`increase(accepted[15m]) == 0` can still see old accepted samples after that series stops scraping. +The green status context was not treated as proof; merge remained blocked. + +The correction adds an aggregate +`sum(count_over_time(sith_auth_attempts_total{outcome="accepted"}[10m])) > 0` guard. Ten minutes +matches the existing telemetry-missing tolerance and the alert hold. The count is summed before +matching, so no source label or new series reaches the fixed alert. + +A direct regression proves the original two-clause expression would be true at the firing boundary +when accepted samples are stale, while the guarded expression and alert stay quiet. Pinned +Prometheus 3.13.1 and the focused observability race test pass. Full final-head gates and repeated +review remain pending before GSTACK checkpoint `#2`. + +Finding: https://github.com/ArdurAI/sith/pull/275#discussion_r3608972947 + +Primary semantics: + +- https://prometheus.io/docs/prometheus/latest/querying/basics/ +- https://prometheus.io/docs/prometheus/latest/querying/functions/ + +## [T] Review-correction full proof + +- Corrected-tree `make ci` passes formatting, zero-issue lint, vet, no reachable vulnerabilities, + repository-wide race/coverage, all safety policies, nine-rule Prometheus proof, performance, + tagged binary e2e, and build. `internal/observability` coverage remains 94.7%. +- PostgreSQL 18.4 forced-RLS passes at 76.2% `hubdb` coverage, followed by both 50,000-case + cross-workspace isolation fuzz campaigns. +- Two reproducible releases, SPDX SBOMs, checksums, Homebrew metadata, release-derived two-platform + OCI, pinned Helm 4.2.3, and standalone two-platform OCI pass. +- The corrected-tree Kubernetes v1.36.1 two-cluster fan-out/OCI/Argo gate passes in 237.298 seconds. + Independent cleanup finds no Kind clusters or Kind containers. +- The unrelated auto-remove `sith-local-ops` container created at `2026-07-18T01:07:24Z` remains + untouched as pre-existing user state. + +README review is complete. The final original-base diff/secret inspection finds zero signature +candidates, and a complete CodeRabbit review of all eight files reports zero findings. +This journal-only evidence update is included in one last full scan and complete review before the +source tree is frozen for signed GSTACK checkpoint `#2`. Push, hosted re-review, thread resolution, +and merge/post-merge proof remain. diff --git a/sessions/2026-07-18-e10-fleet-read-freshness.md b/sessions/2026-07-18-e10-fleet-read-freshness.md new file mode 100644 index 0000000..d715226 --- /dev/null +++ b/sessions/2026-07-18-e10-fleet-read-freshness.md @@ -0,0 +1,43 @@ +# E10 F10.1f fleet-read freshness outcomes + +## Scope + +Issue [#260](https://github.com/ArdurAI/sith/issues/260) adds one bounded request-time freshness +result alongside the existing fleet-read coverage result. It does not add continuous monitoring, +per-spoke labels, an alert, an SLO, or an error budget. + +## Decision + +- Emit one validated `FleetReadObservation` after every authorized persisted read. +- Keep the existing coverage outcomes and add `fresh`, `stale`, `unknown`, `empty`, and `error`. +- Require a structurally valid cluster set with unique identities. Treat a validated stale scope + with retained observation time as `stale`; use `unknown` for mismatched, invalid, or unobserved + results and other non-stale degradation. +- Increment both counters only when the complete pair is valid, and isolate observer panic from the + read result. +- Preinitialize all five freshness series and expose no tenant-proportional label. + +## Cost and security boundary + +The change adds five fixed process-local counter series and one increment per authorized read. It +adds no listener, Service, monitoring CRD, exporter, remote write, persistence, background task, +credential path, network request, or cloud resource. Workspace, identity, trace, request, spoke, +cluster, resource, selector, endpoint, credential, age, and raw-error dimensions remain absent. + +## Primary references + +- [Prometheus instrumentation](https://prometheus.io/docs/practices/instrumentation/) +- [Prometheus metric and label naming](https://prometheus.io/docs/practices/naming/) + +## Verification status + +- Focused race suites and full CI pass with zero lint findings and no reachable vulnerabilities. +- PostgreSQL 18.4 forced-RLS focused coverage is 72.8%; isolation coverage is 76.2%. +- Both 50,000-case workspace-isolation fuzz campaigns pass. +- Reproducible release archives, SPDX SBOMs, release OCI layout, Helm 4.2.3, and cross-platform OCI + pass. +- Kubernetes v1.36.1 two-cluster Kind passes in 237.070 seconds. +- CodeRabbit found and drove fixes for unobserved timestamps, incomplete outcome coverage, and stale + per-spoke documentation. The second complete 10-file review has no findings. + +Hosted exact-head and post-merge gates remain required before closure. diff --git a/sessions/2026-07-18-e10-missing-telemetry-alert.md b/sessions/2026-07-18-e10-missing-telemetry-alert.md new file mode 100644 index 0000000..f45da59 --- /dev/null +++ b/sessions/2026-07-18-e10-missing-telemetry-alert.md @@ -0,0 +1,96 @@ +# Session — 2026-07-18 — E10 missing Hub telemetry alert + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-missing-telemetry-alert` +**Slice:** E10 F10.4d / [#254](https://github.com/ArdurAI/sith/issues/254) · **Status:** corrective commit ready +**Decision record:** [Notion](https://app.notion.com/p/3a12637edb07814a95a1f70de98f1ba1) + +## [G] Goal + +Detect total loss of expected Sith Hub telemetry at the portable rule evaluator without adding a +runtime heartbeat, depending on operator-specific target labels, or claiming that a white-box rule +can prove its own notification path. + +## [R] Research + +- Prometheus documents `absent_over_time` for detecting when no series exists during a bounded + interval. +- Prometheus metamonitoring guidance calls for confidence that monitoring is working and recommends + external black-box coverage for failures invisible to the internal stack. +- The existing `sith_build_info` gauge is set to one at metrics construction, requires no traffic, + and is registered atomically with the isolated Sith registry. + +## [D] Decision + +- Alert on aggregate absence of all `sith_build_info` samples over ten minutes, continuously for a + five-minute hold. +- Emit at most one warning with static labels and annotations; no source labels may propagate. +- Treat loading the portable rule package as the operator's declaration that a Hub scrape/forwarding + path is expected. Environments without an expected Hub must not load it. +- Keep `up`, job/instance naming, Kubernetes metrics and CRDs, alert routing, and full-path synthetic + monitoring outside Sith's portable contract. + +## [S] Security, operability, and cost boundary + +No tenant, workspace, spoke, actor, request, endpoint, credential, target, or raw-error label is +added. The slice adds one range-vector rule evaluation per minute and at most one alert instance. It +adds no listener, Service, ServiceMonitor, PrometheusRule, exporter, remote write, storage, cloud +resource, spoke egress, or action capability. + +## [T] Verification plan + +- Pinned Prometheus 3.13.1 parse and fixture tests for initial absence, hold timing, current/recent + presence, transient gaps, sustained disappearance, multiple series, hostile labels, firing, and + recovery. +- Go contract for the exact expression, hold, static output, six-rule group bound, and runbook link. +- Full race/CI, vulnerability, forced PostgreSQL/RLS isolation, release/SBOM/OCI, Helm, real Kind, + CodeRabbit, CodeQL, security queues, and exact post-merge `dev` proof. + +## [A] Progress + +- Revalidated live `dev`, E10 issue state, duplicates, the traffic-independent sentinel, and primary + Prometheus guidance. +- Created issue 254 as an E10 sub-issue, the Notion decision page, the Obsidian decision/checkpoint, + and this isolated EXTENDED-drive worktree. +- Added the sixth portable rule, deterministic timing fixtures, a static Go contract, README and + EPICS status, and an operator runbook with an explicit expected-environment precondition. + +## [V] Local evidence + +- Pinned Prometheus 3.13.1 parses all six rules. Fixtures prove initial hold timing, current and + recent presence, a bounded transient gap, sustained disappearance, multiple series, hostile + source labels, firing, recovery, and static output. +- Full `make ci` passes formatting, vet, zero-finding lint, reachable-vulnerability scanning, all + race tests, shell policy checks, the six-rule contract, performance, binary E2E, and build. +- Forced PostgreSQL 18.4/RLS isolation passes with `hubdb` coverage at 75.9%; both cross-workspace + fuzz campaigns complete 50,000 executions. +- Reproducible release archives, SPDX SBOMs, Homebrew formula, release-derived multi-architecture + OCI, standalone OCI, and pinned Helm 4.2.3 contracts pass. +- Digest-pinned Kubernetes v1.36.1 two-cluster Kind validation passes in 236.578 seconds. +- CodeRabbit CLI 0.6.5 found one minor structural-test gap: the Go contract did not pin the new + annotation strings independently of the YAML fixture. The contract now pins both exact strings; + focused race and Prometheus tests pass, and the second complete seven-file review has no findings. + +## [C] Checkpoint 1 + +Implementation and all local gates are complete. Next: create one signed DCO/GSTACK commit, push, +open the PR into `dev`, and hold completion until exact-head and post-merge CI, CodeQL, review, and +security queues are green. + +## [C] Checkpoint 2 — hosted shell-policy correction + +- PR 255 exact-head CodeQL and the release job passed, while the main hosted CI job stopped in the + portable Prometheus policy script. +- Root cause: the policy script still expected five alert rules after this slice intentionally added + the sixth. Its bare `[[ ... ]]` assertion also produced no diagnostic and behaved differently + between the macOS Bash 3.2 local gate and the hosted Bash 5 runner. +- Correction: capture the rule count, require exactly six with an explicit conditional failure and + error message, then rerun the focused policy test and the complete local CI gate before pushing a + second signed DCO/GSTACK checkpoint. +- CodeRabbit caught one remaining zero-rule edge case before commit: `grep -c` exits nonzero when it + finds no matches, so `set -e` could still bypass the diagnostic. The final counter uses `awk` and + always emits a numeric count, including zero. +- Its next pass caught an observability wording overclaim in EPICS: sample presence at an evaluator + does not prove that evaluator is healthy. The implementation note now says only that the expected + sample reaches the evaluator, matching the runbook's external-synthetic boundary. +- The zero-rule counter control, focused policy test, and a fresh complete `make ci` pass. The final + eight-file CodeRabbit review against `origin/dev` reports no findings. diff --git a/sessions/2026-07-18-e10-policy-error-alert.md b/sessions/2026-07-18-e10-policy-error-alert.md new file mode 100644 index 0000000..058aa72 --- /dev/null +++ b/sessions/2026-07-18-e10-policy-error-alert.md @@ -0,0 +1,59 @@ +# Session — 2026-07-18 — E10 policy-decision error warning + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-policy-error-alert` +**Slice:** [#264](https://github.com/ArdurAI/sith/issues/264), E10 [#28](https://github.com/ArdurAI/sith/issues/28) · **Status:** local proof complete + +--- + +## [G] Goal + +Add one portable aggregate warning for sustained fail-closed PEP decision errors without claiming +external Ardur PDP latency, dispatch success, an SLO, an error budget, or a page. + +## [A] Decision and implementation + +- Alert when `error / (allow + deny + require-approval + error) > 0.05` over 15 minutes. +- Require at least 20 eligible `allow|deny|require-approval|error` decisions and a continuous + 10-minute hold. +- Treat `deny` and `require-approval` as valid policy results in the denominator only. +- Aggregate away `verb` and every source label; emit only fixed component and severity labels. +- Keep the existing single-event critical policy-audit alert as the immediate sink-failure signal. + +## [S] Security, operability, and cost boundary + +The warning exposes no tenant, workspace, actor, identity, intent, trace, request, verb, reason, +credential, endpoint, selector, or raw-error label. It adds one expression evaluated once per +minute over existing fixed-cardinality series and at most one warning instance. It creates no +runtime path, recording series, listener, exporter, Service, monitoring CRD, storage, remote-write +path, receiver, network request, credential path, or cloud resource. + +## [A] Primary references + +- [Prometheus alerting practices](https://prometheus.io/docs/practices/alerting/) +- [Prometheus recording-rule aggregation guidance](https://prometheus.io/docs/practices/rules/) + +## [T] Proof + +Behavioral fixtures cover sustained firing and resolution, hostile-label aggregation, missing and +low-volume data, strict-threshold and inclusive-volume boundaries, ordinary deny/approval outcomes, +transient recovery, counter resets, and exclusion of a high-volume unknown outcome from the closed +denominator. Pinned Prometheus 3.13.1 validates all eight rules and fixtures. + +- Focused rule, Go contract, and tooling-policy checks: passed. +- The immutable-commit CodeRabbit review found two minor fixture gaps: the ordinary-denial case did + not prove those valid outcomes affected the denominator, and the transient case never entered + the pending state. Both adversarial controls are now explicit; the correction review reports + zero findings and the full CI gate passes again. +- `make ci`: passed, including formatting, lint, vet, `govulncheck`, race tests, policy tests, and + the tagged end-to-end suite. +- PostgreSQL 18.4 forced-RLS, both 50,000-case workspace-isolation fuzz campaigns, reproducible + release/SPDX SBOM, Helm 4.2.3, and cross-platform OCI gates: passed. +- Digest-pinned Kubernetes v1.36.1 Kind fleet, OCI, and Argo projection gate: passed in 239.180s. + +Hosted exact-head CI/CodeQL, empty review/security queues, merge, and exact post-merge `dev` proof +remain required before closure. + +## [C] Checkpoint #1 + +`2026-07-18/e10-policy-error-alert#1` records the complete local proof above on exact base +`00a9489e257c131fa734b9a672c2b1f0af552748` before the signed DCO/GSTACK commit. diff --git a/sessions/2026-07-18-e10-stale-read-alert.md b/sessions/2026-07-18-e10-stale-read-alert.md new file mode 100644 index 0000000..7e1543a --- /dev/null +++ b/sessions/2026-07-18-e10-stale-read-alert.md @@ -0,0 +1,56 @@ +# Session — 2026-07-18 — E10 proven-stale fleet-read warning + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e10-stale-read-alert` +**Slice:** [#262](https://github.com/ArdurAI/sith/issues/262), E10 [#28](https://github.com/ArdurAI/sith/issues/28) · **Status:** locally verified + +--- + +## [G] Goal + +Add one portable aggregate warning over the bounded F10.1f request-time freshness counter without +claiming continuous monitoring, a per-spoke series, an SLO, an error budget, a page, a receiver, or +a new scrape path. + +## [A] Decision and implementation + +- Alert when `stale / (fresh + stale) > 0.05` over 15 minutes. +- Require at least 20 eligible `fresh|stale` reads and a continuous 10-minute hold. +- Exclude `unknown`, `error`, and `empty` from numerator and denominator because none proves age. +- Aggregate away every source label and emit only the fixed `component` and `severity` labels. +- Keep this warning distinct from coverage degradation: a complete result can still be stale, and + an unknown result is not evidence of staleness. + +## [S] Security, operability, and cost boundary + +The rule exposes no tenant, workspace, spoke, cluster, resource, principal, trace, endpoint, age, +credential, or raw-error label. It adds one expression evaluated once per minute over five existing +fixed-cardinality series and at most one warning instance per evaluator. It creates no recording +series, listener, exporter, Service, monitoring CRD, storage, remote-write path, receiver, network +request, credential path, or cloud resource. + +## [A] Primary references + +- [Prometheus alerting practices](https://prometheus.io/docs/practices/alerting/) +- [Prometheus recording rules](https://prometheus.io/docs/practices/rules/) + +## [T] Verification + +Behavioral fixtures cover sustained firing and resolution, hostile-label aggregation, missing and +excluded-only data, minimum-volume and strict-threshold boundaries, transient recovery, and counter +reset behavior. + +- Pinned Prometheus 3.13.1 accepts all seven rules and every deterministic fixture. +- Full CI passes with zero lint findings and no reachable vulnerabilities. +- PostgreSQL 18.4 forced-RLS coverage is 72.8%; the broader isolation suite is 76.2%. +- Both 50,000-case cross-workspace fuzz campaigns pass. +- Reproducible archives, SPDX SBOMs, Helm 4.2.3, and cross-platform OCI checks pass. +- Kubernetes v1.36.1 two-cluster Kind passes in 236.710 seconds. +- CodeRabbit reviewed all eight changed files and returned zero findings. + +Hosted exact-head and post-merge gates remain required before closure. + +## [C] Checkpoint #1 + +Create the signed implementation commit with +`GSTACK-Checkpoint: 2026-07-18/e10-stale-read-alert#1`; next require hosted exact-head CI, CodeQL, +empty review and security queues, merge into `dev`, and exact post-merge proof before closing #262. diff --git a/sessions/2026-07-18-e13-dcgm-gpu-utilization.md b/sessions/2026-07-18-e13-dcgm-gpu-utilization.md new file mode 100644 index 0000000..b9877b1 --- /dev/null +++ b/sessions/2026-07-18-e13-dcgm-gpu-utilization.md @@ -0,0 +1,103 @@ +# Session — 2026-07-18 — E13 bounded DCGM GPU utilization + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e13-dcgm-gpu-utilization` +**Slice:** [#286](https://github.com/ArdurAI/sith/issues/286), E13 +[#31](https://github.com/ArdurAI/sith/issues/31) · **Status:** complete local proof + +## [G] Goal + +Normalize already-authorized DCGM GPU utilization evidence into bounded fleet facts without +guessing the blocked live transport, freshness, coverage, cost-join, or presentation contracts and +without overstating per-workload precision. + +## [S] Scope + +- One exact Prometheus instant-vector expression: `DCGM_FI_DEV_GPU_UTIL`. +- Whole-GPU, paired MIG, and explicitly best-effort workload attribution. +- Deterministic derived TELEMETRY facts, selected-identity SHA-256, atomic failure, and fixed input/ + output bounds. +- No client, endpoint, credentials, network, RBAC, arbitrary PromQL, range aggregation, + persistence, runtime wiring, stale objective, coverage rollup, GPU cost/idle-cost join, team + mapping, UI/API, billing, optimization, mutation, or execution. + +## [A] Decision and implementation + +- Reconciled the live backlog and confirmed that higher-priority residuals remain landed to their + current contracts or explicitly blocked on owner, custody, runtime, package-admin, or upstream + release decisions; no competing PR is open. +- Verified the current Prometheus instant-vector contract and NVIDIA dcgm-exporter 4.6.0-4.8.3 + metric, renderer, MIG, and Kubernetes attribution contracts from primary sources. +- Opened issue 286, linked parent issue 31, and recorded matching Notion and EXTENDED Obsidian + decisions before source work. +- Added a pure projector that revalidates the exact query, source identity, timestamps, values, + disabled API series limit/lookback override, label groups, bounds, and whole response before + returning deterministic facts. +- Discards raw GPU UUID, host, PCI bus, scrape target, and arbitrary labels. Complete workload + evidence is labelled `workload_best_effort`; physical or MIG device scope remains explicit. +- Added an AST structure/import/declaration wall so network, credential, process, persistence, + planning, mutation, or execution capability cannot enter unnoticed. + +## [T] Focused proof + +- Package race tests pass with 92.8% statement coverage on the final query contract. +- Positive fixtures cover physical GPU, MIG, workload-best-effort physical GPU, workload-best-effort + MIG, exact decimal canonicalization, graph attachment, privacy, ordering, and successful-empty + abstention. +- Adversarial fixtures reject malformed/duplicate/deep/oversized JSON, warnings, infos, non-vector + data, label and series overflow, partial or invalid MIG/workload identity, raw metric mismatch, + timestamp mismatch, non-string/non-finite/out-of-range values, projected duplicates, and a late + invalid series without returning partial facts. +- Native Go fuzzing completed exactly 50,000 generated executions with four workers; the projector + did not panic, return partial facts on error, violate bounds, or emit an invalid graph fact. +- The first full CodeRabbit review reported zero findings. A manual second-angle review found that + Prometheus's optional API `limit` can truncate a successful vector. The public contract now + requires `limit=0` and no per-query lookback override, preventing an API-limited prefix from being + asserted as complete while preserving the explicit F13.4 freshness gap. Focused race tests and + another exact 50,000-execution fuzz run passed after the change; the second full CodeRabbit review + also reported zero findings. +- The final unchanged source passes full repository CI with zero lint findings and no reachable + vulnerabilities, forced-RLS PostgreSQL isolation and both at-least-50,000-execution cross-workspace + fuzzers, reproducible four-platform archives plus SPDX SBOM/checksum/Homebrew/two-platform OCI + proof, and the pinned Kubernetes 1.36.1 two-cluster suite in 236.293 seconds. Teardown left no + Kind cluster or matching container. +- Manual red-team confirms atomic late failure, exact query-time binding, complete MIG/workload + groups, stable selected identity, duplicate protection for discarded labels, no raw hardware or + scrape identity retention, and no I/O/authority seam. It also records two nonclaims: Prometheus + evaluation time does not prove scrape age, and the value-only core cannot prove that central + Prometheus data belongs only to the caller-asserted scope. +- The changed-file high-signal credential scan is clean. Live GitHub queues are 0 open Dependabot, + 0 code-scanning, and 0 secret-scanning alerts; no competing PR is open; exact `origin/dev` remains + `f3e00f492e69235fd4df2de9b802332eab6d9793`. + +## [S] Security, reliability, and cost + +The parser treats source JSON and labels as untrusted, bounds memory/cardinality, rejects ambiguous +or partial identity, and preserves only fields needed for the reviewed claim. It introduces no new +authority or recurring cloud cost. A future live query path will add Prometheus compute/cardinality +cost and must be scoped, rate-limited, and observed separately. + +## [P] Primary sources + +- [Prometheus HTTP API](https://prometheus.io/docs/prometheus/latest/querying/api/) +- [NVIDIA dcgm-exporter 4.6.0-4.8.3](https://github.com/NVIDIA/dcgm-exporter/releases/tag/4.6.0-4.8.3) +- [NVIDIA default counters](https://github.com/NVIDIA/dcgm-exporter/blob/4.6.0-4.8.3/etc/default-counters.csv) +- [NVIDIA DCGM exporter Kubernetes and MIG documentation](https://docs.nvidia.com/datacenter/dcgm/latest/gpu-telemetry/dcgm-exporter.html) +- [ADR 0013](../docs/adr/0013-dcgm-gpu-utilization-facts.md) + +## [N] Next + +Run the full local matrix, complete CodeRabbit and manual red-team review, fix every finding, then +create one signed DCO/GSTACK commit and require exact-head plus exact post-merge `dev` proof. + +## [C] Checkpoint #1 + +Issue, primary-source decision, implementation, adversarial tests, AST boundary, exact +50,000-execution fuzz proof, README, mirrored E13 roadmap, ADR, and GSTACK journal are present in the +EXTENDED worktree. Full local and hosted proof remain fail-closed gates. + +## [C] Checkpoint #2 + +The tightened query-completeness contract and complete final local matrix are green from the +EXTENDED worktree. Two complete CodeRabbit reviews end at zero findings; manual red-team, +credential scan, exact base, no-competing-PR check, and GitHub 0/0/0 security queues are clean. +Signed publication and hosted exact-head/post-merge proof remain the fail-closed gates. diff --git a/sessions/2026-07-18-e13-opencost-namespace-costs.md b/sessions/2026-07-18-e13-opencost-namespace-costs.md new file mode 100644 index 0000000..6f07ed6 --- /dev/null +++ b/sessions/2026-07-18-e13-opencost-namespace-costs.md @@ -0,0 +1,108 @@ +# Session — 2026-07-18 — E13 OpenCost namespace cost facts + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e13-opencost-namespace-costs` +**Slice:** [#282](https://github.com/ArdurAI/sith/issues/282), E13 +[#31](https://github.com/ArdurAI/sith/issues/31) · **Status:** complete local proof; ready for signed commit + +## [G] Goal + +Define the first F13.1 normalization boundary: project one already-authorized exact OpenCost +namespace-allocation response into deterministic Sith cost facts without adding transport, +credentials, persistence, billing, optimization, or mutation. + +## [S] Scope + +- One explicit UTC window, one allocation set, `aggregate=namespace`, step equal to the window, and + disabled filter, accumulation, idle, sharing, proportional-asset, and metadata options. +- Trusted USD-only source assertion because allocation JSON has no currency field. +- Exact cluster and Kubernetes namespace identity, allowlisted component/adjustment/total amounts, + source window, and closed provenance. +- Response, row, payload, decimal, identity, time, count, and depth bounds with atomic failure. +- Keep live endpoint discovery, HTTP/TLS/auth, OpenCost availability coverage, fleet/team rollup, + currency conversion, freshness objectives, GPU utilization/DCGM/MIG, billing, optimization, + recommendations, persistence, and every write path out of scope. + +## [A] Decision and implementation + +- Verified the current contract against the official OpenCost API, v1.120.2 release, allocation + response and total-cost source, HTTP envelope, and Swagger schema. +- Opened #282 after a live duplicate check, linked it from #31, and based the isolated EXTENDED + worktree on exact `origin/dev` `df28654b87ecfa1e491901ba8a3d718ba0825a49`. +- Recorded the design, blockers, primary sources, security boundary, costs, alternatives, and + nonclaims in Notion and the EXTENDED Obsidian vault before source work. +- Added `internal/connector/opencost`: case-exact and duplicate-safe envelope decoding, exact query + metadata, UTC window validation, Kubernetes namespace correlation, rational five-decimal cost + validation, total recomputation, deterministic ordering and identity, privacy-minimized graph + facts, and no-partial-result behavior. +- Added a bidirectional AST boundary that pins the value-only public API, imports, declarations, + production structure, and the sole permitted `io.EOF` use. +- Added ADR 0011 plus README and mirrored E13 roadmap updates. The documentation states that no live + CLI or Hub reader exists and does not mark F13.1 or E13 complete. + +## [T] Focused proof + +- Focused race tests pass with 93.2% statement coverage. +- Positive coverage proves sorted namespace facts, exact graph attachment, canonical USD amounts, + true window-end observation time, graph validation, and raw provider/label/annotation/controller/ + endpoint/collection-time non-retention. +- Adversarial coverage proves exact query flags, window/step/clock-skew bounds, success-envelope and + one-set requirements, namespace and cluster matching, synthetic-row refusal, exact-case and + duplicate JSON handling, decimal/magnitude/total checks, depth/size/count limits, atomic errors, + deterministic identity, and complete-empty abstention. +- Native Go fuzzing completed exactly 50,000 generated executions with four workers and no panic, + invalid fact, duplicate identity, taxonomy escape, or partial error result. +- The first complete CodeRabbit pass found two valid mirrored-roadmap ambiguities around empty + results. The second found two valid README contract gaps around exact fact attachment and + whole-response failure. All four documentation findings were corrected; the third complete + review reports zero findings across all nine changed files. +- `make ci` passes formatting, vet, lint with zero findings, `govulncheck` with no reachable + vulnerabilities, the full race suite, operator policy and alert-rule tests, warm-view + performance, subprocess E2E, and the production build. OpenCost coverage remains 93.2%. +- `make e2e-isolation` passes PostgreSQL 18.4 forced-RLS tests and both 50,000-execution + cross-workspace fuzzers. +- `make release-check` verifies modules, two byte-identical four-platform release snapshots, + archive contents, SPDX SBOMs, Homebrew output, and the release-derived amd64/arm64 distroless Hub + OCI layout. +- `make e2e-kind` passes the pinned Kubernetes 1.36.1 two-cluster fan-out, OCI, and Argo suite in + 236.706 seconds. Teardown leaves no Kind cluster or matching Kind/Sith container. +- Manual red-team review rechecked whole-response atomicity, case aliases, duplicate/trailing/deep + JSON, source/query/window/currency/namespace confusion, decimal bounds and rounding tolerance, + historical freshness, secret-bearing unknown fields, synthetic namespace rows, identity + determinism, concurrency, and memory caps; no unresolved path remains. The complete changed-file + high-signal credential scan reports zero candidates. +- Pre-publication GitHub queues are `0` open Dependabot alerts, `0` code-scanning alerts, and `0` + secret-scanning alerts. A final fetch confirms exact base `origin/dev` remains `df28654b`. +- `README.md` was reviewed and updated because the new public library contract needs explicit + exact attachment, decimal, whole-response failure, and no-live-reader boundaries. The roadmap, + ADR, and README all preserve the incomplete F13.1/E13 status. +- Hosted PR and exact post-merge evidence remain pending. + +## [S] Security, reliability, and cost + +Raw allocation data remains in bounded memory and only an allowlisted namespace monetary payload +survives. Unknown and privacy-sensitive fields are discarded; ambiguity fails before any fact is +returned. Runtime cost is bounded local CPU and memory only, with no network, cloud API, egress, +storage, telemetry-volume, credential, privilege, or recurring-service cost. A future live reader +must use least-privilege read-only OpenCost access and preserve the exact request contract. + +## [P] Primary sources + +- [OpenCost allocation API](https://opencost.io/docs/integrations/api/) +- [OpenCost v1.120.2](https://github.com/opencost/opencost/releases/tag/v1.120.2) +- [Allocation response implementation](https://github.com/opencost/opencost/blob/v1.120.2/core/pkg/opencost/allocation_json.go) +- [Total-cost implementation](https://github.com/opencost/opencost/blob/v1.120.2/core/pkg/opencost/allocation.go) +- [HTTP response envelope](https://github.com/opencost/opencost/blob/v1.120.2/core/pkg/protocol/http.go) +- [OpenCost API schema](https://github.com/opencost/opencost/blob/v1.120.2/docs/swagger.json) + +## [N] Next + +Complete the final high-signal credential and GitHub security-queue checks, synchronize Notion and +Obsidian proof, create one signed DCO/GSTACK commit, and require exact-head plus exact post-merge +`dev` evidence before closing #282. + +## [C] Checkpoint #1 + +Pending signed implementation commit. The reviewed issue, decision, projector, adversarial tests, +boundary, README, roadmap, ADR, session record, 50,000-execution projector fuzz proof, full local CI, +isolation, reproducible release/SPDX/Homebrew/multi-platform OCI proof, zero-finding repeated review, +and clean two-cluster teardown are frozen in the EXTENDED worktree. diff --git a/sessions/2026-07-18-e13-opencost-workspace-rollup.md b/sessions/2026-07-18-e13-opencost-workspace-rollup.md new file mode 100644 index 0000000..ff08fb5 --- /dev/null +++ b/sessions/2026-07-18-e13-opencost-workspace-rollup.md @@ -0,0 +1,107 @@ +# Session — 2026-07-18 — E13 OpenCost coverage-aware workspace rollup + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e13-opencost-workspace-rollup` +**Slice:** [#284](https://github.com/ArdurAI/sith/issues/284), E13 +[#31](https://github.com/ArdurAI/sith/issues/31) · **Status:** complete local proof + +## [G] Goal + +Preserve successful-empty per-cluster OpenCost coverage and compute one deterministic workspace USD +total for an explicit window without guessing the blocked live transport or team-attribution +contracts. + +## [S] Scope + +- One value-only snapshot envelope around each successful F13.1a projection. +- One explicit expected cluster set and zero or one successful snapshot per reporting cluster. +- Exact workspace, scope, UTC window, USD, fact taxonomy, entity, provenance, payload, and native + identity revalidation. +- Exact component and total aggregation plus sorted expected/reported/empty/missing coverage. +- No client, endpoint, discovery, credentials, port-forward, Kubernetes Service proxy, OCM + transport, persistence, runtime wiring, team grouping, UI/API, stale objective, conversion, + billing, optimization, GPU efficiency, mutation, or execution. + +## [A] Decision and implementation + +- Reconciled every open issue and current `dev`; all higher-priority residuals remain + landed-to-current-contract or explicitly human/upstream blocked, and no competing PR is open. +- Verified current access guidance in the official OpenCost API and installation documentation. + OpenCost recommends operator port forwarding and permits deployment-specific Service/Ingress + exposure; Sith has no accepted owner for discovery, authentication, or TLS. +- Recorded the live-transport options and recommended local-versus-brokered split on parent issue + 31. F13.1 transport remains held pending GR confirmation; no URL, credential, RBAC grant, or + network path was guessed. +- Opened issue 284 and recorded the accepted contract in Notion and the EXTENDED Obsidian Sith + project before source work. +- Added a successful projection snapshot that preserves an empty fact set as reported coverage. +- Added a deterministic workspace rollup with exact decimal aggregation, explicit coverage, nil + observation time when nothing reported, whole-input atomic failure, privacy-minimized output, and + fixed scope/fact/byte/magnitude bounds. +- Extended the AST boundary so the package remains value-only and cannot acquire network, + credential, persistence, process, planning, mutation, or execution capability unnoticed. + +## [T] Focused proof + +- Package tests pass with the race detector and 91.9% statement coverage. +- Positive coverage proves sorted partial coverage, successful-empty versus missing distinction, + exact component totals, deterministic order independence, absent observation time when no scope + reports, and non-retention of namespace/source metadata beyond authorized cluster coverage. +- Adversarial coverage revalidates workspace, scope, UTC window, currency, taxonomy, entity, + provenance, native identity, canonical JSON, namespace identity, amount precision, component + totals, duplication, count, and size bounds with zero partial rollup on every error. +- Native Go fuzzing completed exactly 50,000 generated rollup executions with four workers after + correcting one fuzz-harness-only invalid-RawMessage identity helper; the production path did not + panic or emit a partial/invalid rollup. +- The projection and rollup fuzz boundaries each pass exactly 50,000 generated executions with four + workers. Full repository CI passes formatting, vet, lint with zero issues, `govulncheck` with no + reachable vulnerabilities, race coverage, shell policy tests, Prometheus rules, performance, + end-to-end tests, and the production build. +- PostgreSQL forced-RLS isolation and two 50,000-execution cross-workspace fuzzers pass. The + reproducible four-platform release, SPDX SBOM, checksum, Homebrew, and multi-platform OCI proof + passes. The pinned Kubernetes 1.36.1 kind suite passes in 238.257 seconds. +- CodeRabbit's first complete review found one minor ADR result-shape omission. The ADR now names + the retained coverage categories, `complete`, and optional `observed_at`; source behavior was + unchanged. The second complete review covers all changed files and reports zero findings. +- Manual red-team confirms that `complete` is relative to the caller-authoritative expected set and + that successful-empty presence is a trusted caller claim; the transport-free core cannot prove + either claim independently and does not pretend to. Late invalid input remains atomic, authorized + cluster coverage is the only retained identity, total and component fields are summed + independently, limits fail closed, errors omit raw inputs, race proof is green, and the AST wall + admits no I/O or authority seam. +- The changed-file credential sweep has zero high-signal matches. Live GitHub queues are 0 open + Dependabot, 0 code-scanning, and 0 secret-scanning alerts; no competing PR is open; exact + `origin/dev` remains `3d45529624a275652d4c6793271859dfe5add152`. + +## [S] Security, reliability, and cost + +The rollup revalidates normalized facts and returns no prefix on an invalid later scope or fact. +Only aggregate amounts plus explicit authorized cluster coverage survive; namespace names and raw +provider/workload/source metadata do not. Work is bounded in memory and CPU and creates no network, +cloud API, storage, egress, logging-volume, credential, privilege, or recurring-service cost. + +## [P] Primary sources + +- [OpenCost allocation API](https://opencost.io/docs/integrations/api/) +- [OpenCost installation and access](https://opencost.io/docs/installation/install/) +- [OpenCost v1.120.2](https://github.com/opencost/opencost/releases/tag/v1.120.2) +- [ADR 0011](../docs/adr/0011-opencost-namespace-cost-facts.md) +- [ADR 0012](../docs/adr/0012-opencost-coverage-aware-workspace-rollup.md) + +## [N] Next + +Create one signed DCO/GSTACK commit, publish a PR to `dev`, require exact-head CI/CodeQL/hosted +CodeRabbit, merge while preserving the signed head, and require exact post-merge `dev` evidence +before closing issue 284. + +## [C] Checkpoint #1 + +Issue, decision, code, focused race coverage, adversarial tests, exact 50,000-execution fuzz proof, +README, mirrored roadmap, ADR, and GSTACK journal are present in the EXTENDED worktree. + +## [C] Checkpoint #2 + +The complete local CI, isolation, reproducible release, and real-cluster gates pass from the +EXTENDED worktree. Both OpenCost fuzz boundaries pass 50,000 executions. The first complete +CodeRabbit review's sole documentation finding is corrected and the second review reports zero +findings. Manual red-team, credential scan, security queues, exact base, and no-competing-PR checks +are clean; signed publication remains the fail-closed gate. diff --git a/sessions/2026-07-18-e14-argocd-sync-failure-rule.md b/sessions/2026-07-18-e14-argocd-sync-failure-rule.md new file mode 100644 index 0000000..6e851c7 --- /dev/null +++ b/sessions/2026-07-18-e14-argocd-sync-failure-rule.md @@ -0,0 +1,171 @@ +# E14 R8 honest Argo CD sync-operation failure rule + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e14-argocd-sync-rule` + +**Slice:** E14 R8 / [#270](https://github.com/ArdurAI/sith/issues/270) · **Status:** landed on `dev` via [#271](https://github.com/ArdurAI/sith/pull/271) + +**Base:** `origin/dev` at `22c6caa834442120d61add38009bacc154eda4fd` + +## [G] Goal + +Add a deterministic, evidence-cited rule for an Argo CD Application sync operation whose reviewed +operation phase is `Failed` or `Error`, without turning drift or health into a guessed failure. + +## [S] Boundary + +- Consume only attached, workspace-valid Argo TIMELINE `FactChange` evidence from projector + protocol `1.0.0`, with matching source/provenance/entity identity and internally consistent + change kind, phase, and event time. +- Project only canonical `change.kind=sync-failed`; discard revision, repository, operation + message, conditions, raw CRD data, credentials, and all other payload fields. +- Preserve caller-declared coverage without inferring it from fact presence. Missing, unavailable, + stale, or observation-stale TIMELINE evidence remains `unconfirmed`. +- State only that Argo CD reported a failed operation. Do not select a rendering, validation, + authorization, hook, network, Kubernetes API, resource, health, or other root cause. +- Emit only a sensitive-marked, shell-quoted, read-only Application `kubectl describe` advisory. +- Do not fetch Argo, add a client or network call, retain data, alert, create an SLO, correlate + fleet-wide, emit an intent, enter the PEP, dispatch, mutate, execute, complete F14.6, or claim the + E12 connector framework is complete. + +## [T] Verification plan + +- Exercise the real Argo projector for both failed phases plus drift, successful/running, + malformed, ambiguous, oversized, stale, unavailable, shell-hostile, and unrelated inputs. +- Require R8 in the sanitized deterministic replay corpus and round-trip text/JSON without + discarded source fields. +- Run focused race tests, full CI/vulnerability checks, PostgreSQL forced-RLS and fuzz isolation, + release/SBOM, Helm/OCI, and real two-cluster Kind gates. +- Secret-scan the complete diff, complete CodeRabbit review to zero actionable findings, sign the + DCO/GSTACK commits, pass exact-head hosted checks, merge into `dev`, and verify exact post-merge + CI/CodeQL plus empty review and security queues. + +## [A] Primary-source decision + +- Argo CD's stable trigger contract uses `app.status?.operationState.phase in ['Error', 'Failed']` + for failed synchronization: + . +- The stable notification catalog describes `on-sync-failed` as Application synchronization + failure and exposes operation details for investigation: + . +- Argo CD describes `OutOfSync` as live state differing from desired state; it is not by itself + proof that a sync operation was attempted and failed: + . + +These contracts support the failed-operation symptom and read-only inspection boundary. They do +not identify a root cause, so R8 retains the complete uncertainty set. + +## Cost and operability + +R8 reuses an existing bounded projector, workspace graph, and pure in-memory evaluator. It adds no +cloud resource, API request, watch, credential, storage, telemetry volume, or egress cost. A human +who runs the advisory may see sensitive Application status and must review it under their existing +kubeconfig authorization; Sith marks the command sensitive and never executes it. + +## [V] Local implementation checkpoint — 2026-07-18 + +- Both `Failed` and `Error` pass through the real Argo projector and produce one canonical, + entity-local R8 observation and verdict; revision and phase are absent from brain output. +- `OutOfSync`, `Unknown`, `Synced`, successful/running/terminating operations, near-miss values, + malformed/oversized payloads, unknown fields, mismatched identity/provenance/protocol/time, + history-shaped failure facts, and unattached facts fail closed. +- Coverage is cloned rather than aliased or inferred. Stale evidence and missing, unavailable, or + stale TIMELINE coverage produce one cited `unconfirmed` R8 verdict with the gap named. +- Shell-hostile identifiers stay inert; the advisory remains sensitive, read-only, and has no PR + diff. Two identical cross-cluster signals remain separate and never become fleet-wide. +- Focused `go test -race ./internal/brain ./internal/cli` passes. + +## [V2] Full local and review proof — 2026-07-18 + +- `make ci` passes on the final source tree: formatting, vet, zero lint findings, `govulncheck` + with no known reachable vulnerabilities, full repository race coverage, shell/pin policies, + eight pinned Prometheus rules, performance, compiled E2E, and binary build. +- Digest-pinned PostgreSQL 18.4 forced-RLS tests pass with 76.2% `hubdb` coverage. Both fixed + 50,000-case workspace-isolation fuzz campaigns pass with four workers. +- Pinned Helm 4.2.3 and standalone linux/amd64 plus linux/arm64 OCI contracts pass. +- `make release-check` passes twice on the final source: module verification, two reproducible + Darwin/Linux amd64/arm64 snapshots, archive and SPDX SBOM validation, Homebrew formula, and the + immutable two-platform hub OCI layout built from release archives. +- The real digest-pinned Kubernetes v1.36.1 two-cluster Kind gate passes in 233.419 seconds under + the race detector, including fleet fan-out, immutable OCI, and live Argo Application projection. + Cleanup confirms zero Kind clusters and no Sith test containers. +- The first complete CodeRabbit CLI 0.6.5 review of all 13 changed/new files against exact base + `22c6caa834442120d61add38009bacc154eda4fd` found one valid provenance edge case: a native ID that + ended exactly at `#operation/` passed the prefix check without identifying an operation. +- The bridge now requires a non-empty operation identifier after the prefix while continuing to + discard that identifier. A focused regression, full CI, isolation/fuzz, Helm/OCI, reproducible + release, and real Kind reruns all pass after the fix. The second complete review reports zero + findings. +- The corrected 13-file tree scans 225,127 bytes with zero recognized credential, private-key, + token, JWT, authenticated-URL, or generic secret-assignment candidates. +- README review is complete. It accurately distinguishes cache-backed R1-R7 behavior from the + graph-fed R8 surface, preserves every cause/non-execution nonclaim, and states that the current + CLI does not fetch Argo or infer TIMELINE coverage. +- A later documentation-inclusive review found two minor README precision gaps: it abbreviated the + advisory and did not enumerate every fail-closed graph gate. README now names the exact + target-bound command and the workspace, provenance, protocol, identity, closed-payload, + phase/change-kind, event-time, and explicit-coverage requirements. At this checkpoint, a final + clean review remained required before staging; the landed closure below records its completion. + +## [C] Checkpoint #1 + +`2026-07-18/e14-argocd-sync-failure-rule#1` records the review-clean implementation and complete +local proof above on exact base `22c6caa834442120d61add38009bacc154eda4fd`. Next: create the +SSH-signed DCO commit, push one narrow PR into `dev`, require exact-head CI/CodeQL and empty review +and security queues, merge preserving the tested head, verify exact post-merge `dev` proof, close +the child issue, and synchronize the landed Notion and Obsidian checkpoints. + +## [V3] Hosted-review hardening and final local proof — 2026-07-18 + +- Hosted CodeRabbit reviewed exact initial head `03fdc34dcb4b90f3ec2f8c61b5ef71c5b6b55794` and found + two valid gaps: the roadmap omitted the explicit TIMELINE-coverage boundary, and direct + evaluator inputs could produce the Argo Application advisory without exact Argo/Application + applicability checks. +- The roadmap now states that callers must explicitly declare TIMELINE coverage and that fact + presence never implies coverage. R8 filters its evidence view before trigger, signal, and + coverage evaluation to exact `source_kind=argocd` and `kind=Application` observations, so + mixed-source input order cannot change the result or citation source. +- The complete local follow-up review found and fixed a related canonicalization gap: R8 now + requires exact `change.kind=sync-failed`; case variants cannot pass through the shared + case-insensitive matcher used by the Kubernetes-oriented rules. Tests cover non-Argo sources, + non-Application kinds, case variants, reversed mixed-source input, and the exact `alpha` plus + `beta` entity-local scope set. +- A repeated complete CodeRabbit CLI 0.6.5 review of all 14 changed/new files against exact base + `22c6caa834442120d61add38009bacc154eda4fd` reports zero findings. The pre-check candidate scanned + 239,208 bytes with zero recognized credential, private-key, provider-token, JWT, + authenticated-URL, or generic secret-assignment candidates. +- Focused brain/CLI race tests and full `make ci` pass after the fixes: formatting, vet, zero lint + findings, `govulncheck` with no reachable vulnerabilities, 87.9% brain coverage, full race + suite, policy checks, eight Prometheus rules, performance, compiled E2E, and build. +- Digest-pinned PostgreSQL 18.4 forced-RLS tests pass with 76.2% `hubdb` coverage; both 50,000-case + workspace-isolation fuzzers pass with four workers. Pinned Helm 4.2.3 and standalone + linux/amd64 plus linux/arm64 OCI contracts pass. +- The release gate produces two identical four-platform snapshots with verified archives, SPDX + SBOMs, Homebrew formula, and a release-derived two-platform hub OCI layout. The real + digest-pinned Kubernetes v1.36.1 two-cluster gate passes in 239.320 seconds under the race + detector; independent cleanup confirms zero Kind clusters and zero Sith containers. + +## [C] Checkpoint #2 + +`2026-07-18/e14-argocd-sync-failure-rule#2` records the hosted-review fixes and repeated local +proof above. At this checkpoint, the next steps were the final exact-tree secret scan and review, a +second SSH-signed DCO/GSTACK commit, fresh exact-head CI/CodeQL, clean incremental hosted review, +thread resolution, and exact post-merge proof. The landed closure below records the hosted and +review outcomes; the later audit note records a fresh scan of the exact landed tree. + +## [L] Landed closure — 2026-07-18 + +- [PR #271](https://github.com/ArdurAI/sith/pull/271) merged exact signed DCO/GSTACK feature commit + `7f243bb57b4f98aba878c3fc1eee917ef234a539` as + `89cffa5cb6f4e51c8c5f4ef9410f323ee044f493`. +- Exact-head [CI](https://github.com/ArdurAI/sith/actions/runs/29652392300) and + [CodeQL](https://github.com/ArdurAI/sith/actions/runs/29652391036) passed before merge. +- Exact post-merge `dev` [CI](https://github.com/ArdurAI/sith/actions/runs/29652751808) and + [CodeQL](https://github.com/ArdurAI/sith/actions/runs/29652752052) passed on the merge commit. +- Final-head hosted CodeRabbit reviewed the delta from + `03fdc34dcb4b90f3ec2f8c61b5ef71c5b6b55794` to the landed feature commit and generated no + actionable comments; the final unresolved review-thread count was zero. +- Issue [#270](https://github.com/ArdurAI/sith/issues/270) closed after verification, with zero + open Dependabot, code-scanning, or secret-scanning alerts. +- The retained pre-merge record documents zero secret candidates in both changed-tree scans. A + 2026-07-22 retrospective Trivy scan of exact landed feature tree + `7f243bb57b4f98aba878c3fc1eee917ef234a539` also returned zero secret findings. diff --git a/sessions/2026-07-18-e14-elasticsearch-r3-bridge.md b/sessions/2026-07-18-e14-elasticsearch-r3-bridge.md new file mode 100644 index 0000000..14af441 --- /dev/null +++ b/sessions/2026-07-18-e14-elasticsearch-r3-bridge.md @@ -0,0 +1,121 @@ +# Session — 2026-07-18 — E14 Elasticsearch R3 graph bridge + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e14-elasticsearch-r3-bridge` +**Slice:** [#280](https://github.com/ArdurAI/sith/issues/280), E14 +[#46](https://github.com/ArdurAI/sith/issues/46) · **Status:** complete local proof; ready for signed commit + +## [G] Goal + +Connect the reviewed, bounded Elasticsearch `search/ecs-v1` log-cause facts to the existing R3 +CrashLoop rule without widening the source contract, retaining raw logs, inferring coverage, +correlating Pods, or adding any read, credential, storage, mutation, or execution capability. + +## [S] Scope + +- Route only exact Elasticsearch `LogSignal` TELEMETRY `FactDerived` values through a fail-closed + graph bridge. +- Revalidate source/provenance, exact Pod attachment, native/resource identity, closed payload, + count, event interval, clock skew, and optional container text. +- Normalize only the exact `logs.cause` value and last classified event time. Preserve source and + staleness; copy caller-declared coverage without inference. +- Exercise the actual projector-to-graph-to-R3 path, deterministic replay, and text/JSON rendering. +- Keep live Elasticsearch querying, TLS/auth, endpoint/index configuration, mappings, pagination, + retention, negative evidence, dependency identity, new rules, alerts, typed intents, mutation, + dispatch, and execution out of scope. + +## [A] Design and implementation + +- `FromGraphFacts` recognizes the exact `elasticsearch` / `search/ecs-v1` protocol before generic + derived-fact filtering, so malformed exact claims cannot bypass validation. +- Accepted facts must be derived TELEMETRY, have exact source and provenance, carry no attributes + or display fields, identify one Pod in the same scope and namespace, and use a lowercase + `sha256:` native ID whose prefix exactly names the `LogSignal` resource. The projector now binds + that digest only to retained sanitized workspace, Pod, aggregate, and collection fields, so the + bridge can recompute it and fail closed on scope, namespace, Pod, or payload retargeting. +- The JSON object is duplicate-safe, exact-case, single-valued, size-bounded, and closed to `key`, + `value`, `count`, `first_event_at`, `last_event_at`, and optional non-null `container`. +- Only `logs.cause` values `panic`, `missing-config`, and `dependency-failure` survive. Count, + container, source payload, and all raw source material are discarded after validation. +- The emitted observation uses the attached Pod identity, the last event time, evidence source, + and fact stale flag. Different Pods remain different evaluator entities; no fleet correlation is + introduced. + +## [T] Focused proof + +- The brain and CLI packages pass after formatting. +- All three cause classes pass through the real Elasticsearch projector and become one exact + Pod-scoped R3 observation and citation. +- Missing, unavailable, stale, omitted, and observation-stale TELEMETRY coverage remain honest; + fact presence never creates coverage. +- Adversarial cases cover workspace/source/scope/namespace/entity mismatch, ambiguous entity + dimensions, wrong kind/lens/protocol, malformed native/resource identity, unexpected metadata, + missing/duplicate/mixed-case/unknown/trailing/malformed JSON, invalid count/time/window, and + invalid optional container values. +- A cross-Pod regression proves Elasticsearch evidence for one Pod cannot strengthen another. +- The deterministic replay corpus now includes a sanitized Elasticsearch R3 case. +- Text and JSON renderer tests start with a raw-message-bearing Search response, pass through the + source projector and graph bridge, and prove the raw marker plus count/container/time metadata + never appears in output. +- Native Go fuzzing completed 653,689 post-hardening executions in fifteen seconds without a panic + or closed contract escape. +- Focused race coverage on the original feature-head local run is green: Elasticsearch 95.7%, + brain 88.6%, and CLI 62.7%. +- CodeRabbit's first complete uncommitted review found one valid defense-in-depth improvement: make + the non-empty source scope requirement explicit at the Elasticsearch bridge even though graph + validation already enforces it. The explicit check and regression are present, focused race tests + remain green, and the repeated full review reports zero findings across all changed files. +- `make ci` passes formatting, vet, lint with zero findings, `govulncheck` with no reachable + vulnerabilities, the complete race suite, policy and alert-rule checks, performance, subprocess + E2E, and production build. Brain coverage is 88.6% and Elasticsearch remains 95.7%. +- `make e2e-isolation` passes PostgreSQL 18.4 forced-RLS tests and both 50,000-execution + cross-workspace fuzzers. +- `make release-check` verifies modules, two byte-identical four-platform release builds, archive + contents, SPDX SBOMs, Homebrew output, and the release-derived amd64/arm64 distroless Hub OCI + layout. +- `make e2e-kind` passes the pinned Kubernetes 1.36.1 two-cluster fan-out, OCI, and Argo suite in + 234.972 seconds. Teardown leaves no Kind cluster or matching Kind/Sith container. +- `README.md` was reviewed in full and updated because this slice adds a public graph-fact behavior + even though the cache-backed CLI still performs no Elasticsearch fetch. A final high-signal + credential scan finds only the deliberately fake `.invalid` raw-message marker used by the CLI + non-retention test; no real credential or private-key material is present. + +## [S] Security, reliability, and cost + +Raw logs are classified and discarded by the existing source projector before the bridge executes. +The bridge accepts only the closed sanitized fact and fails closed on ambiguous provenance, +identity, or payload. It creates one bounded in-memory observation per accepted fact and performs no +I/O. There is no new cloud resource, API request, egress, retention, credential, privilege, write +path, or recurring cost. A future live reader remains responsible for TLS, least-privilege index +`read`, an allowlisted target, finite request/query budgets, and complete-shard responses. + +## [P] Primary sources + +- [Elasticsearch Search API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search) +- [Elasticsearch selected fields](https://www.elastic.co/docs/reference/elasticsearch/rest-apis/retrieve-selected-fields) +- [Elastic Common Schema orchestrator fields](https://www.elastic.co/docs/reference/ecs/ecs-orchestrator) +- [Filebeat Kubernetes processor fields](https://www.elastic.co/docs/reference/beats/filebeat/exported-fields-kubernetes-processor) + +## [N] Next + +Create one signed DCO/GSTACK commit, publish and verify the exact PR head, require hosted CI, +CodeQL, and CodeRabbit, merge without rewriting the signed head, prove exact post-merge `dev`, close +issue 280, update #46 without claiming E14 complete, recheck all GitHub security queues, and synchronize +Notion and Obsidian. + +## [C] Checkpoint #1 + +Pending signed implementation commit — issue, design, production bridge, identity hardening, +adversarial tests, replay, CLI privacy proof, documentation, full local gate matrix, repeated +zero-finding review, and clean Kind teardown are frozen in the EXTENDED worktree. + +## [C] Checkpoint #2 + +The explicit hosted review identified two valid fail-closed gaps and one Markdown lint defect. +The bridge now rejects unreviewed provenance metadata and malformed UTF-8 before decoding; dedicated +regressions cover both paths. The session note no longer starts a line with a malformed issue token. +The complete post-remediation local matrix is green: `make ci` with brain coverage at 88.7% and no +lint or vulnerability findings; 296,650 native fuzz executions; PostgreSQL forced-RLS and both +50,000-execution workspace-isolation fuzzers; reproducible release, SPDX SBOM, Homebrew, and +multi-platform distroless OCI proof; and pinned Kubernetes 1.36.1 two-cluster Kind in 246.857 +seconds with clean teardown. Two consecutive complete local CodeRabbit reviews report zero findings +across all 13 changed files. A new signed commit and fresh exact-head hosted proof remain required. diff --git a/sessions/2026-07-18-e14-github-actions-failure-rule.md b/sessions/2026-07-18-e14-github-actions-failure-rule.md new file mode 100644 index 0000000..319c742 --- /dev/null +++ b/sessions/2026-07-18-e14-github-actions-failure-rule.md @@ -0,0 +1,84 @@ +# Session — 2026-07-18 — E14 GitHub Actions workflow-run failure rule + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e14-github-actions-failure-rule` +**Slice:** [#278](https://github.com/ArdurAI/sith/issues/278), E14 +[#46](https://github.com/ArdurAI/sith/issues/46) · **Status:** complete local proof; ready for signed commit + +## [G] Goal + +Add an honest adjacent R9 rule that recognizes a GitHub Actions workflow-run failure from one +already-authorized GitHub REST response without inventing a root cause or adding a fetch, credential, +write, alert, or execution path. + +## [D] Design + +- Keep host, owner, repository, run ID, collection time, response retrieval, and TIMELINE coverage + caller-owned. +- Normalize only an internally consistent `Get a workflow run` response under GitHub REST API + version `2026-03-10`. +- Emit one unattached source-native TIMELINE fact only for exact status `completed` and conclusion + `failure`, `timed_out`, or `startup_failure`. +- Abstain for incomplete and completed non-failure runs. Fail closed for unknown states, duplicate + JSON members, malformed or oversized responses, invalid timestamps, and identity disagreement. +- Bridge only exact `github` / `WorkflowRun` / `workflow-runs/2026-03-10` facts whose closed payload, + run/attempt/native identity, source, and event time agree. +- Expose only canonical `change.kind=workflow-run-failed` to the deterministic evaluator. R9 stays + entity-local and requires explicit caller-declared TIMELINE coverage. + +## [S] Security and non-claims + +The projection retains no job, step, log, actor, branch, commit, URL, token, annotation, unknown +field, or raw response. It does not infer repository-to-workload identity, correlate across hosts, +diagnose code/configuration/credential/permission/capacity/dependency causes, create an alert or SLO, +load a token, call GitHub, persist data, form a typed intent, cross the PEP, mutate, dispatch, rerun, +or execute. The advisory is sensitive human inspection guidance only. + +## [T] Focused proof + +- Positive, abstention, malformed/duplicate/oversized/invalid-UTF-8, identity, status, conclusion, + timestamp, graph-boundary, stale-coverage, exact applicability, replay, renderer, and non-correlation + tests pass under the race detector. +- A 50,000-execution workflow-projector fuzz campaign preserves bounded valid-output invariants. +- Complete repository CI passes with zero lint findings, no reachable vulnerabilities, 87.1% brain + coverage, 92.8% GitHub connector coverage, policy scripts, alert contracts, performance, e2e smoke, + and production build. +- PostgreSQL 18.4 forced-RLS isolation passes, as do both 50,000-execution cross-workspace fuzzers. +- Helm and multi-platform OCI contracts pass. Four-platform release archives rebuild identically; + SPDX SBOM, Homebrew, and release-derived hub OCI verification passes. +- The pinned Kubernetes 1.36.1 two-cluster fleet/image/Argo suite passes in 231.116 seconds; teardown + leaves no Kind clusters or Kind containers. +- CodeRabbit's first 17-file review found two valid issues: ambiguous correlation wording and an + exact workflow-protocol fact that could bypass provenance validation when both source fields were + malformed. A later synchronized-diff review found Go's case-insensitive JSON field matching could + accept mixed-case aliases. All three are fixed in the response projector and graph bridge with + regression tests; the final complete review reports zero findings. +- Manual red-team review additionally makes exact workflow-protocol facts with non-TIMELINE fact + types fail explicitly. README review is complete. +- A high-signal scan of the complete changed-file set reports zero credential or private-key + candidates. + +## [O] Operability and cost + +The slice adds bounded in-memory JSON validation and one small normalized fact per accepted response. +It introduces no network request, retained store, queue, controller, hosted service, telemetry series, +or recurring cloud cost. A future reader would still incur GitHub API quota and any log-download +egress; neither is implemented here. + +## [P] Primary sources + +- [GitHub REST workflow runs, API 2026-03-10](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2026-03-10) +- [GitHub Checks status and conclusion contract](https://docs.github.com/en/rest/guides/using-the-rest-api-to-interact-with-checks) +- [GitHub workflow-run history](https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/monitoring-workflows/viewing-workflow-run-history) + +## [N] Next + +Create one signed DCO/GSTACK commit, publish a PR to `dev`, require +exact-head CI/CodeQL/hosted CodeRabbit, merge without rewriting the signed head, prove exact +post-merge `dev` checks, close #278, update E14 #46 without claiming the epic is complete, and recheck +the GitHub security queues. + +## [C] Checkpoint #1 + +Pending signed implementation commit — bounded R9, documentation, complete local proof, repeated +zero-finding final review, and knowledge checkpoints are frozen; next: verify the signed commit, +push, and require the exact hosted gates before merge. diff --git a/sessions/2026-07-18-e14-image-pull-rule.md b/sessions/2026-07-18-e14-image-pull-rule.md new file mode 100644 index 0000000..b2c4947 --- /dev/null +++ b/sessions/2026-07-18-e14-image-pull-rule.md @@ -0,0 +1,89 @@ +# E14 R7 honest image-pull failure rule + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e14-image-pull-rule` + +**Slice:** E14 R7 / [#268](https://github.com/ArdurAI/sith/issues/268) · **Status:** review fixes verified + +**Base:** `origin/dev` at `91f4a47da7691843fd9e00b257849e63243646fd` + +## [G] Goal + +Add deterministic, evidence-cited detection for exact Kubernetes `ImagePullBackOff` and +`ErrImagePull` waiting reasons without guessing the underlying cause. + +## [S] Boundary + +- Reuse only sanitized LIVE `pod.reason` evidence already projected from bounded Pod status. +- Emit a sensitive-marked, read-only `kubectl describe pod` advisory that Sith never runs. +- Do not retain image references, registry credentials, Secrets, Event messages, or raw payloads. +- Do not claim registry auth, inspect credentials, probe a registry/network, add a connector, + correlate fleet-wide, create a typed intent, or bypass blocked F14.6. + +## [T] Verification plan + +- Exact-reason, near-miss, stale-LIVE abstention, cache projection, replay determinism/schema, CLI, + and no-write-boundary tests. +- Full local, security, CodeRabbit, signed DCO/GSTACK, exact-head, merge, and exact post-merge gates. + +## [R] Primary-source check + +- Kubernetes documents `Waiting` as the container state used while startup operations such as an + image pull are incomplete, with a `Reason` that summarizes the state: + . +- Kubernetes recommends `kubectl describe pods` to inspect container state and recent Events and + notes that image-pull failure is a common reason for a waiting Pod: + . + +These sources support symptom detection and the read-only inspection command. They do not support +choosing a cause from the waiting reason alone, so R7 retains the full uncertainty boundary. + +## [C] Cost and operability + +R7 reuses the existing bounded cache and pure in-memory evaluator. It adds no cloud resource, +credential, network call, watch, storage, telemetry volume, or egress cost. Operator-run +`kubectl describe pod` may expose sensitive detail, so the advisory is explicitly marked sensitive +and is never executed by the brain. + +## [V] Local verification — 2026-07-18 + +- Exact-reason, canonical/lowercase case-folding, hostile near-miss, stale/unavailable LIVE, + citation identity, shell-quoted advisory, no-PR-diff, cache projection, replay, CLI text/JSON, + and no-fleet-correlation tests pass under the race detector. +- Full `make ci` passes: formatting, vet, lint, `govulncheck`, repository race coverage, shell + policies, pinned Prometheus rule fixtures, performance, end-to-end tests, and binary build. +- PostgreSQL 18.4 forced-RLS/isolation tests plus two 50,000-case workspace fuzz campaigns pass. +- Reproducible release/archive/SPDX-SBOM and immutable two-platform OCI layout verification pass. +- Helm 4.2.3, standalone cross-platform OCI, and digest-pinned Kubernetes v1.36.1 two-cluster kind + gates pass; kind completed in 238.980 seconds and left no clusters or Sith test containers. +- GitHub secret scanning found zero secrets in 22,207 changed bytes. +- CodeRabbit CLI 0.6.5 reviewed all 13 changed/new files against exact base + `91f4a47da7691843fd9e00b257849e63243646fd`. Its valid R7-identity assertion finding is fixed; + its request to reject lowercase `errimagepull` was declined because the issue explicitly requires + exact case-insensitive matching. The second complete review reports zero findings. +- README review is complete. The investigation section documents R7's uncertainty, sensitive + read-only advisory, retained-data exclusions, and explicit exemption from fleet correlation. + +## [V2] Hosted review remediation — 2026-07-18 + +- Exact-head CI run [29647811006](https://github.com/ArdurAI/sith/actions/runs/29647811006) + and CodeQL run [29647810480](https://github.com/ArdurAI/sith/actions/runs/29647810480) + passed for signed head `604eb1778252f9e8e1a0444d98546d52e2da0f07`. +- Hosted CodeRabbit reviewed the exact 13-file base-to-head diff and identified one documentation + inconsistency plus three test-hardening opportunities. Validation exposed two real fail-open + behaviors: whitespace-padded rule values were normalized before matching, and undeclared lens + coverage could be inferred from an observation. +- Matching now remains case-insensitive but does not trim caller-controlled values. Missing, + unavailable, or stale coverage now fails closed to `unconfirmed`; cache tests assert the full + Pod citation boundary; and both authoritative rule inventories distinguish implemented R7 from + future adjacent rules. +- Focused brain and CLI race tests pass. The full `make ci` gate passes with zero lint findings and + zero known Go vulnerabilities. PostgreSQL 18.4 forced-RLS coverage and both 50,000-case + workspace-isolation fuzz campaigns also pass on the review-fixed working tree. +- The pre-commit base-to-working-tree secret scan reports zero recognized credential or private-key + patterns. CodeRabbit CLI 0.6.5 then re-reviewed the complete 13-file diff and reported zero + findings. README was rechecked and already contains the required operator-facing boundary; no + additional README change is needed for the review fix. + +Remaining before completion: signed DCO/GSTACK review-fix commit, push, fresh exact-head +CI/CodeQL and hosted review with empty review/security queues, merge preserving the tested head, +and exact post-merge `dev` CI/CodeQL proof. diff --git a/sessions/2026-07-18-e6-bounded-audit-export.md b/sessions/2026-07-18-e6-bounded-audit-export.md new file mode 100644 index 0000000..d2bdfe3 --- /dev/null +++ b/sessions/2026-07-18-e6-bounded-audit-export.md @@ -0,0 +1,58 @@ +# Session — 2026-07-18 — E6 bounded verified audit export + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e6-bounded-audit-export` +**Slice:** [#256](https://github.com/ArdurAI/sith/issues/256), E6 [#24](https://github.com/ArdurAI/sith/issues/24) · **Status:** locally verified + +## [G] Goal + +Expose the existing privacy-minimized policy and approval hash chain as one portable workspace +compliance record without claiming the future unified action/Ardur decision ledger. + +## [D] Design + +- Add the closed `export-audit` action and `audit.export` verb. Only a signed workspace admin may + pass both the PEP and the independent store-side role check. +- Accept one exact bearer-only, query-free, body-free + `GET /v1/workspaces/{workspace}/audit/export` route. Browser cookies, filters, selectors, raw + payloads, non-canonical paths, and spoofed identity/correlation carriers are not accepted. +- Durably audit the authorization decision before reading the export. The successful document + therefore contains the decision that authorized its own disclosure. +- Read and verify the complete retained chain in one forced-RLS repeatable-read transaction, return + only after commit, then structurally revalidate and encode the finished document at the HTTP + boundary. Network backpressure cannot pin the database snapshot. +- Limit one online document to 512 entries and four concurrent process requests. Return one generic + unavailable response for saturation, oversize, tamper, uninitialized state, or store failure. + +## [S] Security and non-claims + +The export contains only actor/role, closed operation and verdict metadata, approval evidence +digests, timestamps, trace identifiers, and SHA-256 chain links already retained by Sith. It adds no +target, arguments, selector, justification, credential, policy digest, request/response payload, +connector, KMS, filesystem, process, Kubernetes client, background job, object store, or mutation +surface. It is not an intent-correlated decision ledger, WORM storage, external anchoring, +pagination, or E6 completion. + +## [T] Local proof + +- Unit and race suites pass for the portable schema, tenant role matrix, PEP, store, handler, + runtime composition, and structural privacy boundary. +- PostgreSQL 18.4 proves forced-RLS isolation, own-authorization inclusion, complete-chain + verification, tamper refusal, exact 512-entry success, exact 513-entry refusal, and admin-only + store access. +- Full `make ci` passes after the final payload-free request hardening; `govulncheck` reports no + reachable vulnerability. +- Reproducible release/SBOM, multi-platform OCI, Helm, two-cluster Kind, and two 50,000-case + cross-workspace plus two 50,000-case audit-framing fuzz gates pass. +- CodeRabbit 0.6.5 found documentation/test-policy improvements and one body-framing omission. All + were corrected; the final complete 17-file review reports no findings. + +## [O] Operability and cost + +Each successful request adds one audit row, scans at most 512 retained rows, buffers a bounded JSON +document, and incurs normal HTTP egress. There is no new cloud resource or recurring stream. Larger +workspaces need a separately reviewed pagination or asynchronous export protocol. + +## [N] Next + +Create one signed DCO/GSTACK commit and PR into `dev`; require exact-head CI, CodeQL, empty review +threads and security queues, merge, and exact post-merge `dev` CI/CodeQL before closing #256. diff --git a/sessions/2026-07-18-e6-offline-audit-verifier.md b/sessions/2026-07-18-e6-offline-audit-verifier.md new file mode 100644 index 0000000..ea5987c --- /dev/null +++ b/sessions/2026-07-18-e6-offline-audit-verifier.md @@ -0,0 +1,61 @@ +# Session — 2026-07-18 — E6 offline portable audit verifier + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e6-offline-audit-verifier` +**Slice:** [#258](https://github.com/ArdurAI/sith/issues/258), E6 [#24](https://github.com/ArdurAI/sith/issues/24) · **Status:** locally verified + +## [G] Goal + +Let an incident responder or compliance recipient verify an F6.4a export from the serialized JSON +alone, without contacting a hub or claiming external authenticity. + +## [D] Design + +- Centralize the format-1 and format-2 canonical SHA-256 framing in `internal/auditrecord`; the + PostgreSQL writer, retained-chain verifier, HTTP projection check, and offline verifier share it. +- Add `sith audit verify ` as a local-only command. It accepts exactly one stable, + non-symlink regular file, bounds the read to 1 MiB, and retains the 512-entry document ceiling. +- Parse JSON case-sensitively and reject duplicate, unknown, malformed, or trailing content before + checking the closed schema and every hash-bound field, link, and head. +- Emit only schema, workspace, count, head, and the explicit `internally-consistent` status. + +## [S] Security and non-claims + +The command opens no network connection, hub session, database, credential store, temporary file, +or telemetry path. It refuses directories, devices, FIFOs, symlinks, unstable files, oversized +inputs, unsupported formats, malformed shapes, and any hash mismatch. Internal consistency does +not prove origin and cannot detect wholesale replacement by a privileged store owner without an +external anchor. No Ardur decision-ledger, intent lifecycle, WORM, pagination, proposal, dispatch, +or execution claim is added. + +## [T] Proof plan + +- Immutable golden hashes for policy and approval formats plus negative controls for all bound + fields and the closed invalid-request sentinel. +- CLI tests for bounded summaries, strict JSON, tamper, file type, symlink, FIFO, size, and race. +- PostgreSQL mixed-version export-to-offline verification, full CI, vulnerability, release/SBOM, + Helm, OCI, two-cluster Kind, independent review, hosted exact-head, and post-merge proof. + +## [V] Local proof + +- Focused unit and race suites pass for `auditrecord`, CLI, Hub HTTP export, and storage. +- Immutable format-1 and format-2 golden hashes pass; 50,000 canonical-field mutations cannot + verify without rehashing. +- PostgreSQL 18.4 forced-RLS integration passes at 72.8% focused and 76.2% isolation coverage, + including a mixed-version database export verified by the portable algorithm. +- Full `make ci` passes with zero lint findings and no reachable vulnerabilities. +- Reproducible release/SPDX SBOM, release OCI layout, Helm 4.2.3, cross-platform OCI, and both + 50,000-case workspace isolation campaigns pass. +- Kubernetes v1.36.1 two-cluster Kind passes in 237.938 seconds. +- CodeRabbit 0.6.5 identified sub-microsecond timestamp malleability and invalid-UTF-8 handling; + both were fixed with negative controls. The final complete 15-file review has no findings. + +## [O] Operability and cost + +One bounded local read and at most 512 SHA-256 computations. No cloud resource, object storage, +egress, background process, or recurring telemetry cost. + +## [N] Next + +Complete the full local gate matrix and adversarial review, then create one signed DCO/GSTACK PR +into `dev`. Do not close #258 before exact post-merge `dev` CI and CodeQL pass and all security +queues are empty. diff --git a/sessions/2026-07-18-e6-snapshot-audit-pages.md b/sessions/2026-07-18-e6-snapshot-audit-pages.md new file mode 100644 index 0000000..262de82 --- /dev/null +++ b/sessions/2026-07-18-e6-snapshot-audit-pages.md @@ -0,0 +1,76 @@ +# Session — 2026-07-18 — E6 snapshot-bound audit pages + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e6-snapshot-audit-pages` +**Slice:** [#276](https://github.com/ArdurAI/sith/issues/276), E6 +[#24](https://github.com/ArdurAI/sith/issues/24) · **Status:** complete local proof; ready for signed commit + +## [G] Goal + +Export retained Sith audit chains beyond the existing 512-entry complete-document ceiling as +bounded pages tied to one immutable workspace snapshot, then verify the full page sequence offline. + +## [D] Design + +- Preserve the exact query-free complete export route and schema. +- Add a distinct page route: no query for page one; exactly one canonical `cursor` query for a + continuation; reject every other query, body, cookie, selector, or caller trace input. +- Durably authorize every page request with the existing admin-only `export-audit` action and + `audit.export` verb before storage work. +- Fix page one to the current head after its authorization row. Later page-request authorization + rows remain durable in the live chain but outside the original snapshot. +- Carry a fixed-size versioned base64url continuation bound to a domain-separated workspace digest, + snapshot head sequence/hash, next sequence, and expected previous hash. It is not a credential. +- Validate live/snapshot head and page-boundary anchors under forced RLS and Repeatable Read, read + and validate at most 512 consecutive rows, commit, then encode the finished page. +- Verify page files incrementally with `sith audit verify-pages`; require one same-workspace, + same-snapshot, genesis-to-head sequence without gaps, replay, or reordering. + +## [S] Security and non-claims + +The route accepts only a signed workspace-admin session. Cursor possession grants nothing. +Malformed, foreign, altered, skipped, replayed, swapped, stale, saturated, or integrity-invalid +requests fail without disclosing store details. Page documents retain only the already-sanitized +policy and approval events. This is not asynchronous export, WORM retention, external anchoring or +authenticity, the Ardur decision ledger, a complete action lifecycle, or E6 completion. + +## [T] Focused proof + +- Portable contract and cursor tests pass under the race detector. +- A 50,000-execution cursor mutation fuzz campaign cannot bind a changed continuation to the + original page. +- CLI, storage, HTTP, and runtime focused race suites pass. +- PostgreSQL 18.4 forced-RLS integration passes at 76.6% package coverage. It proves exact 512/513 + paging, an append after page one that cannot move the fixed snapshot, offline full-sequence + verification, role/workspace refusal, and altered head/previous-anchor rejection. +- Complete repository CI passes with zero lint findings, no reachable vulnerabilities, race + coverage, policy scripts, e2e tests, and a production build. +- Both 50,000-execution tenant-isolation fuzz campaigns pass. +- Reproducible four-platform release archives, SPDX SBOMs, the Homebrew artifact, and the + multi-platform release-hub OCI layout pass verification. +- Helm and cross-platform OCI contract suites pass under the race detector. +- The pinned Kubernetes 1.36.1 two-cluster fleet, image, and Argo projection suite passes in + 243.827 seconds; teardown leaves no Kind clusters or Sith/Kind containers. +- CodeRabbit's first complete-diff pass found two valid minor issues: canonical timestamp + structural validation and transaction-order wording. Both were corrected; regression coverage + passes and the second complete-diff pass reports zero findings across all 16 changed files. +- A high-signal secret scan reports no credential or private-key material in the review scope. + +## [O] Operability and cost + +Each page adds one audit row, reads at most 512 entries plus fixed anchors, performs bounded SHA-256 +work, and incurs one response of egress. Total work is linear in retained entries and page count. +No object store, background job, queue, external service, or recurring cloud resource is added. + +## [P] Primary sources + +- [PostgreSQL 18 Repeatable Read](https://www.postgresql.org/docs/18/transaction-iso.html#XACT-REPEATABLE-READ) +- [PostgreSQL 18 row security](https://www.postgresql.org/docs/18/ddl-rowsecurity.html) +- [RFC 4648 base64url](https://www.rfc-editor.org/rfc/rfc4648.html#section-5) +- [Go encoding/base64](https://pkg.go.dev/encoding/base64) + +## [N] Next + +Create one signed DCO/GSTACK commit with the mandated author, push the branch, and open the PR to +`dev`. Require exact-head CI, CodeQL, and hosted CodeRabbit; merge without rewriting the signed +head, prove the exact post-merge `dev` checks, close #276, update E6 #24 without claiming the epic is +complete, and recheck the GitHub security queues. diff --git a/sessions/2026-07-19-e12-wire-adapter-version-split.md b/sessions/2026-07-19-e12-wire-adapter-version-split.md new file mode 100644 index 0000000..cb92230 --- /dev/null +++ b/sessions/2026-07-19-e12-wire-adapter-version-split.md @@ -0,0 +1,97 @@ +# Session — 2026-07-19 — E12 wire and adapter version split + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e12-wire-version-split` +**Slice:** [#288](https://github.com/ArdurAI/sith/issues/288), E12 +[#30](https://github.com/ArdurAI/sith/issues/30) · **Status:** local proof complete; hosted proof pending + +## [G] Goal + +Split framework transport compatibility from opaque adapter/evidence provenance and prove a +deterministic fail-closed negotiation seam before adding protobuf, gRPC transport, subprocesses, +credentials, or networking. + +## [S] Scope + +- Structured framework `WireVersion{Major, Minor}` values. +- Explicit supported-version offers with exact-set negotiation. +- A separately named opaque connector `AdapterVersion`. +- Atomic registry validation and deterministic descriptor inspection. +- Existing fleet evidence `protocol_version` serialization remains unchanged. +- No protobuf, generated code, gRPC dependency, process launch, listener, credentials, network, + persistence migration, adapter port, execution, package, release, cluster, or cloud change. + +## [A] Decision and implementation + +- GR approved the recommended E12 version split on 2026-07-19. The decision is recorded on + [parent issue 30](https://github.com/ArdurAI/sith/issues/30#issuecomment-5016946815), in Notion, + and in the EXTENDED-backed Obsidian Sith decision trail. +- Opened bounded child issue 288 after confirming no duplicate issue or competing pull request. +- Added a structured wire-version type and initial version 1.0. +- Added exact-set negotiation that selects the highest mutually advertised version and + distinguishes invalid offers, major mismatch, and unsupported minor overlap. +- Registry validation rejects empty, zero-major, duplicate, or missing wire offers and empty + adapter versions before registration, then stores a sorted defensive copy. +- Migrated the local kubeconfig and GitHub planning descriptors plus test connectors while leaving + all evidence `ProtocolV` values untouched. +- Updated the source-adapter spec, E12 mirrors, accepted ADR 0014, and this GSTACK journal. + +## [T] Proof + +- Connector, kubeconfig, GitHub planner, CLI, hydrator, and fleet package tests pass. +- Negotiation tests cover exact 1.0, explicit minor fallback, highest explicit major, input order, + empty offers, zero major, duplicates, major mismatch, and same-major/no-common-minor refusal. +- Registry tests cover atomic malformed-version refusal, deterministic sorting, descriptor JSON + domain separation, and defensive copying. +- A native Go fuzz target asserts that every successful result was explicitly offered by both + endpoints and that no higher common version was skipped. +- The first complete CodeRabbit review found two documentation/contract clarity gaps: make `Kind` + normatively the canonical target-tool identity and make `Major > 0` explicit in the source spec. + Both were fixed, including a cross-taxonomy duplicate-tool registry test. +- Manual red-team review found that future peer-controlled offers needed an explicit cardinality + bound. Offers are now limited to 32 versions and oversized input is covered by refusal tests. +- `go mod verify` passed and pinned `govulncheck` v1.6.0 reported no vulnerabilities. +- The full `make ci` gate passed formatting, vet, zero-issue lint, race tests, shell-policy tests, + nine Prometheus rule tests, warm-cache performance, a 30.724-second binary end-to-end run, and + the production build. +- `make e2e-isolation` passed the forced PostgreSQL RLS suites and both isolation fuzz targets at + exactly 50,000 executions each. +- `make release-check` passed GoReleaser validation, dual snapshot reproducibility, SBOM and + checksum generation, Homebrew formula generation, and the multi-architecture OCI layout check. +- The pinned kind gate passed the fleet fan-out, OCI image contract, and Argo Application + projection tests under the race detector in 244.090 seconds. Teardown left no kind clusters. +- The final whole-diff CodeRabbit review covered all 18 changed files and returned zero findings. +- `README.md` was reviewed before commit. No update is needed because this slice changes an + internal connector compatibility seam; the operator-visible contract is documented in the + source-adapter spec and ADR 0014. + +## [S] Security, reliability, and cost + +The version boundary fails closed and never parses opaque provenance as transport compatibility. +It adds no authority, process, listener, credential, I/O, telemetry cardinality, or recurring cloud +cost. The remaining transport security and supervision design stays separately reviewable. + +## [P] Primary sources + +- [Protocol Buffers proto3 compatibility rules](https://protobuf.dev/programming-guides/proto3/) +- [Protocol Buffers best practices](https://protobuf.dev/best-practices/dos-donts/) +- [gRPC core concepts](https://grpc.io/docs/what-is-grpc/core-concepts/) +- [ADR 0014](../docs/adr/0014-connector-wire-adapter-version-split.md) + +## [N] Next + +Create one signed DCO/GSTACK commit, push the branch, and require exact-head review/CI/CodeQL plus +exact post-merge `dev` proof before closing issue 288. E12 remains open for the separately approved +transport and supervision slices. + +## [C] Checkpoint #1 + +The owner decision, child issue, Notion record, EXTENDED Obsidian decision, implementation, +adversarial unit tests, fuzz target, spec correction, E12 mirrors, ADR, and GSTACK journal are +present. Full local and hosted proof remain fail-closed gates. + +## [C] Checkpoint #2 + +All required local gates are green: module integrity, pinned vulnerability scanning, full CI, +isolation fuzzing, reproducible release assembly, and pinned real-cluster end-to-end coverage. +Disposable cluster cleanup and a zero-finding final whole-diff review are verified. The remaining +gates are the signed commit, exact-head hosted proof, merge, and exact post-merge `dev` proof. diff --git a/sessions/2026-07-20-e14-remediation-candidate-contract.md b/sessions/2026-07-20-e14-remediation-candidate-contract.md new file mode 100644 index 0000000..939ddde --- /dev/null +++ b/sessions/2026-07-20-e14-remediation-candidate-contract.md @@ -0,0 +1,107 @@ +# Session — 2026-07-20 — E14 remediation candidate contract + +**Builder:** Gnani Rahul · **Branch:** `gnanirahulnutakki/feat/e14-remediation-candidate` +**Slice:** [#292](https://github.com/ArdurAI/sith/issues/292), E14 +[#46](https://github.com/ArdurAI/sith/issues/46) · **Status:** local proof complete; hosted proof pending + +## [G] Goal + +Implement GR-approved F14.6 option 1 as a contract-only child: let reviewed Brain rules emit an +inert typed remediation requirement template without resolving a target, constructing handler +arguments, proposing to PEP, approving, dispatching, mutating, or executing anything. + +## [S] Scope + +- Add a `RemediationCandidate` containing only one closed `intent.Verb` and ordered closed + provenance requirements. +- Map R1 only to `argocd.rollback`, requiring an authoritative Argo Application target and exact + revision. +- Map R2 and R4 only to `gitops.open-pr`, requiring repository, base ref, expected base commit, + file path, observed blob identity, and exact bounded desired content. +- Keep R3, R5, R6, R7, R8, and R9 advisory-only and preserve every existing advisory. +- Update deterministic replay expectations, public documentation, and the no-write package + boundary. +- Exclude PEP imports or proposals, provenance resolution, handler invocation, approval, + persistence, credentials, signatures, network, dispatch, mutation, execution, and cloud changes. + +## [A] Decision and implementation + +- GR approved the two-stage structured-candidate design on 2026-07-20. The decision is recorded on + [parent issue 46](https://github.com/ArdurAI/sith/issues/46#issuecomment-5017157418), bounded child + issue 292, Notion, and the EXTENDED-backed Obsidian Sith decision trail. +- Added a rule-owned candidate with exactly two public fields: the existing closed verb type and a + slice of closed provenance requirement identifiers. +- Encoded the two approved verb-to-requirements mappings in a closed switch. Unsupported or empty + verbs produce no candidate. +- Candidate JSON marshaling and unmarshaling fail closed on missing, reordered, duplicated, + unknown-field, unknown-verb, forged, and trailing forms. +- Each evaluation returns fresh candidate state, and fleet correlation deep-copies its candidate + so a caller cannot mutate an entity verdict through a fleet-wide alias. +- Replay fixtures now lock R1, R2, and R4 to the approved mappings and prove every other canonical + rule remains candidate-free. +- The Brain import boundary now rejects connector, Hub DB, local-ops, MCP, PEP, SQL, network, OS, + process, and gRPC paths. The package imports only the side-effect-free intent vocabulary. +- Updated the E2 source specification, README, this GSTACK journal, Notion, and Obsidian. F14.6 + intentionally remains open for a separately reviewed authoritative provenance and governed-handoff child. + +## [T] Proof + +- `go mod verify` passed. +- `go test -race -count=20 ./internal/brain`, focused vet, and the exact linter passed with zero + issues. Adversarial tests cover public shape, exact catalog mappings, strict JSON, repeated-call + mutation isolation, fleet/entity alias isolation, unconfirmed R1 behavior, and advisory-only R3. +- `make ci` passed formatting, vet, zero-issue lint, `govulncheck ./...` with no vulnerabilities, + the complete race suite, script policies, nine Prometheus rules, performance, binary end-to-end, + and production build. Brain coverage was 88.2%. +- `make e2e-isolation` passed PostgreSQL 18.4 forced-RLS tests and both cross-workspace fuzz targets + at exactly 50,000 executions each. +- `make release-check` passed module verification, dual four-platform release reproducibility, + archive checks, SPDX SBOMs, checksums, Homebrew formula generation, and the release-derived + amd64/arm64 distroless OCI layout. +- The isolated EXTENDED Docker configuration initially hid Docker Desktop's Buildx plugin. A + plugin-only symlink restored Buildx without copying credentials or auth state; the complete gate + then passed. One later registry TLS timeout was treated as external and the full gate passed on retry. +- The pinned Kubernetes 1.36.1 Kind gate passed fleet fan-out, OCI image contract, and Argo + Application projection under the race detector in 238.591 seconds. Cleanup left no matching Kind + cluster or Buildx builder. +- `README.md` was reviewed before commit and updated because verdict JSON now exposes an optional + public candidate field and the prior no-intent-import statement would have become false. +- The complete diff received a manual security and correctness review. Hosted review, exact-head + CI/CodeQL, and exact post-merge `dev` proof remain mandatory. + +## [S] Security, reliability, and cost + +The candidate is a requirement template, not evidence readiness or authorization. It carries no +target, handler arguments, actor, workspace, credential, signature, policy decision, or execution +state. Human advisory prose never becomes an implicit action contract. The slice performs no I/O +and adds no privilege, API request, egress, storage, telemetry cardinality, cloud resource, or +recurring cost. + +## [N] Next + +Create one SSH-signed DCO/GSTACK commit, push the branch, open a PR to `dev`, require exact-head +hosted review, CI, and CodeQL, merge without rewriting the signed feature commit, and prove the exact +post-merge `dev` SHA. Then close child issue 292, update parent issue 46 without claiming F14.6 or +E14 complete, recheck all GitHub security queues, and synchronize Notion and Obsidian. + +## [C] Checkpoint #1 + +The approved decision, bounded issue, implementation, adversarial tests, replay contract, +import-boundary guard, README/spec updates, Notion/Obsidian records, and complete local gate matrix +are frozen on exact base `76f8f8fb3c630d755eb47d4cc006fa1decae551a`. Remaining gates are the signed +commit, exact-head hosted proof and review, merge, and exact post-merge `dev` proof. + +## [C] Checkpoint #2 + +The complete committed-diff review found two valid fail-closed gaps. Candidate JSON now rejects +case-variant field aliases before Go's case-insensitive struct matching, and the Brain side-effect +import classifier now rejects every listed package tree, including `net/rpc`, `os/user`, +`database/sql/driver`, and gRPC subpackages. Dedicated regressions cover both boundaries. The +remediation-only independent review reports zero findings across all three changed files. + +The complete post-remediation matrix is green: twenty focused race repetitions, focused vet and +zero-issue lint, full CI with no vulnerability findings and Brain coverage at 88.3%, PostgreSQL +forced-RLS plus both 50,000-execution isolation fuzzers, reproducible release/SBOM/formula/OCI +proof, and pinned Kubernetes 1.36.1 Kind in 237.037 seconds with clean teardown. A new signed +checkpoint commit, fresh exact-head hosted CI/CodeQL/review, merge, and exact post-merge `dev` proof +remain required. diff --git a/sessions/2026-07-21-approval-grant-expiry.md b/sessions/2026-07-21-approval-grant-expiry.md new file mode 100644 index 0000000..1e3120a --- /dev/null +++ b/sessions/2026-07-21-approval-grant-expiry.md @@ -0,0 +1,65 @@ +# E5 F5.9b — immutable approval-grant expiry + +**Builder:** gnanirahulnutakki · **Effort:** deep · **Branch:** +`gnanirahulnutakki/approval-expiry-20260721` +**Slice(s):** E5 / F5.9b · #299 · **Status:** ready for review + +--- + +[G] Goal: give every new approval grant one immutable, server-enforced 10-minute absolute +lifetime while preserving exact single-use, tenant isolation, privacy-minimized evidence, and +offline audit verification. + +[D] Decision: PostgreSQL statement time is the only approval and consumption clock. The public +approval API accepts no timestamp, and the one conditional consumption update enforces the +half-open `approved_at <= consumed_at < expires_at` interval. Expired, legacy, unknown, foreign, +mismatched, and replayed grants share `ErrApprovalGrantUnavailable`. + +[D] Decision: new rows use evidence version 2 and audit format 3. The evidence digest uses a new +domain and binds `expires_at`; the audit entry hash also uses a distinct format-3 domain. Existing +format-1 policy records and format-2 approval records remain independently rehashable. + +[A] Action: added migration 0013 with immutable `expires_at = approved_at + interval '10 minutes'` +and evidence-version constraints. Legacy rows are retained, backfilled as evidence version 1, and +excluded from the new consume predicate. Their `consumed_at` value is not fabricated. + +[A] Action: the legacy backfill temporarily removes FORCE RLS only inside the serializable +migration transaction. PostgreSQL holds an access-exclusive table lock, FORCE RLS is restored +immediately after the update, and any error rolls back the entire relaxation. The application +role remains unable to update expiry/evidence fields or delete rows. + +[T] Test: pure Go unit tests and PostgreSQL-tag compilation pass. The real digest-pinned PostgreSQL +18.4 race suite proves an incremental 0012-to-0013 migration, fixed lifetime, legacy invalidation, +pre-approval/expiry refusal without row or audit mutation, one-winner concurrent consumption, +audit rollback, forced RLS, immutable columns, and mixed format-1/2/3 offline verification. + +[T] Test: focused race suites and two 50,000-execution fuzz campaigns cover expiry-evidence framing +and portable format-3 chain integrity. Full `make ci` passes formatting, vet, lint, reachable +vulnerability scanning, all repository race tests, operator policy checks, performance budget, +subprocess E2E, and build. The real isolation gate reaches 76.7% `hubdb` coverage and adds two +100,000-execution tenant-isolation fuzz campaigns. + +[T] Test: `make release-check` passes two reproducible builds, four platform archives, SPDX SBOMs, +Homebrew generation, and the two-platform distroless OCI layout. The pinned real two-cluster kind +suite passes in 242.456 seconds. CodeRabbit CLI 0.6.5 found one minor fixture-clarity issue; the +format-3 fixture now uses a real expiry-bound golden digest, and the second full review reports no +findings. + +[S] Scope boundary: no MCP transport, Ardur PDP, multi-approver policy, configurable lifetime, +renewal, credential minting, connector execution, dispatch, shell/filesystem access, generic apply, +or production mutation is introduced. + +[C] Cost: one timestamp and one small version field per grant, with no new service, queue, poller, +egress, or cloud resource. The consume path remains one indexed transactional update. + +[R] Primary references: +- https://modelcontextprotocol.io/specification/2025-11-25/client/elicitation +- https://cheatsheetseries.owasp.org/cheatsheets/Transaction_Authorization_Cheat_Sheet.html +- https://www.rfc-editor.org/rfc/rfc6749.html#section-4.1.2 +- https://www.postgresql.org/docs/current/sql-update.html +- https://www.postgresql.org/docs/current/functions-datetime.html +- https://www.postgresql.org/docs/current/ddl-rowsecurity.html + +--- + +**Session close:** ready for review · **Open questions touched:** renewal remains outside F5.9b diff --git a/sessions/2026-07-21-ci-release-policy-sync.md b/sessions/2026-07-21-ci-release-policy-sync.md new file mode 100644 index 0000000..48aa6ca --- /dev/null +++ b/sessions/2026-07-21-ci-release-policy-sync.md @@ -0,0 +1,60 @@ +# Session — 2026-07-21 — CI release policy sync + +**Builder:** Gnani Rahul Nutakki · **Branch:** `gnanirahulnutakki/ci-release-policy-sync-20260721` +**Scope:** release-policy enforcement prerequisite · **Status:** local proof complete + +## [G] Goal + +Restore one trustworthy local and hosted CI contract after GitHub Actions maintenance changed the +release action commits without updating a policy test that hosted CI did not execute. + +## [S] Scope + +- Run the complete operator shell-policy suite in hosted CI. +- Test immutable Docker action commit pins without duplicating version-specific commit values. +- Fail the release-image verifier immediately when repository or distribution path + canonicalization fails. +- Exclude dependencies, desktop tooling, product behavior, release publication, and cloud changes. + +## [A] Action + +- Added `make test-scripts` to the hosted build job and removed a duplicate Prometheus policy call. +- Replaced four stale hard-coded action commits with a structural assertion requiring exactly one + full 40-character commit pin and version comment for each expected Docker action. +- Split command substitutions from `readonly` declarations so `set -e` observes failed `cd` + operations, and added a dynamic regression proving an invalid dist path stops before archive or + Docker work. + +## [T] Proof + +- `make test-scripts` passed every Helm, M0, release-tag, release-PR, release-image, and Prometheus + policy assertion. +- ShellCheck is clean for the changed scripts, with intentional literal workflow expressions + explicitly classified as `SC2016` test fixtures. +- Actionlint is clean for the changed workflow. +- Complete local CI passed formatting, vet, zero-issue lint, `govulncheck`, the repository-wide race + suite, all operator policy scripts, nine Prometheus rules, performance, binary end-to-end, and + production build. +- `make e2e-isolation` passed PostgreSQL 18.4 forced-RLS coverage and both cross-workspace fuzzers + at 50,000 executions each. +- `make release-check` passed dual four-platform reproducibility, SPDX SBOMs, checksums, formula + generation, and release-derived amd64/arm64 OCI verification through the changed script. +- Exact-head hosted CI and CodeQL plus exact post-merge `dev` proof remain mandatory. +- `README.md` was reviewed. No update is needed because this changes CI enforcement and verifier + failure handling, not an operator or product contract. + +## [S] Security, reliability, and cost + +Hosted CI can no longer report green while skipping the aggregate release/operator policy suite. +The verifier fails before using an unresolved path. The change adds no API call, cloud resource, +runtime privilege, artifact, storage, telemetry, or recurring cost. + +## [N] Next + +Create one SSH-signed DCO/GSTACK commit, obtain exact-head hosted proof, merge into `dev`, and +rebase the dependent audit-hardening branch onto the verified post-merge SHA. + +## [C] Checkpoint #1 + +The hosted/local policy alignment, durable action-pin assertion, fail-fast verifier change, +regression tests, and local focused proof are frozen for signed review. diff --git a/sessions/2026-07-21-deep-quality-audit.md b/sessions/2026-07-21-deep-quality-audit.md new file mode 100644 index 0000000..844fa40 --- /dev/null +++ b/sessions/2026-07-21-deep-quality-audit.md @@ -0,0 +1,66 @@ +# Session — 2026-07-21 — deep quality audit + +**Builder:** Gnani Rahul Nutakki · **Branch:** `gnanirahulnutakki/audit-hardening-20260721` +**Scope:** repository-wide deep audit and open-issue completion · **Status:** in progress + +## [G] Goal + +Audit exact `dev` with current primary-source tooling, fix every confirmed defect in narrow signed +changes, then complete every independently buildable open issue without weakening fail-closed +boundaries. + +## [S] Scope + +- This checkpoint covers malformed UTF-8 rejection at the tenancy and policy-audit boundary, + audit-hash fuzz-oracle correctness, and lifecycle ownership for coalesced spoke refreshes. +- Release-policy drift, dependency/toolchain updates, desktop-build version drift, approval expiry, + and remaining open-issue slices stay separate so each can be reviewed and reverted independently. +- No credential, authorization, schema, cloud resource, external package, or API surface changes. + +## [A] Action + +- Repaired the audit-hash fuzz target so it compares two valid records with the same unframed bytes + split at different actor/reason boundaries instead of comparing validation-error sentinels. +- Added explicit UTF-8 validation for policy actors and tenancy identities. This aligns the durable + writer with the portable audit verifier and fails before malformed bytes reach hashing or storage. +- Made the collector lifecycle context mandatory. Shared refreshes remain detached from individual + callers but now inherit hub shutdown cancellation and deadlines through a value-free boundary. +- Updated every collector construction site and the README's current refresh-lifecycle contract. + +## [T] Proof + +- `go test -race -count=10 ./internal/hubfleet ./internal/pep ./internal/tenancy ./internal/hubdb` + passed. +- `FuzzPolicyAuditEntryHashUsesLengthFraming` passed 250,000 executions after the regression fix; + all 23 repository fuzz targets separately passed 50,000 executions each. +- The first full local CI run exposed a stale release-policy assertion and a hosted coverage gap. + The separately signed prerequisite landed through PR #294, and exact merge SHA `f0e071a` passed + expanded CI, reproducible release, real kind, and all three CodeQL analyses. +- After rebasing onto that verified SHA, complete local CI passed formatting, vet, zero-issue lint, + `govulncheck`, every race test and shell policy, nine alert rules, performance, binary end-to-end, + and production build. +- `make e2e-isolation` passed PostgreSQL 18.4 forced-RLS coverage and both workspace-isolation + fuzzers at 50,000 executions each. +- `make release-check` passed dual four-platform reproducibility, SPDX SBOMs, checksums, Homebrew + formula generation, and release-derived amd64/arm64 OCI verification. +- The Kubernetes 1.36.1 kind gate passed fleet fan-out, OCI image, and Argo Application projection + under the race detector in 260.143 seconds; teardown left no kind cluster or Buildx builder. +- `README.md` was reviewed and updated because refresh lifetime is now bounded by hub shutdown. + +## [S] Security, reliability, and cost + +Malformed identities now fail consistently across online and offline audit paths. Detached work no +longer survives process cancellation, while lifecycle values and request credentials remain outside +the refresh context. The change adds no cloud resource, API call, storage, telemetry cardinality, +or recurring cost. + +## [N] Next + +Push the rebased signed branch, obtain exact-head hosted CI and CodeQL proof, merge without rewriting +the signed commit, and verify the exact post-merge `dev` SHA. + +## [C] Checkpoint #1 + +The validation alignment, lifecycle boundary, adversarial tests, fuzz repair, construction-site +migration, README correction, prerequisite merge, and complete local gate matrix are frozen for +signed review. Hosted exact-head and exact post-merge `dev` proof remain mandatory. diff --git a/sessions/2026-07-21-release-tooling.md b/sessions/2026-07-21-release-tooling.md new file mode 100644 index 0000000..87fc945 --- /dev/null +++ b/sessions/2026-07-21-release-tooling.md @@ -0,0 +1,45 @@ +# Release supply-chain tooling refresh — 2026-07-21 + +## [S] Scope + +Refresh the explicitly pinned SBOM and signing executables used by pull-request reproducibility and +tag release. The release identity, permissions, action commit pins, artifact graph, publish order, +and consumer verification contract remain unchanged. + +## [D] Decision + +- Syft advances from v1.46.0 to v1.49.0. The upstream release fixes Go `replace`-directive + interpretation and adds root OCI-layout index support, both relevant to Sith's Go SBOM and + multi-architecture image boundary. +- Cosign advances from v3.0.6 to v3.1.2. The upstream release fixes malformed-input panics and + bundle signing/verification defects. Sith already emits bundles and uses none of the newly + deprecated `--payload` or `--output-attestation` flags. +- A repository policy test now keeps CI, tag release, README prerequisites, and the supply-chain ADR + synchronized instead of relying on manual version searches. + +## [V] Verification + +- Rebased onto exact `dev` merge `ae2d28de2d7fc6e6661098d9b1bb07e1b9381cad`, after its CI and + CodeQL workflows completed successfully. The one shared Makefile insertion retained both the + Wails and release-tooling policy gates. +- Official release assets are checksum-verified before local use. Both selected versions are stable + upstream releases: [Syft v1.49.0](https://github.com/anchore/syft/releases/tag/v1.49.0), published + 2026-07-21, and [Cosign v3.1.2](https://github.com/sigstore/cosign/releases/tag/v3.1.2), published + 2026-07-17. +- Actionlint, ShellCheck, and the focused policy test pass. Its 9 assertions validate every + synchronized pin, installer binding, documentation reference, and deprecated-flag exclusion. +- The real installed tools report Syft 1.49.0, Cosign 3.1.2, and GoReleaser 2.17.0. +- `go mod verify`, `govulncheck ./...`, and `make ci` pass, including race and all operator-facing + policy tests. +- `make e2e-isolation` passes against PostgreSQL plus 100,000 tenant-boundary fuzz executions. +- `make release-check` passes two independently rebuilt archive/SBOM distributions, Homebrew + formula validation, and the dual-architecture OCI layout contract using Syft 1.49.0. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` passes the fleet fan-out, immutable + OCI image, and Argo application projection contracts in 241.080 seconds. + +## [C] Security, operations, and cost + +The update repairs producer-side parsing and verification behavior without adding credentials, +permissions, services, storage, egress, or recurring cloud cost. Pull-request and tag CPU duration +may vary slightly with the newer scanners, but the number of builds, SBOMs, signatures, and +attestations is unchanged. diff --git a/sessions/2026-07-21-wails-version-policy.md b/sessions/2026-07-21-wails-version-policy.md new file mode 100644 index 0000000..29f4cfc --- /dev/null +++ b/sessions/2026-07-21-wails-version-policy.md @@ -0,0 +1,40 @@ +# Wails desktop version policy — 2026-07-21 + +## [S] Scope + +The desktop module was already upgraded to Wails v2.13.0, but `make desktop-build` still required +v2.12.0. Its substring check also accepted lookalike version strings. This slice aligns the tool +gate with `go.mod` and makes exact-version enforcement independently testable on every CI runner. + +## [D] Decision + +Treat the `github.com/wailsapp/wails/v2` requirement in `go.mod` as the authoritative compatibility +version. A fail-closed verifier resolves the configured executable, requires a strict `vX.Y.Z` +expected value, executes the version command successfully, and compares its first output line +exactly. Additional informational lines remain compatible; prerelease, vendor, whitespace, empty, +and failed-command variants are rejected. + +## [V] Verification + +- Rebased without conflict onto the exact post-merge `dev` commit + `9e135de08cab047386b0a948311e7443e2741404`, whose CI and CodeQL runs completed successfully. +- The policy test derives both pins and passed 12 assertions covering valid, lookalike, + missing-command, command-failure, and malformed-expectation cases; both scripts pass ShellCheck. +- The checksummed Wails v2.13.0 CLI passes the verifier. The corresponding upstream release is + stable and was published on 2026-07-06: + . +- `go mod verify`, `govulncheck ./...`, and `make ci` pass, including the race detector and all + operator-facing policy tests. +- `make e2e-isolation` passes against PostgreSQL plus 100,000 tenant-boundary fuzz executions. +- `make release-check` passes two independently rebuilt archive/SBOM distributions, Homebrew + formula validation, and the dual-architecture OCI layout contract. +- `make desktop-build` produces a strictly code-sign-valid `com.ardurai.sith` bundle containing a + Mach-O arm64 executable. +- `make e2e-kind KIND=/Volumes/EXTENDED/MacData/tools/bin/kind` passes the fleet fan-out, immutable + OCI image, and Argo application projection contracts in 241.937 seconds. + +## [C] Security, operations, and cost + +Exact matching prevents an unintended prerelease or vendor binary from satisfying the local build +gate. The change adds no runtime dependency, service, permission, network path, storage, or recurring +cloud cost. diff --git a/sessions/2026-07-21-xtext-security.md b/sessions/2026-07-21-xtext-security.md new file mode 100644 index 0000000..eae130a --- /dev/null +++ b/sessions/2026-07-21-xtext-security.md @@ -0,0 +1,46 @@ +# x/text malformed-input security update — 2026-07-21 + +## [S] Scope + +This slice starts from exact `dev` merge `ff435833816ddc3a523686304ef52cd8a4444f24` and changes +only the vulnerable Go module resolution. No production API, configuration, schema, IAM, network, +or release contract changes. + +## [D] Decision + +Upgrade `golang.org/x/text` from v0.38.0 to v0.39.0, the first release containing the fix for +[GO-2026-5970](https://pkg.go.dev/vuln/GO-2026-5970) / CVE-2026-56852. The affected normalization +package is reachable through `internal/hubdb -> pgx/pgconn -> x/text/secure/precis -> +x/text/unicode/norm`; malformed external identity or credential text therefore must not retain the +known infinite-loop implementation. + +The minimum fixed version is intentional. v0.40.0 is available, but adopting a later pre-v1 module +than required would widen compatibility risk without improving this remediation. + +## [V] Verification + +- `go mod verify` passed. +- `govulncheck ./...` reported no reachable vulnerabilities. +- `go test -race -count=1 ./...` passed. +- Current Trivy and Grype scans no longer report CVE-2026-56852. +- Full `make ci` passed formatting, vet, zero-issue lint, vulnerability analysis, every race test, + operator policy, alert rules, performance, binary integration, and production build. +- `make e2e-isolation` passed PostgreSQL 18.4 forced-RLS coverage and both 50,000-execution + workspace-isolation fuzzers. +- `make release-check` passed dual four-platform reproducibility, SPDX SBOMs, checksums, Homebrew, + and release-derived amd64/arm64 OCI verification. +- The Kubernetes 1.36.1 kind gate passed fleet fan-out, OCI image, and Argo Application projection + under the race detector in 242.332 seconds; teardown left no kind cluster or Sith Buildx builder. + +Both broad scanners still list GO-2026-5932 against `x/crypto/openpgp`, for which the Go advisory +publishes no fixed version. Sith does not import the affected OpenPGP cleartext-signature packages, +and call-graph-aware `govulncheck` reports no reachable vulnerability; this is a scanner-level, +not-affected disposition rather than an ignored reachable finding. + +## [C] Cost and blast radius + +The module checksum and selected library implementation change at build time only. There is no new +service, storage, egress, credential, permission, or recurring cloud cost. + +`README.md` was reviewed and remains accurate because no user-visible or operator-facing contract +changes. diff --git a/sessions/2026-07-22-e14-desired-change.md b/sessions/2026-07-22-e14-desired-change.md new file mode 100644 index 0000000..5587c16 --- /dev/null +++ b/sessions/2026-07-22-e14-desired-change.md @@ -0,0 +1,100 @@ +# Session — 2026-07-22 — E14 immutable desired change + +**Builder:** Gnani Rahul Nutakki · **Branch:** `gnanirahulnutakki/desired-change-20260722` +**Slice:** [#307](https://github.com/ArdurAI/sith/issues/307), E14 +[#46](https://github.com/ArdurAI/sith/issues/46) · **Status:** complete local proof; hosted proof pending + +## [G] Goal + +Implement the owner-approved second half of the F14.6 provenance split: one immutable +`DesiredChange` that binds exact proposed file bytes to an exact `GitSourceSnapshot`, cited +evidence, and a transformer version without enabling R2/R4 writes. + +## [S] Scope + +- Add a versioned opaque desired-change contract outside `internal/brain`. +- Bind one validated snapshot, one canonical transformer version, exact desired bytes, and a + bounded unique evidence set. +- Keep construction package-private until a concrete deterministic transformer or declarative + renderer policy receives separate review. +- Exclude R2/R4 transformation logic, PR metadata, handler binding, actor, intent, policy, + approval, credential, endpoint, persistence, dispatch, mutation, and execution behavior. + +## [A] Decision and implementation + +- The owner selected the snapshot-plus-change decomposition in + [issue comment 5051813734](https://github.com/ArdurAI/sith/issues/46#issuecomment-5051813734). + Snapshot child #303 landed first through #305; child #307 locks this separate output contract. +- `desired-change/v1` privately contains a defensive copy of one valid + `git-source-snapshot/v1`, a lowercase canonical `/` identity, exact desired + content, and copied evidence references. +- Desired content is a non-NUL UTF-8 sequence capped at 64 KiB. Empty output, CRLF, tabs, Unicode, + and trailing whitespace remain exact bytes; no Unicode, line-ending, whitespace, YAML, Helm, or + Kustomize normalization occurs. +- Exact output equal to the snapshot current bytes is rejected as a no-op. +- Evidence is capped at 32 unique stable references, canonically sorted, and must attach both the + affected resource and the exact observed Git blob. The snapshot's own nested resource and + evidence values are deep-copied. +- The only exported operation is `Version`. The constructor and validator remain package-private; + an AST boundary test rejects an exported construction function. +- No transformer implementation or allowlisted R2/R4 policy exists in this slice. A future + reviewed transformer must live at the trusted package boundary before it can construct the + contract. + +## [T] Proof + +- Focused remediation tests pass under the race detector, including 50 consecutive repetitions. +- Adversarial coverage rejects zero and forged snapshots, malformed transformer versions, + invalid/NUL/oversized/no-op output, missing/duplicate/unsafe/foreign evidence, private-field + forgery, input alias mutation, and nondeterministic evidence ordering. +- The dedicated constructor fuzzer passes 50,000 executions. Fuzz and 64-reader concurrent tests + preserve exact bytes and source binding without mutation. +- Reflection and AST tests lock the five-field private shape, single-method public surface, and + absence of an exported constructor. The recursive production import guard continues to exclude + I/O, policy, persistence, and runtime authority. +- `make ci` passes formatting, vet, zero-issue lint, current vulnerability scanning, every race + test, shell/tooling policies, all nine Prometheus rules, performance, binary end-to-end, and the + production build. The remediation package reaches 94.8% statement coverage. +- `make e2e-isolation` passes PostgreSQL 18.4 forced RLS and both 50,000-execution cross-workspace + fuzzers. +- `make release-check` passes module verification, two reproducible four-platform builds, SPDX + SBOMs, formula generation, and the amd64/arm64 distroless OCI layout. +- The pinned Kubernetes 1.36.1 Kind gate passes two-cluster fleet fan-out, OCI image, and Argo + Application projection under the race detector in 295.351 seconds. Teardown leaves no Kind + cluster or isolated release builder. +- The independent CodeRabbit loop reviewed all six files. Two documentation/boundary findings and + one concurrent-test proof finding were incorporated; the final full-diff pass reports zero + findings. +- Hosted exact-head proof and exact post-merge proof remain required before closure. + +## [S] Security, reliability, and cost + +The contract is pure and offline. It cannot be constructed by request/runtime packages, stores no +secret or authority, and adds no API request, egress, storage, cloud resource, telemetry +cardinality, or recurring cost. Before a transformer is approved, its review must cover parser +ambiguity, multi-document YAML, Helm/Kustomize semantics, file mapping, safety bounds, rollback, +and deterministic evidence-to-output binding. + +## [R] Primary references + +- [GitHub REST Git blobs](https://docs.github.com/en/rest/git/blobs?apiVersion=2026-03-10) +- [GitHub REST Git trees](https://docs.github.com/en/rest/git/trees?apiVersion=2026-03-10) +- [GitHub REST Git commits](https://docs.github.com/en/rest/git/commits?apiVersion=2026-03-10) + +## [N] Next + +Complete local fuzz, full CI, isolation, release, and real Kind gates; run independent review; +then create one signed DCO/GSTACK commit and require exact-head hosted CI/CodeQL plus exact +post-merge `dev` proof. Close only child #307. F14.6 and E14 remain open; R2/R4 transformer policy, +live Git reads, resolver composition, Hub identity, PEP, approval, dispatch, and execution are +separate slices. + +## [C] Checkpoint #1 + +The desired-change contract, package-private construction boundary, adversarial tests, docs, and +complete local gate matrix are frozen on exact base +`01b5ae7a0d329e19d11696bb24a8f73babb0449b`. README was reviewed and updated before commit because +the public architecture now includes the separately gated output contract. The self-managed QA +kubeconfig was not used; this slice is pure/offline and the real-cluster requirement was satisfied +by disposable local Kind clusters. Remaining gates are the signed commit, exact-head hosted proof, +merge without rewriting the signed feature commit, and exact post-merge `dev` proof. diff --git a/sessions/2026-07-22-e14-git-source-snapshot.md b/sessions/2026-07-22-e14-git-source-snapshot.md new file mode 100644 index 0000000..7480d25 --- /dev/null +++ b/sessions/2026-07-22-e14-git-source-snapshot.md @@ -0,0 +1,105 @@ +# Session — 2026-07-22 — E14 immutable Git source snapshot + +**Builder:** Gnani Rahul Nutakki · **Branch:** `gnanirahulnutakki/git-source-snapshot-20260722` +**Slice:** [#303](https://github.com/ArdurAI/sith/issues/303), E14 +[#46](https://github.com/ArdurAI/sith/issues/46) · **Status:** complete local proof; hosted proof pending + +## [G] Goal + +Implement the owner-approved first half of the F14.6 provenance split: one immutable +`GitSourceSnapshot` containing only canonical observed Git state. Defer `DesiredChange` to a later +separately reviewed transformer/renderer and keep R2/R4 advisory-only. + +## [S] Scope + +- Add a versioned source snapshot outside `internal/brain`, bound to one workspace, affected + resource, pinned GitHub source identity, repository, configured base ref, exact resolved commit, + one path, exact current bytes, matching blob identity, evidence set, and short validity interval. +- Keep validated fields private, copy mutable inputs, canonicalize evidence ordering, and expose + only the contract version plus a trusted-time freshness classification. +- Exclude desired bytes, PR metadata, handler binding, actor, intent, policy, approval, credential, + endpoint, persistence, dispatch, mutation, and execution behavior. + +## [A] Decision and implementation + +- The owner decision is recorded on E14 in + [issue comment 5051813734](https://github.com/ArdurAI/sith/issues/46#issuecomment-5051813734), + and child issue 303 locks this bounded acceptance contract as an E14 sub-issue. +- `git-source-snapshot/v1` accepts exactly one + `github-git-source-snapshot/2026-03-10` source whose native repository identity must match the + separately validated host/owner/repository tuple. +- The configured base ref rejects symbolic `HEAD`, full `refs/...`, object-ID-shaped, + option-shaped, reflog, lock, traversal-like, whitespace, and malformed Git ref names. Commit and + blob identities are canonical lowercase 40-hex SHA-1 or 64-hex SHA-256 values and must use the + same object format. +- The snapshot recomputes the exact Git blob identity over + `blob \0` and rejects a blob/content mismatch. SHA-1 is used only for + current GitHub Git-object interoperability; SHA-256 is supported under Git's transition format. +- Current content is an exact non-NUL UTF-8 byte sequence capped at 64 KiB. No Unicode, + line-ending, whitespace, or YAML normalization occurs; empty files and CRLF are preserved. +- The repository-relative path rejects absolute, unclean, traversal, backslash-ambiguous, and Git + metadata paths while retaining otherwise valid Unicode file names. +- Evidence is capped at 32 unique stable references, defensively copied, and canonically sorted. + It must attach both the affected resource and the exact repository blob. +- Construction normalizes observation times to UTC and permits at most five minutes. `Freshness` + uses only a supplied trusted time: before observation is future, at/after expiry is stale, and the + half-open interval between is fresh. Zero time or a forged invalid snapshot fails closed. +- Reflection tests lock the exact input, private snapshot, and two-method public shapes. A recursive + import guard prevents I/O, policy, persistence, authority, connector-runtime, or Brain imports. + +## [T] Proof + +- Focused remediation race tests pass; 50 repeated snapshot runs remain green and the package + reaches 94.0% statement coverage in the full race suite. +- Snapshot fuzzing passes 50,000 executions. Adversarial tests cover zero/multiple/mismatched + sources, invalid workspace/subject/repository/evidence, symbolic and malformed refs, malformed or + mixed-format object IDs, blob/content mismatch, unsafe paths, invalid/oversized content, invalid + validity windows, future/stale/zero clocks, input alias mutation, deterministic ordering, and 64 + concurrent readers. +- `make ci` passes formatting, vet, zero-issue lint, current vulnerability scanning, every race + test, shell/tooling policy, nine Prometheus rules, performance, binary end-to-end, and production + build gates. +- `make e2e-isolation` passes PostgreSQL 18.4 forced RLS and two 50,000-execution cross-workspace + fuzzers. +- One intervening repeat exposed an existing host-clock/database-clock exact-boundary race in the + approval-expiry test fixture (`expired approval consume error = `). The production predicate + remained database-clock authoritative, a clean repeat passed, and follow-up issue + [#304](https://github.com/ArdurAI/sith/issues/304) tracks the test-only repair outside this PR. +- `make release-check` passes module verification, two reproducible four-platform builds, SPDX + SBOMs, formula generation, and the amd64/arm64 distroless OCI layout. +- The pinned Kubernetes 1.36.1 Kind gate passes two-cluster fleet fan-out, OCI image, and Argo + Application projection under the race detector in 242.424 seconds. Teardown leaves no Kind + cluster or isolated release builder. +- Final isolated-GOPATH module verification and `govulncheck` v1.6.0 report no failures or reachable + vulnerabilities. + +## [S] Security, reliability, and cost + +The snapshot is pure and offline. It stores neither secrets nor authority and adds no API request, +egress, storage, cloud resource, telemetry cardinality, or recurring cost. A later live adapter +must separately own least-privilege contents-read credentials, GitHub rate limits, egress, +content-size policy, and remote-state freshness. A later `DesiredChange` contract must bind this +snapshot version and evidence without gaining implicit authorization. + +## [R] Primary references + +- [GitHub REST Git references](https://docs.github.com/en/rest/git/refs?apiVersion=2026-03-10) +- [GitHub REST Git commits](https://docs.github.com/en/rest/git/commits?apiVersion=2026-03-10) +- [GitHub REST Git blobs](https://docs.github.com/en/rest/git/blobs?apiVersion=2026-03-10) +- [Git hash transition](https://git-scm.com/docs/hash-function-transition) + +## [N] Next + +Create one signed DCO/GSTACK commit, open a PR to `dev`, and require exact-head hosted CI, CodeQL, +and review plus exact post-merge `dev` proof. Close only child issue 303. F14.6 and E14 remain open; +`DesiredChange`, live Git reads, Hub composition, PEP, and runtime execution are separate slices. +After this snapshot lands, repair the independently tracked approval-expiry test flake in #304. + +## [C] Checkpoint #1 + +The snapshot contract, hash binding, adversarial tests, import/public-shape guards, README/spec +documentation, local review, and complete local gate matrix are frozen on exact base +`430eea3faff4c889c8435b155b042e2104b1aeda`. README was reviewed and updated before commit because +the public architecture now includes the observed-only provenance stage. Remaining gates are the +signed commit, exact-head hosted proof, merge without rewriting the signed feature commit, and exact +post-merge `dev` proof. diff --git a/sessions/2026-07-22-e14-gitops-provenance-resolver.md b/sessions/2026-07-22-e14-gitops-provenance-resolver.md new file mode 100644 index 0000000..f7d374e --- /dev/null +++ b/sessions/2026-07-22-e14-gitops-provenance-resolver.md @@ -0,0 +1,113 @@ +# Session — 2026-07-22 — E14 GitOps provenance resolver + +**Builder:** Gnani Rahul Nutakki · **Branch:** `gnanirahulnutakki/gitops-provenance-20260722` +**Slice:** [#301](https://github.com/ArdurAI/sith/issues/301), E14 +[#46](https://github.com/ArdurAI/sith/issues/46) · **Status:** complete local proof; hosted proof pending + +## [G] Goal + +Implement the owner-approved F14.6b contract-only GitOps stage: resolve one confirmed canonical +Brain candidate against one fresh, source-owned provenance bundle without allowing a caller to +invent target or handler arguments and without creating a PEP proposal or write capability. + +## [S] Scope + +- Add one versioned immutable provenance bundle outside `internal/brain`, owned by the canonical + GitHub source contract and bound to one workspace, cited subject, repository, exact Git objects, + desired content, evidence set, and bounded validity interval. +- Pin provenance to the exact planning handler adapter version and argument-schema digest. +- Resolve confirmed entity-local R2/R4 `gitops.open-pr` candidates only; keep R1 and all other rules + candidate-only or advisory-only. +- Reuse the GitHub planner's pure schema and semantic validation boundary and verify that its + canonical output preserves every source-owned value. +- Exclude endpoint, PEP, persistence, database, credential, network, Git access, PR creation, + dispatch, shell, filesystem, cluster mutation, or execution behavior. + +## [A] Decision and implementation + +- Parent issue 46 records GR's approval of the source-owned-bundle plus pure-resolver design; child + issue 301 locks the bounded acceptance contract. +- `GitOpsProvenanceBundle` has private fields and is constructible only after exact source count, + source version, workspace, subject, repository, object-ID, validity, content-bound, and evidence + validation. Mutable references and slices are defensively copied and evidence ordering compares + the complete resource-identity tuple. +- The bundle expires no later than five minutes after observation and rejects symbolic `HEAD`, full + `refs/...`, option-shaped, and commit-shaped base branch identities. The shared GitHub planner now + rejects those ambiguous configured base branches too. +- `OpenPRPlanner.CanonicalizeOpenPRArgs` exposes the planner's existing I/O-free schema and semantic + checks without constructing an action plan. `Plan` now calls the same internal seam, so policy + cannot drift between resolver and planner. +- The resolver requires one confirmed non-fleet R2/R4 verdict with an attached fresh citation and + exactly one fresh matching bundle. It verifies the live descriptor, wire version, adapter + version, schema digest, repository target, canonical schema, and exact commit/blob/content output, + then rechecks the descriptor to close an in-process time-of-check/time-of-use gap. +- Ready output is limited to normalized target, canonical arguments, SHA-256 argument digest, and + copied evidence references. All expected refusals use bounded closed abstention reasons. +- A recursive import guard prevents I/O, policy, persistence, or authority packages from entering + the remediation package, and reflection tests lock authority-free public shapes plus opaque bundle + fields. + +## [T] Proof + +- Focused package tests pass under the race detector for the GitHub planner and remediation resolver. +- Twenty consecutive focused resolver race runs pass. The exact focused linter passes with zero + issues, and the resolver fuzz target passes 50,000 executions. +- Positive R2/R4 tests prove stable canonical arguments and digests across repeated calls and + evidence input ordering. Sixty-four concurrent resolutions remain byte-identical. +- Adversarial tests cover missing/mutated/unsupported candidates; stale/unattached verdict evidence; + zero/multiple/stale/future/foreign/unattached/multi-source provenance; unsafe/floating inputs; + validity, content, repository, base, commit, and blob failures; descriptor, wire, adapter, and + schema drift; handler target/output mutation; nil dependencies; cancellation before and after + handler validation; output aliases; and fuzzed paths/content/base refs. +- `make ci` passes formatting, vet, zero-issue lint, `govulncheck` with no vulnerabilities, every + race test, shell/tooling policies, nine Prometheus rules, performance, binary end-to-end, and the + production build. The new resolver package reaches 94.1% statement coverage. +- `make e2e-isolation` passes PostgreSQL 18.4 forced-RLS coverage and both cross-workspace fuzzers at + 50,000 executions each. +- `make release-check` passes module verification, two reproducible four-platform snapshots, SPDX + SBOMs, checksums, formula generation, and the release-derived amd64/arm64 distroless OCI layout. +- The pinned Kubernetes 1.36.1 Kind gate passes two-cluster fleet fan-out, OCI image, and Argo + Application projection under the race detector in 243.913 seconds. Teardown leaves no Kind + cluster, Sith container, or isolated release builder. +- The first CodeRabbit pass found two valid documentation omissions: evidence references in the + README result list and exact byte/digest/freshness semantics in the spec. Both were corrected. + A second whole-diff review covering all nine changed files reports zero findings. +- A final manual execution trace then found a last-check time-of-check/time-of-use gap: cancellation + or expiry during the post-canonicalization descriptor check could otherwise return ready. The + resolver now rechecks both immediately before output, with dedicated regressions. +- The exact post-hardening focused fuzz, full CI, isolation, release, and real-cluster matrices all + pass again, and the third whole-diff CodeRabbit review reports zero findings. +- Hosted review, exact-head CI/CodeQL, and post-merge `dev` proofs remain pending. + +## [S] Security, reliability, and cost + +The resolver is pure and offline. Actor, role, authenticated scope, server-owned intent ID, policy +decision, approval state, credential, endpoint, and execution state are absent from the bundle and +result. A ready result is provenance-complete, not authorized. This slice adds no API request, +egress, storage, cloud resource, telemetry cardinality, or recurring cost. A future GitHub read +adapter must separately account for credential custody, rate limits, egress, and remote-state +freshness. + +## [R] Primary references + +- [GitHub REST references](https://docs.github.com/en/rest/git/refs?apiVersion=2026-03-10) +- [GitHub REST commits](https://docs.github.com/en/rest/git/commits?apiVersion=2026-03-10) +- [GitHub REST trees](https://docs.github.com/en/rest/git/trees?apiVersion=2026-03-10) +- [Argo CD Application specification](https://argo-cd.readthedocs.io/en/latest/user-guide/application-specification/) + +## [N] Next + +Create one SSH-signed DCO/GSTACK commit, open a PR to `dev`, and require exact-head hosted +CI/CodeQL/review plus exact post-merge proof. Close only child issue 301. F14.6 and E14 remain open +for the separately reviewed authenticated Hub-to-PEP composition and the live canonical provenance +adapter. + +## [C] Checkpoint #1 + +The resolver contract, shared handler canonicalization seam, ambiguous-base hardening, adversarial +tests, import/public-shape guards, README/spec correction, session record, clean independent review, +and complete local gate matrix are frozen on exact base +`4b562abe6a16cf5f7ba77b6b16b682361d43f23b`. README was reviewed and updated before commit because +the public architecture now includes a provenance-resolution stage. Remaining gates are the signed +commit, exact-head hosted CI/CodeQL/review, merge without rewriting the signed feature commit, and +exact post-merge `dev` proof. diff --git a/sessions/2026-07-23-demo-launch.md b/sessions/2026-07-23-demo-launch.md new file mode 100644 index 0000000..6e0edd9 --- /dev/null +++ b/sessions/2026-07-23-demo-launch.md @@ -0,0 +1,18 @@ +# Session — 2026-07-23 — demo-launch + +**Builder:** Gnani Rahul Nutakki · **Model/effort:** Codex, high · **Branch:** gnanirahulnutakki/feat/demo-launch +**Slice(s):** Phase-L demo readiness · #315 · **Status:** done + +--- + +[G] Goal: Make the completed local fleet client discoverable through one safe graphical launch command for issue #315. +[S] Scope: `internal/cli`, README first-run guidance, command and launch smoke tests, and this journal. Out of scope: demo fixture deployment, Hub release policy, GHCR visibility, governed writes, and cluster mutation. +[A] Action: Reconciled live `dev`, releases, roadmap/build-sequence demo criteria, existing command surfaces, duplicate issues, and release blockers; selected orchestration over the existing desktop and loopback UI surfaces. +[A] Action: Added `sith launch` with closed `auto|desktop|ui` selection, explicit UI-only flag validation, shared bounded directory import, root registration, and first-run documentation. Refactored the existing UI command to accept an explicit directory without requiring a default kubeconfig backend. +[T] Test: Focused CLI race tests and twenty repeated package runs passed. The built binary launched the loopback UI from the provided self-managed test kubeconfig directory, served HTTP 200, and shut down cleanly without creating a cluster workload. +[T] Test: `make ci`, `make e2e-isolation`, both 50,000-case tenancy fuzzers, Helm and multi-architecture OCI contracts, the real two-cluster Kind suite, `go mod tidy -diff`, `go mod verify`, and the double-build `make release-check` all passed; Kind cleanup left no clusters. +[C] Checkpoint #1: this commit — one-command local demo launcher and complete local verification; next: publish PR into `dev`, require green PR and exact post-merge checks, then resume the broader demo-readiness gap audit. + +--- + +**Session close:** implementation and local verification complete · **Open questions touched:** Q12 — preserve the existing graphical surfaces and add an auto-selecting entry point rather than creating a new UI diff --git a/tests/e2e/helm_chart_test.go b/tests/e2e/helm_chart_test.go index ff5c344..69d2639 100644 --- a/tests/e2e/helm_chart_test.go +++ b/tests/e2e/helm_chart_test.go @@ -12,6 +12,7 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "strings" "testing" "time" @@ -21,10 +22,14 @@ import ( ) const ( - helmContractVersion = "v4.2.2" + helmContractVersion = "v4.2.3" validHubImage = "registry.example.invalid/sith/hub@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ) +var helmContractVersionPattern = regexp.MustCompile( + `^` + regexp.QuoteMeta(helmContractVersion) + `(?:\+g[0-9a-f]{7,40})?$`, +) + type helmProfileResources struct { requests map[string]string limits map[string]string @@ -42,6 +47,8 @@ var hubProfileResources = map[string]helmProfileResources{ } func TestHelmHubChartContract(t *testing.T) { + assertHelmVersionPolicy(t) + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) defer cancel() root := repositoryRoot(t) @@ -52,7 +59,7 @@ func TestHelmHubChartContract(t *testing.T) { if _, err := exec.LookPath(helm); err != nil { t.Fatalf("find Helm binary %q: %v", helm, err) } - if output, err := runHelm(ctx, t, helm, root, "version", "--short"); err != nil || !strings.HasPrefix(strings.TrimSpace(output), helmContractVersion) { + if output, err := runHelm(ctx, t, helm, root, "version", "--short"); err != nil || !helmVersionIsPinnedRelease(output) { t.Fatalf("Helm version/output = %q / %v, want %s", output, err, helmContractVersion) } @@ -65,13 +72,30 @@ func TestHelmHubChartContract(t *testing.T) { if err != nil { t.Fatalf("helm template light profile: %v\n%s", err, lightRendered) } - lightObjects := assertHelmHubRender(t, lightRendered, "light") + lightObjects := assertHelmHubRender(t, lightRendered, "light", false) heavyRendered, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, profileHubValues("heavy"))) if err != nil { t.Fatalf("helm template heavy profile: %v\n%s", err, heavyRendered) } - heavyObjects := assertHelmHubRender(t, heavyRendered, "heavy") + heavyObjects := assertHelmHubRender(t, heavyRendered, "heavy", false) assertProfileOnlyChangesResources(t, lightObjects, heavyObjects) + browserRendered, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, browserOIDCHubValues())) + if err != nil { + t.Fatalf("helm template browser OIDC profile: %v\n%s", err, browserRendered) + } + assertBrowserOIDCDeployment(t, requiredHelmObject(t, assertHelmHubRender(t, browserRendered, "light", false), "Deployment")) + metricsRendered, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, metricsHubValues())) + if err != nil { + t.Fatalf("helm template metrics profile: %v\n%s", err, metricsRendered) + } + assertLoopbackMetricsDeployment(t, requiredHelmObject(t, assertHelmHubRender(t, metricsRendered, "light", true), "Deployment")) + browserMetricsRendered, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, browserMetricsHubValues())) + if err != nil { + t.Fatalf("helm template browser OIDC plus metrics profile: %v\n%s", err, browserMetricsRendered) + } + browserMetricsDeployment := requiredHelmObject(t, assertHelmHubRender(t, browserMetricsRendered, "light", true), "Deployment") + assertBrowserOIDCDeployment(t, browserMetricsDeployment) + assertLoopbackMetricsDeployment(t, browserMetricsDeployment) for name, invalid := range map[string]string{ "mutable tag": strings.Replace(validHubValues(), validHubImage, "registry.example.invalid/sith/hub:latest", 1), @@ -80,6 +104,10 @@ func TestHelmHubChartContract(t *testing.T) { "unknown profile": strings.Replace(validHubValues(), "profile: light", "profile: unbounded", 1), "unexpected image pull secret": validHubValues() + "\nimagePullSecrets:\n password: must-not-render\n", "unexpected resources": validHubValues() + "\nresources:\n requests:\n cpu: 999\n", + "partial browser OIDC": strings.Replace(validHubValues(), ` issuer: ""`, " issuer: https://idp.sith.example", 1), + "metrics hostname": strings.Replace(validHubValues(), " proxyAddress:", " metrics:\n listenAddress: localhost:9464\n proxyAddress:", 1), + "metrics wildcard": strings.Replace(validHubValues(), " proxyAddress:", " metrics:\n listenAddress: 0.0.0.0:9464\n proxyAddress:", 1), + "metrics bad port": strings.Replace(validHubValues(), " proxyAddress:", " metrics:\n listenAddress: 127.0.0.1:65536\n proxyAddress:", 1), } { t.Run(name, func(t *testing.T) { if output, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--values", writeHelmValues(t, invalid)); err == nil { @@ -92,6 +120,8 @@ func TestHelmHubChartContract(t *testing.T) { "mutable image": strings.Replace(validHubValues(), validHubImage, "registry.example.invalid/sith/hub:latest", 1), "unknown profile": strings.Replace(validHubValues(), "profile: light", "profile: unbounded", 1), "unexpected resources": validHubValues() + "\nresources:\n requests:\n cpu: 999\n", + "metrics wildcard": strings.Replace(validHubValues(), " proxyAddress:", " metrics:\n listenAddress: 0.0.0.0:9464\n proxyAddress:", 1), + "metrics bad port": strings.Replace(validHubValues(), " proxyAddress:", " metrics:\n listenAddress: 127.0.0.1:65536\n proxyAddress:", 1), } { t.Run("skip schema validation "+name, func(t *testing.T) { if output, err := runHelm(ctx, t, helm, root, "template", "sith-hub", chart, "--namespace", "sith-system", "--skip-schema-validation", "--values", writeHelmValues(t, invalid)); err == nil { @@ -101,6 +131,36 @@ func TestHelmHubChartContract(t *testing.T) { } } +func helmVersionIsPinnedRelease(output string) bool { + return helmContractVersionPattern.MatchString(strings.TrimSuffix(output, "\n")) +} + +func assertHelmVersionPolicy(t *testing.T) { + t.Helper() + + for _, test := range []struct { + name string + output string + want bool + }{ + {name: "plain release", output: "v4.2.3", want: true}, + {name: "official build metadata", output: "v4.2.3+g43e8b7f\n", want: true}, + {name: "lookalike patch", output: "v4.2.30", want: false}, + {name: "prerelease", output: "v4.2.3-rc.1", want: false}, + {name: "arbitrary suffix", output: "v4.2.3+vendor", want: false}, + {name: "short commit", output: "v4.2.3+g123456", want: false}, + {name: "uppercase commit", output: "v4.2.3+g43E8B7F", want: false}, + {name: "leading whitespace", output: " v4.2.3+g43e8b7f\n", want: false}, + {name: "extra output", output: "warning\nv4.2.3+g43e8b7f\n", want: false}, + } { + t.Run("version policy/"+test.name, func(t *testing.T) { + if got := helmVersionIsPinnedRelease(test.output); got != test.want { + t.Fatalf("helmVersionIsPinnedRelease(%q) = %t, want %t", test.output, got, test.want) + } + }) + } +} + func validHubValues() string { return profileHubValues("light") } @@ -114,6 +174,10 @@ runtime: sessionIssuer: https://issuer.sith.example sessionAudience: https://hub.sith.example sessionKeyID: session-2026-07 + browserOIDC: + issuer: "" + clientID: "" + redirectURI: "" proxyAddress: cluster-proxy.open-cluster-management.svc:8090 proxyServerName: cluster-proxy.open-cluster-management.svc kubeAPIServerName: kubernetes @@ -123,6 +187,24 @@ migration: `, profile, validHubImage) } +func browserOIDCHubValues() string { + return strings.Replace(validHubValues(), ` browserOIDC: + issuer: "" + clientID: "" + redirectURI: ""`, ` browserOIDC: + issuer: https://idp.sith.example + clientID: sith-browser + redirectURI: https://hub.sith.example/v1/console/oidc/callback`, 1) +} + +func metricsHubValues() string { + return strings.Replace(validHubValues(), " proxyAddress:", " metrics:\n listenAddress: 127.0.0.1:9464\n proxyAddress:", 1) +} + +func browserMetricsHubValues() string { + return strings.Replace(browserOIDCHubValues(), " proxyAddress:", " metrics:\n listenAddress: 127.0.0.1:9464\n proxyAddress:", 1) +} + func writeHelmValues(t *testing.T, contents string) string { t.Helper() path := filepath.Join(t.TempDir(), "values.yaml") @@ -147,7 +229,7 @@ func runHelm(ctx context.Context, t *testing.T, helm, root string, args ...strin return string(output), err } -func assertHelmHubRender(t *testing.T, rendered, profile string) []*unstructured.Unstructured { +func assertHelmHubRender(t *testing.T, rendered, profile string, metricsEnabled bool) []*unstructured.Unstructured { t.Helper() if strings.Contains(rendered, "kind: Secret") || strings.Contains(rendered, "stringData:") || strings.Contains(rendered, "\ndata:") { t.Fatal("rendered chart created or embedded secret data") @@ -162,7 +244,7 @@ func assertHelmHubRender(t *testing.T, rendered, profile string) []*unstructured } } - serviceAccountName, selectorLabels := assertHubDeployment(t, requiredHelmObject(t, objects, "Deployment"), profile) + serviceAccountName, selectorLabels := assertHubDeployment(t, requiredHelmObject(t, objects, "Deployment"), profile, metricsEnabled) assertHubRBAC(t, requiredHelmObject(t, objects, "ClusterRole"), requiredHelmObject(t, objects, "ClusterRoleBinding"), serviceAccountName) assertMigrationJob(t, requiredHelmObject(t, objects, "Job"), profile) assertHubService(t, requiredHelmObject(t, objects, "Service"), selectorLabels) @@ -211,7 +293,7 @@ func requiredHelmObject(t *testing.T, objects []*unstructured.Unstructured, kind return found } -func assertHubDeployment(t *testing.T, deployment *unstructured.Unstructured, profile string) (string, map[string]any) { +func assertHubDeployment(t *testing.T, deployment *unstructured.Unstructured, profile string, metricsEnabled bool) (string, map[string]any) { t.Helper() podSpec := nestedHelmMap(t, deployment.Object, "spec", "template", "spec") if value, found, _ := unstructured.NestedBool(podSpec, "automountServiceAccountToken"); !found || !value { @@ -224,10 +306,23 @@ func assertHubDeployment(t *testing.T, deployment *unstructured.Unstructured, pr } assertHelmContainerSecurity(t, container) assertHelmProfileResources(t, container, profile) + assertHubProbes(t, container) environment := helmEnvironment(t, container) - if len(environment) != 14 || environment["SITH_HUB_DATABASE_URL"] != "secret:sith-runtime/database-url" { + if !matchesHubEnvironmentMode(environment, metricsEnabled) || environment["SITH_HUB_DATABASE_URL"] != "secret:sith-runtime/database-url" { t.Fatalf("hub environment = %#v", environment) } + ports, found, err := unstructured.NestedSlice(container, "ports") + if err != nil || !found || len(ports) != map[bool]int{false: 1, true: 2}[metricsEnabled] { + t.Fatalf("hub container ports = %#v / %v", ports, err) + } + if !metricsEnabled { + for _, port := range ports { + entry, ok := port.(map[string]any) + if ok && entry["name"] == "metrics" { + t.Fatalf("metrics container port unexpectedly rendered: %#v", ports) + } + } + } for _, name := range []string{"SITH_HUB_SESSION_PUBLIC_KEY_FILE", "SITH_HUB_SERVER_TLS_CERT_FILE", "SITH_HUB_SERVER_TLS_KEY_FILE", "SITH_HUB_PROXY_CA_FILE", "SITH_HUB_PROXY_CERT_FILE", "SITH_HUB_PROXY_KEY_FILE"} { if !strings.HasPrefix(environment[name], "/var/run/sith/runtime/") { t.Fatalf("hub mounted path %s = %q", name, environment[name]) @@ -252,6 +347,105 @@ func assertHubDeployment(t *testing.T, deployment *unstructured.Unstructured, pr return serviceAccountName, nestedHelmMap(t, deployment.Object, "spec", "template", "metadata", "labels") } +func assertHubProbes(t *testing.T, container map[string]any) { + t.Helper() + for _, probe := range []struct { + name string + path string + initialDelay int64 + period int64 + }{ + {name: "livenessProbe", path: "/healthz", initialDelay: 10, period: 10}, + {name: "readinessProbe", path: "/readyz", initialDelay: 2, period: 5}, + } { + configuration := nestedHelmMap(t, container, probe.name) + if _, found := configuration["tcpSocket"]; found { + t.Fatalf("%s regressed to a socket-only check: %#v", probe.name, configuration) + } + httpGet := nestedHelmMap(t, configuration, "httpGet") + if httpGet["path"] != probe.path || httpGet["port"] != "https" || httpGet["scheme"] != "HTTPS" || + helmInt(t, configuration["initialDelaySeconds"]) != probe.initialDelay || + helmInt(t, configuration["periodSeconds"]) != probe.period || + helmInt(t, configuration["timeoutSeconds"]) != 1 || + helmInt(t, configuration["successThreshold"]) != 1 || + helmInt(t, configuration["failureThreshold"]) != 3 { + t.Fatalf("%s contract = %#v", probe.name, configuration) + } + } +} + +func matchesHubEnvironmentMode(environment map[string]string, metricsEnabled bool) bool { + _, configured := environment["SITH_HUB_METRICS_LISTEN_ADDR"] + if configured != metricsEnabled { + return false + } + if metricsEnabled { + return len(environment) == 15 || len(environment) == 19 + } + return len(environment) == 14 || len(environment) == 18 +} + +func assertLoopbackMetricsDeployment(t *testing.T, deployment *unstructured.Unstructured) { + t.Helper() + podSpec := nestedHelmMap(t, deployment.Object, "spec", "template", "spec") + container := onlyHelmContainer(t, podSpec) + environment := helmEnvironment(t, container) + if environment["SITH_HUB_METRICS_LISTEN_ADDR"] != "127.0.0.1:9464" { + t.Fatalf("loopback metrics environment = %#v", environment) + } + ports, found, err := unstructured.NestedSlice(container, "ports") + if err != nil || !found || len(ports) != 2 { + t.Fatalf("loopback metrics ports = %#v / %v", ports, err) + } + wantPorts := map[string]int64{"https": 8443, "metrics": 9464} + for _, port := range ports { + entry, ok := port.(map[string]any) + if !ok { + t.Fatalf("loopback metrics port entry = %#v", port) + } + name, ok := entry["name"].(string) + wantPort, found := wantPorts[name] + if !ok || !found || helmInt(t, entry["containerPort"]) != wantPort || entry["protocol"] != "TCP" { + t.Fatalf("loopback metrics port entry = %#v", entry) + } + delete(wantPorts, name) + } + if len(wantPorts) != 0 { + t.Fatalf("loopback metrics ports missing = %#v", wantPorts) + } +} + +func assertBrowserOIDCDeployment(t *testing.T, deployment *unstructured.Unstructured) { + t.Helper() + podSpec := nestedHelmMap(t, deployment.Object, "spec", "template", "spec") + container := onlyHelmContainer(t, podSpec) + environment := helmEnvironment(t, container) + if (len(environment) != 18 && len(environment) != 19) || environment["SITH_HUB_BROWSER_OIDC_ISSUER"] != "https://idp.sith.example" || + environment["SITH_HUB_BROWSER_OIDC_CLIENT_ID"] != "sith-browser" || environment["SITH_HUB_BROWSER_OIDC_REDIRECT_URI"] != "https://hub.sith.example/v1/console/oidc/callback" || + environment["SITH_HUB_SESSION_PRIVATE_KEY_FILE"] != "/var/run/sith/runtime/session-private.pem" { + t.Fatalf("browser OIDC environment = %#v", environment) + } + volumes, found, err := unstructured.NestedSlice(podSpec, "volumes") + if err != nil || !found || len(volumes) != 1 { + t.Fatalf("browser OIDC volumes = %#v / %v", volumes, err) + } + volume, ok := volumes[0].(map[string]any) + if !ok { + t.Fatalf("browser OIDC volume = %#v", volumes[0]) + } + items, found, err := unstructured.NestedSlice(nestedHelmMap(t, volume, "secret"), "items") + if err != nil || !found { + t.Fatalf("browser OIDC secret items = %#v / %v", volume, err) + } + for _, item := range items { + entry, ok := item.(map[string]any) + if ok && entry["key"] == "session-private.pem" && entry["path"] == "session-private.pem" { + return + } + } + t.Fatalf("browser OIDC session-private.pem mount missing: %#v", items) +} + func assertHubService(t *testing.T, service *unstructured.Unstructured, wantSelector map[string]any) { t.Helper() spec := nestedHelmMap(t, service.Object, "spec") @@ -259,9 +453,13 @@ func assertHubService(t *testing.T, service *unstructured.Unstructured, wantSele t.Fatalf("service spec = %#v", spec) } ports, found, err := unstructured.NestedSlice(spec, "ports") - if err != nil || !found || len(ports) == 0 { + if err != nil || !found || len(ports) != 1 { t.Fatalf("service ports = %#v / %v", ports, err) } + port, ok := ports[0].(map[string]any) + if !ok || port["name"] != "https" || helmInt(t, port["port"]) != 8443 || port["targetPort"] != "https" || port["protocol"] != "TCP" { + t.Fatalf("service port = %#v", ports[0]) + } if !reflect.DeepEqual(nestedHelmMap(t, spec, "selector"), wantSelector) { t.Fatalf("service selector = %#v, want pod labels %#v", spec["selector"], wantSelector) } diff --git a/tests/e2e/kind_argocd_projection_test.go b/tests/e2e/kind_argocd_projection_test.go new file mode 100644 index 0000000..15028d5 --- /dev/null +++ b/tests/e2e/kind_argocd_projection_test.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +//go:build e2e && kind + +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/ArdurAI/sith/internal/connector/argocd" + "github.com/ArdurAI/sith/internal/fleet" +) + +func TestKindArgoApplicationProjection(t *testing.T) { + kindBinary := environmentOr("KIND_BIN", "kind") + if _, err := exec.LookPath(kindBinary); err != nil { + t.Fatalf("find kind binary %q: %v", kindBinary, err) + } + if _, err := exec.LookPath("docker"); err != nil { + t.Fatalf("find docker: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + clusterName := fmt.Sprintf("sith-argocd-e2e-%d", time.Now().UnixNano()) + image := environmentOr("KIND_NODE_IMAGE", defaultKindNodeImage) + runCommand(ctx, t, "", kindBinary, "create", "cluster", "--name", clusterName, "--image", image, "--wait", "180s") + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cleanupCancel() + command := exec.CommandContext(cleanupCtx, kindBinary, "delete", "cluster", "--name", clusterName) + if output, err := command.CombinedOutput(); err != nil { + t.Errorf("delete kind cluster %s: %v\n%s", clusterName, err, output) + } + }) + + kubeconfigPath := filepath.Join(t.TempDir(), "kubeconfig") + kubeconfig := runCommand(ctx, t, "", kindBinary, "get", "kubeconfig", "--name", clusterName) + if err := os.WriteFile(kubeconfigPath, []byte(kubeconfig), 0o600); err != nil { + t.Fatalf("write kind kubeconfig: %v", err) + } + contextName := "kind-" + clusterName + client := dynamicClientForContext(t, kubeconfigPath, contextName) + + crdResource := client.Resource(schema.GroupVersionResource{ + Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions", + }) + crd := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apiextensions.k8s.io/v1", + "kind": "CustomResourceDefinition", + "metadata": map[string]any{"name": "applications.argoproj.io"}, + "spec": map[string]any{ + "group": "argoproj.io", "scope": "Namespaced", + "names": map[string]any{ + "plural": "applications", "singular": "application", "kind": "Application", "listKind": "ApplicationList", + }, + "versions": []any{map[string]any{ + "name": "v1alpha1", "served": true, "storage": true, + "schema": map[string]any{"openAPIV3Schema": map[string]any{ + "type": "object", "x-kubernetes-preserve-unknown-fields": true, + }}, + }}, + }, + }} + if _, err := crdResource.Create(ctx, crd, metav1.CreateOptions{}); err != nil { + t.Fatalf("create minimal Argo Application CRD: %v", err) + } + waitForCRDEstablished(ctx, t, crdResource, "applications.argoproj.io") + + applications := client.Resource(schema.GroupVersionResource{ + Group: "argoproj.io", Version: "v1alpha1", Resource: "applications", + }).Namespace("default") + application := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "argoproj.io/v1alpha1", + "kind": "Application", + "metadata": map[string]any{"name": "payments", "namespace": "default"}, + "spec": map[string]any{ + "project": "default", + "source": map[string]any{ + "repoURL": "https://kind-token:kind-password@git.example/ardur/payments.git?token=do-not-retain", + "path": "clusters/kind/payments", "targetRevision": "main", + "helm": map[string]any{"parameters": []any{map[string]any{"name": "secret", "value": "raw-kind-secret"}}}, + }, + "destination": map[string]any{"name": "in-cluster", "namespace": "payments"}, + }, + "status": map[string]any{ + "health": map[string]any{"status": "Healthy"}, + "sync": map[string]any{"status": "OutOfSync", "revision": "abc123"}, + "history": []any{map[string]any{ + "id": int64(1), "revision": "abc123", "deployedAt": "2026-07-16T20:00:00Z", + }}, + }, + }} + if _, err := applications.Create(ctx, application, metav1.CreateOptions{}); err != nil { + t.Fatalf("create real kind Argo Application: %v", err) + } + observed, err := applications.Get(ctx, "payments", metav1.GetOptions{}) + if err != nil { + t.Fatalf("read real kind Argo Application: %v", err) + } + facts, err := argocd.ProjectApplication(argocd.Projection{ + Workspace: fleet.LocalWorkspace, Scope: contextName, ObservedAt: time.Now().UTC(), Application: *observed, + }) + if err != nil { + t.Fatalf("project real kind Argo Application: %v", err) + } + if len(facts) != 4 { + t.Fatalf("real kind Argo facts = %#v, want desired, health, drift, and timeline", facts) + } + graph, err := fleet.NewGraph(fleet.LocalWorkspace, facts) + if err != nil || len(graph.Nodes) != 1 || graph.Nodes[0].Entity.Cluster != contextName || + graph.Nodes[0].Entity.Namespace != "default" || graph.Nodes[0].Entity.Kind != "Application" || + graph.Nodes[0].Entity.Name != "payments" || len(graph.Nodes[0].Facts) != len(facts) { + t.Fatalf("real kind Argo graph = %#v, error = %v", graph, err) + } + encoded, err := json.Marshal(graph) + if err != nil { + t.Fatalf("marshal real kind Argo graph: %v", err) + } + for _, secret := range []string{"kind-token", "kind-password", "do-not-retain", "raw-kind-secret"} { + if strings.Contains(string(encoded), secret) { + t.Fatalf("real kind Argo graph retained secret marker %q: %s", secret, encoded) + } + } +} + +func waitForCRDEstablished( + ctx context.Context, + t *testing.T, + resource interface { + Get(context.Context, string, metav1.GetOptions, ...string) (*unstructured.Unstructured, error) + }, + name string, +) { + t.Helper() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + crd, err := resource.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read CRD %s: %v", name, err) + } + conditions, _, _ := unstructured.NestedSlice(crd.Object, "status", "conditions") + for _, raw := range conditions { + condition, ok := raw.(map[string]any) + if !ok { + continue + } + if condition["type"] == "Established" && condition["status"] == "True" { + return + } + } + select { + case <-ctx.Done(): + t.Fatalf("wait for CRD %s: %v", name, ctx.Err()) + case <-ticker.C: + } + } +} diff --git a/tests/e2e/kind_fanout_test.go b/tests/e2e/kind_fanout_test.go index e3c8b4f..cfd73b6 100644 --- a/tests/e2e/kind_fanout_test.go +++ b/tests/e2e/kind_fanout_test.go @@ -15,6 +15,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "time" @@ -28,6 +29,7 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/ArdurAI/sith/internal/brain" + "github.com/ArdurAI/sith/internal/connector" "github.com/ArdurAI/sith/internal/connector/kubeconfig" "github.com/ArdurAI/sith/internal/fleet" "github.com/ArdurAI/sith/internal/fleetcache" @@ -125,6 +127,25 @@ func TestKindFleetFanout(t *testing.T) { t.Errorf("query did not return a source-stamped namespace from %s", scope) } } + exerciseWorkspaceIsolatedFleetCache(ctx, t, adapter, "kind-"+clusterNames[0]) + liveContextNames := []string{"kind-" + clusterNames[0], "kind-" + clusterNames[1]} + paged, err := adapter.Query(ctx, fleet.Query{ + Kinds: []fleet.FactKind{fleet.FactInventory}, + Scopes: liveContextNames, + Selector: fleet.Selector{ResourceKind: "Namespace"}, + Limit: 2, + }) + if err != nil { + t.Fatalf("query paginated namespaces across kind contexts: %v", err) + } + if len(paged.Facts) != 2 || paged.Coverage.Requested != 2 || paged.Coverage.Reachable != 2 || + len(paged.Coverage.Truncated) != 0 || !paged.Coverage.Complete() { + t.Fatalf("paginated kind query = %#v, want a fleet-limited result from complete bounded scans", paged) + } + if paged.Facts[0].Ref.String() >= paged.Facts[1].Ref.String() || + paged.Facts[0].Ref.Scope != liveContextNames[0] || paged.Facts[1].Ref.Scope != liveContextNames[0] { + t.Fatalf("paginated kind facts = %#v, want the first two globally sorted refs", paged.Facts) + } assertRealKindEntityGraph(ctx, t, adapter, clusterNames) exerciseReadFederationSnapshots(ctx, t, adapter, clusterNames) @@ -200,6 +221,7 @@ func TestKindFleetFanout(t *testing.T) { "kind-" + clusterNames[0]: false, "kind-" + clusterNames[1]: false, } + paginatedDisplay := false for _, record := range genericSnapshot.Records { if record.Kind == "ConfigMap" && record.Name == "sith-generic-sample" { for _, field := range record.Display { @@ -208,12 +230,23 @@ func TestKindFleetFanout(t *testing.T) { } } } + if record.Kind == "ConfigMap" && record.Cluster == "kind-"+clusterNames[0] && + record.Name == "sith-table-page-259" { + for _, field := range record.Display { + if field.Name == "Data" { + paginatedDisplay = true + } + } + } } for scope, seen := range genericScopes { if !seen { t.Errorf("generic lens did not return a ConfigMap from %s", scope) } } + if !paginatedDisplay { + t.Error("generic lens did not retain server columns from the second bounded Table page") + } if genericSnapshot.Coverage.Reachable != 2 || !strings.Contains(genericStderr, "warning: covered 2/3 clusters") { t.Fatalf("generic coverage/stderr = %#v/%q, want partial two-of-three", genericSnapshot.Coverage, genericStderr) } @@ -222,6 +255,18 @@ func TestKindFleetFanout(t *testing.T) { t.Fatalf("generic server-column text/error = %q/%v", genericText, err) } + bootstrapCtx, bootstrapCancel := context.WithCancel(ctx) + bootstrapEvents, err := adapter.Watch(bootstrapCtx, "ConfigMap") + if err != nil { + bootstrapCancel() + t.Fatalf("open paginated ConfigMap watch: %v", err) + } + waitForWatchSnapshotFact( + ctx, t, bootstrapEvents, "ConfigMap", "kind-"+clusterNames[0], "sith-table-page-259", + ) + bootstrapCancel() + waitForWatchClose(ctx, t, bootstrapEvents) + watchStore := fleetcache.New() watchHydrator, err := hydrate.New(adapter, watchStore, hydrate.WithResyncInterval(10*time.Minute)) if err != nil { @@ -230,7 +275,7 @@ func TestKindFleetFanout(t *testing.T) { watchCtx, watchCancel := context.WithCancel(ctx) watchDone := make(chan error, 1) go func() { watchDone <- watchHydrator.Run(watchCtx) }() - waitForCacheRecord(ctx, t, watchStore, "kind-"+clusterNames[0], "sith-vuln-sample", true) + waitForCacheRecord(ctx, t, watchStore, "Pod", "kind-"+clusterNames[0], "sith-vuln-sample", true) watchClient := dynamicClientForContext(t, kubeconfigPath, "kind-"+clusterNames[0]) watchPod := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "v1", @@ -244,12 +289,12 @@ func TestKindFleetFanout(t *testing.T) { Namespace("default").Create(ctx, watchPod, metav1.CreateOptions{}); err != nil { t.Fatalf("create watched pod: %v", err) } - waitForCacheRecord(ctx, t, watchStore, "kind-"+clusterNames[0], "sith-watch-sample", true) + waitForCacheRecord(ctx, t, watchStore, "Pod", "kind-"+clusterNames[0], "sith-watch-sample", true) if err := watchClient.Resource(schema.GroupVersionResource{Version: "v1", Resource: "pods"}). Namespace("default").Delete(ctx, "sith-watch-sample", metav1.DeleteOptions{}); err != nil { t.Fatalf("delete watched pod: %v", err) } - waitForCacheRecord(ctx, t, watchStore, "kind-"+clusterNames[0], "sith-watch-sample", false) + waitForCacheRecord(ctx, t, watchStore, "Pod", "kind-"+clusterNames[0], "sith-watch-sample", false) watchCancel() if err := <-watchDone; err != nil { t.Fatalf("watch hydrator shutdown: %v", err) @@ -307,6 +352,67 @@ func TestKindFleetFanout(t *testing.T) { } } +func exerciseWorkspaceIsolatedFleetCache( + ctx context.Context, + t *testing.T, + adapter *kubeconfig.Adapter, + scope string, +) { + t.Helper() + result, err := adapter.Query(ctx, fleet.Query{ + Kinds: []fleet.FactKind{fleet.FactInventory}, + Scopes: []string{scope}, + Selector: fleet.Selector{ + ResourceKind: "Pod", + Name: "sith-vuln-sample", + Namespace: "default", + }, + }) + if err != nil || len(result.Facts) != 1 || !result.Coverage.Complete() { + t.Fatalf("query real pod for workspace isolation = %#v, error = %v", result, err) + } + + store := fleetcache.New() + fact := result.Facts[0] + discovery := connector.Discovery{Scopes: []connector.Scope{{ + Name: fact.Ref.Scope, Reachable: true, ObservedAt: fact.ObservedAt, + }}} + for _, workspace := range []string{"workspace-a", "workspace-b"} { + if err := store.SetDiscovery(workspace, discovery); err != nil { + t.Fatalf("set %s discovery: %v", workspace, err) + } + workspaceFact := fact + workspaceFact.Workspace = workspace + if err := store.Replace(workspace, "Pod", fleet.QueryResult{ + Facts: []fleet.Fact{workspaceFact}, Coverage: fleet.Coverage{Requested: 1, Reachable: 1}, + }); err != nil { + t.Fatalf("replace real fact in %s: %v", workspace, err) + } + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchError, Workspace: "workspace-a", Kind: "Pod", Scope: fact.Ref.Scope, + Err: fmt.Errorf("simulated workspace-a watch failure"), + }); err != nil { + t.Fatalf("apply workspace A watch failure: %v", err) + } + if err := store.ApplyWatchEvent(connector.WatchEvent{ + Type: connector.WatchDelete, Workspace: "workspace-b", Kind: "Pod", Scope: fact.Ref.Scope, + Ref: fact.Ref, ObservedAt: fact.ObservedAt, + }); err != nil { + t.Fatalf("apply workspace B watch delete: %v", err) + } + + snapshotA := store.Query("workspace-a", fleetcache.Query{Kind: "Pod"}) + if len(snapshotA.Records) != 1 || snapshotA.Records[0].Workspace != "workspace-a" || + snapshotA.LastError == "" || !slices.Equal(snapshotA.Coverage.Unreachable, []string{fact.Ref.Scope}) { + t.Fatalf("workspace A real-fact snapshot = %#v, want isolated degraded record", snapshotA) + } + snapshotB := store.Query("workspace-b", fleetcache.Query{Kind: "Pod"}) + if len(snapshotB.Records) != 0 || snapshotB.LastError != "" || !snapshotB.Coverage.Complete() { + t.Fatalf("workspace B real-fact snapshot = %#v, want independent successful delete", snapshotB) + } +} + func assertRealKindEntityGraph(ctx context.Context, t *testing.T, adapter *kubeconfig.Adapter, clusterNames []string) { t.Helper() result, err := adapter.Query(ctx, fleet.Query{ @@ -530,6 +636,9 @@ func seedKindResources(ctx context.Context, t *testing.T, kubeconfigPath string, Namespace("default").Create(ctx, configMap, metav1.CreateOptions{}); err != nil { t.Fatalf("create configmap in %s: %v", contextName, err) } + if index == 0 { + seedPaginatedTableResources(ctx, t, client, contextName) + } replicas := int64(index) deployment := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "apps/v1", @@ -553,6 +662,51 @@ func seedKindResources(ctx context.Context, t *testing.T, kubeconfigPath string, } } +func seedPaginatedTableResources( + ctx context.Context, + t *testing.T, + client dynamic.Interface, + contextName string, +) { + t.Helper() + const count = 260 + const concurrency = 16 + semaphore := make(chan struct{}, concurrency) + errors := make(chan error, count) + var waitGroup sync.WaitGroup + for index := range count { + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + select { + case semaphore <- struct{}{}: + defer func() { <-semaphore }() + case <-ctx.Done(): + errors <- ctx.Err() + return + } + name := fmt.Sprintf("sith-table-page-%03d", index) + configMap := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{"name": name, "namespace": "default"}, + "data": map[string]any{"page": name}, + }} + if _, err := client.Resource(schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}). + Namespace("default").Create(ctx, configMap, metav1.CreateOptions{}); err != nil { + errors <- fmt.Errorf("create paginated table fixture %s in %s: %w", name, contextName, err) + } + }() + } + waitGroup.Wait() + close(errors) + for err := range errors { + if err != nil { + t.Fatal(err) + } + } +} + func seedBrainResources( ctx context.Context, t *testing.T, @@ -662,7 +816,7 @@ func waitForCacheRecord( ctx context.Context, t *testing.T, store *fleetcache.Store, - cluster, name string, + kind, cluster, name string, want bool, ) { t.Helper() @@ -672,7 +826,7 @@ func waitForCacheRecord( defer deadline.Stop() for { found := false - for _, record := range store.Query(fleet.LocalWorkspace, fleetcache.Query{Kind: "Pod"}).Records { + for _, record := range store.Query(fleet.LocalWorkspace, fleetcache.Query{Kind: kind}).Records { if record.Cluster == cluster && record.Name == name { found = true break @@ -691,6 +845,62 @@ func waitForCacheRecord( } } +func waitForWatchSnapshotFact( + ctx context.Context, + t *testing.T, + events <-chan connector.WatchEvent, + kind, cluster, name string, +) { + t.Helper() + deadline := time.NewTimer(30 * time.Second) + defer deadline.Stop() + for { + select { + case event, open := <-events: + if !open { + t.Fatalf("watch closed before %s/%s appeared in %s snapshot", cluster, name, kind) + } + if event.Kind != kind || event.Scope != cluster { + continue + } + if event.Type == connector.WatchError { + t.Fatalf("watch bootstrap for %s/%s failed: %v", kind, cluster, event.Err) + } + if event.Type != connector.WatchSnapshot { + continue + } + for _, fact := range event.Facts { + if fact.Ref.Name == name { + return + } + } + t.Fatalf("%s snapshot for %s omitted late-page object %s", kind, cluster, name) + case <-ctx.Done(): + t.Fatalf("wait for %s snapshot in %s: %v", kind, cluster, ctx.Err()) + case <-deadline.C: + t.Fatalf("timed out waiting for %s snapshot in %s", kind, cluster) + } + } +} + +func waitForWatchClose(ctx context.Context, t *testing.T, events <-chan connector.WatchEvent) { + t.Helper() + deadline := time.NewTimer(10 * time.Second) + defer deadline.Stop() + for { + select { + case _, open := <-events: + if !open { + return + } + case <-ctx.Done(): + t.Fatalf("wait for watch shutdown: %v", ctx.Err()) + case <-deadline.C: + t.Fatal("watch did not close after cancellation") + } + } +} + func runSith(ctx context.Context, binary, kubeconfigPath string, args ...string) ([]byte, string, error) { command := exec.CommandContext(ctx, binary, args...) command.Env = append(os.Environ(), diff --git a/tests/e2e/kind_read_federation_test.go b/tests/e2e/kind_read_federation_test.go index 3c13df8..21b0ce1 100644 --- a/tests/e2e/kind_read_federation_test.go +++ b/tests/e2e/kind_read_federation_test.go @@ -39,9 +39,10 @@ func exerciseReadFederationSnapshots( failures: make(map[string]hubfleet.FailureKind), } collector, err := hubfleet.NewCollector(hubfleet.CollectorConfig{ - Store: store, - Transport: kindSnapshotTransport{adapter: adapter}, - PEP: e2eReadPEP(t), + LifecycleContext: ctx, + Store: store, + Transport: kindSnapshotTransport{adapter: adapter}, + PEP: e2eReadPEP(t), }) if err != nil { t.Fatalf("construct read-federation collector: %v", err) diff --git a/tests/scripts/helm_tooling_policy_test.sh b/tests/scripts/helm_tooling_policy_test.sh new file mode 100755 index 0000000..4067010 --- /dev/null +++ b/tests/scripts/helm_tooling_policy_test.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# shellcheck disable=SC2016 + +set -Eeuo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly REPO_ROOT +readonly EXPECTED_VERSION="v4.2.3" +readonly EXPECTED_LINUX_AMD64_SHA256="e9b88b4ee95b18c706839c28d3a0220e5bc470e9cd9262410c90793c45ff8b7c" + +PASS_COUNT=0 + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + printf '[helm-policy] PASS: %s\n' "$1" +} + +assert_equal() { + local actual=$1 + local expected=$2 + local description=$3 + + if [[ "${actual}" != "${expected}" ]]; then + printf '[helm-policy] FAIL: %s = %q, want %q\n' \ + "${description}" "${actual}" "${expected}" >&2 + return 1 + fi + pass "${description}" +} + +ci_version="$(awk -F '"' '/^ HELM_VERSION: / { print $2 }' "${REPO_ROOT}/.github/workflows/ci.yml")" +ci_checksum="$(awk -F '"' '/^ HELM_LINUX_AMD64_SHA256: / { print $2 }' "${REPO_ROOT}/.github/workflows/ci.yml")" +runner_version="$(awk -F '"' '/^readonly HELM_VERSION=/ { print $2 }' "${REPO_ROOT}/hack/experiments/m0-ocm-falsification.sh")" +contract_version="$(awk -F '"' '/^[[:space:]]*helmContractVersion = / { print $2 }' "${REPO_ROOT}/tests/e2e/helm_chart_test.go")" + +assert_equal "${ci_version}" "${EXPECTED_VERSION}" "CI Helm pin is current" +assert_equal "${runner_version}" "${EXPECTED_VERSION}" "M0 runner Helm pin matches CI" +assert_equal "${contract_version}" "${EXPECTED_VERSION}" "hub chart contract Helm pin matches CI" +assert_equal "${ci_checksum}" "${EXPECTED_LINUX_AMD64_SHA256}" "CI Linux amd64 checksum matches the official archive" + +grep -Fqx '| Helm | `v4.2.3` |' "${REPO_ROOT}/docs/experiments/M0-ocm-falsification.md" +pass "M0 experiment documentation matches the executable pin" + +grep -Fq 'helm_version_is_pinned_release "${helm_version}"' \ + "${REPO_ROOT}/hack/experiments/m0-ocm-falsification.sh" +pass "M0 tool validation uses the tested exact-version policy" + +if grep -Fq 'strings.HasPrefix(strings.TrimSpace(output), helmContractVersion)' \ + "${REPO_ROOT}/tests/e2e/helm_chart_test.go"; then + printf '[helm-policy] FAIL: hub chart contract still accepts Helm version prefixes\n' >&2 + exit 1 +fi +pass "hub chart contract has no prefix-version fallback" + +printf '[helm-policy] %d assertions passed\n' "${PASS_COUNT}" diff --git a/tests/scripts/m0_ocm_falsification_safety_test.sh b/tests/scripts/m0_ocm_falsification_safety_test.sh index 3c24e80..42edd93 100644 --- a/tests/scripts/m0_ocm_falsification_safety_test.sh +++ b/tests/scripts/m0_ocm_falsification_safety_test.sh @@ -173,6 +173,20 @@ expect_failure "kind enumeration failure is not treated as cluster absence" \ env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/kind-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ KIND_BIN=/usr/bin/false bash -c 'source "$1"; cluster_exists sith-m0-hub' _ "${SCRIPT}" +for helm_version in v4.2.3 v4.2.3+g43e8b7f; do + env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/helm-version-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + HELM_VERSION_OUTPUT="${helm_version}" bash -c \ + 'source "$1"; helm_version_is_pinned_release "${HELM_VERSION_OUTPUT}"' _ "${SCRIPT}" +done +pass "Helm version policy accepts the pinned release and official build metadata" + +for helm_version in v4.2.30 v4.2.3-rc.1 v4.2.3+vendor v4.2.3+g123456 v4.2.3+g43E8B7F; do + expect_failure "Helm version policy rejects lookalike ${helm_version}" \ + env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/helm-version-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + HELM_VERSION_OUTPUT="${helm_version}" bash -c \ + 'source "$1"; helm_version_is_pinned_release "${HELM_VERSION_OUTPUT}"' _ "${SCRIPT}" +done + token_flag_marker="${TEST_ROOT}/token-flag" expect_failure "malformed token output still requires invalidation" \ env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/token-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ @@ -194,30 +208,83 @@ expect_failure "malformed token output still requires invalidation" \ [[ "$(cat "${token_flag_marker}")" == "1" ]] pass "token acquisition boundary is conservative" -addon_wait_marker="${TEST_ROOT}/addon-wait" +addon_wait_state="${TEST_ROOT}/addon-wait-state" +printf '0\n' >"${addon_wait_state}" env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/addon-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ - KUBECTL_BIN=fake_kubectl ADDON_WAIT_MARKER="${addon_wait_marker}" bash -c ' - attempts=0 + KUBECTL_BIN=fake_kubectl ADDON_WAIT_STATE="${addon_wait_state}" bash -c ' + fake_kubectl() { + [[ " $* " == *" get "* && " $* " != *" wait "* ]] || return 90 + local attempts + attempts=$(( $(cat "${ADDON_WAIT_STATE}") + 1 )) + printf "%s\n" "${attempts}" >"${ADDON_WAIT_STATE}" + if (( attempts >= 3 )); then + printf "uid-created\tTrue|" + fi + } + sleep() { SECONDS=$((SECONDS + 1)); } + source "$1" + wait_for_addon_creation spoke-a cluster-proxy + ' _ "${SCRIPT}" +[[ "$(cat "${addon_wait_state}")" == "4" ]] +pass "addon wait tolerates delayed creation within one deadline" + +addon_recreate_state="${TEST_ROOT}/addon-recreate-state" +printf '0\n' >"${addon_recreate_state}" +env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/addon-recreate-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + KUBECTL_BIN=fake_kubectl ADDON_RECREATE_STATE="${addon_recreate_state}" bash -c ' fake_kubectl() { - for argument in "$@"; do - if [[ "${argument}" == "get" ]]; then - attempts=$((attempts + 1)) - [[ "${attempts}" -ge 3 ]] - return - fi - if [[ "${argument}" == "wait" ]]; then - printf "%s\n" "${attempts}" >"${ADDON_WAIT_MARKER}" - return 0 - fi - done + [[ " $* " == *" get "* && " $* " != *" wait "* ]] || return 90 + local attempts + attempts=$(( $(cat "${ADDON_RECREATE_STATE}") + 1 )) + printf "%s\n" "${attempts}" >"${ADDON_RECREATE_STATE}" + case "${attempts}" in + 1) printf "uid-old\tFalse|" ;; + 2) printf "uid-old\tTrue|" ;; + 3) : ;; + 4) printf "uid-new\tFalse|" ;; + 5 | 6) printf "uid-new\tTrue|" ;; + *) return 91 ;; + esac + } + sleep() { SECONDS=$((SECONDS + 1)); } + source "$1" + wait_for_addon_creation spoke-a managed-serviceaccount + ' _ "${SCRIPT}" +[[ "$(cat "${addon_recreate_state}")" == "6" ]] +pass "addon wait rejects a deleted old instance and confirms the recreated UID" + +terminal_output="$(env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/addon-terminal-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + KUBECTL_BIN=fake_kubectl bash -c ' + fake_kubectl() { + printf "SENSITIVE-SERVER-BODY\n" >&2 return 1 } - sleep() { :; } + source "$1" + wait_for_addon_creation spoke-a cluster-proxy + ' _ "${SCRIPT}" 2>&1)" && { + printf '[m0-safety] FAIL: terminal addon read unexpectedly succeeded\n' >&2 + exit 1 +} +[[ "${terminal_output}" == *"cannot read current spoke-a managedclusteraddon/cluster-proxy state"* ]] +[[ "${terminal_output}" != *"SENSITIVE-SERVER-BODY"* ]] +pass "addon wait fails closed without logging terminal response bodies" + +expect_failure "addon wait rejects malformed or duplicate Available conditions" \ + env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/addon-malformed-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + KUBECTL_BIN=fake_kubectl bash -c ' + fake_kubectl() { printf "uid-malformed\tFalse|True|"; } + source "$1" + wait_for_addon_creation spoke-a cluster-proxy + ' _ "${SCRIPT}" + +expect_failure "addon creation and availability consume one absolute deadline" \ + env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/addon-deadline-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ + KUBECTL_BIN=fake_kubectl bash -c ' + fake_kubectl() { printf "uid-pending\tFalse|"; } + sleep() { SECONDS=$((SECONDS + 300)); } source "$1" wait_for_addon_creation spoke-a cluster-proxy ' _ "${SCRIPT}" -[[ "$(cat "${addon_wait_marker}")" == "3" ]] -pass "addon wait tolerates asynchronous creation before availability" expect_failure "unrelated hub exec failure cannot satisfy the active deny" \ env SITH_M0_SCRATCH_ROOT="${TEST_ROOT}/probe-root" SITH_M0_ALLOW_NON_EXTENDED=1 \ diff --git a/tests/scripts/prometheus_tooling_policy_test.sh b/tests/scripts/prometheus_tooling_policy_test.sh new file mode 100644 index 0000000..b390c52 --- /dev/null +++ b/tests/scripts/prometheus_tooling_policy_test.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 + +set -Eeuo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly REPO_ROOT +readonly EXPECTED_VERSION="v3.13.1" +readonly EXPECTED_LINUX_AMD64_SHA256="962b812371aff838d152b6ff2d56fdb7a6396f5542f48ebf73421b9721f0d103" + +ci_version="$(awk -F '"' '/^ PROMETHEUS_VERSION: / { print $2 }' "${REPO_ROOT}/.github/workflows/ci.yml")" +ci_checksum="$(awk -F '"' '/^ PROMETHEUS_LINUX_AMD64_SHA256: / { print $2 }' "${REPO_ROOT}/.github/workflows/ci.yml")" + +[[ "${ci_version}" == "${EXPECTED_VERSION}" ]] || { + printf '[prometheus-policy] FAIL: CI version = %q, want %q\n' "${ci_version}" "${EXPECTED_VERSION}" >&2 + exit 1 +} +[[ "${ci_checksum}" == "${EXPECTED_LINUX_AMD64_SHA256}" ]] || { + printf '[prometheus-policy] FAIL: CI checksum = %q, want official archive checksum %q\n' \ + "${ci_checksum}" "${EXPECTED_LINUX_AMD64_SHA256}" >&2 + exit 1 +} + +rules="${REPO_ROOT}/monitoring/sith-hub.rules.yml" +alert_count="$(awk '/^[[:space:]]*- alert:/ { count++ } END { print count + 0 }' "${rules}")" +[[ "${alert_count}" == 9 ]] || { + printf '[prometheus-policy] FAIL: portable rule count = %q, want 9\n' "${alert_count}" >&2 + exit 1 +} +if grep -Fq '{{' "${rules}"; then + printf '[prometheus-policy] FAIL: dynamic templates are prohibited\n' >&2 + exit 1 +fi +if grep -Eq 'kind:[[:space:]]*(ServiceMonitor|PrometheusRule)|apiVersion:[[:space:]]*monitoring\.coreos\.com/' "${rules}"; then + printf '[prometheus-policy] FAIL: monitoring CRDs are prohibited\n' >&2 + exit 1 +fi +go test -count=1 -run '^TestPortableAlertRulesStayBoundedAndStatic$' ./internal/observability + +printf '[prometheus-policy] pinned promtool and static portable-rule boundaries verified\n' diff --git a/tests/scripts/release_hub_image_policy_test.sh b/tests/scripts/release_hub_image_policy_test.sh index 6afe6fe..e5919d7 100644 --- a/tests/scripts/release_hub_image_policy_test.sh +++ b/tests/scripts/release_hub_image_policy_test.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# shellcheck disable=SC2016 + set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -63,11 +65,17 @@ release_job_step() { assert_contains "$workflow_contents" 'HUB_IMAGE: ghcr.io/ardurai/sith-hub' 'uses one fixed GHCR hub image name' assert_text_contains "$release_job" 'packages: write' 'grants package publication permission to the release job' +for action in setup-qemu setup-buildx login build-push; do + action_lines="$(grep -E "^[[:space:]]+uses: docker/${action}-action@" <<<"$release_job" || true)" + if [[ "$(wc -l <<<"$action_lines" | tr -d ' ')" != 1 || + ! "$action_lines" =~ docker/${action}-action@[0-9a-f]{40}[[:space:]]+#[[:space:]]+v[0-9] ]]; then + printf '[release-hub-image] FAIL: %s action must use one full commit pin with a version comment\n' "$action" >&2 + exit 1 + fi + printf '[release-hub-image] PASS: %s action uses one full commit pin\n' "$action" +done + for assertion in \ - 'docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130|pins QEMU setup action' \ - 'docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f|pins Buildx setup action' \ - 'docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9|pins registry login action' \ - 'docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8|pins image build action' \ 'tar -xzf "dist/sith_${VERSION}_linux_amd64.tar.gz"|stages the released linux amd64 binary' \ 'tar -xzf "dist/sith_${VERSION}_linux_arm64.tar.gz"|stages the released linux arm64 binary' \ 'dist/sith_${VERSION}_hub.image|attaches the digest address to the release' \ @@ -78,6 +86,17 @@ for assertion in \ assert_contains "$release_job" "$needle" "$description" done +missing_dist="${repo_root}/tests/scripts/.missing-release-dist" +set +e +missing_dist_output="$(DOCKER_BIN=true "$verifier" --dist "$missing_dist" 2>&1)" +missing_dist_status="$?" +set -e +if [[ "$missing_dist_status" == 0 || "$missing_dist_output" == *'expected exactly one Linux archive'* ]]; then + printf '[release-hub-image] FAIL: invalid dist path did not stop at canonicalization\n' >&2 + exit 1 +fi +printf '[release-hub-image] PASS: invalid dist path fails before archive or Docker work\n' + publish_step="$(release_job_step 'Publish immutable multi-platform hub image')" platforms="$(awk '/^[[:space:]]+platforms:/{ print $2 }' <<<"$publish_step")" if [[ "$platforms" != 'linux/amd64,linux/arm64' ]]; then diff --git a/tests/scripts/release_tooling_policy_test.sh b/tests/scripts/release_tooling_policy_test.sh new file mode 100755 index 0000000..642544a --- /dev/null +++ b/tests/scripts/release_tooling_policy_test.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# shellcheck disable=SC2016 + +set -Eeuo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly REPO_ROOT +readonly EXPECTED_GORELEASER_VERSION="v2.17.0" +readonly EXPECTED_SYFT_VERSION="v1.49.0" +readonly EXPECTED_COSIGN_VERSION="v3.1.2" + +PASS_COUNT=0 + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + printf '[release-tooling] PASS: %s\n' "$1" +} + +assert_equal() { + local actual=$1 + local expected=$2 + local description=$3 + + if [[ "${actual}" != "${expected}" ]]; then + printf '[release-tooling] FAIL: %s = %q, want %q\n' \ + "${description}" "${actual}" "${expected}" >&2 + return 1 + fi + pass "${description}" +} + +workflow_value() { + local workflow=$1 + local name=$2 + awk -F '"' -v key="${name}:" '$1 ~ "^[[:space:]]*" key "[[:space:]]*$" { print $2 }' "${workflow}" +} + +ci_workflow="${REPO_ROOT}/.github/workflows/ci.yml" +release_workflow="${REPO_ROOT}/.github/workflows/release.yml" + +ci_goreleaser="$(workflow_value "${ci_workflow}" GORELEASER_VERSION)" +ci_syft="$(workflow_value "${ci_workflow}" SYFT_VERSION)" +release_goreleaser="$(workflow_value "${release_workflow}" GORELEASER_VERSION)" +release_syft="$(workflow_value "${release_workflow}" SYFT_VERSION)" +release_cosign="$(workflow_value "${release_workflow}" COSIGN_VERSION)" + +assert_equal "${ci_goreleaser}" "${EXPECTED_GORELEASER_VERSION}" "CI GoReleaser pin is current" +assert_equal "${release_goreleaser}" "${ci_goreleaser}" "release GoReleaser pin matches CI" +assert_equal "${ci_syft}" "${EXPECTED_SYFT_VERSION}" "CI Syft pin is current" +assert_equal "${release_syft}" "${ci_syft}" "release Syft pin matches CI" +assert_equal "${release_cosign}" "${EXPECTED_COSIGN_VERSION}" "release Cosign pin is current" + +grep -Fq "GoReleaser ${EXPECTED_GORELEASER_VERSION} and Syft ${EXPECTED_SYFT_VERSION}" \ + "${REPO_ROOT}/README.md" +pass "README release prerequisites match executable pins" + +grep -Fq "Syft ${EXPECTED_SYFT_VERSION}" "${REPO_ROOT}/docs/adr/0009-release-supply-chain.md" +grep -Fq "Cosign ${EXPECTED_COSIGN_VERSION}" "${REPO_ROOT}/docs/adr/0009-release-supply-chain.md" +pass "release-supply-chain decision records current pins" + +grep -Fq 'syft-version: ${{ env.SYFT_VERSION }}' "${ci_workflow}" +grep -Fq 'syft-version: ${{ env.SYFT_VERSION }}' "${release_workflow}" +grep -Fq 'cosign-release: ${{ env.COSIGN_VERSION }}' "${release_workflow}" +pass "installer actions consume the synchronized pins" + +if grep -Eq -- '--(payload|output-attestation)([=[:space:]]|$)' "${release_workflow}"; then + printf '[release-tooling] FAIL: release workflow uses a Cosign v3.1-deprecated flag\n' >&2 + exit 1 +fi +pass "release workflow avoids newly deprecated Cosign flags" + +printf '[release-tooling] %d assertions passed\n' "${PASS_COUNT}" diff --git a/tests/scripts/wails_tooling_policy_test.sh b/tests/scripts/wails_tooling_policy_test.sh new file mode 100755 index 0000000..bc891ac --- /dev/null +++ b/tests/scripts/wails_tooling_policy_test.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# shellcheck disable=SC2016 + +set -Eeuo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly REPO_ROOT +readonly EXPECTED_VERSION="v2.13.0" +readonly VERIFIER="${REPO_ROOT}/hack/verify-wails-version.sh" + +PASS_COUNT=0 + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + printf '[wails-policy] PASS: %s\n' "$1" +} + +assert_equal() { + local actual=$1 + local expected=$2 + local description=$3 + + if [[ "${actual}" != "${expected}" ]]; then + printf '[wails-policy] FAIL: %s = %q, want %q\n' \ + "${description}" "${actual}" "${expected}" >&2 + return 1 + fi + pass "${description}" +} + +expect_failure() { + local description=$1 + shift + if "$@" >/dev/null 2>&1; then + printf '[wails-policy] FAIL: %s\n' "${description}" >&2 + exit 1 + fi + pass "${description}" +} + +makefile_version="$(awk '$1 == "WAILS_VERSION" && $2 == "?=" { print $3 }' "${REPO_ROOT}/Makefile")" +module_version="$(awk '$1 == "github.com/wailsapp/wails/v2" { print $2 }' "${REPO_ROOT}/go.mod")" + +assert_equal "${module_version}" "${EXPECTED_VERSION}" "Wails module is current" +assert_equal "${makefile_version}" "${module_version}" "desktop tool pin matches go.mod" + +scratch="$(mktemp -d)" +readonly scratch +trap 'rm -rf "${scratch}"' EXIT + +fake_wails="${scratch}/wails" +readonly fake_wails +printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'if [[ "${WAILS_FAKE_EXIT:-0}" != "0" ]]; then exit "${WAILS_FAKE_EXIT}"; fi' \ + 'printf "%b" "${WAILS_FAKE_OUTPUT:-}"' >"${fake_wails}" +chmod 0700 "${fake_wails}" + +WAILS_FAKE_OUTPUT=$'v2.13.0\nadditional upstream text\n' \ + "${VERIFIER}" "${fake_wails}" "${EXPECTED_VERSION}" +pass "exact version accepts additional lines after the version" + +for lookalike in 'v2.13.0-rc.1' 'v2.13.00' 'v2.13.0+vendor' ' v2.13.0' ''; do + expect_failure "rejects lookalike version ${lookalike:-}" \ + env WAILS_FAKE_OUTPUT="${lookalike}" "${VERIFIER}" "${fake_wails}" "${EXPECTED_VERSION}" +done + +expect_failure "rejects a failed version command" \ + env WAILS_FAKE_EXIT=1 "${VERIFIER}" "${fake_wails}" "${EXPECTED_VERSION}" +expect_failure "rejects a missing Wails command" \ + "${VERIFIER}" "${scratch}/missing-wails" "${EXPECTED_VERSION}" +expect_failure "rejects a malformed expected version" \ + "${VERIFIER}" "${fake_wails}" 'v2.13' + +grep -Fq 'hack/verify-wails-version.sh "$(WAILS)" "$(WAILS_VERSION)"' "${REPO_ROOT}/Makefile" +pass "desktop build invokes the tested exact-version verifier" + +printf '[wails-policy] %d assertions passed\n' "${PASS_COUNT}"