diff --git a/.gitignore b/.gitignore index 985ae0b..75a7008 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ docs/assets/* !docs/concepts/ docs/concepts/* !docs/concepts/canonical-store.md +!docs/concepts/generation-bound-retrieval.md !docs/contributing/ docs/contributing/* !docs/contributing/adapter-contributor.md diff --git a/README.md b/README.md index f3a95ee..80c593c 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ requests: | --- | --- | --- | | Records | SQLite | Firestore, AlloyDB/pgvector, Azure Cosmos DB, DynamoDB | | Objects | Filesystem | Google Cloud Storage, Cloudflare R2, Azure Blob Storage, S3 | -| Retrieval | SQLite/local providers | pgvector and provider-shaped adapters | +| Retrieval | SQLite/local providers | pgvector, OpenSearch, and Cloudflare Worker/R2 projections | | Traces | SQLite | Firestore and deployment-specific sinks | Exercise the portable data boundary without cloud credentials: @@ -316,6 +316,13 @@ The deterministic local embedding provider is for mechanics and repeatability, not semantic quality. ONNX and provider-backed models require explicit model or credential configuration. +For derived indexes that need stronger evidence, the optional +[generation-bound retrieval contract](docs/concepts/generation-bound-retrieval.md) +pins every result and continuation to one immutable descriptor and fails closed +unless all requested logical partitions are covered. Local, OpenSearch, and +Cloudflare Worker/R2 implementations share the candidate-only boundary; +applications still authorize and hydrate canonical records before use. + RAG ingestion separates planning from commit, persists manifests, and admits large mutations through durable execution. RAG context responses preserve citations, source spans, truncation diagnostics, and stable context hashes. @@ -406,6 +413,7 @@ Start with the [execution design](design/execution-runtime.md), | Google execution | Live qualified; other Google adapters retain their separate qualification posture | | Temporal coordinator and projection | Prototype; disposable and operator-provisioned qualification gates | | Cloudflare R2 | Preview object-store adapter | +| Generation-bound retrieval projections | Local reference plus OpenSearch and Worker/R2 shapes; maturity is topology- and evidence-scoped | An adapter is `live_qualified` only when the versioned [qualification matrix](qualification/README.md) records the required evidence. @@ -423,6 +431,7 @@ runtimes/ Peer runtime implementations, currently Python contracts/ OpenAPI-derived public SDK catalog and JSON schemas conformance/ Language-neutral cross-runtime fixtures examples/ Short HTTP client recipes +experiments/ Consumer-neutral provider proofs that are not published packages samples/ Runnable .NET and execution examples tests/ Unit, integration, conformance, and adapter suites qualification/ Versioned adapter qualification report and policy diff --git a/conformance/runtime/v1/manifest.json b/conformance/runtime/v1/manifest.json index ea1c30e..2b291ad 100644 --- a/conformance/runtime/v1/manifest.json +++ b/conformance/runtime/v1/manifest.json @@ -13,6 +13,7 @@ "vyral.runtime.external-worker.v1", "vyral.runtime.providers.v1", "vyral.runtime.canonical.v1", + "vyral.runtime.retrieval-generation.v1", "vyral.runtime.execution.local.v1", "vyral.runtime.rest.v1", "vyral.runtime.mcp.stateless-2026-07-28.v1", @@ -54,6 +55,20 @@ "profile": "vyral.runtime.data-rag.v1", "kind": "golden" }, + { + "id": "records.projection-generation.v1", + "path": "scenarios/goldens/record-search-projection-generation.json", + "sha256": "sha256:98e7a37d649b8008d29118a8cfe51c1ca3a178e58c3a602145717fb26fd8f0ca", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "golden" + }, + { + "id": "records.projection-generation-lifecycle.v1", + "path": "scenarios/retrieval/generation-bound-lifecycle.json", + "sha256": "sha256:7a5433eb745637004155322a76abbb0a5eaf6cf26148b7e18397c1e26c194875", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "stateful" + }, { "id": "embeddings.vectors.v1", "path": "scenarios/goldens/embedding-vectors.json", diff --git a/conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json b/conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json new file mode 100644 index 0000000..9ec058a --- /dev/null +++ b/conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json @@ -0,0 +1,60 @@ +{ + "$schema": "../../scenario.schema.json", + "id": "records.projection-generation.v1", + "fixtureVersion": "1.0.0", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "golden", + "description": "Immutable record-search projection descriptor hashing shared by embedded and hosted adapters.", + "steps": [ + { + "id": "descriptor-hash-two-partitions", + "operation": "records.projection-generation-descriptor-hash", + "arguments": { + "descriptor": { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "gen-2026-08-27-a", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recordRevisionSetDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": [ + "public-a", + "public-b" + ], + "capabilities": [ + "completeCoverage", + "generationPinnedContinuation", + "lexical" + ], + "artifacts": [ + { + "id": "partition-public-a", + "kind": "index-part", + "contentHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "sizeBytes": 128, + "mediaType": "application/octet-stream" + }, + { + "id": "partition-public-b", + "kind": "index-part", + "contentHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "sizeBytes": 96, + "mediaType": "application/octet-stream" + } + ], + "createdAtUtc": "2026-08-27T12:00:00Z", + "descriptorDigest": "sha256:3b92c8927a2a93f12d99e64e03d118e29ac57284663943fa257568ae178d31d5" + } + }, + "expect": { + "value": "sha256:3b92c8927a2a93f12d99e64e03d118e29ac57284663943fa257568ae178d31d5" + } + } + ] +} diff --git a/conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json b/conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json new file mode 100644 index 0000000..cef46c6 --- /dev/null +++ b/conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json @@ -0,0 +1,273 @@ +{ + "$schema": "../../scenario.schema.json", + "id": "records.projection-generation-lifecycle.v1", + "fixtureVersion": "1.0.0", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "stateful", + "description": "Generation activation, retained continuations, fail-closed coverage, descriptor fences, tamper rejection, and retirement.", + "steps": [ + { + "id": "publish-generation-a", + "operation": "records.projection-generation-publish", + "arguments": { + "descriptor": { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "generation-a", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recordRevisionSetDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": ["public-a", "public-b"], + "capabilities": ["completeCoverage", "generationPinnedContinuation", "lexical"], + "artifacts": [], + "createdAtUtc": "2026-08-27T12:00:00Z", + "descriptorDigest": "sha256:01666b57bb8922e1360b5662219fb7955cb1ba5f8e8865f4fd46eecddaeb50de" + }, + "documents": [ + { "partitionKey": "public-a", "id": "a-1", "revision": 1, "searchText": "portable contracts" }, + { "partitionKey": "public-a", "id": "a-2", "revision": 1, "searchText": "provider native semantics" }, + { "partitionKey": "public-b", "id": "b-1", "revision": 1, "searchText": "portable retrieval" } + ] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "activate-generation-a", + "operation": "records.projection-generation-activate", + "arguments": { "collection": "library", "generationId": "generation-a" }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "inspect-active-generation-a", + "operation": "records.projection-generation-inspect", + "arguments": { "collection": "library" }, + "expect": { + "value": { + "generationId": "generation-a", + "state": "active", + "coverageStatus": "complete", + "availablePartitions": ["public-a", "public-b"] + } + } + }, + { + "id": "search-generation-a-first-page", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "saveContinuationAs": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "succeeded", + "generationId": "generation-a", + "ids": ["a-1"], + "continuation": "present", + "coverageStatus": "complete", + "coveredPartitions": ["public-a", "public-b"], + "missingPartitions": [], + "failureCode": null + } + } + }, + { + "id": "publish-generation-b", + "operation": "records.projection-generation-publish", + "arguments": { + "descriptor": { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "generation-b", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recordRevisionSetDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": ["public-a", "public-b"], + "capabilities": ["completeCoverage", "generationPinnedContinuation", "lexical"], + "artifacts": [], + "createdAtUtc": "2026-08-27T12:05:00Z", + "descriptorDigest": "sha256:a025b15641dbacea0af4bffa84c707bd4171dc48b8d58fce1135c8402d7f1d3b" + }, + "documents": [ + { "partitionKey": "public-a", "id": "a-1", "revision": 2, "searchText": "portable execution" }, + { "partitionKey": "public-a", "id": "a-3", "revision": 1, "searchText": "provider evidence" }, + { "partitionKey": "public-b", "id": "b-2", "revision": 1, "searchText": "portable qualification" } + ] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "activate-generation-b", + "operation": "records.projection-generation-activate", + "arguments": { "collection": "library", "generationId": "generation-b" }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "continue-retained-generation-a", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "continuationRef": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "succeeded", + "generationId": "generation-a", + "ids": ["b-1"], + "continuation": "absent", + "coverageStatus": "complete", + "coveredPartitions": ["public-a", "public-b"], + "missingPartitions": [], + "failureCode": null + } + } + }, + { + "id": "reject-tampered-continuation", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "tamperContinuationRef": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "failed", + "generationId": null, + "ids": [], + "continuation": "absent", + "coverageStatus": "unavailable", + "coveredPartitions": [], + "missingPartitions": ["public-a", "public-b"], + "failureCode": "invalidContinuation" + } + } + }, + { + "id": "remove-generation-b-partition", + "operation": "records.projection-generation-set-available", + "arguments": { + "collection": "library", + "generationId": "generation-b", + "availablePartitions": ["public-a"] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "fail-closed-on-incomplete-generation-b", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 10 + }, + "expect": { + "value": { + "status": "failed", + "generationId": "generation-b", + "ids": [], + "continuation": "absent", + "coverageStatus": "incomplete", + "coveredPartitions": ["public-a"], + "missingPartitions": ["public-b"], + "failureCode": "coverageIncomplete" + } + } + }, + { + "id": "inspect-incomplete-generation-b", + "operation": "records.projection-generation-inspect", + "arguments": { "collection": "library" }, + "expect": { + "value": { + "generationId": "generation-b", + "state": "active", + "coverageStatus": "incomplete", + "availablePartitions": ["public-a"] + } + } + }, + { + "id": "restore-generation-b-coverage", + "operation": "records.projection-generation-set-available", + "arguments": { + "collection": "library", + "generationId": "generation-b", + "availablePartitions": ["public-a", "public-b"] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "reject-wrong-descriptor-fence", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 10, + "expectedDescriptorDigest": "sha256:9999999999999999999999999999999999999999999999999999999999999999" + }, + "expect": { + "value": { + "status": "failed", + "generationId": "generation-b", + "ids": [], + "continuation": "absent", + "coverageStatus": "complete", + "coveredPartitions": ["public-a", "public-b"], + "missingPartitions": [], + "failureCode": "generationDescriptorMismatch" + } + } + }, + { + "id": "retire-generation-a", + "operation": "records.projection-generation-retire", + "arguments": { "collection": "library", "generationId": "generation-a" }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "reject-retired-generation-a-continuation", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "continuationRef": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "failed", + "generationId": "generation-a", + "ids": [], + "continuation": "absent", + "coverageStatus": "unavailable", + "coveredPartitions": [], + "missingPartitions": ["public-a", "public-b"], + "failureCode": "generationRetired" + } + } + } + ] +} diff --git a/docs/README.md b/docs/README.md index 7a7988c..bc7deec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,9 @@ here so each document has a clear audience and owner. readiness, common workflows, and failure interpretation. - [CanonicalStore](concepts/canonical-store.md) — strong-storage transactions, identity policy, export/restore, and provider qualification. +- [Generation-bound retrieval](concepts/generation-bound-retrieval.md) — + immutable index generations, complete logical coverage, candidate hydration, + and provider-specific evidence boundaries. - [Execution runtime limitations](reference/execution-runtime-limitations.md) — portable guarantees and adapter-specific boundaries. - [Extropic execution](guides/extropic-execution.md) — experimental, diff --git a/docs/concepts/generation-bound-retrieval.md b/docs/concepts/generation-bound-retrieval.md new file mode 100644 index 0000000..9821ba6 --- /dev/null +++ b/docs/concepts/generation-bound-retrieval.md @@ -0,0 +1,87 @@ +# Generation-bound retrieval projections + +Vyral's optional generation-bound projection contract lets a derived search index prove exactly +which immutable generation it is serving and whether that generation completely covers every +requested logical partition. It is intended for retrieval paths where a healthy endpoint or a +successful query is not enough evidence that the index is complete, current, or internally +consistent. + +The baseline `IRecordSearchProjection` remains the simpler eventual-projection seam. Implement +`IGenerationBoundRecordSearchProjection` only when the provider can support the stronger evidence +model. + +## Portable boundary + +A generation descriptor binds: + +- the collection, generation, provider, profile, and strategy; +- source-manifest and record-revision-set digests; +- projection-schema, analyzer, and configuration digests; +- expected item count and logical partitions; +- capabilities and content-addressed artifact declarations; and +- the descriptor's own canonical SHA-256 digest. + +Search selects the active generation or an explicitly named retained generation. A caller can also +fence the expected descriptor digest. Incomplete, unavailable, retired, late, or substituted +generations fail closed with no candidates. An adapter that emits a continuation must authenticate +it and bind it to the exact generation, descriptor, invariant request, expiry, and page boundary. + +Search results contain candidate identities, canonical record revisions, and scores—not canonical +records or an authorization decision. `SearchGenerationAndHydrateAsync` reads candidates from the +canonical store and discards any whose stored revision no longer matches. The application still +owns tenant authorization, eligibility, final ranking, abstention, and adoption policy. + +Provider index names, SDK objects, endpoints, credentials, health checks, and physical shard or +replica topology remain outside the portable descriptor. A provider-private index format may change +without changing the public contract. + +## Building a generation + +`IRecordSearchProjectionGenerationBuilder` is a build-and-verify seam, not a portable lifecycle +manager. It writes provider-native parts through an `IObjectStore` and returns a compact verified +descriptor receipt. `ExecutionRuntimeProjectionGenerationAdapter` can admit that build as durable, +idempotent Vyral work, retain progress and terminal evidence, and avoid adding a Vyral network hop +to the query path. + +Activation, rollback, and retirement remain host-owned. More provider evidence is required before +those operations can be promoted into a credible portable contract. + +## Implementations + +| Shape | Current evidence boundary | +| --- | --- | +| Local exhaustive projection | Deterministic reference implementation and shared lifecycle conformance | +| OpenSearch exact-index projection | Local OpenSearch data-plane conformance; managed AWS IAM, networking, scaling, and availability remain unqualified | +| Cloudflare Worker/R2 with direct R2 binding | Deterministic Miniflare conformance for the exact Worker source | +| Cloudflare Worker/R2 with isolated service reader | Deterministic Miniflare conformance plus private, opaque consumer validation of the exact Worker and reader components | + +The Cloudflare service-reader topology gives the public query Worker no R2 binding. A separate +non-public Worker alone binds the bucket and exposes only an authenticated, allowlisted read +operation over a service binding. The query Worker fails closed when neither or both read modes are +configured. See the [reference implementation and proof harness](../../experiments/worker-r2-generation-projection). + +The consumer validation used a small synthetic corpus and does not establish representative +traffic, production capacity, availability, cost, or provider observability. The public evidence +reference is intentionally opaque: +`urn:vyral:private-consumer-evidence:sha256:cdc973c71ac0abba92dd2ec0fa3eb016b4e35233e9a059bf96f6b97b93a197da`. +The exact live-exercised source components are: + +- query Worker: `sha256:e9311869f2e671397b6f2a017882002bde33b46b9a78d4367d3e7ea192f0d6fc` +- object reader: `sha256:9fe0cba9c743210314ef6a91d9306a867527a6267068f75108fd5d868bd15a28` + +Consumer identities, repositories, commands, deployment identifiers, hostnames, generation names, +and private receipt locations are not public qualification data. Maintainers keep the identity map +outside the repository unless a consumer separately authorizes disclosure. + +## API status + +The descriptor, request, result, inspection, build, and receipt shapes are covered by JSON Schema +and language-neutral conformance fixtures. The .NET abstraction and local/OpenSearch implementations +are available for host composition. Python reproduces the shared descriptor and lifecycle fixtures. +The generation-aware retrieval-evaluation target remains an in-process host seam and is deliberately +excluded from REST and peer-SDK serialization until a versioned public wire surface is reviewed. + +Adapter qualification and generation rollout are separate evidence planes: adapter qualification +says a topology can implement the contract, while each generation receipt proves one exact +generation was built and verified. Neither a healthy process nor an adapter's qualification makes +an individual generation complete. diff --git a/docs/contributing/adapter-contributor.md b/docs/contributing/adapter-contributor.md index 51bdabb..c012e56 100644 --- a/docs/contributing/adapter-contributor.md +++ b/docs/contributing/adapter-contributor.md @@ -321,6 +321,10 @@ Rules: - `advertisedCapabilities` must match runtime descriptor capabilities. - Every advertised capability must appear in the qualification capability list. - Evidence ages out (90-day freshness by default). +- Consumer validation may raise the maturity claim without naming the validating consumer. Record + it publicly as `private_opaque` with an opaque evidence digest; keep consumer identities, + repositories, commands, result paths, deployment identifiers, and generation names in private + maintainer evidence unless separately authorized for publication. - **Do not** mark `live_qualified` because a script exists or passed once on a laptop. - Release generation: diff --git a/docs/maintainers/releasing.md b/docs/maintainers/releasing.md index d452e49..8214279 100644 --- a/docs/maintainers/releasing.md +++ b/docs/maintainers/releasing.md @@ -15,7 +15,11 @@ retained as release evidence. SBOM, and `qualification/adapter-qualification.json` as build evidence. 2. Review dependency, secret, and release-ownership scan results, the SBOM and third-party notices, package metadata, adapter qualification matrix, and clean consumer validation for - .NET, JavaScript, Python, and Go. + .NET, JavaScript, Python, and Go. A public `consumer_validation` item must use + `private_opaque` disclosure, an opaque `urn:vyral:private-consumer-evidence:sha256:...` + reference, and no consumer command, result path, generation identifier, repository, deployment + identity, or name. Keep the identity-to-receipt mapping in ignored operator evidence. Naming a + validating consumer requires separate, explicit publication authorization from that consumer. 3. Record contract additions, behavior changes, qualification changes, migrations, and known limits in the release notes. Package availability or an advertised capability must not be presented as `live_qualified` without a current live receipt in the release artifact. diff --git a/experiments/worker-r2-generation-projection/README.md b/experiments/worker-r2-generation-projection/README.md new file mode 100644 index 0000000..cfc2f97 --- /dev/null +++ b/experiments/worker-r2-generation-projection/README.md @@ -0,0 +1,67 @@ +# Worker/R2 generation-bound projection + +This experiment implements Vyral's optional generation-bound candidate contract on Cloudflare +Workers and R2. It is consumer-neutral reference code, not a published package or a portable index +file format. + +The query Worker: + +- resolves active or explicitly selected generations from R2-backed catalog records; +- verifies the descriptor, manifest, and every required shard before returning candidates; +- applies bounded lexical scoring and filters inside the Worker; +- fails closed on missing, corrupt, stale, retired, late, or incomplete generations; and +- signs continuations that remain bound to the retained generation and exact request. + +The JSON manifest and shard schemas beginning with `vyral.private.worker-r2` are deliberately +adapter-private. The portable boundary is the public Vyral generation descriptor, request, result, +coverage, failure, and continuation behavior. + +## Immutable-object access modes + +The same query Worker supports two mutually exclusive deployment shapes: + +1. `direct-r2` binds `INDEX` directly to the query Worker. This is the simplest single-Worker + shape, but the query Worker receives the R2 binding's full technical authority. +2. `service-reader` gives the query Worker no R2 binding. Its `OBJECT_READER` service binding points + to `src/object-reader.mjs`, which alone binds `INDEX`. The reader accepts only authenticated + `POST /read` requests for allowlisted generation keys, exposes no mutation method, and should + have neither a public route nor a `workers.dev` address. + +The query Worker fails closed unless exactly one of `INDEX` and `OBJECT_READER` is present. Do not +configure both as a fallback: ambiguity is treated as provider unavailability. + +Query Worker configuration requires independently generated values of at least 32 bytes for +`AUTHORIZATION_SECRET` and `CONTINUATION_SECRET`. Service-reader mode additionally requires the same +independently generated `OBJECT_READER_SECRET` in both Workers and a service binding shaped like: + +```json +{ + "services": [ + { "binding": "OBJECT_READER", "service": "" } + ] +} +``` + +Bind the reader Worker to the R2 bucket as `INDEX`, set `workers_dev = false`, and declare no +routes. Put the query Worker behind the intended authenticated ingress, rate limits, and +request-size controls. Bearer checks are defense in depth, not substitutes for route isolation or +service identity. + +## Local proof + +Install the exact locked dependency graph and exercise both topologies against a deterministic, +consumer-neutral fixture: + +```shell +npm ci --ignore-scripts --prefix experiments/worker-r2-generation-projection +python3 scripts/verify-worker-r2-generation-projection.py \ + --output /tmp/vyral-worker-r2-proof.json +``` + +The harness proves exact candidate/revision/score parity, generation lifecycle and continuation +behavior, content verification, authentication and body bounds, reader non-mutation guards, and +fail-closed handling for missing, ambiguous, corrupt, or incomplete configuration and artifacts. + +Public qualification is scoped to the exact source and evidence named in the qualification +materials. A local proof does not establish live Cloudflare IAM, latency, cache eviction, billing, +or production capacity. diff --git a/experiments/worker-r2-generation-projection/package-lock.json b/experiments/worker-r2-generation-projection/package-lock.json new file mode 100644 index 0000000..f08db87 --- /dev/null +++ b/experiments/worker-r2-generation-projection/package-lock.json @@ -0,0 +1,1001 @@ +{ + "name": "vyral-worker-r2-generation-proof", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "vyral-worker-r2-generation-proof", + "version": "0.0.0", + "devDependencies": { + "miniflare": "4.20260730.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@cloudflare/workerd-darwin-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260730.1.tgz", + "integrity": "sha512-+MBHmPaiTe2KajryW0T24rZvWFxb41hD3d8anNzQqHzft6vSEb18+sp0znSwxgij7ApPhSM1+vhkNg4f3YMguA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-darwin-arm64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260730.1.tgz", + "integrity": "sha512-SBHKntPkKvNPgaCrTe99xC1CAl8ygJDzlYfK0LbuJ1muKadIw35WnhO0wu894fKBtllsVQdNzDLee+cm0ppLSQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260730.1.tgz", + "integrity": "sha512-ouyPOSMbiKPeSwUJUvxtMcxGAXs2J4aPE4T5ABIYX5ClcQx5j5bbHTmnqOQEY8sAuLTPjH7dY+iB6UI5ISlwwA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-linux-arm64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260730.1.tgz", + "integrity": "sha512-YQ+Mi78U3TPdgBPtwq+Sm6rJU+Ihl2y0pjYtuuKkdmUbYzL7oLR6Xqq9wljhasnuCFICssDJaqhMep5WizYoEQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cloudflare/workerd-windows-64": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260730.1.tgz", + "integrity": "sha512-27fAN+vUECW1oYVc1KOcHYpkL8COM2Uxtxql7TL595kxbjoqS5yckw7NLz7bTf2pALFCZWjqXDjZGJ/xbG4ZKQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.2.tgz", + "integrity": "sha512-eEieHsMksAW4IiO5NzauESRl2D2qz3J/kwUxUrSfV06A93eEaRfMpHXyUb1mAqrR7i8U9A0GRqE9pjn6u1Jjpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.2.tgz", + "integrity": "sha512-BaktuGPCeHJMARpodR8jK4uKiZrPAy9WrfQW0sdI37clracq8Bp01AYS3SZgi5FS/y5twa9t4+LIuuxQjqRrWw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.2.tgz", + "integrity": "sha512-YoAxdnd8hPUkvLHd3bWY+YA8nw3xM/RyRopYucNsWHVSan8NLVM3X2volsfoRDcXdUJPg6tXahSd7HXPK7lRnw==", + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.1.tgz", + "integrity": "sha512-4V/M3roRMTYjiwZY9IOVQOE8OyeCxFAkYmyZDrZl51uOKjibm3oeEJ4WAmLxutAfzFbC9jqUiPs2gbnGflH+7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.1.tgz", + "integrity": "sha512-c0/DxItpJv2+dGhgycJBBgotdqruGYDvA79drdh0MD1dFpy7JzJ/PlXwi1H4rFf0eTy8tgbI91aHDnZIceY3jQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.1.tgz", + "integrity": "sha512-aGGy9aWzXgHBG7HNyQPWorZthlp7+x6fDRoPAQbGO3ThcttuTyKIx3NuSHb6zb4gBNq6/yNn9f1cy9nFKS/Vmg==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.1.tgz", + "integrity": "sha512-JznefmcK9j1JKPz8AkQDh89kjojubyfOasWBPKfzMIhPwsgDy9evpE/naJTXXXmghS1iFwR8u/kTwh/I2/+GCw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.1.tgz", + "integrity": "sha512-1EkwGNCZk6iWNCMWqrvdJ+r1j0PT1zIz60CNPhYnJlK/zyeWqlsPZIe+ocBVqPF8k/Ssee/NCk+tE9Ryrko6ng==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.1.tgz", + "integrity": "sha512-Ilays+w2bXdnxzxtQdmXR62u8o8GYa3eL4+Gr+1KiE4xperMZUslRaVPJwwPkzlHEjGfXAfRVAa/7CYCtSqsBw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.1.tgz", + "integrity": "sha512-VfBwVHQTbRoj4XlpA/KLZ7ltgMpz+4WSejFzQ+GnoImjo1PtEJ59QB2qR1xQEeRPYIkNrPIm2L4cICMvz4C2ew==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.1.tgz", + "integrity": "sha512-+c8ukgwU62DS54nCAjw7keOfHUkmr0B5QHEdcOqRnodF/MNXJbVI8Eopoj4B/0H8Asr65I+A4Amrn7a85/md6A==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.1.tgz", + "integrity": "sha512-qlKb/pwbkAi1WMsJrYHk7CuDrd12s27U2QnRhFYUoJNrRCmkosMTttuRFat/DDB3IlDm5qE1TJgZ4JDnHX8Ldw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.1.tgz", + "integrity": "sha512-yO21HwoUVLN8Qa+/SBjQLMYwBWAVJjeGPNe+hc0OUeMeifEtJqu5a1c4HayE1nNpDih9y3/KkoltfkDodmKAlg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.2.tgz", + "integrity": "sha512-SE4kzF2mepn6z+6E7L6lsV8FzuLL6IPQdyX8ZiwROAG/G8td+hP/m7FsFPwidtrF19gvajuC9l6TxAVcsA4S7A==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.2.tgz", + "integrity": "sha512-af12Pnd0ZGu2HfP8NayB0kk6eC/lrfbQE6HlR4jD+34wdJ1Vw9TF6TMn6ZvffT+WgqVsl0hRbmNvz2u/23VmwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.2.tgz", + "integrity": "sha512-hYSBm7zcNtDCozCxQHYZJiu63b/bXsgRZuOxCIBZsStMM9Vap47iFHdbX4kCvQsblPB/k+clhELpdQJHQLSHvg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.2.tgz", + "integrity": "sha512-qQt0Kc13+Hoan/Awq/qMSQw3L+RI1NCRPgD5cUJ/1WSSmIoysLOc72jlRM3E0OHN9Yr313jgeQ2T+zW+F03QFA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.2.tgz", + "integrity": "sha512-E4fLLfRPzDLlEeDaTzI98OFLcv++WL5ChLLMwPoVd0CIoZQqupBSNbOisPL5am9XsbQ9T84+iiMpUvbFtkunbA==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.1" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.2.tgz", + "integrity": "sha512-gi0zFJJRLswfCZmHtJdikXPOc5u7qamSOS3NHedLqLd4W8Q0NqjdBr6TTRIgsfFjqfTsHFgdfvJ9LwqSgcHiAA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.2.tgz", + "integrity": "sha512-siWbOW1u6HFnFLrp0waKyW7VEf7jYvcDWdrXEFa8AkdAQgEvuu5Fz8/Y70w9EeqAdwDtfU012BhEHHaDqvQNzg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.2.tgz", + "integrity": "sha512-YBqMMcjDi4QGYiSn4vNOYBhmlC4z5AXqkOUUqI2e0AFA4urNv4ESgOgwNl3K+4etQhha0twXlzeF20bbULm9Yg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.1" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.2.tgz", + "integrity": "sha512-Mrv4JQNYVQ94xH+jzZ9r+gowleN8mv2FTgKT+PI6bx5C0G8TdNYndu161pg2i7uoBwxy2ImPMHrJOM2LZef7Bw==", + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.2.tgz", + "integrity": "sha512-QNV27pxs9wpApEiCfvHM1RDoP1w1+2KrUWWDPEhEwg+latvOrfuhWrHWZKwdSFwU6jh3myjw/yOCRsUIuOft3g==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.2" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.2.tgz", + "integrity": "sha512-BiVRYc/t6/Vl3e1hBx0hugG4oN9Pydf4fgMSpxTQJmwGUg/YoXTWHiFeRymHfCZzifxu4F4rpk/I67D0LQ20wQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.2.tgz", + "integrity": "sha512-YYEhx9PImCC7T0tI8JDMi4DB9LwLCXCU5OWNYEXAxh5Q1ShKkyC6byxzoBJ3gEFDnH2lQckWuDe70G7mB2XJog==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.2.tgz", + "integrity": "sha512-imoOyBcoM/iiUr4J6VPpCNjPnjvP/Gks95898yB8YqoGGYmHYbOyCuNv9FMhFgtaiHFGbHW8bxKqRV6VjtXThQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@poppinss/colors": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", + "integrity": "sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^4.1.5" + } + }, + "node_modules/@poppinss/dumper": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/@poppinss/dumper/-/dumper-0.6.5.tgz", + "integrity": "sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@sindresorhus/is": "^7.0.2", + "supports-color": "^10.0.0" + } + }, + "node_modules/@poppinss/exception": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@poppinss/exception/-/exception-1.2.3.tgz", + "integrity": "sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sindresorhus/is": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-7.2.0.tgz", + "integrity": "sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@speed-highlight/core": { + "version": "1.2.24", + "resolved": "https://registry.npmjs.org/@speed-highlight/core/-/core-1.2.24.tgz", + "integrity": "sha512-qeW2e1l78afw8VhRPfPQ1Gjj+KU5XFQ/OFV5ti6eTa9bruO7mJyZtA4vw0ofqmA3tKCkROE9xLk3VZoeRc98nw==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/error-stack-parser-es": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/error-stack-parser-es/-/error-stack-parser-es-1.0.5.tgz", + "integrity": "sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/kleur": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", + "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/miniflare": { + "version": "4.20260730.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260730.0.tgz", + "integrity": "sha512-1Z9SB9r/o//80UA02Re3QhtcecSHAyAjf5EcKBfQVlQrCg7Miy79hl2PvtkwFLIaJ5rcrOPdDcRr577okwZPsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "0.8.1", + "sharp": "0.35.2", + "undici": "7.28.0", + "workerd": "1.20260730.1", + "ws": "8.21.0", + "youch": "4.1.0-beta.10" + }, + "bin": { + "miniflare": "bootstrap.js" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.35.2", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.2.tgz", + "integrity": "sha512-FVtFjtBCMiJS6yb5CX7Sop45WFMpeGw6oRKuJnXYgf/f1ms/D7LE/ZUSNxnW7rZ/dbslQWYkoqFHGPaDBtaK4w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.4" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.2", + "@img/sharp-darwin-x64": "0.35.2", + "@img/sharp-freebsd-wasm32": "0.35.2", + "@img/sharp-libvips-darwin-arm64": "1.3.1", + "@img/sharp-libvips-darwin-x64": "1.3.1", + "@img/sharp-libvips-linux-arm": "1.3.1", + "@img/sharp-libvips-linux-arm64": "1.3.1", + "@img/sharp-libvips-linux-ppc64": "1.3.1", + "@img/sharp-libvips-linux-riscv64": "1.3.1", + "@img/sharp-libvips-linux-s390x": "1.3.1", + "@img/sharp-libvips-linux-x64": "1.3.1", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.1", + "@img/sharp-libvips-linuxmusl-x64": "1.3.1", + "@img/sharp-linux-arm": "0.35.2", + "@img/sharp-linux-arm64": "0.35.2", + "@img/sharp-linux-ppc64": "0.35.2", + "@img/sharp-linux-riscv64": "0.35.2", + "@img/sharp-linux-s390x": "0.35.2", + "@img/sharp-linux-x64": "0.35.2", + "@img/sharp-linuxmusl-arm64": "0.35.2", + "@img/sharp-linuxmusl-x64": "0.35.2", + "@img/sharp-webcontainers-wasm32": "0.35.2", + "@img/sharp-win32-arm64": "0.35.2", + "@img/sharp-win32-ia32": "0.35.2", + "@img/sharp-win32-x64": "0.35.2" + } + }, + "node_modules/supports-color": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz", + "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/undici": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/workerd": { + "version": "1.20260730.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260730.1.tgz", + "integrity": "sha512-zmfNIjwYSWFY5chGBOjWtH3xAE7p97FTC6vR4Ep98290ho6AeAR/NVcBD274YCLEUYzqm8yxdtZlxMybU8a3jA==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "workerd": "bin/workerd" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "@cloudflare/workerd-darwin-64": "1.20260730.1", + "@cloudflare/workerd-darwin-arm64": "1.20260730.1", + "@cloudflare/workerd-linux-64": "1.20260730.1", + "@cloudflare/workerd-linux-arm64": "1.20260730.1", + "@cloudflare/workerd-windows-64": "1.20260730.1" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/youch": { + "version": "4.1.0-beta.10", + "resolved": "https://registry.npmjs.org/youch/-/youch-4.1.0-beta.10.tgz", + "integrity": "sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/colors": "^4.1.5", + "@poppinss/dumper": "^0.6.4", + "@speed-highlight/core": "^1.2.7", + "cookie": "^1.0.2", + "youch-core": "^0.3.3" + } + }, + "node_modules/youch-core": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/youch-core/-/youch-core-0.3.3.tgz", + "integrity": "sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@poppinss/exception": "^1.2.2", + "error-stack-parser-es": "^1.0.5" + } + } + } +} diff --git a/experiments/worker-r2-generation-projection/package.json b/experiments/worker-r2-generation-projection/package.json new file mode 100644 index 0000000..f9dee99 --- /dev/null +++ b/experiments/worker-r2-generation-projection/package.json @@ -0,0 +1,22 @@ +{ + "name": "vyral-worker-r2-generation-proof", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "check": "node --check src/worker.mjs && node --check src/object-reader.mjs && node --check verify.mjs", + "verify": "node verify.mjs" + }, + "engines": { + "node": ">=22" + }, + "devDependencies": { + "miniflare": "4.20260730.0" + }, + "overrides": { + "undici": "7.29.0" + }, + "allowScripts": { + "workerd": false + } +} diff --git a/experiments/worker-r2-generation-projection/src/object-reader.mjs b/experiments/worker-r2-generation-projection/src/object-reader.mjs new file mode 100644 index 0000000..55bbad0 --- /dev/null +++ b/experiments/worker-r2-generation-projection/src/object-reader.mjs @@ -0,0 +1,69 @@ +const MAX_BODY_BYTES = 1_024; +const MAX_OBJECT_BYTES = 33_554_432; +const textEncoder = new TextEncoder(); + +function response(status, body, contentType = "application/json") { + return new Response(body, { + status, + headers: { + "cache-control": "no-store", + "content-type": contentType, + "referrer-policy": "no-referrer", + "x-content-type-options": "nosniff", + }, + }); +} + +function constantTimeEqual(left, right) { + let difference = left.length ^ right.length; + const length = Math.max(left.length, right.length); + for (let index = 0; index < length; index += 1) difference |= (left[index] ?? 0) ^ (right[index] ?? 0); + return difference === 0; +} + +function authorized(request, env) { + if (typeof env.OBJECT_READER_SECRET !== "string" || textEncoder.encode(env.OBJECT_READER_SECRET).byteLength < 32) return null; + const supplied = request.headers.get("authorization") ?? ""; + if (supplied.length > 1_024) return false; + return constantTimeEqual(textEncoder.encode(supplied), textEncoder.encode(`Bearer ${env.OBJECT_READER_SECRET}`)); +} + +async function boundedBody(request) { + const declared = Number(request.headers.get("content-length") ?? "0"); + if (Number.isFinite(declared) && declared > MAX_BODY_BYTES) throw new Error("body-too-large"); + const bytes = new Uint8Array(await request.arrayBuffer()); + if (bytes.byteLength > MAX_BODY_BYTES) throw new Error("body-too-large"); + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); +} + +function validKey(value) { + if (typeof value !== "string" || value.length < 1 || value.length > 500 || value.includes("..")) return false; + return ( + /^active\/[A-Za-z0-9._%~-]+\.json$/u.test(value) || + /^catalog\/[A-Za-z0-9._%~-]+\/[A-Za-z0-9._%~-]+\.json$/u.test(value) || + /^objects\/sha256\/[0-9a-f]{64}\.json$/u.test(value) + ); +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + if (request.method !== "POST" || url.pathname !== "/read" || url.search || url.hash) return response(404, '{"status":"not-found"}'); + const authorization = authorized(request, env); + if (authorization === null) return response(503, '{"status":"unavailable"}'); + if (!authorization) return response(401, '{"status":"unauthorized"}'); + if (env.INDEX === undefined || typeof env.INDEX?.get !== "function") return response(503, '{"status":"unavailable"}'); + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") return response(415, '{"status":"invalid-request"}'); + let body; + try { + body = await boundedBody(request); + } catch { + return response(400, '{"status":"invalid-request"}'); + } + if (body === null || typeof body !== "object" || Array.isArray(body) || Object.keys(body).length !== 1 || !validKey(body.key)) return response(400, '{"status":"invalid-request"}'); + const object = await env.INDEX.get(body.key); + if (object === null) return response(404, '{"status":"not-found"}'); + if (object.size > MAX_OBJECT_BYTES) return response(413, '{"status":"object-too-large"}'); + return response(200, await object.arrayBuffer(), "application/json"); + }, +}; diff --git a/experiments/worker-r2-generation-projection/src/worker.mjs b/experiments/worker-r2-generation-projection/src/worker.mjs new file mode 100644 index 0000000..a3f4844 --- /dev/null +++ b/experiments/worker-r2-generation-projection/src/worker.mjs @@ -0,0 +1,819 @@ +const DESCRIPTOR_SCHEMA = "vyral.record-search-projection-generation.v1"; +const REQUEST_SCHEMA = "vyral.record-search-projection-request.v1"; +const RESULT_SCHEMA = "vyral.record-search-projection-result.v1"; +const INSPECTION_SCHEMA = "vyral.record-search-projection-inspection.v1"; +const CATALOG_SCHEMA = "vyral.private.worker-r2-catalog.v1"; +const ACTIVE_SCHEMA = "vyral.private.worker-r2-active.v1"; +const MANIFEST_SCHEMA = "vyral.private.worker-r2-manifest.v1"; +const SHARD_SCHEMA = "vyral.private.worker-r2-shard.v1"; +const CONTINUATION_SCHEMA = "vyral.private.worker-r2-continuation.v1"; +const MAX_BODY_BYTES = 131_072; +const MAX_QUERY_BYTES = 2_048; +const MAX_FILTER_NODES = 64; +const MAX_FILTER_DEPTH = 8; +const MAX_CACHE_ENTRIES = 32; +const MAX_REQUEST_MILLISECONDS = 5_000; +const MAX_MANIFEST_BYTES = 1_048_576; +const MAX_CONTROL_OBJECT_BYTES = 1_048_576; +const MAX_IMMUTABLE_OBJECT_BYTES = 33_554_432; +const MAX_SELECTED_SHARD_BYTES = 33_554_432; +const CONTINUATION_LIFETIME_MILLISECONDS = 15 * 60 * 1_000; +const MAX_PAGE_COUNT = 100; +const textEncoder = new TextEncoder(); +const textDecoder = new TextDecoder("utf-8", { fatal: true }); +const contentCache = new Map(); + +class ProjectionError extends Error { + constructor(code, message, retryable = false) { + super(message); + this.code = code; + this.retryable = retryable; + } +} + +function isObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function requireExactKeys(value, keys, label) { + if (!isObject(value)) throw new ProjectionError("invalidArtifact", `${label} must be an object.`); + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new ProjectionError("invalidArtifact", `${label} has unsupported fields.`); + } +} + +function requireIdentifier(value, label) { + if (typeof value !== "string" || value.length < 1 || value.length > 200 || value.trim() !== value || /[\0\r\n]/u.test(value)) { + throw new ProjectionError("invalidArtifact", `${label} is not a bounded identifier.`); + } + return value; +} + +function requireDigest(value, label) { + if (typeof value !== "string" || !/^sha256:[0-9a-f]{64}$/.test(value)) { + throw new ProjectionError("invalidArtifact", `${label} is not a SHA-256 digest.`); + } + return value; +} + +function canonicalJson(value) { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new ProjectionError("invalidArtifact", "Non-finite numbers are not canonical JSON."); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (isObject(value)) { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`; + } + throw new ProjectionError("invalidArtifact", "Unsupported canonical JSON value."); +} + +async function sha256Bytes(bytes) { + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); + return `sha256:${[...digest].map((value) => value.toString(16).padStart(2, "0")).join("")}`; +} + +async function sha256Json(value) { + return sha256Bytes(textEncoder.encode(canonicalJson(value))); +} + +function canonicalSet(value, label, allowEmpty = false) { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.some((item) => typeof item !== "string" || !item)) { + throw new ProjectionError("invalidArtifact", `${label} must be a bounded string set.`); + } + const sorted = [...new Set(value)].sort(); + if (sorted.length !== value.length || sorted.some((item, index) => item !== value[index])) { + throw new ProjectionError("invalidArtifact", `${label} must be canonical and unique.`); + } + return sorted; +} + +function descriptorMaterial(descriptor) { + return { + schema: descriptor.schema, + collection: descriptor.collection, + generationId: descriptor.generationId, + providerId: descriptor.providerId, + profileId: descriptor.profileId, + strategyVersion: descriptor.strategyVersion, + sourceManifestDigest: descriptor.sourceManifestDigest, + recordRevisionSetDigest: descriptor.recordRevisionSetDigest, + projectionSchemaDigest: descriptor.projectionSchemaDigest, + analyzerDigest: descriptor.analyzerDigest, + configurationDigest: descriptor.configurationDigest, + expectedItemCount: descriptor.expectedItemCount, + expectedPartitions: descriptor.expectedPartitions, + capabilities: descriptor.capabilities, + artifacts: descriptor.artifacts, + createdAtUtc: descriptor.createdAtUtc, + }; +} + +async function validateDescriptor(descriptor) { + requireExactKeys(descriptor, [ + "schema", "collection", "generationId", "providerId", "profileId", "strategyVersion", + "sourceManifestDigest", "recordRevisionSetDigest", "projectionSchemaDigest", "analyzerDigest", + "configurationDigest", "expectedItemCount", "expectedPartitions", "capabilities", "artifacts", + "createdAtUtc", "descriptorDigest", + ], "generation descriptor"); + if (descriptor.schema !== DESCRIPTOR_SCHEMA) throw new ProjectionError("invalidArtifact", "Unsupported descriptor schema."); + for (const field of ["collection", "generationId", "providerId", "profileId", "strategyVersion"]) requireIdentifier(descriptor[field], field); + for (const field of ["sourceManifestDigest", "recordRevisionSetDigest", "projectionSchemaDigest", "configurationDigest", "descriptorDigest"]) requireDigest(descriptor[field], field); + if (descriptor.analyzerDigest !== null) requireDigest(descriptor.analyzerDigest, "analyzerDigest"); + if (!Number.isSafeInteger(descriptor.expectedItemCount) || descriptor.expectedItemCount < 0) throw new ProjectionError("invalidArtifact", "expectedItemCount is invalid."); + canonicalSet(descriptor.expectedPartitions, "expectedPartitions"); + canonicalSet(descriptor.capabilities, "capabilities"); + if (!descriptor.capabilities.includes("completeCoverage") || !descriptor.capabilities.includes("generationPinnedContinuation") || !descriptor.capabilities.includes("lexical")) { + throw new ProjectionError("invalidArtifact", "Worker/R2 descriptor lacks required capabilities."); + } + if (!Array.isArray(descriptor.artifacts)) throw new ProjectionError("invalidArtifact", "artifacts must be an array."); + if (descriptor.artifacts.length < 1 || descriptor.artifacts.length > 65) throw new ProjectionError("invalidArtifact", "Descriptor artifact count exceeds the adapter bound."); + const artifactIds = []; + for (const artifact of descriptor.artifacts) { + requireExactKeys(artifact, ["id", "kind", "contentHash", "sizeBytes", "mediaType"], "descriptor artifact"); + artifactIds.push(requireIdentifier(artifact.id, "artifact id")); + requireIdentifier(artifact.kind, "artifact kind"); + requireDigest(artifact.contentHash, "artifact contentHash"); + if (!Number.isSafeInteger(artifact.sizeBytes) || artifact.sizeBytes < 0) throw new ProjectionError("invalidArtifact", "artifact sizeBytes is invalid."); + if (artifact.mediaType !== null && (typeof artifact.mediaType !== "string" || artifact.mediaType.length > 200)) throw new ProjectionError("invalidArtifact", "artifact mediaType is invalid."); + } + canonicalSet(artifactIds, "artifact IDs", true); + if (typeof descriptor.createdAtUtc !== "string" || !Number.isFinite(Date.parse(descriptor.createdAtUtc))) throw new ProjectionError("invalidArtifact", "createdAtUtc is invalid."); + if (await sha256Json(descriptorMaterial(descriptor)) !== descriptor.descriptorDigest) throw new ProjectionError("invalidArtifact", "Descriptor digest mismatch."); + return descriptor; +} + +function keyComponent(value) { + return encodeURIComponent(requireIdentifier(value, "key component")); +} + +function activeKey(collection) { + return `active/${keyComponent(collection)}.json`; +} + +function catalogKey(collection, generationId) { + return `catalog/${keyComponent(collection)}/${keyComponent(generationId)}.json`; +} + +function cachePut(key, value) { + if (contentCache.has(key)) contentCache.delete(key); + contentCache.set(key, value); + while (contentCache.size > MAX_CACHE_ENTRIES) contentCache.delete(contentCache.keys().next().value); +} + +function immutableReaderMode(env) { + const direct = env.INDEX !== undefined && typeof env.INDEX?.get === "function"; + const service = env.OBJECT_READER !== undefined && typeof env.OBJECT_READER?.fetch === "function"; + if (direct === service) throw new ProjectionError("providerUnavailable", "Exactly one immutable-object reader binding is required.", true); + return direct ? "direct-r2" : "service-reader"; +} + +function objectByteLimit(key) { + return key.startsWith("objects/sha256/") ? MAX_IMMUTABLE_OBJECT_BYTES : MAX_CONTROL_OBJECT_BYTES; +} + +async function readImmutableBytes(env, key) { + const limit = objectByteLimit(key); + if (immutableReaderMode(env) === "direct-r2") { + const object = await env.INDEX.get(key); + if (object === null) throw new ProjectionError("generationUnavailable", "A required immutable generation object is unavailable.", true); + if (object.size > limit) throw new ProjectionError("workLimitExceeded", "A required immutable generation object exceeds the adapter byte bound."); + const bytes = new Uint8Array(await object.arrayBuffer()); + if (bytes.byteLength > limit) throw new ProjectionError("workLimitExceeded", "A required immutable generation object exceeds the adapter byte bound."); + return bytes; + } + + if (typeof env.OBJECT_READER_SECRET !== "string" || textEncoder.encode(env.OBJECT_READER_SECRET).byteLength < 32) { + throw new ProjectionError("providerUnavailable", "Immutable-object reader authorization is unavailable.", true); + } + let response; + try { + response = await env.OBJECT_READER.fetch("https://objects.internal/read", { + method: "POST", + headers: { + "authorization": `Bearer ${env.OBJECT_READER_SECRET}`, + "content-type": "application/json", + }, + body: canonicalJson({ key }), + }); + } catch { + throw new ProjectionError("providerUnavailable", "The immutable-object reader request failed.", true); + } + if (response.status === 404) throw new ProjectionError("generationUnavailable", "A required immutable generation object is unavailable.", true); + if (response.status === 413) throw new ProjectionError("workLimitExceeded", "A required immutable generation object exceeds the adapter byte bound."); + if (!response.ok) throw new ProjectionError("providerUnavailable", "The immutable-object reader failed.", response.status === 429 || response.status >= 500); + if (response.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") throw new ProjectionError("invalidArtifact", "The immutable-object reader returned an unsupported media type."); + const declared = Number(response.headers.get("content-length") ?? "0"); + if (Number.isFinite(declared) && declared > limit) throw new ProjectionError("workLimitExceeded", "A required immutable generation object exceeds the adapter byte bound."); + const bytes = new Uint8Array(await response.arrayBuffer()); + if (bytes.byteLength > limit) throw new ProjectionError("workLimitExceeded", "A required immutable generation object exceeds the adapter byte bound."); + return bytes; +} + +async function readObjectJson(env, key, expectedDigest = null) { + const cacheKey = expectedDigest === null ? null : `${key}:${expectedDigest}`; + if (cacheKey !== null && contentCache.has(cacheKey)) return { value: contentCache.get(cacheKey), cacheHit: true }; + const bytes = await readImmutableBytes(env, key); + if (expectedDigest !== null && await sha256Bytes(bytes) !== expectedDigest) throw new ProjectionError("artifactDigestMismatch", "A required immutable generation object failed digest verification."); + let value; + try { + value = JSON.parse(textDecoder.decode(bytes)); + } catch { + throw new ProjectionError("invalidArtifact", "A required immutable generation object is invalid JSON."); + } + if (cacheKey !== null) cachePut(cacheKey, value); + return { value, cacheHit: false, sizeBytes: bytes.byteLength }; +} + +async function resolveCatalog(env, collection, requestedGeneration) { + let generationId = requestedGeneration; + let activeDigest = null; + let fromActive = false; + if (generationId === null) { + fromActive = true; + const activeRead = await readObjectJson(env, activeKey(collection)); + const active = activeRead.value; + requireExactKeys(active, ["schemaVersion", "collection", "generationId", "descriptorDigest"], "active generation pointer"); + if (active.schemaVersion !== ACTIVE_SCHEMA || active.collection !== collection) throw new ProjectionError("invalidArtifact", "Active generation pointer is invalid."); + generationId = requireIdentifier(active.generationId, "active generationId"); + activeDigest = requireDigest(active.descriptorDigest, "active descriptorDigest"); + } + const catalogRead = await readObjectJson(env, catalogKey(collection, generationId)); + const catalog = catalogRead.value; + requireExactKeys(catalog, ["schemaVersion", "collection", "generationId", "state", "descriptor", "manifestKey", "availablePartitions"], "generation catalog record"); + if (catalog.schemaVersion !== CATALOG_SCHEMA || catalog.collection !== collection || catalog.generationId !== generationId) throw new ProjectionError("invalidArtifact", "Generation catalog identity mismatch."); + if (!["active", "retained", "retired"].includes(catalog.state)) throw new ProjectionError("invalidArtifact", "Generation catalog state is invalid."); + const descriptor = await validateDescriptor(catalog.descriptor); + if (descriptor.collection !== collection || descriptor.generationId !== generationId) throw new ProjectionError("invalidArtifact", "Descriptor and catalog identity differ."); + if (activeDigest !== null && activeDigest !== descriptor.descriptorDigest) throw new ProjectionError("invalidArtifact", "Active pointer descriptor fence mismatch."); + if (fromActive && catalog.state !== "active") throw new ProjectionError("invalidArtifact", "Active pointer names a generation that is not active."); + if (typeof catalog.manifestKey !== "string" || !catalog.manifestKey.startsWith("objects/sha256/")) throw new ProjectionError("invalidArtifact", "Manifest key is not content-addressed."); + canonicalSet(catalog.availablePartitions, "availablePartitions", true); + if (catalog.availablePartitions.some((partition) => !descriptor.expectedPartitions.includes(partition))) throw new ProjectionError("invalidArtifact", "Catalog availability exceeds descriptor coverage."); + return { catalog, descriptor, generationId }; +} + +function validateManifest(manifest, descriptor, manifestArtifact) { + requireExactKeys(manifest, [ + "schemaVersion", "generationId", "sourceManifestDigest", "recordRevisionSetDigest", + "projectionSchemaDigest", "analyzerDigest", "scoringContract", "tieBreak", "k1", "b", + "tokenPattern", "stopWords", "queryAliases", "expectedItemCount", "expectedPartitions", + "averageDocumentLength", "candidateCapacity", "maxWorkUnits", "shards", + ], "Worker/R2 manifest"); + if (manifest.schemaVersion !== MANIFEST_SCHEMA || manifest.generationId !== descriptor.generationId) throw new ProjectionError("invalidArtifact", "Manifest generation identity mismatch."); + for (const field of ["sourceManifestDigest", "recordRevisionSetDigest", "projectionSchemaDigest", "analyzerDigest"]) requireDigest(manifest[field], `manifest ${field}`); + for (const field of ["sourceManifestDigest", "recordRevisionSetDigest", "projectionSchemaDigest", "analyzerDigest"]) { + if (manifest[field] !== descriptor[field]) throw new ProjectionError("invalidArtifact", `Manifest ${field} differs from the descriptor.`); + } + if (manifest.scoringContract !== "global-bm25-like-card-v1" || manifest.tieBreak !== "score-desc-partition-id-asc-v1" || manifest.tokenPattern !== "[a-z0-9]+") throw new ProjectionError("invalidArtifact", "Manifest search contract is unsupported."); + if (manifest.k1 !== 1.2 || manifest.b !== 0.75) throw new ProjectionError("invalidArtifact", "Manifest BM25 constants are unsupported."); + canonicalSet(manifest.stopWords, "manifest stopWords", true); + if (!Array.isArray(manifest.queryAliases) || manifest.queryAliases.some((entry) => !Array.isArray(entry) || entry.length !== 2 || entry.some((item) => typeof item !== "string" || !item))) throw new ProjectionError("invalidArtifact", "Manifest query aliases are invalid."); + if (!Number.isSafeInteger(manifest.expectedItemCount) || manifest.expectedItemCount !== descriptor.expectedItemCount) throw new ProjectionError("invalidArtifact", "Manifest expected item count mismatch."); + canonicalSet(manifest.expectedPartitions, "manifest expectedPartitions"); + if (canonicalJson(manifest.expectedPartitions) !== canonicalJson(descriptor.expectedPartitions)) throw new ProjectionError("invalidArtifact", "Manifest expected partitions mismatch."); + if (!Number.isFinite(manifest.averageDocumentLength) || manifest.averageDocumentLength <= 0) throw new ProjectionError("invalidArtifact", "Manifest average document length is invalid."); + if (!Number.isSafeInteger(manifest.candidateCapacity) || manifest.candidateCapacity < 1 || manifest.candidateCapacity > 10_000) throw new ProjectionError("invalidArtifact", "Manifest candidate capacity is invalid."); + if (!Number.isSafeInteger(manifest.maxWorkUnits) || manifest.maxWorkUnits < 1 || manifest.maxWorkUnits > 10_000_000) throw new ProjectionError("invalidArtifact", "Manifest work bound is invalid."); + if (!Array.isArray(manifest.shards) || manifest.shards.length < 1 || manifest.shards.length > 64) throw new ProjectionError("invalidArtifact", "Manifest shard set is invalid."); + const shardIds = []; + let itemCount = 0; + for (const shard of manifest.shards) { + requireExactKeys(shard, ["id", "key", "contentHash", "sizeBytes", "itemCount", "partitions"], "manifest shard"); + shardIds.push(requireIdentifier(shard.id, "shard id")); + if (typeof shard.key !== "string" || !shard.key.startsWith("objects/sha256/")) throw new ProjectionError("invalidArtifact", "Shard key is not content-addressed."); + requireDigest(shard.contentHash, "shard contentHash"); + if (!Number.isSafeInteger(shard.sizeBytes) || shard.sizeBytes < 1 || !Number.isSafeInteger(shard.itemCount) || shard.itemCount < 0) throw new ProjectionError("invalidArtifact", "Shard size/count is invalid."); + canonicalSet(shard.partitions, "shard partitions"); + if (shard.partitions.some((partition) => !manifest.expectedPartitions.includes(partition))) throw new ProjectionError("invalidArtifact", "Shard covers an unexpected partition."); + itemCount += shard.itemCount; + } + canonicalSet(shardIds, "manifest shard IDs"); + if (itemCount !== descriptor.expectedItemCount) throw new ProjectionError("invalidArtifact", "Manifest shard item counts are incomplete."); + if (manifestArtifact.kind !== "worker-r2-generation-manifest" || manifestArtifact.sizeBytes > MAX_MANIFEST_BYTES) throw new ProjectionError("invalidArtifact", "Manifest artifact declaration is invalid or exceeds the adapter bound."); + return manifest; +} + +function validateShard(shard, declaration, manifest, descriptor) { + requireExactKeys(shard, ["schemaVersion", "generationId", "sourceManifestDigest", "shardId", "partitions", "itemCount", "records", "directMap", "terms"], "Worker/R2 shard"); + if (shard.schemaVersion !== SHARD_SCHEMA || shard.generationId !== descriptor.generationId || shard.sourceManifestDigest !== descriptor.sourceManifestDigest || shard.shardId !== declaration.id) throw new ProjectionError("invalidArtifact", "Shard identity mismatch."); + canonicalSet(shard.partitions, "shard partitions"); + if (canonicalJson(shard.partitions) !== canonicalJson(declaration.partitions)) throw new ProjectionError("invalidArtifact", "Shard partition declaration mismatch."); + if (!Number.isSafeInteger(shard.itemCount) || shard.itemCount !== declaration.itemCount || !Array.isArray(shard.records) || shard.records.length !== shard.itemCount) throw new ProjectionError("invalidArtifact", "Shard item count mismatch."); + const identifiers = new Set(); + for (const record of shard.records) { + requireExactKeys(record, ["partitionKey", "id", "revision", "length", "metadata"], "shard record"); + requireIdentifier(record.partitionKey, "record partitionKey"); + requireIdentifier(record.id, "record id"); + if (!shard.partitions.includes(record.partitionKey) || identifiers.has(record.id)) throw new ProjectionError("invalidArtifact", "Shard record identity or partition is invalid."); + identifiers.add(record.id); + if (!Number.isSafeInteger(record.revision) || record.revision < 1 || !Number.isFinite(record.length) || record.length <= 0 || !isObject(record.metadata)) throw new ProjectionError("invalidArtifact", "Shard record evidence is invalid."); + } + if (!isObject(shard.directMap) || !isObject(shard.terms)) throw new ProjectionError("invalidArtifact", "Shard directories are invalid."); + for (const [key, ordinals] of Object.entries(shard.directMap)) { + if (!key || !Array.isArray(ordinals) || ordinals.some((ordinal) => !Number.isSafeInteger(ordinal) || ordinal < 0 || ordinal >= shard.records.length)) throw new ProjectionError("invalidArtifact", "Shard direct map is invalid."); + } + for (const [term, value] of Object.entries(shard.terms)) { + if (!term || !Array.isArray(value) || value.length !== 2 || !Number.isFinite(value[0]) || value[0] < 0 || !Array.isArray(value[1])) throw new ProjectionError("invalidArtifact", "Shard term directory is invalid."); + let previous = -1; + for (const posting of value[1]) { + if (!Array.isArray(posting) || posting.length !== 2 || !Number.isSafeInteger(posting[0]) || posting[0] <= previous || posting[0] >= shard.records.length || !Number.isFinite(posting[1]) || posting[1] <= 0) throw new ProjectionError("invalidArtifact", "Shard posting is invalid."); + previous = posting[0]; + } + } + return shard; +} + +function requestedPartitions(query, descriptor) { + const value = query.partitionKeys === undefined || query.partitionKeys === null + ? descriptor.expectedPartitions + : query.partitionKeys; + if (!Array.isArray(value) || value.length < 1 || value.some((item) => typeof item !== "string" || !item)) throw new ProjectionError("invalidRequest", "partitionKeys must be a non-empty string set."); + const canonical = [...new Set(value)].sort(); + if (canonical.length !== value.length) throw new ProjectionError("invalidRequest", "partitionKeys must be unique."); + if (canonical.some((partition) => !descriptor.expectedPartitions.includes(partition))) throw new ProjectionError("invalidRequest", "The request names a partition outside the generation."); + return canonical; +} + +function valueAtPath(record, path) { + if (typeof path !== "string" || !path || path.length > 200 || path.split(".").some((part) => !part)) throw new ProjectionError("invalidRequest", "Filter path is invalid."); + let value = { ...record.metadata, partitionKey: record.partitionKey, id: record.id, revision: record.revision }; + for (const part of path.split(".")) { + if (!isObject(value) || !Object.hasOwn(value, part)) return undefined; + value = value[part]; + } + return value; +} + +function compareFilter(actual, op, expected) { + if (op === "exists") return expected === false ? actual === undefined : actual !== undefined; + if (op === "eq") return actual === expected; + if (op === "neq") return actual !== expected; + if (op === "in") return Array.isArray(expected) && expected.some((item) => actual === item || (Array.isArray(actual) && actual.includes(item))); + if (op === "contains") return (Array.isArray(actual) && actual.includes(expected)) || (typeof actual === "string" && typeof expected === "string" && actual.includes(expected)); + if (!["gt", "gte", "lt", "lte"].includes(op) || (typeof actual !== "number" && typeof actual !== "string") || typeof expected !== typeof actual) throw new ProjectionError("invalidRequest", "Filter comparison is invalid."); + if (op === "gt") return actual > expected; + if (op === "gte") return actual >= expected; + if (op === "lt") return actual < expected; + return actual <= expected; +} + +function evaluateFilter(record, filter, state = { nodes: 0 }, depth = 0) { + if (filter === null || filter === undefined) return true; + if (!isObject(filter) || depth > MAX_FILTER_DEPTH || ++state.nodes > MAX_FILTER_NODES) throw new ProjectionError("invalidRequest", "Filter tree exceeds its structural bound."); + const hasCompound = Object.hasOwn(filter, "combine") || Object.hasOwn(filter, "children"); + if (hasCompound) { + if (!Object.keys(filter).every((key) => ["combine", "children"].includes(key)) || !["all", "any"].includes(filter.combine) || !Array.isArray(filter.children) || filter.children.length < 1) throw new ProjectionError("invalidRequest", "Compound filter is invalid."); + const values = filter.children.map((child) => evaluateFilter(record, child, state, depth + 1)); + return filter.combine === "all" ? values.every(Boolean) : values.some(Boolean); + } + if (!Object.keys(filter).every((key) => ["path", "op", "value"].includes(key)) || typeof filter.op !== "string") throw new ProjectionError("invalidRequest", "Leaf filter is invalid."); + return compareFilter(valueAtPath(record, filter.path), filter.op, filter.value); +} + +function queryWithoutContinuation(query) { + const value = structuredClone(query); + value.continuationToken = null; + return value; +} + +function validateSearchRequest(request, descriptor, manifest) { + if (!isObject(request) || request.schema !== REQUEST_SCHEMA || !isObject(request.query)) throw new ProjectionError("invalidRequest", "Search request is invalid."); + if (!["schema", "generationId", "expectedDescriptorDigest", "query", "deadlineUtc"].every((key) => Object.hasOwn(request, key))) throw new ProjectionError("invalidRequest", "Search request is missing required fields."); + if (Object.keys(request).some((key) => !["schema", "generationId", "expectedDescriptorDigest", "query", "deadlineUtc"].includes(key))) throw new ProjectionError("invalidRequest", "Search request has unsupported fields."); + if (request.generationId !== null) requireIdentifier(request.generationId, "request generationId"); + if (request.expectedDescriptorDigest !== null) requireDigest(request.expectedDescriptorDigest, "request expectedDescriptorDigest"); + const query = request.query; + if (Object.keys(query).some((key) => !["partitionKeys", "filter", "vector", "lexical", "orderBy", "limit", "continuationToken"].includes(key))) throw new ProjectionError("invalidRequest", "Query envelope has unsupported fields."); + if (query.vector !== undefined && query.vector !== null) throw new ProjectionError("capabilityUnsupported", "Worker/R2 proof supports lexical search only."); + if (query.orderBy !== undefined && query.orderBy !== null) throw new ProjectionError("capabilityUnsupported", "Worker/R2 proof uses its bound score order."); + if (!isObject(query.lexical) || typeof query.lexical.query !== "string" || query.lexical.query.trim() === "" || textEncoder.encode(query.lexical.query).byteLength > MAX_QUERY_BYTES) throw new ProjectionError("invalidRequest", "A bounded lexical query is required."); + const lexical = query.lexical; + const lexicalFields = ["query", "fields", "top", "scanLimit", "minScore", "scoring", "matchMode", "fieldBoosts", "bm25K1", "bm25B", "phraseBoost", "exactBoost", "metadataBoost", "prefixMatching", "prefixMinChars", "requiredPhraseGroups"]; + if (Object.keys(lexical).some((key) => !lexicalFields.includes(key))) throw new ProjectionError("invalidRequest", "Lexical options have unsupported fields."); + if (lexical.fields !== undefined && lexical.fields !== null) throw new ProjectionError("capabilityUnsupported", "Custom lexical fields are not supported by this profile."); + if (lexical.scoring !== undefined && lexical.scoring !== "bm25") throw new ProjectionError("capabilityUnsupported", "The scoring contract is BM25-like."); + if (lexical.matchMode !== undefined && lexical.matchMode !== "any") throw new ProjectionError("capabilityUnsupported", "Only any-term matching is supported."); + if ( + lexical.prefixMatching === true || + (lexical.fieldBoosts !== undefined && lexical.fieldBoosts !== null) || + (lexical.requiredPhraseGroups !== undefined && lexical.requiredPhraseGroups !== null) || + (lexical.bm25K1 !== undefined && lexical.bm25K1 !== manifest.k1) || + (lexical.bm25B !== undefined && lexical.bm25B !== manifest.b) || + (lexical.phraseBoost !== undefined && lexical.phraseBoost !== 0.15) || + (lexical.exactBoost !== undefined && lexical.exactBoost !== 0.25) || + (lexical.metadataBoost !== undefined && lexical.metadataBoost !== 0.1) || + (lexical.prefixMinChars !== undefined && lexical.prefixMinChars !== 3) + ) throw new ProjectionError("capabilityUnsupported", "Lexical options differ from the immutable profile."); + const limit = query.limit ?? lexical.top ?? 10; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > manifest.candidateCapacity) throw new ProjectionError("invalidRequest", "Candidate limit exceeds the generation bound."); + const scanLimit = lexical.scanLimit ?? manifest.maxWorkUnits; + if (!Number.isSafeInteger(scanLimit) || scanLimit < 1 || scanLimit > manifest.maxWorkUnits) throw new ProjectionError("invalidRequest", "scanLimit exceeds the generation work bound."); + if (lexical.minScore !== undefined && lexical.minScore !== null && !Number.isFinite(lexical.minScore)) throw new ProjectionError("invalidRequest", "minScore must be finite."); + if (query.continuationToken !== undefined && query.continuationToken !== null && (typeof query.continuationToken !== "string" || query.continuationToken.length > 8192)) throw new ProjectionError("invalidRequest", "Continuation token is invalid."); + const partitions = requestedPartitions(query, descriptor); + return { query, lexical, limit, scanLimit, partitions }; +} + +function base64Url(bytes) { + let binary = ""; + for (const value of bytes) binary += String.fromCharCode(value); + return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replace(/=+$/u, ""); +} + +function fromBase64Url(value) { + if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/u.test(value)) throw new ProjectionError("invalidContinuation", "Continuation token is invalid."); + const padded = value.replaceAll("-", "+").replaceAll("_", "/") + "=".repeat((4 - value.length % 4) % 4); + try { + return Uint8Array.from(atob(padded), (character) => character.charCodeAt(0)); + } catch { + throw new ProjectionError("invalidContinuation", "Continuation token is invalid."); + } +} + +async function hmac(secret, bytes) { + if (typeof secret !== "string" || textEncoder.encode(secret).byteLength < 32) throw new ProjectionError("providerUnavailable", "Continuation signing is unavailable.", true); + const key = await crypto.subtle.importKey("raw", textEncoder.encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]); + return new Uint8Array(await crypto.subtle.sign("HMAC", key, bytes)); +} + +function constantTimeEqual(left, right) { + let difference = left.length ^ right.length; + const length = Math.max(left.length, right.length); + for (let index = 0; index < length; index += 1) difference |= (left[index] ?? 0) ^ (right[index] ?? 0); + return difference === 0; +} + +function scoreHex(score) { + const bytes = new Uint8Array(8); + new DataView(bytes.buffer).setFloat64(0, score, false); + return [...bytes].map((value) => value.toString(16).padStart(2, "0")).join(""); +} + +function scoreFromHex(value) { + if (typeof value !== "string" || !/^[0-9a-f]{16}$/u.test(value)) throw new ProjectionError("invalidContinuation", "Continuation score boundary is invalid."); + const bytes = Uint8Array.from(value.match(/../gu), (pair) => Number.parseInt(pair, 16)); + const score = new DataView(bytes.buffer).getFloat64(0, false); + if (!Number.isFinite(score)) throw new ProjectionError("invalidContinuation", "Continuation score boundary is invalid."); + return score; +} + +async function issueContinuation(env, payload) { + const bytes = textEncoder.encode(canonicalJson(payload)); + const signature = await hmac(env.CONTINUATION_SECRET, bytes); + return `${base64Url(bytes)}.${base64Url(signature)}`; +} + +async function parseContinuation(env, token) { + const parts = typeof token === "string" ? token.split(".") : []; + if (parts.length !== 2) throw new ProjectionError("invalidContinuation", "Continuation token is invalid."); + const bytes = fromBase64Url(parts[0]); + const supplied = fromBase64Url(parts[1]); + const expected = await hmac(env.CONTINUATION_SECRET, bytes); + if (!constantTimeEqual(supplied, expected)) throw new ProjectionError("invalidContinuation", "Continuation authentication failed."); + let payload; + try { + payload = JSON.parse(textDecoder.decode(bytes)); + } catch { + throw new ProjectionError("invalidContinuation", "Continuation payload is invalid."); + } + try { + requireExactKeys(payload, ["schemaVersion", "collection", "generationId", "descriptorDigest", "requestFingerprint", "scoreHex", "partitionKey", "identifier", "page", "expiresAtUnixMs"], "continuation payload"); + if (payload.schemaVersion !== CONTINUATION_SCHEMA || !Number.isSafeInteger(payload.page) || payload.page < 1 || payload.page >= MAX_PAGE_COUNT || !Number.isSafeInteger(payload.expiresAtUnixMs) || Date.now() >= payload.expiresAtUnixMs) throw new ProjectionError("invalidContinuation", "Continuation payload is expired or invalid."); + requireIdentifier(payload.collection, "continuation collection"); + requireIdentifier(payload.generationId, "continuation generationId"); + requireIdentifier(payload.partitionKey, "continuation partitionKey"); + requireIdentifier(payload.identifier, "continuation identifier"); + requireDigest(payload.descriptorDigest, "continuation descriptorDigest"); + requireDigest(payload.requestFingerprint, "continuation requestFingerprint"); + payload.score = scoreFromHex(payload.scoreHex); + } catch { + throw new ProjectionError("invalidContinuation", "Continuation payload is expired or invalid."); + } + return payload; +} + +function failureResult({ generationId = null, descriptorDigest = null, requested = [], covered = [], missing = requested, coverageStatus = "unavailable", code, message, retryable = false, details = null }) { + return { + schema: RESULT_SCHEMA, + status: "failed", + generationId, + generationDescriptorDigest: descriptorDigest, + items: [], + continuationToken: null, + consistency: "immutableGeneration", + coverage: { status: coverageStatus, requestedPartitions: requested, coveredPartitions: covered, missingPartitions: missing }, + diagnostics: { workLimit: null, workUnits: null, candidateBound: null, candidateCount: null, returnedCount: 0, cacheStatus: "notApplicable", details }, + failure: { code, message, retryable }, + }; +} + +function jsonResponse(status, body) { + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" } }); +} + +async function readBody(request) { + const contentLength = Number(request.headers.get("content-length") ?? "0"); + if (Number.isFinite(contentLength) && contentLength > MAX_BODY_BYTES) throw new ProjectionError("invalidRequest", "Request body exceeds its bound."); + const text = await request.text(); + if (textEncoder.encode(text).byteLength > MAX_BODY_BYTES) throw new ProjectionError("invalidRequest", "Request body exceeds its bound."); + try { + return JSON.parse(text); + } catch { + throw new ProjectionError("invalidRequest", "Request body is invalid JSON."); + } +} + +function availableCoverage(catalog, requested) { + const covered = requested.filter((partition) => catalog.availablePartitions.includes(partition)); + const missing = requested.filter((partition) => !catalog.availablePartitions.includes(partition)); + return { covered, missing, complete: missing.length === 0 }; +} + +async function loadGeneration(env, catalog, descriptor) { + const manifestArtifact = descriptor.artifacts.find((artifact) => artifact.id === "worker-r2-manifest"); + if (manifestArtifact === undefined) throw new ProjectionError("invalidArtifact", "Descriptor lacks the Worker/R2 manifest artifact."); + const manifestRead = await readObjectJson(env, catalog.manifestKey, manifestArtifact.contentHash); + if (manifestRead.sizeBytes !== undefined && manifestRead.sizeBytes !== manifestArtifact.sizeBytes) throw new ProjectionError("artifactDigestMismatch", "Manifest size differs from its descriptor evidence."); + const manifest = validateManifest(manifestRead.value, descriptor, manifestArtifact); + return { manifest, manifestCacheHit: manifestRead.cacheHit }; +} + +async function loadRequiredShards(env, manifest, descriptor, partitions, deadlineMs) { + const declarations = manifest.shards.filter((shard) => shard.partitions.some((partition) => partitions.includes(partition))); + if (declarations.reduce((total, declaration) => total + declaration.sizeBytes, 0) > MAX_SELECTED_SHARD_BYTES) throw new ProjectionError("workLimitExceeded", "Selected immutable shards exceed the adapter memory bound.", true); + const represented = new Set(declarations.flatMap((shard) => shard.partitions)); + if (partitions.some((partition) => !represented.has(partition))) throw new ProjectionError("coverageIncomplete", "The immutable manifest does not cover every requested partition.", true); + const reads = await Promise.allSettled(declarations.map(async (declaration) => { + if (Date.now() >= deadlineMs) throw new ProjectionError("deadlineExceeded", "The shared search deadline elapsed.", true); + const read = await readObjectJson(env, declaration.key, declaration.contentHash); + if (read.sizeBytes !== undefined && read.sizeBytes !== declaration.sizeBytes) throw new ProjectionError("artifactDigestMismatch", "Shard size differs from its manifest evidence."); + if (Date.now() >= deadlineMs) throw new ProjectionError("deadlineExceeded", "The shared search deadline elapsed.", true); + return { shard: validateShard(read.value, declaration, manifest, descriptor), cacheHit: read.cacheHit }; + })); + const failed = reads.find((result) => result.status === "rejected"); + if (failed !== undefined) throw failed.reason instanceof ProjectionError ? failed.reason : new ProjectionError("providerUnavailable", "A required shard could not be read.", true); + return reads.map((result) => result.value); +} + +function analyze(query, manifest) { + let expanded = query.toLowerCase(); + for (const [alias, canonical] of manifest.queryAliases) expanded = expanded.replaceAll(alias, canonical); + const stopWords = new Set(manifest.stopWords); + return [...new Set(expanded.match(/[a-z0-9]+/gu) ?? [])].filter((term) => !stopWords.has(term)).sort(); +} + +function afterBoundary(candidate, boundary) { + return boundary === null || + candidate.score < boundary.score || + (candidate.score === boundary.score && candidate.partitionKey > boundary.partitionKey) || + (candidate.score === boundary.score && candidate.partitionKey === boundary.partitionKey && candidate.id > boundary.identifier); +} + +function searchShards(shards, manifest, lexical, filter, limit, scanLimit, boundary, deadlineMs, requested) { + let workUnits = 0; + const directKey = lexical.query.toLowerCase().trim(); + const direct = []; + const seenIdentities = new Set(); + for (const { shard } of shards) { + for (const record of shard.records) { + const identity = `${record.partitionKey}\n${record.id}`; + if (seenIdentities.has(identity)) throw new ProjectionError("invalidArtifact", "A record appears in multiple immutable shards."); + seenIdentities.add(identity); + } + for (const ordinal of shard.directMap[directKey] ?? []) { + const record = shard.records[ordinal]; + if (requested.includes(record.partitionKey) && evaluateFilter(record, filter)) { + direct.push({ partitionKey: record.partitionKey, id: record.id, revision: record.revision, score: record.id === directKey ? 1000 : 900 }); + } + } + } + let candidates; + if (direct.length > 0) { + candidates = direct; + } else { + const terms = analyze(lexical.query, manifest); + candidates = []; + const termIdfs = new Map(); + for (const { shard } of shards) { + const scores = new Map(); + for (const term of terms) { + const termValue = shard.terms[term]; + if (termValue === undefined) continue; + const [idf, postings] = termValue; + if (termIdfs.has(term) && termIdfs.get(term) !== idf) throw new ProjectionError("invalidArtifact", "Shards disagree on global term statistics."); + termIdfs.set(term, idf); + for (const [ordinal, frequency] of postings) { + workUnits += 1; + if (workUnits > scanLimit) throw new ProjectionError("workLimitExceeded", "Lexical work exceeded the admitted scan limit.", true); + if ((workUnits & 1023) === 0 && Date.now() >= deadlineMs) throw new ProjectionError("deadlineExceeded", "The shared search deadline elapsed.", true); + const record = shard.records[ordinal]; + if (!requested.includes(record.partitionKey) || !evaluateFilter(record, filter)) continue; + const denominator = frequency + manifest.k1 * (1 - manifest.b + manifest.b * record.length / manifest.averageDocumentLength); + const contribution = idf * frequency * (manifest.k1 + 1) / denominator; + scores.set(ordinal, (scores.get(ordinal) ?? 0) + contribution); + } + } + for (const [ordinal, score] of scores) { + const record = shard.records[ordinal]; + candidates.push({ partitionKey: record.partitionKey, id: record.id, revision: record.revision, score }); + } + } + } + const minimum = lexical.minScore ?? Number.NEGATIVE_INFINITY; + candidates = candidates.filter((candidate) => candidate.score >= minimum && afterBoundary(candidate, boundary)); + candidates.sort((left, right) => right.score - left.score || + (left.partitionKey < right.partitionKey ? -1 : left.partitionKey > right.partitionKey ? 1 : 0) || + (left.id < right.id ? -1 : left.id > right.id ? 1 : 0)); + const returned = candidates.slice(0, limit); + return { returned, hasMore: candidates.length > limit, candidateCount: candidates.length, workUnits }; +} + +async function inspect(env, body) { + if (!isObject(body) || Object.keys(body).some((key) => !["collection", "generationId", "expectedDescriptorDigest"].includes(key))) throw new ProjectionError("invalidRequest", "Inspection request is invalid."); + let collection; + try { + collection = requireIdentifier(body.collection, "collection"); + } catch { + throw new ProjectionError("invalidRequest", "Inspection collection is invalid."); + } + const generationId = body.generationId === undefined || body.generationId === null ? null : requireIdentifier(body.generationId, "generationId"); + const resolved = await resolveCatalog(env, collection, generationId); + if (body.expectedDescriptorDigest !== undefined && body.expectedDescriptorDigest !== null) { + try { + requireDigest(body.expectedDescriptorDigest, "expectedDescriptorDigest"); + } catch { + throw new ProjectionError("invalidRequest", "Inspection expectedDescriptorDigest is invalid."); + } + if (body.expectedDescriptorDigest !== resolved.descriptor.descriptorDigest) throw new ProjectionError("generationDescriptorMismatch", "Generation descriptor fence mismatch."); + } + let available = resolved.catalog.state === "retired" ? [] : resolved.catalog.availablePartitions; + let coverageStatus = resolved.catalog.state === "retired" ? "unavailable" : available.length === resolved.descriptor.expectedPartitions.length ? "complete" : "incomplete"; + if (resolved.catalog.state !== "retired") { + try { + const loaded = await loadGeneration(env, resolved.catalog, resolved.descriptor); + await loadRequiredShards(env, loaded.manifest, resolved.descriptor, resolved.descriptor.expectedPartitions, Date.now() + MAX_REQUEST_MILLISECONDS); + } catch { + available = []; + coverageStatus = "unavailable"; + } + } + return { + schema: INSPECTION_SCHEMA, + descriptor: resolved.descriptor, + state: resolved.catalog.state, + availablePartitions: available, + coverageStatus, + observedAtUtc: new Date().toISOString(), + }; +} + +async function search(env, body) { + if (!isObject(body) || Object.keys(body).some((key) => !["collection", "request"].includes(key)) || !isObject(body.request)) throw new ProjectionError("invalidRequest", "Search envelope is invalid."); + let collection; + try { + collection = requireIdentifier(body.collection, "collection"); + } catch { + throw new ProjectionError("invalidRequest", "Search collection is invalid."); + } + const request = body.request; + if (request.generationId !== null && request.generationId !== undefined) { + try { + requireIdentifier(request.generationId, "request generationId"); + } catch { + return failureResult({ code: "invalidRequest", message: "Search generationId is invalid." }); + } + } + if (request.expectedDescriptorDigest !== null && request.expectedDescriptorDigest !== undefined) { + try { + requireDigest(request.expectedDescriptorDigest, "request expectedDescriptorDigest"); + } catch { + return failureResult({ code: "invalidRequest", message: "Search expectedDescriptorDigest is invalid." }); + } + } + let continuation = null; + try { + if (request.query?.continuationToken !== undefined && request.query.continuationToken !== null) continuation = await parseContinuation(env, request.query.continuationToken); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("invalidContinuation", "Continuation token is invalid."); + return failureResult({ code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + if (continuation !== null && continuation.collection !== collection) return failureResult({ code: "invalidContinuation", message: "Continuation collection mismatch." }); + const requestedGeneration = continuation?.generationId ?? (request.generationId ?? null); + let resolved; + try { + resolved = await resolveCatalog(env, collection, requestedGeneration); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("providerUnavailable", "Generation resolution failed.", true); + return failureResult({ generationId: requestedGeneration, code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + const { catalog, descriptor, generationId } = resolved; + let preliminaryPartitions; + try { + preliminaryPartitions = requestedPartitions(request.query, descriptor); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("invalidRequest", "Search request is invalid."); + return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: descriptor.expectedPartitions, code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + if (catalog.state === "retired") return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: preliminaryPartitions, code: "generationRetired", message: "The selected generation is retired." }); + const expectedDescriptor = continuation?.descriptorDigest ?? request.expectedDescriptorDigest; + if (expectedDescriptor !== undefined && expectedDescriptor !== null && expectedDescriptor !== descriptor.descriptorDigest) return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: preliminaryPartitions, covered: preliminaryPartitions, missing: [], coverageStatus: "complete", code: "generationDescriptorMismatch", message: "Generation descriptor fence mismatch." }); + const deadlineValue = request.deadlineUtc === undefined || request.deadlineUtc === null ? Date.now() + MAX_REQUEST_MILLISECONDS : Date.parse(request.deadlineUtc); + if (!Number.isFinite(deadlineValue)) return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: preliminaryPartitions, code: "invalidRequest", message: "deadlineUtc is invalid." }); + const deadlineMs = Math.min(deadlineValue, Date.now() + MAX_REQUEST_MILLISECONDS); + if (Date.now() >= deadlineMs) return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: preliminaryPartitions, code: "deadlineExceeded", message: "The shared search deadline elapsed.", retryable: true }); + let loaded; + try { + loaded = await loadGeneration(env, catalog, descriptor); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("providerUnavailable", "Manifest loading failed.", true); + return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: preliminaryPartitions, code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + let validated; + try { + validated = validateSearchRequest(request, descriptor, loaded.manifest); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("invalidRequest", "Search request is invalid."); + return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: preliminaryPartitions, code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + const { query, lexical, limit, scanLimit, partitions } = validated; + const coverage = availableCoverage(catalog, partitions); + if (!coverage.complete) return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: partitions, covered: coverage.covered, missing: coverage.missing, coverageStatus: "incomplete", code: "coverageIncomplete", message: "The selected generation lacks complete requested coverage.", retryable: true }); + const fingerprint = await sha256Json({ collection, generationId, descriptorDigest: descriptor.descriptorDigest, query: queryWithoutContinuation(query) }); + if (continuation !== null && continuation.requestFingerprint !== fingerprint) return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: partitions, covered: partitions, missing: [], coverageStatus: "complete", code: "invalidContinuation", message: "Continuation request binding mismatch." }); + let shards; + try { + shards = await loadRequiredShards(env, loaded.manifest, descriptor, partitions, deadlineMs); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("providerUnavailable", "Required shard loading failed.", true); + return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: partitions, code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + let page; + try { + page = searchShards(shards, loaded.manifest, lexical, query.filter, limit, scanLimit, continuation, deadlineMs, partitions); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("providerFailure", "Candidate evaluation failed.", true); + const coverageStatus = wrapped.code === "workLimitExceeded" ? "complete" : "unavailable"; + return failureResult({ generationId, descriptorDigest: descriptor.descriptorDigest, requested: partitions, covered: coverageStatus === "complete" ? partitions : [], missing: coverageStatus === "complete" ? [] : partitions, coverageStatus, code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + let next = null; + if (page.hasMore && page.returned.length > 0) { + const last = page.returned.at(-1); + next = await issueContinuation(env, { + schemaVersion: CONTINUATION_SCHEMA, + collection, + generationId, + descriptorDigest: descriptor.descriptorDigest, + requestFingerprint: fingerprint, + scoreHex: scoreHex(last.score), + partitionKey: last.partitionKey, + identifier: last.id, + page: (continuation?.page ?? 0) + 1, + expiresAtUnixMs: Date.now() + CONTINUATION_LIFETIME_MILLISECONDS, + }); + } + const cacheHit = loaded.manifestCacheHit && shards.every((value) => value.cacheHit); + return { + schema: RESULT_SCHEMA, + status: "succeeded", + generationId, + generationDescriptorDigest: descriptor.descriptorDigest, + items: page.returned, + continuationToken: next, + consistency: "immutableGeneration", + coverage: { status: "complete", requestedPartitions: partitions, coveredPartitions: partitions, missingPartitions: [] }, + diagnostics: { + workLimit: scanLimit, + workUnits: page.workUnits, + candidateBound: limit, + candidateCount: page.candidateCount, + returnedCount: page.returned.length, + cacheStatus: cacheHit ? "hit" : "miss", + details: { adapter: "private-worker-r2", immutableReaderMode: immutableReaderMode(env), shardCount: shards.length, scoringContract: loaded.manifest.scoringContract }, + }, + failure: null, + }; +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + if (request.method !== "POST" || !["/inspect", "/search"].includes(url.pathname) || url.search || url.hash) return jsonResponse(404, { status: "not-found" }); + if (typeof env.AUTHORIZATION_SECRET !== "string" || textEncoder.encode(env.AUTHORIZATION_SECRET).byteLength < 32) return jsonResponse(503, { status: "failed", code: "providerUnavailable", message: "Worker authorization is unavailable.", retryable: true }); + const authorization = request.headers.get("authorization") ?? ""; + const expectedAuthorization = `Bearer ${env.AUTHORIZATION_SECRET}`; + if (authorization.length > 1024 || !constantTimeEqual(textEncoder.encode(authorization), textEncoder.encode(expectedAuthorization))) return jsonResponse(401, { status: "failed", code: "unauthorized", message: "Worker authorization failed.", retryable: false }); + if (request.headers.get("content-type")?.split(";", 1)[0].trim().toLowerCase() !== "application/json") return jsonResponse(415, { status: "failed", code: "invalidRequest", message: "Content-Type must be application/json.", retryable: false }); + try { + const body = await readBody(request); + const value = url.pathname === "/inspect" ? await inspect(env, body) : await search(env, body); + return jsonResponse(200, value); + } catch (error) { + const wrapped = error instanceof ProjectionError ? error : new ProjectionError("providerFailure", "The projection request failed.", true); + return jsonResponse(wrapped.code === "invalidRequest" ? 400 : 503, { status: "failed", code: wrapped.code, message: wrapped.message, retryable: wrapped.retryable }); + } + }, +}; diff --git a/experiments/worker-r2-generation-projection/verify.mjs b/experiments/worker-r2-generation-projection/verify.mjs new file mode 100644 index 0000000..b21c24b --- /dev/null +++ b/experiments/worker-r2-generation-projection/verify.mjs @@ -0,0 +1,468 @@ +#!/usr/bin/env node + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { Miniflare } from "miniflare"; + +const root = path.dirname(fileURLToPath(import.meta.url)); +const workerPath = path.join(root, "src", "worker.mjs"); +const objectReaderPath = path.join(root, "src", "object-reader.mjs"); +const secret = "worker-r2-proof-continuation-secret-20260827"; +const authorizationSecret = "worker-r2-proof-authorization-secret-20260827"; +const objectReaderSecret = "worker-r2-proof-object-reader-secret-20260827"; +let bindingMode = "direct-r2"; + +function progress(message) { + process.stderr.write(`[worker-r2-proof] ${message}\n`); +} + +function scoreHex(score) { + const bytes = new Uint8Array(8); + new DataView(bytes.buffer).setFloat64(0, score, false); + return [...bytes].map((value) => value.toString(16).padStart(2, "0")).join(""); +} + +function requireArguments() { + const argumentsByName = new Map(); + for (let index = 2; index < process.argv.length; index += 2) { + const name = process.argv[index]; + const value = process.argv[index + 1]; + if (!name?.startsWith("--") || value === undefined) throw new Error("Usage: verify.mjs --bundle [--mode direct-r2|service-reader]"); + argumentsByName.set(name, value); + } + const bundle = argumentsByName.get("--bundle"); + const mode = argumentsByName.get("--mode") ?? "direct-r2"; + if (bundle === undefined || argumentsByName.size > 2 || [...argumentsByName.keys()].some((name) => !["--bundle", "--mode"].includes(name)) || !["direct-r2", "service-reader"].includes(mode)) { + throw new Error("Usage: verify.mjs --bundle [--mode direct-r2|service-reader]"); + } + return { bundle, mode }; +} + +async function createRuntime(objects, omittedKeys = new Set()) { + progress(`creating ${bindingMode} runtime with ${Object.keys(objects).length - omittedKeys.size} R2 objects`); + const runtime = bindingMode === "service-reader" + ? new Miniflare({ + workers: [ + { + name: "query", + modules: [{ type: "ESModule", path: workerPath }], + compatibilityDate: "2026-08-06", + compatibilityFlags: ["nodejs_compat"], + serviceBindings: { OBJECT_READER: "reader" }, + bindings: { + CONTINUATION_SECRET: secret, + AUTHORIZATION_SECRET: authorizationSecret, + OBJECT_READER_SECRET: objectReaderSecret, + }, + }, + { + name: "reader", + modules: [{ type: "ESModule", path: objectReaderPath }], + compatibilityDate: "2026-08-06", + compatibilityFlags: ["nodejs_compat"], + r2Buckets: { INDEX: "vyral-worker-r2-proof" }, + bindings: { OBJECT_READER_SECRET: objectReaderSecret }, + }, + ], + }) + : new Miniflare({ + modules: [{ type: "ESModule", path: workerPath }], + compatibilityDate: "2026-08-06", + compatibilityFlags: ["nodejs_compat"], + r2Buckets: { INDEX: "vyral-worker-r2-proof" }, + bindings: { CONTINUATION_SECRET: secret, AUTHORIZATION_SECRET: authorizationSecret }, + }); + try { + const bucket = await runtime.getR2Bucket("INDEX", bindingMode === "service-reader" ? "reader" : undefined); + for (const [key, value] of Object.entries(objects)) { + if (!omittedKeys.has(key)) await bucket.put(key, value); + } + progress("runtime and R2 fixture ready"); + return { + runtime, + bucket, + worker: await runtime.getWorker(bindingMode === "service-reader" ? "query" : undefined), + reader: bindingMode === "service-reader" ? await runtime.getWorker("reader") : null, + }; + } catch (error) { + await runtime.dispose(); + throw error; + } +} + +async function post(worker, route, body) { + const response = await worker.fetch(`https://worker.invalid${route}`, { + method: "POST", + headers: { "authorization": `Bearer ${authorizationSecret}`, "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const value = await response.json(); + assert.equal(response.status, 200, `unexpected ${route} status: ${response.status} ${JSON.stringify(value)}`); + return value; +} + +function compareCandidates(actual, expected, label) { + assert.deepEqual(actual.map((item) => [item.partitionKey, item.id, item.revision]), expected.map((item) => [item.partitionKey, item.id, item.revision]), `${label}: candidate identity/revision divergence`); + assert.deepEqual(actual.map((item) => scoreHex(item.score)), expected.map((item) => item.scoreHex), `${label}: candidate score divergence`); +} + +async function withRuntime(objects, action, omittedKeys = new Set()) { + const context = await createRuntime(objects, omittedKeys); + try { + return await action(context); + } finally { + progress("disposing runtime"); + await context.runtime.dispose(); + progress("runtime disposed"); + } +} + +async function proveReaderConfigurationFailsClosed(bundle) { + for (const configuration of ["missing", "ambiguous", "query-secret-missing", "reader-secret-missing"]) { + const query = { + name: "query", + modules: [{ type: "ESModule", path: workerPath }], + compatibilityDate: "2026-08-06", + compatibilityFlags: ["nodejs_compat"], + bindings: { CONTINUATION_SECRET: secret, AUTHORIZATION_SECRET: authorizationSecret }, + }; + const workers = [query]; + if (configuration !== "missing") { + query.serviceBindings = { OBJECT_READER: "reader" }; + if (configuration !== "query-secret-missing") query.bindings.OBJECT_READER_SECRET = objectReaderSecret; + if (configuration === "ambiguous") query.r2Buckets = { INDEX: "vyral-worker-r2-proof" }; + workers.push({ + name: "reader", + modules: [{ type: "ESModule", path: objectReaderPath }], + compatibilityDate: "2026-08-06", + compatibilityFlags: ["nodejs_compat"], + r2Buckets: { INDEX: "vyral-worker-r2-proof" }, + bindings: configuration === "reader-secret-missing" ? {} : { OBJECT_READER_SECRET: objectReaderSecret }, + }); + } + const runtime = new Miniflare({ workers }); + try { + const worker = await runtime.getWorker("query"); + const result = await post(worker, "/search", { + collection: bundle.collection, + request: bundle.queries[0].request, + }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "providerUnavailable"); + assert.deepEqual(result.items, []); + } finally { + await runtime.dispose(); + } + } + return true; +} + +function clone(value) { + return structuredClone(value); +} + +async function main() { + const argumentsValue = requireArguments(); + const bundlePath = argumentsValue.bundle; + bindingMode = argumentsValue.mode; + const bundle = JSON.parse(await readFile(bundlePath, "utf8")); + assert.equal(bundle.schemaVersion, "vyral.worker-r2-proof-bundle.v1"); + assert.ok(Array.isArray(bundle.queries) && bundle.queries.length >= 4); + + const sampleResults = []; + progress("starting primary lifecycle and parity proof"); + const mainProof = await withRuntime(bundle.objects, async ({ worker, bucket, reader }) => { + let serviceReaderGuards = null; + if (reader !== null) { + const missingReaderAuthorization = await reader.fetch("https://objects.internal/read", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ key: bundle.activeKey }), + }); + assert.equal(missingReaderAuthorization.status, 401); + const wrongReaderAuthorization = await reader.fetch("https://objects.internal/read", { + method: "POST", + headers: { "authorization": "Bearer wrong", "content-type": "application/json" }, + body: JSON.stringify({ key: bundle.activeKey }), + }); + assert.equal(wrongReaderAuthorization.status, 401); + const readerMutation = await reader.fetch("https://objects.internal/read", { + method: "PUT", + headers: { "authorization": `Bearer ${objectReaderSecret}`, "content-type": "application/json" }, + body: JSON.stringify({ key: bundle.activeKey }), + }); + assert.equal(readerMutation.status, 404); + const invalidReaderKey = await reader.fetch("https://objects.internal/read", { + method: "POST", + headers: { "authorization": `Bearer ${objectReaderSecret}`, "content-type": "application/json" }, + body: JSON.stringify({ key: "../outside" }), + }); + assert.equal(invalidReaderKey.status, 400); + serviceReaderGuards = true; + } + const unauthorized = await worker.fetch("https://worker.invalid/search", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ collection: bundle.collection, request: bundle.queries[0].request }), + }); + assert.equal(unauthorized.status, 401); + assert.equal((await unauthorized.json()).code, "unauthorized"); + const wrongAuthorization = await worker.fetch("https://worker.invalid/search", { + method: "POST", + headers: { "authorization": "Bearer wrong", "content-type": "application/json" }, + body: JSON.stringify({ collection: bundle.collection, request: bundle.queries[0].request }), + }); + assert.equal(wrongAuthorization.status, 401); + assert.equal((await wrongAuthorization.json()).code, "unauthorized"); + const wrongMediaType = await worker.fetch("https://worker.invalid/search", { + method: "POST", + headers: { "authorization": `Bearer ${authorizationSecret}`, "content-type": "text/plain" }, + body: "{}", + }); + assert.equal(wrongMediaType.status, 415); + assert.equal((await wrongMediaType.json()).code, "invalidRequest"); + const oversizedBody = await worker.fetch("https://worker.invalid/search", { + method: "POST", + headers: { "authorization": `Bearer ${authorizationSecret}`, "content-type": "application/json" }, + body: JSON.stringify({ padding: "x".repeat(131_072) }), + }); + assert.equal(oversizedBody.status, 400); + assert.equal((await oversizedBody.json()).code, "invalidRequest"); + + const inspection = await post(worker, "/inspect", { + collection: bundle.collection, + generationId: null, + expectedDescriptorDigest: bundle.descriptor.descriptorDigest, + }); + assert.equal(inspection.schema, "vyral.record-search-projection-inspection.v1"); + assert.equal(inspection.state, "active"); + assert.equal(inspection.coverageStatus, "complete"); + assert.deepEqual(inspection.availablePartitions, bundle.descriptor.expectedPartitions); + const namedInspection = await post(worker, "/inspect", { + collection: bundle.collection, + generationId: bundle.generationId, + expectedDescriptorDigest: bundle.descriptor.descriptorDigest, + }); + assert.equal(namedInspection.descriptor.descriptorDigest, bundle.descriptor.descriptorDigest); + assert.equal(namedInspection.coverageStatus, "complete"); + + let parity = 0; + for (const query of bundle.queries) { + progress(`running parity query ${query.id}`); + const result = await post(worker, "/search", { collection: bundle.collection, request: query.request }); + assert.equal(result.status, "succeeded", `${query.id}: ${JSON.stringify(result.failure)}`); + assert.equal(result.generationId, bundle.generationId); + assert.equal(result.generationDescriptorDigest, bundle.descriptor.descriptorDigest); + assert.equal(result.coverage.status, "complete"); + assert.equal(result.diagnostics.details.immutableReaderMode, bindingMode); + compareCandidates(result.items, query.expected, query.id); + parity += 1; + if (sampleResults.length < 2) sampleResults.push(result); + } + + const exactRequest = clone(bundle.queries[0].request); + exactRequest.generationId = bundle.generationId; + exactRequest.expectedDescriptorDigest = bundle.descriptor.descriptorDigest; + const exact = await post(worker, "/search", { collection: bundle.collection, request: exactRequest }); + assert.equal(exact.status, "succeeded"); + compareCandidates(exact.items, bundle.queries[0].expected, "exact generation selection"); + + const malformedDigestRequest = clone(bundle.queries[0].request); + malformedDigestRequest.expectedDescriptorDigest = "sha256:not-a-digest"; + const malformedDigest = await post(worker, "/search", { collection: bundle.collection, request: malformedDigestRequest }); + assert.equal(malformedDigest.status, "failed"); + assert.equal(malformedDigest.failure.code, "invalidRequest"); + assert.deepEqual(malformedDigest.items, []); + sampleResults.push(malformedDigest); + + const paged = bundle.pagedQuery; + const firstRequest = clone(paged.request); + firstRequest.query.limit = 5; + firstRequest.query.lexical.top = 5; + const first = await post(worker, "/search", { collection: bundle.collection, request: firstRequest }); + assert.equal(first.status, "succeeded"); + assert.equal(typeof first.continuationToken, "string"); + compareCandidates(first.items, paged.expected.slice(0, 5), "first page"); + + const retainedCatalog = JSON.parse(bundle.objects[bundle.catalogKey]); + retainedCatalog.state = "retained"; + await bucket.put(bundle.catalogKey, JSON.stringify(retainedCatalog)); + const replacement = { + schemaVersion: "vyral.private.worker-r2-active.v1", + collection: bundle.collection, + generationId: "replacement-generation", + descriptorDigest: `sha256:${"9".repeat(64)}`, + }; + await bucket.put(bundle.activeKey, JSON.stringify(replacement)); + + const secondRequest = clone(firstRequest); + secondRequest.query.continuationToken = first.continuationToken; + const second = await post(worker, "/search", { collection: bundle.collection, request: secondRequest }); + assert.equal(second.status, "succeeded"); + assert.equal(second.generationId, bundle.generationId); + compareCandidates(second.items, paged.expected.slice(5, 10), "retained second page"); + + const changedRequest = clone(secondRequest); + changedRequest.query.lexical.query += " changed"; + const changed = await post(worker, "/search", { collection: bundle.collection, request: changedRequest }); + assert.equal(changed.status, "failed"); + assert.equal(changed.failure.code, "invalidContinuation"); + assert.deepEqual(changed.items, []); + + const tamperedRequest = clone(secondRequest); + const token = tamperedRequest.query.continuationToken; + tamperedRequest.query.continuationToken = `${token[0] === "A" ? "B" : "A"}${token.slice(1)}`; + const tampered = await post(worker, "/search", { collection: bundle.collection, request: tamperedRequest }); + assert.equal(tampered.status, "failed"); + assert.equal(tampered.failure.code, "invalidContinuation"); + assert.deepEqual(tampered.items, []); + + retainedCatalog.state = "retired"; + retainedCatalog.availablePartitions = []; + await bucket.put(bundle.catalogKey, JSON.stringify(retainedCatalog)); + const retired = await post(worker, "/search", { collection: bundle.collection, request: secondRequest }); + assert.equal(retired.status, "failed"); + assert.equal(retired.failure.code, "generationRetired"); + assert.deepEqual(retired.items, []); + sampleResults.push(changed, tampered, retired); + + return { + parity, + authenticationRejected: true, + wrongAuthenticationRejected: true, + nonJsonRejected: true, + oversizedBodyRejected: true, + malformedDescriptorDigestRejected: true, + exactGenerationSelection: true, + serviceReaderGuards, + retainedContinuation: true, + tamperRejected: true, + changedRequestRejected: true, + retirementRejected: true, + }; + }); + + progress("starting verified-content cache proof"); + const cacheProof = await withRuntime(bundle.objects, async ({ worker }) => { + const first = await post(worker, "/search", { collection: bundle.collection, request: bundle.queries[0].request }); + const second = await post(worker, "/search", { collection: bundle.collection, request: bundle.queries[0].request }); + assert.equal(first.status, "succeeded"); + assert.equal(second.status, "succeeded"); + assert.equal(second.diagnostics.cacheStatus, "hit"); + return true; + }); + + progress("starting missing-shard proof"); + const missingShard = await withRuntime(bundle.objects, async ({ worker }) => { + const result = await post(worker, "/search", { collection: bundle.collection, request: bundle.queries[0].request }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "generationUnavailable"); + assert.deepEqual(result.items, []); + sampleResults.push(result); + return true; + }, new Set([bundle.shardKeys[0]])); + + progress("starting corrupt-shard proof"); + const corruptShard = await withRuntime(bundle.objects, async ({ worker, bucket }) => { + await bucket.put(bundle.shardKeys[0], "{\"corrupt\":true}"); + const result = await post(worker, "/search", { collection: bundle.collection, request: bundle.queries[0].request }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "artifactDigestMismatch"); + assert.deepEqual(result.items, []); + sampleResults.push(result); + return true; + }); + + const incompleteCoverage = await withRuntime(bundle.objects, async ({ worker, bucket }) => { + const catalog = JSON.parse(bundle.objects[bundle.catalogKey]); + catalog.availablePartitions = []; + await bucket.put(bundle.catalogKey, JSON.stringify(catalog)); + const result = await post(worker, "/search", { collection: bundle.collection, request: bundle.queries[0].request }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "coverageIncomplete"); + assert.equal(result.coverage.status, "incomplete"); + assert.deepEqual(result.items, []); + sampleResults.push(result); + return true; + }); + + const descriptorFence = await withRuntime(bundle.objects, async ({ worker }) => { + const request = clone(bundle.queries[0].request); + request.expectedDescriptorDigest = `sha256:${"8".repeat(64)}`; + const result = await post(worker, "/search", { collection: bundle.collection, request }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "generationDescriptorMismatch"); + assert.deepEqual(result.items, []); + sampleResults.push(result); + return true; + }); + + const deadlineFence = await withRuntime(bundle.objects, async ({ worker }) => { + const request = clone(bundle.queries[0].request); + request.deadlineUtc = "2000-01-01T00:00:00Z"; + const result = await post(worker, "/search", { collection: bundle.collection, request }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "deadlineExceeded"); + assert.deepEqual(result.items, []); + sampleResults.push(result); + return true; + }); + + const workFence = await withRuntime(bundle.objects, async ({ worker }) => { + const request = clone(bundle.workBoundQuery.request); + request.query.lexical.scanLimit = 1; + const result = await post(worker, "/search", { collection: bundle.collection, request }); + assert.equal(result.status, "failed"); + assert.equal(result.failure.code, "workLimitExceeded"); + assert.equal(result.coverage.status, "complete"); + assert.deepEqual(result.items, []); + sampleResults.push(result); + return true; + }); + + const missingInspection = await withRuntime(bundle.objects, async ({ worker }) => { + const inspection = await post(worker, "/inspect", { collection: bundle.collection, generationId: null }); + assert.equal(inspection.coverageStatus, "unavailable"); + assert.deepEqual(inspection.availablePartitions, []); + return true; + }, new Set([bundle.shardKeys[0]])); + + const readerConfigurationFence = await proveReaderConfigurationFailsClosed(bundle); + + process.stdout.write(`${JSON.stringify({ + schemaVersion: "vyral.worker-r2-proof-report.v1", + status: "complete", + bindingMode, + queryCount: bundle.queries.length, + exactCandidateAndScoreParityCount: mainProof.parity, + unauthenticatedRequestRejected: mainProof.authenticationRejected, + wrongAuthenticationRejected: mainProof.wrongAuthenticationRejected, + nonJsonContentRejected: mainProof.nonJsonRejected, + oversizedBodyRejected: mainProof.oversizedBodyRejected, + malformedDescriptorDigestRejected: mainProof.malformedDescriptorDigestRejected, + exactGenerationSelectionPassed: mainProof.exactGenerationSelection, + serviceReaderGuardsPassed: mainProof.serviceReaderGuards, + readerBindingConfigurationFailedClosed: readerConfigurationFence, + retainedGenerationContinuationPassed: mainProof.retainedContinuation, + continuationTamperRejected: mainProof.tamperRejected, + continuationRequestSubstitutionRejected: mainProof.changedRequestRejected, + retiredGenerationRejected: mainProof.retirementRejected, + verifiedContentCacheHit: cacheProof, + missingShardFailedClosed: missingShard, + corruptShardFailedClosed: corruptShard, + incompleteCoverageFailedClosed: incompleteCoverage, + descriptorFenceRejected: descriptorFence, + expiredDeadlineRejected: deadlineFence, + workLimitFailedClosed: workFence, + inspectionVerifiesArtifacts: missingInspection, + sampleResults, + })}\n`); +} + +main().catch((error) => { + process.stderr.write(`${error.stack ?? error}\n`); + process.exitCode = 1; +}); diff --git a/qualification/README.md b/qualification/README.md index 4df061f..0527b3f 100644 --- a/qualification/README.md +++ b/qualification/README.md @@ -29,4 +29,9 @@ The checked evidence baseline was generated on `2026-08-14`. Evidence is current | Live qualified | Current provider version, isolated live gate, redacted result artifact, and cleanup evidence. | | Consumer validated | Live qualification plus evidence from a representative consumer environment. | -Provider endpoints, account or tenant identifiers, credentials, and consumer identities do not belong in this public artifact. A workflow run or package presence does not promote an adapter automatically; the checked qualification record is the claim boundary. +Provider endpoints, account or tenant identifiers, credentials, and consumer identities do not belong in this public artifact. Consumer-validation evidence is represented by an opaque content digest; the private identity-to-receipt mapping remains outside the repository unless the consumer separately authorizes disclosure. A workflow run or package presence does not promote an adapter automatically; the checked qualification record is the claim boundary. + +Generation-bound retrieval projections use a separate +[qualification artifact](retrieval-projection-qualification.json) because adapter/profile evidence +and per-generation rollout receipts are distinct from execution-runtime admission evidence. Its +implementation digests keep opaque live evidence bound to the exact public source components. diff --git a/qualification/adapter-qualification.schema.json b/qualification/adapter-qualification.schema.json index 75aaec1..dc7ba7c 100644 --- a/qualification/adapter-qualification.schema.json +++ b/qualification/adapter-qualification.schema.json @@ -244,6 +244,12 @@ "minLength": 1, "maxLength": 300 }, + "disclosure": { + "enum": [ + "public", + "private_opaque" + ] + }, "command": { "type": "string", "minLength": 1, diff --git a/qualification/retrieval-projection-qualification.json b/qualification/retrieval-projection-qualification.json new file mode 100644 index 0000000..17f253e --- /dev/null +++ b/qualification/retrieval-projection-qualification.json @@ -0,0 +1,277 @@ +{ + "$schema": "https://openvyral.com/qualification/retrieval-projection-qualification.schema.json", + "schemaVersion": "1.0", + "coreContractVersion": "0.3.0", + "generatedAtUtc": "2026-08-27T18:41:00Z", + "qualificationExpiresAfterDays": 90, + "adapters": [ + { + "adapterId": "local-exhaustive-generation", + "displayName": "Local exhaustive generation-bound projection", + "implementation": "Vyral.Local.LocalGenerationBoundRecordSearchProjection", + "implementationArtifacts": [ + { + "path": "src/Vyral.Local/LocalGenerationBoundRecordSearchProjection.cs", + "sha256": "sha256:5168c9e1d491528d15448c1b0733a995af9f6809681f86298c6c1871a990caae" + } + ], + "provider": "local", + "topology": "in-process-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "qualification": "local_conformant", + "status": "current", + "capabilities": [ + "complete-coverage", + "generation-pinned-continuation", + "lexical" + ], + "generationEvidencePolicy": { + "descriptorSchema": "vyral.record-search-projection-generation.v1", + "requiresExactGenerationBinding": true, + "requiresCompleteRequestedCoverage": true, + "mutableHealthEstablishesCompleteness": false + }, + "evidence": [ + { + "kind": "unit_gate", + "observedAtUtc": "2026-08-27T18:40:00Z", + "expiresAtUtc": "2026-11-25T18:40:00Z", + "reference": "tests/Vyral.Tests.Local/LocalGenerationBoundRecordSearchProjectionTests.cs", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "local_gate", + "observedAtUtc": "2026-08-27T18:40:00Z", + "expiresAtUtc": "2026-11-25T18:40:00Z", + "reference": "tests/Vyral.Tests.Local/GenerationBoundRetrievalEvaluationTargetTests.cs", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "shared_conformance", + "observedAtUtc": "2026-08-27T18:38:00Z", + "expiresAtUtc": "2026-11-25T18:38:00Z", + "reference": "conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + } + ] + }, + { + "adapterId": "opensearch-generation", + "displayName": "OpenSearch generation-bound projection", + "implementation": "Vyral.Aws.OpenSearchGenerationBoundRecordSearchProjection", + "implementationArtifacts": [ + { + "path": "src/Vyral.Aws/OpenSearchGenerationBoundRecordSearchProjection.cs", + "sha256": "sha256:b9fbd3d2c7004b9cd3e7c99515bd7d748cb3cf92f13e3bc2e80b318ab5935c38" + } + ], + "provider": "opensearch", + "topology": "single-node-container", + "profileId": "vector-v1", + "strategyVersion": "opensearch-3.8", + "qualification": "local_conformant", + "status": "current", + "capabilities": [ + "complete-coverage", + "generation-pinned-continuation", + "vector" + ], + "generationEvidencePolicy": { + "descriptorSchema": "vyral.record-search-projection-generation.v1", + "requiresExactGenerationBinding": true, + "requiresCompleteRequestedCoverage": true, + "mutableHealthEstablishesCompleteness": false + }, + "evidence": [ + { + "kind": "unit_gate", + "observedAtUtc": "2026-08-27T18:40:00Z", + "expiresAtUtc": "2026-11-25T18:40:00Z", + "reference": "tests/Vyral.Tests.Aws/OpenSearchGenerationBoundRecordSearchProjectionTests.cs", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "local_gate", + "observedAtUtc": "2026-08-27T13:31:58Z", + "expiresAtUtc": "2026-11-25T13:31:58Z", + "reference": "urn:vyral:private-evidence:sha256:2b85bfdf7253c3c58d8dede99e8a0abb2c37c9dbb3a0c0b6fdc118daefd56f48", + "disclosure": "private_opaque", + "sourceCommit": "10636ae6aa0e31e05a2cc908a5038ed99ada1504", + "sourceTreeDigest": "sha256:6ef183e8e29f1b42158f28a53a99ede7ed01f37a3c88b9f52c24c4bfa4095fd4", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + } + ] + }, + { + "adapterId": "worker-r2-generation-direct", + "displayName": "Cloudflare Worker/R2 generation-bound projection (direct R2)", + "implementation": "experiments/worker-r2-generation-projection/src/worker.mjs", + "implementationArtifacts": [ + { + "path": "experiments/worker-r2-generation-projection/src/worker.mjs", + "sha256": "sha256:e9311869f2e671397b6f2a017882002bde33b46b9a78d4367d3e7ea192f0d6fc" + } + ], + "provider": "cloudflare", + "topology": "worker-r2-miniflare-direct", + "profileId": "lexical-global-bm25-v1", + "strategyVersion": "private-json-shards-v1", + "qualification": "local_conformant", + "status": "current", + "capabilities": [ + "complete-coverage", + "generation-pinned-continuation", + "lexical" + ], + "generationEvidencePolicy": { + "descriptorSchema": "vyral.record-search-projection-generation.v1", + "requiresExactGenerationBinding": true, + "requiresCompleteRequestedCoverage": true, + "mutableHealthEstablishesCompleteness": false + }, + "evidence": [ + { + "kind": "unit_gate", + "observedAtUtc": "2026-08-27T18:38:00Z", + "expiresAtUtc": "2026-11-25T18:38:00Z", + "reference": "experiments/worker-r2-generation-projection/verify.mjs", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "local_gate", + "observedAtUtc": "2026-08-27T18:38:00Z", + "expiresAtUtc": "2026-11-25T18:38:00Z", + "reference": "scripts/verify-worker-r2-generation-projection.py", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + } + ] + }, + { + "adapterId": "worker-r2-generation-service-reader", + "displayName": "Cloudflare Worker/R2 generation-bound projection (service reader)", + "implementation": "experiments/worker-r2-generation-projection/src/worker.mjs + src/object-reader.mjs", + "implementationArtifacts": [ + { + "path": "experiments/worker-r2-generation-projection/src/object-reader.mjs", + "sha256": "sha256:9fe0cba9c743210314ef6a91d9306a867527a6267068f75108fd5d868bd15a28" + }, + { + "path": "experiments/worker-r2-generation-projection/src/worker.mjs", + "sha256": "sha256:e9311869f2e671397b6f2a017882002bde33b46b9a78d4367d3e7ea192f0d6fc" + } + ], + "provider": "cloudflare", + "topology": "worker-r2-miniflare-service-reader", + "profileId": "lexical-global-bm25-v1", + "strategyVersion": "private-json-shards-v1", + "qualification": "consumer_validated", + "status": "current", + "capabilities": [ + "complete-coverage", + "generation-pinned-continuation", + "lexical" + ], + "generationEvidencePolicy": { + "descriptorSchema": "vyral.record-search-projection-generation.v1", + "requiresExactGenerationBinding": true, + "requiresCompleteRequestedCoverage": true, + "mutableHealthEstablishesCompleteness": false + }, + "evidence": [ + { + "kind": "unit_gate", + "observedAtUtc": "2026-08-27T18:38:00Z", + "expiresAtUtc": "2026-11-25T18:38:00Z", + "reference": "experiments/worker-r2-generation-projection/src/object-reader.mjs", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "local_gate", + "observedAtUtc": "2026-08-27T18:38:00Z", + "expiresAtUtc": "2026-11-25T18:38:00Z", + "reference": "scripts/verify-worker-r2-generation-projection.py", + "disclosure": "public", + "sourceCommit": "3f35a88b5e9d7e4e6840ab64dfe786fe612d14a2", + "sourceTreeDigest": "sha256:2663cd143a60fb2f54522116f9b63c9c5549f36073855e6b230d5ec770bd28f7", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "live_gate", + "observedAtUtc": "2026-08-27T17:53:27Z", + "expiresAtUtc": "2026-11-25T17:53:27Z", + "reference": "urn:vyral:private-consumer-evidence:sha256:cdc973c71ac0abba92dd2ec0fa3eb016b4e35233e9a059bf96f6b97b93a197da", + "disclosure": "private_opaque", + "sourceCommit": "04612e9d528e901ede19c182068bbd782008a2ae", + "sourceTreeDigest": "sha256:f5c682b5ff2dbfcc30d4ece24aa9d6c1c268165e3fcb27b696a084a64e961c54", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "cleanup", + "observedAtUtc": "2026-08-27T17:53:27Z", + "expiresAtUtc": "2026-11-25T17:53:27Z", + "reference": "urn:vyral:private-consumer-evidence:sha256:cdc973c71ac0abba92dd2ec0fa3eb016b4e35233e9a059bf96f6b97b93a197da", + "disclosure": "private_opaque", + "sourceCommit": "04612e9d528e901ede19c182068bbd782008a2ae", + "sourceTreeDigest": "sha256:f5c682b5ff2dbfcc30d4ece24aa9d6c1c268165e3fcb27b696a084a64e961c54", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + }, + { + "kind": "consumer_validation", + "observedAtUtc": "2026-08-27T17:53:27Z", + "expiresAtUtc": "2026-11-25T17:53:27Z", + "reference": "urn:vyral:private-consumer-evidence:sha256:cdc973c71ac0abba92dd2ec0fa3eb016b4e35233e9a059bf96f6b97b93a197da", + "disclosure": "private_opaque", + "sourceCommit": "04612e9d528e901ede19c182068bbd782008a2ae", + "sourceTreeDigest": "sha256:f5c682b5ff2dbfcc30d4ece24aa9d6c1c268165e3fcb27b696a084a64e961c54", + "sourceDirty": false, + "generationIds": [], + "descriptorDigests": [] + } + ] + } + ] +} diff --git a/qualification/retrieval-projection-qualification.schema.json b/qualification/retrieval-projection-qualification.schema.json new file mode 100644 index 0000000..0c07546 --- /dev/null +++ b/qualification/retrieval-projection-qualification.schema.json @@ -0,0 +1,171 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openvyral.com/qualification/retrieval-projection-qualification.schema.json", + "title": "Vyral retrieval projection adapter qualification artifact", + "description": "Aging evidence for immutable candidate-only retrieval projection implementations. This adapter-level artifact is distinct from the rollout receipt for each concrete generation.", + "type": "object", + "additionalProperties": false, + "required": [ + "schemaVersion", + "coreContractVersion", + "generatedAtUtc", + "qualificationExpiresAfterDays", + "adapters" + ], + "properties": { + "$schema": { "type": "string" }, + "schemaVersion": { "const": "1.0" }, + "coreContractVersion": { "$ref": "#/$defs/version" }, + "generatedAtUtc": { "type": "string", "format": "date-time" }, + "qualificationExpiresAfterDays": { "type": "integer", "minimum": 1, "maximum": 365 }, + "adapters": { + "type": "array", + "items": { "$ref": "#/$defs/adapter" } + } + }, + "$defs": { + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$" + }, + "identifier": { + "type": "string", + "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$", + "maxLength": 150 + }, + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "adapter": { + "type": "object", + "additionalProperties": false, + "required": [ + "adapterId", + "displayName", + "implementation", + "implementationArtifacts", + "provider", + "topology", + "profileId", + "strategyVersion", + "qualification", + "status", + "capabilities", + "generationEvidencePolicy", + "evidence" + ], + "properties": { + "adapterId": { "$ref": "#/$defs/identifier" }, + "displayName": { "type": "string", "minLength": 1, "maxLength": 200 }, + "implementation": { "type": "string", "minLength": 1, "maxLength": 300 }, + "implementationArtifacts": { + "type": "array", + "minItems": 1, + "maxItems": 16, + "items": { "$ref": "#/$defs/implementationArtifact" } + }, + "provider": { "$ref": "#/$defs/identifier" }, + "topology": { "$ref": "#/$defs/identifier" }, + "profileId": { "$ref": "#/$defs/identifier" }, + "strategyVersion": { "type": "string", "minLength": 1, "maxLength": 100 }, + "qualification": { + "enum": ["prototype", "local_conformant", "live_qualified", "consumer_validated"] + }, + "status": { "enum": ["current", "stale"] }, + "capabilities": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "generationEvidencePolicy": { "$ref": "#/$defs/generationEvidencePolicy" }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/evidence" } + } + } + }, + "implementationArtifact": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha256"], + "properties": { + "path": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._/-]*$", + "maxLength": 300 + }, + "sha256": { "$ref": "#/$defs/sha256" } + } + }, + "generationEvidencePolicy": { + "type": "object", + "additionalProperties": false, + "required": [ + "descriptorSchema", + "requiresExactGenerationBinding", + "requiresCompleteRequestedCoverage", + "mutableHealthEstablishesCompleteness" + ], + "properties": { + "descriptorSchema": { "const": "vyral.record-search-projection-generation.v1" }, + "requiresExactGenerationBinding": { "const": true }, + "requiresCompleteRequestedCoverage": { "const": true }, + "mutableHealthEstablishesCompleteness": { "const": false } + } + }, + "evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "observedAtUtc", + "expiresAtUtc", + "reference", + "sourceCommit", + "sourceTreeDigest", + "sourceDirty", + "generationIds", + "descriptorDigests" + ], + "properties": { + "kind": { + "enum": [ + "unit_gate", + "shared_conformance", + "local_gate", + "fault_restart", + "live_gate", + "cleanup", + "consumer_validation" + ] + }, + "observedAtUtc": { "type": "string", "format": "date-time" }, + "expiresAtUtc": { "type": "string", "format": "date-time" }, + "reference": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "disclosure": { + "enum": ["public", "private_opaque"] + }, + "sourceCommit": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "sourceTreeDigest": { "$ref": "#/$defs/sha256" }, + "sourceDirty": { "type": "boolean" }, + "generationIds": { + "type": "array", + "uniqueItems": true, + "items": { "type": "string", "minLength": 1, "maxLength": 200 } + }, + "descriptorDigests": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/sha256" } + } + } + } + } +} diff --git a/runtimes/python/src/vyral_runtime/__init__.py b/runtimes/python/src/vyral_runtime/__init__.py index b561e68..68b2d3a 100644 --- a/runtimes/python/src/vyral_runtime/__init__.py +++ b/runtimes/python/src/vyral_runtime/__init__.py @@ -14,8 +14,10 @@ ConformanceError, ConformanceManifest, GoldenResult, + ProjectionGenerationScenarioResult, load_conformance_manifest, run_bundled_goldens, + run_bundled_projection_generation_scenario, ) from .contracts import ( ContractBundle, @@ -477,6 +479,7 @@ "MAX_EXPECTED_GRAPH_ITEMS", "MAX_RAG_EVALUATION_CASES", "OrderExpression", + "ProjectionGenerationScenarioResult", "ObjectDeleteRequest", "ObjectInfo", "ObjectListRequest", @@ -607,6 +610,7 @@ "normalize_embedding_purpose", "prepare_embedding_text", "run_bundled_goldens", + "run_bundled_projection_generation_scenario", "run_bundled_external_worker_scenario", "run_bundled_native_execution_scenario", "run_bundled_record_store_scenario", diff --git a/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/manifest.json b/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/manifest.json index ea1c30e..2b291ad 100644 --- a/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/manifest.json +++ b/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/manifest.json @@ -13,6 +13,7 @@ "vyral.runtime.external-worker.v1", "vyral.runtime.providers.v1", "vyral.runtime.canonical.v1", + "vyral.runtime.retrieval-generation.v1", "vyral.runtime.execution.local.v1", "vyral.runtime.rest.v1", "vyral.runtime.mcp.stateless-2026-07-28.v1", @@ -54,6 +55,20 @@ "profile": "vyral.runtime.data-rag.v1", "kind": "golden" }, + { + "id": "records.projection-generation.v1", + "path": "scenarios/goldens/record-search-projection-generation.json", + "sha256": "sha256:98e7a37d649b8008d29118a8cfe51c1ca3a178e58c3a602145717fb26fd8f0ca", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "golden" + }, + { + "id": "records.projection-generation-lifecycle.v1", + "path": "scenarios/retrieval/generation-bound-lifecycle.json", + "sha256": "sha256:7a5433eb745637004155322a76abbb0a5eaf6cf26148b7e18397c1e26c194875", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "stateful" + }, { "id": "embeddings.vectors.v1", "path": "scenarios/goldens/embedding-vectors.json", diff --git a/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json b/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json new file mode 100644 index 0000000..9ec058a --- /dev/null +++ b/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json @@ -0,0 +1,60 @@ +{ + "$schema": "../../scenario.schema.json", + "id": "records.projection-generation.v1", + "fixtureVersion": "1.0.0", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "golden", + "description": "Immutable record-search projection descriptor hashing shared by embedded and hosted adapters.", + "steps": [ + { + "id": "descriptor-hash-two-partitions", + "operation": "records.projection-generation-descriptor-hash", + "arguments": { + "descriptor": { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "gen-2026-08-27-a", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recordRevisionSetDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": [ + "public-a", + "public-b" + ], + "capabilities": [ + "completeCoverage", + "generationPinnedContinuation", + "lexical" + ], + "artifacts": [ + { + "id": "partition-public-a", + "kind": "index-part", + "contentHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "sizeBytes": 128, + "mediaType": "application/octet-stream" + }, + { + "id": "partition-public-b", + "kind": "index-part", + "contentHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "sizeBytes": 96, + "mediaType": "application/octet-stream" + } + ], + "createdAtUtc": "2026-08-27T12:00:00Z", + "descriptorDigest": "sha256:3b92c8927a2a93f12d99e64e03d118e29ac57284663943fa257568ae178d31d5" + } + }, + "expect": { + "value": "sha256:3b92c8927a2a93f12d99e64e03d118e29ac57284663943fa257568ae178d31d5" + } + } + ] +} diff --git a/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json b/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json new file mode 100644 index 0000000..cef46c6 --- /dev/null +++ b/runtimes/python/src/vyral_runtime/_conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json @@ -0,0 +1,273 @@ +{ + "$schema": "../../scenario.schema.json", + "id": "records.projection-generation-lifecycle.v1", + "fixtureVersion": "1.0.0", + "profile": "vyral.runtime.retrieval-generation.v1", + "kind": "stateful", + "description": "Generation activation, retained continuations, fail-closed coverage, descriptor fences, tamper rejection, and retirement.", + "steps": [ + { + "id": "publish-generation-a", + "operation": "records.projection-generation-publish", + "arguments": { + "descriptor": { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "generation-a", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recordRevisionSetDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": ["public-a", "public-b"], + "capabilities": ["completeCoverage", "generationPinnedContinuation", "lexical"], + "artifacts": [], + "createdAtUtc": "2026-08-27T12:00:00Z", + "descriptorDigest": "sha256:01666b57bb8922e1360b5662219fb7955cb1ba5f8e8865f4fd46eecddaeb50de" + }, + "documents": [ + { "partitionKey": "public-a", "id": "a-1", "revision": 1, "searchText": "portable contracts" }, + { "partitionKey": "public-a", "id": "a-2", "revision": 1, "searchText": "provider native semantics" }, + { "partitionKey": "public-b", "id": "b-1", "revision": 1, "searchText": "portable retrieval" } + ] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "activate-generation-a", + "operation": "records.projection-generation-activate", + "arguments": { "collection": "library", "generationId": "generation-a" }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "inspect-active-generation-a", + "operation": "records.projection-generation-inspect", + "arguments": { "collection": "library" }, + "expect": { + "value": { + "generationId": "generation-a", + "state": "active", + "coverageStatus": "complete", + "availablePartitions": ["public-a", "public-b"] + } + } + }, + { + "id": "search-generation-a-first-page", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "saveContinuationAs": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "succeeded", + "generationId": "generation-a", + "ids": ["a-1"], + "continuation": "present", + "coverageStatus": "complete", + "coveredPartitions": ["public-a", "public-b"], + "missingPartitions": [], + "failureCode": null + } + } + }, + { + "id": "publish-generation-b", + "operation": "records.projection-generation-publish", + "arguments": { + "descriptor": { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "generation-b", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "recordRevisionSetDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": ["public-a", "public-b"], + "capabilities": ["completeCoverage", "generationPinnedContinuation", "lexical"], + "artifacts": [], + "createdAtUtc": "2026-08-27T12:05:00Z", + "descriptorDigest": "sha256:a025b15641dbacea0af4bffa84c707bd4171dc48b8d58fce1135c8402d7f1d3b" + }, + "documents": [ + { "partitionKey": "public-a", "id": "a-1", "revision": 2, "searchText": "portable execution" }, + { "partitionKey": "public-a", "id": "a-3", "revision": 1, "searchText": "provider evidence" }, + { "partitionKey": "public-b", "id": "b-2", "revision": 1, "searchText": "portable qualification" } + ] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "activate-generation-b", + "operation": "records.projection-generation-activate", + "arguments": { "collection": "library", "generationId": "generation-b" }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "continue-retained-generation-a", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "continuationRef": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "succeeded", + "generationId": "generation-a", + "ids": ["b-1"], + "continuation": "absent", + "coverageStatus": "complete", + "coveredPartitions": ["public-a", "public-b"], + "missingPartitions": [], + "failureCode": null + } + } + }, + { + "id": "reject-tampered-continuation", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "tamperContinuationRef": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "failed", + "generationId": null, + "ids": [], + "continuation": "absent", + "coverageStatus": "unavailable", + "coveredPartitions": [], + "missingPartitions": ["public-a", "public-b"], + "failureCode": "invalidContinuation" + } + } + }, + { + "id": "remove-generation-b-partition", + "operation": "records.projection-generation-set-available", + "arguments": { + "collection": "library", + "generationId": "generation-b", + "availablePartitions": ["public-a"] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "fail-closed-on-incomplete-generation-b", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 10 + }, + "expect": { + "value": { + "status": "failed", + "generationId": "generation-b", + "ids": [], + "continuation": "absent", + "coverageStatus": "incomplete", + "coveredPartitions": ["public-a"], + "missingPartitions": ["public-b"], + "failureCode": "coverageIncomplete" + } + } + }, + { + "id": "inspect-incomplete-generation-b", + "operation": "records.projection-generation-inspect", + "arguments": { "collection": "library" }, + "expect": { + "value": { + "generationId": "generation-b", + "state": "active", + "coverageStatus": "incomplete", + "availablePartitions": ["public-a"] + } + } + }, + { + "id": "restore-generation-b-coverage", + "operation": "records.projection-generation-set-available", + "arguments": { + "collection": "library", + "generationId": "generation-b", + "availablePartitions": ["public-a", "public-b"] + }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "reject-wrong-descriptor-fence", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 10, + "expectedDescriptorDigest": "sha256:9999999999999999999999999999999999999999999999999999999999999999" + }, + "expect": { + "value": { + "status": "failed", + "generationId": "generation-b", + "ids": [], + "continuation": "absent", + "coverageStatus": "complete", + "coveredPartitions": ["public-a", "public-b"], + "missingPartitions": [], + "failureCode": "generationDescriptorMismatch" + } + } + }, + { + "id": "retire-generation-a", + "operation": "records.projection-generation-retire", + "arguments": { "collection": "library", "generationId": "generation-a" }, + "expect": { "value": { "status": "ok" } } + }, + { + "id": "reject-retired-generation-a-continuation", + "operation": "records.projection-generation-search", + "arguments": { + "collection": "library", + "query": "portable", + "partitionKeys": ["public-a", "public-b"], + "limit": 1, + "continuationRef": "generation-a-page-2" + }, + "expect": { + "value": { + "status": "failed", + "generationId": "generation-a", + "ids": [], + "continuation": "absent", + "coverageStatus": "unavailable", + "coveredPartitions": [], + "missingPartitions": ["public-a", "public-b"], + "failureCode": "generationRetired" + } + } + } + ] +} diff --git a/runtimes/python/src/vyral_runtime/conformance.py b/runtimes/python/src/vyral_runtime/conformance.py index 0f4e49b..847d260 100644 --- a/runtimes/python/src/vyral_runtime/conformance.py +++ b/runtimes/python/src/vyral_runtime/conformance.py @@ -1,13 +1,15 @@ from __future__ import annotations +import base64 from dataclasses import dataclass from datetime import datetime, timezone from hashlib import sha256 +import hmac from importlib import resources import json from pathlib import Path, PurePosixPath import struct -from typing import Any, Callable, Mapping, Protocol +from typing import Any, Callable, Mapping, Protocol, cast from ._datetime import parse_iso_datetime from ._version import CONTRACT_VERSION, FIXTURE_VERSION, RUNTIME_VERSION @@ -82,6 +84,14 @@ class GoldenResult: value: JSONValue +@dataclass(frozen=True) +class ProjectionGenerationScenarioResult: + scenario_id: str + step_id: str + operation: str + value: JSONValue + + def _safe_parts(relative: str) -> tuple[str, ...]: path = PurePosixPath(relative) if path.is_absolute() or not path.parts or any(part in {"", ".", ".."} for part in path.parts): @@ -521,12 +531,475 @@ def _operation_rag_ingestion_plan( } +def _operation_projection_generation_descriptor_hash( + arguments: Mapping[str, Any], +) -> JSONValue: + raw_descriptor = arguments.get("descriptor") + if not isinstance(raw_descriptor, Mapping): + raise ConformanceError("arguments.descriptor must be an object.") + + descriptor = dict(raw_descriptor) + required = ( + "schema", + "collection", + "generationId", + "providerId", + "profileId", + "strategyVersion", + "sourceManifestDigest", + "recordRevisionSetDigest", + "projectionSchemaDigest", + "configurationDigest", + "expectedItemCount", + "expectedPartitions", + "capabilities", + "artifacts", + "createdAtUtc", + ) + for name in required: + if name not in descriptor: + raise ConformanceError( + f"arguments.descriptor.{name} is required." + ) + + expected_partitions = descriptor["expectedPartitions"] + capabilities = descriptor["capabilities"] + artifacts = descriptor["artifacts"] + if not isinstance(expected_partitions, list) or not all( + isinstance(value, str) and value for value in expected_partitions + ): + raise ConformanceError( + "arguments.descriptor.expectedPartitions must be a string array." + ) + if not isinstance(capabilities, list) or not all( + isinstance(value, str) and value for value in capabilities + ): + raise ConformanceError( + "arguments.descriptor.capabilities must be a string array." + ) + if not isinstance(artifacts, list) or not all( + isinstance(value, Mapping) for value in artifacts + ): + raise ConformanceError( + "arguments.descriptor.artifacts must be an object array." + ) + + material = { + "schema": descriptor["schema"], + "collection": descriptor["collection"], + "generationId": descriptor["generationId"], + "providerId": descriptor["providerId"], + "profileId": descriptor["profileId"], + "strategyVersion": descriptor["strategyVersion"], + "sourceManifestDigest": descriptor["sourceManifestDigest"], + "recordRevisionSetDigest": descriptor["recordRevisionSetDigest"], + "projectionSchemaDigest": descriptor["projectionSchemaDigest"], + "analyzerDigest": descriptor.get("analyzerDigest"), + "configurationDigest": descriptor["configurationDigest"], + "expectedItemCount": descriptor["expectedItemCount"], + "expectedPartitions": sorted(expected_partitions), + "capabilities": sorted(capabilities), + "artifacts": sorted( + (dict(value) for value in artifacts), + key=lambda value: str(value.get("id", "")), + ), + "createdAtUtc": descriptor["createdAtUtc"], + } + encoded = json.dumps( + material, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return "sha256:" + sha256(encoded).hexdigest() + + +class _ProjectionGenerationFixtureRuntime: + """Independent stateful runner for the portable generation lifecycle fixture.""" + + _SIGNING_KEY = sha256(b"portable-generation-fixture-key").digest() + + def __init__(self) -> None: + self._generations: dict[tuple[str, str], dict[str, Any]] = {} + self._active: dict[str, str] = {} + self._continuations: dict[str, str] = {} + + def execute(self, operation: str, arguments: Mapping[str, Any]) -> JSONValue: + if operation == "records.projection-generation-publish": + self._publish(arguments) + return {"status": "ok"} + if operation == "records.projection-generation-activate": + generation = self._generation(arguments) + collection = _require_string(arguments.get("collection"), "arguments.collection") + prior_id = self._active.get(collection) + if prior_id is not None: + self._generations[(collection, prior_id)]["state"] = "retained" + generation["state"] = "active" + self._active[collection] = cast(str, generation["descriptor"]["generationId"]) + return {"status": "ok"} + if operation == "records.projection-generation-retire": + generation = self._generation(arguments) + generation["state"] = "retired" + generation["availablePartitions"] = [] + return {"status": "ok"} + if operation == "records.projection-generation-set-available": + generation = self._generation(arguments) + raw = arguments.get("availablePartitions") + if not isinstance(raw, list) or not all(isinstance(value, str) for value in raw): + raise ConformanceError("availablePartitions must be a string array.") + generation["availablePartitions"] = sorted(raw) + return {"status": "ok"} + if operation == "records.projection-generation-inspect": + return self._inspect(arguments) + if operation == "records.projection-generation-search": + return self._search(arguments) + raise ConformanceError(f"Unsupported projection generation operation {operation!r}.") + + def _publish(self, arguments: Mapping[str, Any]) -> None: + raw_descriptor = arguments.get("descriptor") + raw_documents = arguments.get("documents") + if not isinstance(raw_descriptor, Mapping) or not isinstance(raw_documents, list): + raise ConformanceError("Projection publication requires descriptor and documents.") + descriptor = dict(raw_descriptor) + expected_digest = _require_string( + descriptor.get("descriptorDigest"), + "descriptor.descriptorDigest", + ) + actual_digest = _operation_projection_generation_descriptor_hash( + {"descriptor": descriptor} + ) + if actual_digest != expected_digest: + raise ConformanceError("Published projection descriptor digest is invalid.") + collection = _require_string(descriptor.get("collection"), "descriptor.collection") + generation_id = _require_string( + descriptor.get("generationId"), "descriptor.generationId" + ) + expected_partitions = descriptor.get("expectedPartitions") + if not isinstance(expected_partitions, list) or not all( + isinstance(value, str) for value in expected_partitions + ): + raise ConformanceError("descriptor.expectedPartitions must be a string array.") + documents: list[dict[str, Any]] = [] + for raw_document in raw_documents: + if not isinstance(raw_document, Mapping): + raise ConformanceError("Every projection document must be an object.") + documents.append(dict(raw_document)) + if len(documents) != descriptor.get("expectedItemCount"): + raise ConformanceError("Projection document count does not match the descriptor.") + self._generations[(collection, generation_id)] = { + "descriptor": descriptor, + "documents": documents, + "availablePartitions": sorted(expected_partitions), + "state": "retained", + } + + def _generation(self, arguments: Mapping[str, Any]) -> dict[str, Any]: + collection = _require_string(arguments.get("collection"), "arguments.collection") + generation_id = _require_string( + arguments.get("generationId"), "arguments.generationId" + ) + generation = self._generations.get((collection, generation_id)) + if generation is None: + raise ConformanceError("The fixture generation is unavailable.") + return generation + + def _inspect(self, arguments: Mapping[str, Any]) -> JSONValue: + collection = _require_string(arguments.get("collection"), "arguments.collection") + generation_id = self._active.get(collection) + if generation_id is None: + raise ConformanceError("The fixture collection has no active generation.") + generation = self._generations[(collection, generation_id)] + descriptor = cast(dict[str, Any], generation["descriptor"]) + available = cast(list[str], generation["availablePartitions"]) + expected = cast(list[str], descriptor["expectedPartitions"]) + coverage = "complete" if available == sorted(expected) else "incomplete" + return { + "generationId": generation_id, + "state": cast(str, generation["state"]), + "coverageStatus": coverage, + "availablePartitions": cast(list[JSONValue], list(available)), + } + + def _search(self, arguments: Mapping[str, Any]) -> JSONValue: + collection = _require_string(arguments.get("collection"), "arguments.collection") + query = _require_text(arguments.get("query"), "arguments.query") + raw_partitions = arguments.get("partitionKeys") + if not isinstance(raw_partitions, list) or not all( + isinstance(value, str) for value in raw_partitions + ): + raise ConformanceError("partitionKeys must be a string array.") + partitions = sorted(raw_partitions) + raw_limit = arguments.get("limit") + if isinstance(raw_limit, bool) or not isinstance(raw_limit, int) or raw_limit <= 0: + raise ConformanceError("limit must be a positive integer.") + + continuation: Mapping[str, Any] | None = None + continuation_ref = arguments.get("continuationRef") + tamper_ref = arguments.get("tamperContinuationRef") + if continuation_ref is not None: + token = self._continuations[ + _require_string(continuation_ref, "arguments.continuationRef") + ] + continuation = self._read_continuation(token) + elif tamper_ref is not None: + token = self._continuations[ + _require_string(tamper_ref, "arguments.tamperContinuationRef") + ] + continuation = self._read_continuation(self._tamper(token)) + if continuation is None: + return self._failed( + None, + "unavailable", + [], + partitions, + "invalidContinuation", + ) + + generation_id = ( + _require_string(continuation.get("generationId"), "continuation.generationId") + if continuation is not None + else self._active.get(collection) + ) + if generation_id is None: + return self._failed(None, "unavailable", [], partitions, "generationUnavailable") + generation = self._generations.get((collection, generation_id)) + if generation is None: + return self._failed(generation_id, "unavailable", [], partitions, "generationUnavailable") + if generation["state"] == "retired": + return self._failed(generation_id, "unavailable", [], partitions, "generationRetired") + + descriptor = cast(dict[str, Any], generation["descriptor"]) + available = cast(list[str], generation["availablePartitions"]) + covered = sorted(set(partitions).intersection(available)) + missing = sorted(set(partitions).difference(covered)) + if missing: + return self._failed( + generation_id, + "incomplete", + covered, + missing, + "coverageIncomplete", + ) + expected_digest = arguments.get("expectedDescriptorDigest") + if expected_digest is not None and expected_digest != descriptor["descriptorDigest"]: + return self._failed( + generation_id, + "complete", + covered, + [], + "generationDescriptorMismatch", + ) + + fingerprint = self._request_fingerprint( + collection, + generation_id, + cast(str, descriptor["descriptorDigest"]), + query, + partitions, + raw_limit, + ) + offset = 0 + if continuation is not None: + if continuation.get("fingerprint") != fingerprint: + return self._failed( + generation_id, + "complete", + covered, + [], + "invalidContinuation", + ) + raw_offset = continuation.get("offset") + if isinstance(raw_offset, bool) or not isinstance(raw_offset, int) or raw_offset < 0: + return self._failed( + generation_id, + "complete", + covered, + [], + "invalidContinuation", + ) + offset = raw_offset + + query_tokens = query.casefold().split() + matches: list[dict[str, Any]] = [] + for document in cast(list[dict[str, Any]], generation["documents"]): + if document.get("partitionKey") not in partitions: + continue + text_tokens = _require_text( + document.get("searchText"), "document.searchText" + ).casefold().split() + score = sum(text_tokens.count(token) for token in query_tokens) + if not query_tokens or score > 0: + match = dict(document) + match["score"] = score + matches.append(match) + matches.sort( + key=lambda item: ( + -cast(int, item["score"]), + cast(str, item["partitionKey"]), + cast(str, item["id"]), + ) + ) + page = matches[offset : offset + raw_limit] + next_offset = offset + len(page) + next_token: str | None = None + if next_offset < len(matches): + next_token = self._write_continuation( + { + "generationId": generation_id, + "fingerprint": fingerprint, + "offset": next_offset, + } + ) + save_as = arguments.get("saveContinuationAs") + if save_as is not None: + if next_token is None: + raise ConformanceError("The fixture expected a continuation to save.") + self._continuations[_require_string(save_as, "arguments.saveContinuationAs")] = next_token + return { + "status": "succeeded", + "generationId": generation_id, + "ids": cast(list[JSONValue], [cast(str, item["id"]) for item in page]), + "continuation": "present" if next_token is not None else "absent", + "coverageStatus": "complete", + "coveredPartitions": cast(list[JSONValue], covered), + "missingPartitions": [], + "failureCode": None, + } + + @staticmethod + def _failed( + generation_id: str | None, + coverage: str, + covered: list[str], + missing: list[str], + code: str, + ) -> JSONValue: + return { + "status": "failed", + "generationId": generation_id, + "ids": [], + "continuation": "absent", + "coverageStatus": coverage, + "coveredPartitions": cast(list[JSONValue], covered), + "missingPartitions": cast(list[JSONValue], missing), + "failureCode": code, + } + + @staticmethod + def _request_fingerprint( + collection: str, + generation_id: str, + descriptor_digest: str, + query: str, + partitions: list[str], + limit: int, + ) -> str: + material = { + "collection": collection, + "generationId": generation_id, + "descriptorDigest": descriptor_digest, + "query": query, + "partitionKeys": partitions, + "limit": limit, + } + encoded = json.dumps(material, separators=(",", ":"), sort_keys=True).encode("utf-8") + return "sha256:" + sha256(encoded).hexdigest() + + def _write_continuation(self, payload: Mapping[str, Any]) -> str: + body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + signature = hmac.digest(self._SIGNING_KEY, body, "sha256") + return self._encode(body) + "." + self._encode(signature) + + def _read_continuation(self, token: str) -> Mapping[str, Any] | None: + if len(token) > 8192 or token.count(".") != 1: + return None + raw_body, raw_signature = token.split(".", 1) + try: + body = self._decode(raw_body) + signature = self._decode(raw_signature) + except ValueError: + return None + expected = hmac.digest(self._SIGNING_KEY, body, "sha256") + if not hmac.compare_digest(signature, expected): + return None + try: + value = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + return value if isinstance(value, Mapping) else None + + @staticmethod + def _encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode("ascii") + + @staticmethod + def _decode(value: str) -> bytes: + try: + return base64.b64decode( + value + "=" * (-len(value) % 4), + altchars=b"-_", + validate=True, + ) + except (ValueError, TypeError) as exc: + raise ValueError("Invalid base64url value.") from exc + + @staticmethod + def _tamper(token: str) -> str: + signature_start = token.index(".") + 1 + replacement = "B" if token[signature_start] == "A" else "A" + return token[:signature_start] + replacement + token[signature_start + 1 :] + + +def run_bundled_projection_generation_scenario( + root: str | Path | None = None, +) -> tuple[ProjectionGenerationScenarioResult, ...]: + manifest = load_conformance_manifest(root) + descriptor = next( + ( + item + for item in manifest.scenarios + if item.scenario_id == "records.projection-generation-lifecycle.v1" + ), + None, + ) + if descriptor is None: + raise ConformanceError("The projection generation lifecycle scenario is unavailable.") + scenario = manifest.scenario(descriptor) + runtime = _ProjectionGenerationFixtureRuntime() + results: list[ProjectionGenerationScenarioResult] = [] + for raw_step in scenario["steps"]: + step = cast(Mapping[str, Any], raw_step) + operation = _require_string(step.get("operation"), "step.operation") + arguments = step.get("arguments") + expectation = step.get("expect") + if not isinstance(arguments, Mapping) or not isinstance(expectation, Mapping): + raise ConformanceError("Projection lifecycle steps require arguments and expectations.") + actual = runtime.execute(operation, arguments) + expected = expectation.get("value") + if actual != expected: + raise ConformanceError( + f"Projection lifecycle step {step.get('id')!r} produced {actual!r}, expected {expected!r}." + ) + results.append( + ProjectionGenerationScenarioResult( + scenario_id=descriptor.scenario_id, + step_id=_require_string(step.get("id"), "step.id"), + operation=operation, + value=actual, + ) + ) + return tuple(results) + + _GOLDEN_OPERATIONS: dict[str, Callable[[Mapping[str, Any]], JSONValue]] = { "hash.sha256-utf8": _operation_sha256, "canonical.transaction-id": _operation_transaction_id, "canonical.lease-token-hash": _operation_lease_hash, "admission.receipt": _operation_admission_receipt, "records.snapshot-hash": _operation_snapshot_hash, + "records.projection-generation-descriptor-hash": ( + _operation_projection_generation_descriptor_hash + ), "embeddings.generate": _operation_embedding, "graph.record-map": _operation_graph_record_map, "rag.ingestion-plan": _operation_rag_ingestion_plan, diff --git a/runtimes/python/src/vyral_runtime/profiles.py b/runtimes/python/src/vyral_runtime/profiles.py index cc30a51..bd0f15a 100644 --- a/runtimes/python/src/vyral_runtime/profiles.py +++ b/runtimes/python/src/vyral_runtime/profiles.py @@ -19,6 +19,7 @@ class RuntimeProfileId(str, Enum): EXTERNAL_WORKER = "vyral.runtime.external-worker.v1" PROVIDERS = "vyral.runtime.providers.v1" CANONICAL = "vyral.runtime.canonical.v1" + RETRIEVAL_GENERATION = "vyral.runtime.retrieval-generation.v1" EXECUTION_LOCAL = "vyral.runtime.execution.local.v1" REST = "vyral.runtime.rest.v1" MCP_STATELESS = "vyral.runtime.mcp.stateless-2026-07-28.v1" @@ -92,6 +93,16 @@ def to_dict(self) -> dict[str, JSONValue]: "hash-verified snapshots, and byte-identical .NET archive goldens; " "supported-platform and multi-provider promotion remain incomplete.", ), + ProfileStatus( + RuntimeProfileId.RETRIEVAL_GENERATION, + Maturity.PROTOTYPE, + True, + False, + "Immutable generation descriptors, complete-coverage outcomes, and " + "generation-pinned candidate-search fixtures are available as an " + "optional contract experiment; provider lifecycle semantics remain " + "intentionally unpromoted.", + ), ProfileStatus( RuntimeProfileId.EXECUTION_LOCAL, Maturity.PROTOTYPE, diff --git a/runtimes/python/tests/test_runtime.py b/runtimes/python/tests/test_runtime.py index 17a8923..d6bdd30 100644 --- a/runtimes/python/tests/test_runtime.py +++ b/runtimes/python/tests/test_runtime.py @@ -37,6 +37,7 @@ run_bundled_canonical_scenario, run_bundled_native_execution_scenario, run_bundled_goldens, + run_bundled_projection_generation_scenario, run_bundled_record_store_scenarios, sha256_utf8, ) @@ -366,6 +367,8 @@ def test_manifest_loads_and_runs_all_current_goldens(self) -> None: "records.core-crud.v1", "records.query-semantics.v1", "records.snapshot-hash.v1", + "records.projection-generation.v1", + "records.projection-generation-lifecycle.v1", "embeddings.vectors.v1", "rag.ingestion-plan.v1", "graph.record-mapping.v1", @@ -385,6 +388,7 @@ def test_manifest_loads_and_runs_all_current_goldens(self) -> None: "replayed-receipt-keeps-identity", "rejected-receipt", "snapshot-hash-unicode-float32", + "descriptor-hash-two-partitions", "deterministic-hash-unicode", "token-hash-lexical", "dry-run-plan-hash-and-chunk-boundaries", @@ -396,6 +400,31 @@ def test_manifest_loads_and_runs_all_current_goldens(self) -> None: 30, len(run_bundled_record_store_scenarios(FIXTURE_ROOT)), ) + self.assertEqual( + [ + "publish-generation-a", + "activate-generation-a", + "inspect-active-generation-a", + "search-generation-a-first-page", + "publish-generation-b", + "activate-generation-b", + "continue-retained-generation-a", + "reject-tampered-continuation", + "remove-generation-b-partition", + "fail-closed-on-incomplete-generation-b", + "inspect-incomplete-generation-b", + "restore-generation-b-coverage", + "reject-wrong-descriptor-fence", + "retire-generation-a", + "reject-retired-generation-a-continuation", + ], + [ + result.step_id + for result in run_bundled_projection_generation_scenario( + FIXTURE_ROOT + ) + ], + ) self.assertEqual( [ "handler-success-side-effects", diff --git a/scripts/compute-source-tree-evidence.py b/scripts/compute-source-tree-evidence.py new file mode 100644 index 0000000..122d614 --- /dev/null +++ b/scripts/compute-source-tree-evidence.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Compute a content digest for the exact tracked-plus-untracked source worktree.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import stat +import subprocess + + +def _git(root: Path, *arguments: str) -> bytes: + return subprocess.run( + ["git", "-C", str(root), *arguments], + check=True, + stdout=subprocess.PIPE, + ).stdout + + +def _content_digest(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def compute(root: Path) -> dict[str, object]: + root = root.resolve() + if not (root / ".git").exists(): + raise SystemExit(f"Not a Git worktree: {root}") + + encoded_paths = _git( + root, + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + ).split(b"\0") + paths = sorted( + path.decode("utf-8", errors="strict") + for path in encoded_paths + if path + ) + + tree_digest = hashlib.sha256() + file_count = 0 + for relative in paths: + if "\n" in relative or "\r" in relative: + raise SystemExit("Source-tree evidence does not permit control characters in paths.") + source = root / relative + if not source.exists(): + # A tracked deletion is represented by its absence from the resulting tree. + continue + if source.is_symlink() or not source.is_file(): + raise SystemExit(f"Source-tree evidence requires regular files: {relative}") + mode = "755" if source.stat().st_mode & stat.S_IXUSR else "644" + content_digest = _content_digest(source) + tree_digest.update(f"{mode} {content_digest} {relative}\n".encode("utf-8")) + file_count += 1 + + status = _git(root, "status", "--porcelain=v1", "--untracked-files=all") + commit = _git(root, "rev-parse", "HEAD").decode("ascii").strip() + return { + "schemaVersion": "vyral.source-tree-evidence.v1", + "sourceCommit": commit, + "sourceDirty": bool(status.strip()), + "sourceTreeDigest": "sha256:" + tree_digest.hexdigest(), + "fileCount": file_count, + } + + +def compute_commit(root: Path, commit: str) -> dict[str, object]: + """Compute the same source digest from one immutable Git commit.""" + root = root.resolve() + if not (root / ".git").exists(): + raise SystemExit(f"Not a Git worktree: {root}") + if not commit or any(character not in "0123456789abcdef" for character in commit): + raise SystemExit("Commit evidence requires a lowercase hexadecimal Git object ID.") + resolved = _git(root, "rev-parse", "--verify", f"{commit}^{{commit}}") + resolved_commit = resolved.decode("ascii").strip() + tree = _git(root, "ls-tree", "-r", "-z", "--full-tree", resolved_commit) + entries: list[tuple[str, str, str]] = [] + for encoded in tree.split(b"\0"): + if not encoded: + continue + metadata, encoded_path = encoded.split(b"\t", 1) + mode, object_type, object_id = metadata.decode("ascii").split(" ") + if object_type != "blob": + raise SystemExit("Source-tree evidence permits only Git blobs.") + relative = encoded_path.decode("utf-8", errors="strict") + if "\n" in relative or "\r" in relative: + raise SystemExit("Source-tree evidence does not permit control characters in paths.") + if mode == "120000": + raise SystemExit(f"Source-tree evidence requires regular files: {relative}") + normalized_mode = "755" if mode == "100755" else "644" + content = _git(root, "cat-file", "blob", object_id) + entries.append((relative, normalized_mode, hashlib.sha256(content).hexdigest())) + + tree_digest = hashlib.sha256() + for relative, mode, content_digest in sorted(entries): + tree_digest.update(f"{mode} {content_digest} {relative}\n".encode("utf-8")) + return { + "schemaVersion": "vyral.source-tree-evidence.v1", + "sourceCommit": resolved_commit, + "sourceDirty": False, + "sourceTreeDigest": "sha256:" + tree_digest.hexdigest(), + "fileCount": len(entries), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--commit", help="compute evidence for an immutable commit") + arguments = parser.parse_args() + evidence = ( + compute_commit(arguments.root, arguments.commit) + if arguments.commit + else compute(arguments.root) + ) + print(json.dumps(evidence, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export-public-tree.py b/scripts/export-public-tree.py index 0a83d1e..845e016 100755 --- a/scripts/export-public-tree.py +++ b/scripts/export-public-tree.py @@ -45,6 +45,7 @@ "contracts", "deploy", "examples", + "experiments", "packaging", "qualification", "runtimes", @@ -62,6 +63,7 @@ "docs/assets/vyral-portability-proof.png", "docs/assets/vyral-logo-50.png", "docs/concepts/canonical-store.md", + "docs/concepts/generation-bound-retrieval.md", "docs/contributing/adapter-contributor.md", "docs/guides/consumer-handoff.md", "docs/guides/extropic-execution.md", @@ -151,6 +153,7 @@ "deploy/canonical-cutover/cloudbuild-postgres.yaml", "docs/README.md", "docs/concepts/canonical-store.md", + "docs/concepts/generation-bound-retrieval.md", "docs/contributing/adapter-contributor.md", "docs/guides/consumer-handoff.md", "docs/guides/portable-cutovers.md", @@ -163,6 +166,12 @@ "examples/python/retrieval_migration.py", "examples/python/canonical_store_cutover.py", "examples/python/stateless_mcp_round_robin.py", + "experiments/worker-r2-generation-projection/README.md", + "experiments/worker-r2-generation-projection/package-lock.json", + "experiments/worker-r2-generation-projection/package.json", + "experiments/worker-r2-generation-projection/src/object-reader.mjs", + "experiments/worker-r2-generation-projection/src/worker.mjs", + "experiments/worker-r2-generation-projection/verify.mjs", "LICENSE", "packaging/nuget/README.md", "packaging/container-security-release.json", @@ -178,6 +187,7 @@ "design/public-sdk-surface-and-stateless-mcp.md", "design/python-runtime.md", "qualification/adapter-qualification.json", + "qualification/retrieval-projection-qualification.json", "runtimes/python/pyproject.toml", "runtimes/python/LICENSE", "scripts/benchmark-python-runtime.py", @@ -223,6 +233,7 @@ "scripts/verify-container-security-release.py", "scripts/verify-worker-container-release.py", "scripts/verify-ripgrep-retrieval-report.py", + "scripts/verify-worker-r2-generation-projection.py", "scripts/verify-source-quickstart.py", "scripts/validate-aws-live-qualification.sh", "scripts/validate-azure-live-qualification.sh", diff --git a/scripts/generate-adapter-qualification.sh b/scripts/generate-adapter-qualification.sh index 69177cd..8f862d3 100755 --- a/scripts/generate-adapter-qualification.sh +++ b/scripts/generate-adapter-qualification.sh @@ -73,6 +73,9 @@ portable_capabilities = { commit_pattern = re.compile(r"^[0-9a-f]{40}$") identifier_pattern = re.compile(r"^[a-z0-9]+(?:[._-][a-z0-9]+)*$") version_pattern = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$") +opaque_consumer_reference = re.compile( + r"^urn:vyral:private-consumer-evidence:sha256:[0-9a-f]{64}$" +) def fail(message: str) -> None: @@ -218,15 +221,32 @@ for index, adapter in enumerate(adapters): item_where = f"{where}.qualification.evidence[{evidence_index}]" if not isinstance(item, dict): fail(f"{item_where} must be an object") - check_keys(item, item_where, {"kind", "result", "reference", "command"}, {"resultArtifact"}) + check_keys( + item, + item_where, + {"kind", "result", "reference", "command"}, + {"resultArtifact", "disclosure"}, + ) kind = require_string(item.get("kind"), f"{item_where}.kind", 100) if kind not in {"unit_gate", "shared_conformance", "fault_restart", "public_surface", "live_gate", "cleanup", "consumer_validation"}: fail(f"{item_where}.kind is invalid") evidence_kinds.add(kind) if item.get("result") != "passed": fail(f"{item_where}.result must be passed") - require_string(item.get("reference"), f"{item_where}.reference", 300) - require_string(item.get("command"), f"{item_where}.command", 500) + reference = require_string(item.get("reference"), f"{item_where}.reference", 300) + command = require_string(item.get("command"), f"{item_where}.command", 500) + disclosure = item.get("disclosure", "public") + if disclosure not in {"public", "private_opaque"}: + fail(f"{item_where}.disclosure is invalid") + if kind == "consumer_validation" and disclosure != "private_opaque": + fail(f"{item_where} consumer validation must remain private_opaque") + if disclosure == "private_opaque": + if not opaque_consumer_reference.fullmatch(reference): + fail(f"{item_where} private evidence must use an opaque consumer-evidence reference") + if command != "withheld" or "resultArtifact" in item: + fail(f"{item_where} private evidence must withhold commands and result artifacts") + elif opaque_consumer_reference.fullmatch(reference): + fail(f"{item_where} opaque evidence must declare private_opaque disclosure") missing_evidence = level_requirements[level] - evidence_kinds if missing_evidence: fail(f"{where} lacks required {level} evidence: {', '.join(sorted(missing_evidence))}") diff --git a/scripts/render-adapter-qualification.py b/scripts/render-adapter-qualification.py index 7f76257..72d2ef5 100755 --- a/scripts/render-adapter-qualification.py +++ b/scripts/render-adapter-qualification.py @@ -243,7 +243,12 @@ def render(artifact: object) -> str: "| Live qualified | Current provider version, isolated live gate, redacted result artifact, and cleanup evidence. |", "| Consumer validated | Live qualification plus evidence from a representative consumer environment. |", "", - "Provider endpoints, account or tenant identifiers, credentials, and consumer identities do not belong in this public artifact. A workflow run or package presence does not promote an adapter automatically; the checked qualification record is the claim boundary.", + "Provider endpoints, account or tenant identifiers, credentials, and consumer identities do not belong in this public artifact. Consumer-validation evidence is represented by an opaque content digest; the private identity-to-receipt mapping remains outside the repository unless the consumer separately authorizes disclosure. A workflow run or package presence does not promote an adapter automatically; the checked qualification record is the claim boundary.", + "", + "Generation-bound retrieval projections use a separate", + "[qualification artifact](retrieval-projection-qualification.json) because adapter/profile evidence", + "and per-generation rollout receipts are distinct from execution-runtime admission evidence. Its", + "implementation digests keep opaque live evidence bound to the exact public source components.", "", ) ) diff --git a/scripts/test-compute-source-tree-evidence.py b/scripts/test-compute-source-tree-evidence.py new file mode 100644 index 0000000..33ee69b --- /dev/null +++ b/scripts/test-compute-source-tree-evidence.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Regression checks for exact dirty-worktree evidence binding.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +HELPER = Path(__file__).with_name("compute-source-tree-evidence.py") + + +def _run(*arguments: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [*arguments], + check=check, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + +def _evidence(root: Path) -> dict[str, object]: + result = _run(sys.executable, str(HELPER), "--root", str(root)) + return json.loads(result.stdout) + + +def _commit_evidence(root: Path, commit: str) -> dict[str, object]: + result = _run( + sys.executable, + str(HELPER), + "--root", + str(root), + "--commit", + commit, + ) + return json.loads(result.stdout) + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="vyral-source-tree-evidence-") as directory: + root = Path(directory) + _run("git", "-C", str(root), "init", "--quiet") + _run("git", "-C", str(root), "config", "user.name", "Vyral Test") + _run("git", "-C", str(root), "config", "user.email", "test@example.invalid") + (root / ".gitignore").write_text("ignored.tmp\n", encoding="utf-8") + (root / "source.txt").write_text("one\n", encoding="utf-8") + _run("git", "-C", str(root), "add", ".gitignore", "source.txt") + _run("git", "-C", str(root), "commit", "--quiet", "-m", "fixture") + + clean = _evidence(root) + repeated = _evidence(root) + if clean != repeated or clean["sourceDirty"] is not False or clean["fileCount"] != 2: + raise SystemExit("Clean source-tree evidence is not deterministic and exact.") + committed = _commit_evidence(root, str(clean["sourceCommit"])) + if committed != clean: + raise SystemExit("Commit and clean-worktree evidence do not use the same digest model.") + + (root / "ignored.tmp").write_text("ignored\n", encoding="utf-8") + if _evidence(root) != clean: + raise SystemExit("Ignored local state changed source-tree evidence.") + + (root / "source.txt").write_text("two\n", encoding="utf-8") + modified = _evidence(root) + if modified["sourceDirty"] is not True or modified["sourceTreeDigest"] == clean["sourceTreeDigest"]: + raise SystemExit("A tracked modification did not change dirty source-tree evidence.") + if _commit_evidence(root, str(clean["sourceCommit"])) != clean: + raise SystemExit("Dirty worktree state changed immutable commit evidence.") + + (root / "untracked.txt").write_text("new\n", encoding="utf-8") + untracked = _evidence(root) + if untracked["fileCount"] != 3 or untracked["sourceTreeDigest"] == modified["sourceTreeDigest"]: + raise SystemExit("An untracked source file did not enter source-tree evidence.") + + os.symlink("source.txt", root / "untracked-link") + rejected = _run( + sys.executable, + str(HELPER), + "--root", + str(root), + check=False, + ) + if rejected.returncode == 0 or "requires regular files" not in rejected.stderr: + raise SystemExit("Source-tree evidence accepted a symbolic link.") + + print("source-tree-evidence-test=ok") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-generate-adapter-qualification.sh b/scripts/test-generate-adapter-qualification.sh index 6df71ba..1911fd3 100755 --- a/scripts/test-generate-adapter-qualification.sh +++ b/scripts/test-generate-adapter-qualification.sh @@ -56,4 +56,49 @@ if VYRAL_ADAPTER_QUALIFICATION_SOURCE="$work_root/invalid-capability.json" \ exit 1 fi +python3 - "qualification/adapter-qualification.json" "$work_root/invalid-consumer.json" <<'PY' +import json +import sys +from pathlib import Path + +source = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +qualification = source["adapters"][0]["qualification"] +qualification["level"] = "consumer_validated" +qualification["environmentClass"] = "consumer_environment" +qualification["evidence"].append({ + "kind": "consumer_validation", + "result": "passed", + "reference": "consumer-project/private-receipt.json", + "command": "consumer-project/scripts/qualify.sh", +}) +Path(sys.argv[2]).write_text(json.dumps(source), encoding="utf-8") +PY +if VYRAL_ADAPTER_QUALIFICATION_SOURCE="$work_root/invalid-consumer.json" \ + scripts/generate-adapter-qualification.sh "$work_root/invalid-output.json" >/dev/null 2>&1; then + echo "Qualification generation accepted identity-bearing consumer evidence." >&2 + exit 1 +fi + +python3 - "$work_root/invalid-consumer.json" "$work_root/private-consumer.json" <<'PY' +import json +import sys +from pathlib import Path + +source = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +evidence = source["adapters"][0]["qualification"]["evidence"][-1] +evidence.update({ + "reference": "urn:vyral:private-consumer-evidence:sha256:" + "a" * 64, + "command": "withheld", + "disclosure": "private_opaque", +}) +Path(sys.argv[2]).write_text(json.dumps(source), encoding="utf-8") +PY +VYRAL_ADAPTER_QUALIFICATION_SOURCE="$work_root/private-consumer.json" \ + scripts/generate-adapter-qualification.sh "$work_root/private-consumer-output.json" >/dev/null +jq -e ' + .adapters[0].qualification.level == "consumer_validated" and + .adapters[0].qualification.evidence[-1].disclosure == "private_opaque" and + .adapters[0].qualification.evidence[-1].command == "withheld" +' "$work_root/private-consumer-output.json" >/dev/null + printf 'adapter-qualification-generator-test=ok\n' diff --git a/scripts/test-verify-retrieval-projection-qualification.py b/scripts/test-verify-retrieval-projection-qualification.py new file mode 100644 index 0000000..4249ff1 --- /dev/null +++ b/scripts/test-verify-retrieval-projection-qualification.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Regression checks for retrieval-projection qualification policy.""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime, timezone +import importlib.util +from pathlib import Path + + +VERIFIER = Path(__file__).with_name("verify-retrieval-projection-qualification.py") +SPEC = importlib.util.spec_from_file_location("vyral_retrieval_qualification", VERIFIER) +if SPEC is None or SPEC.loader is None: + raise SystemExit("Could not load the retrieval qualification verifier.") +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +SOURCE = { + "sourceCommit": "e0b666601b199db4e667857027aa6424c64e52d5", + "sourceTreeDigest": "sha256:" + "1" * 64, + "sourceDirty": True, +} + + +def _artifact() -> dict[str, object]: + evidence = { + "kind": "unit_gate", + "observedAtUtc": "2026-08-27T12:00:00Z", + "expiresAtUtc": "2026-11-25T12:00:00Z", + "reference": "tests/Vyral.Tests.Aws/OpenSearchGenerationBoundRecordSearchProjectionTests.cs", + "disclosure": "public", + **SOURCE, + "generationIds": [], + "descriptorDigests": [], + } + return { + "schemaVersion": "1.0", + "coreContractVersion": "0.3.0", + "generatedAtUtc": "2026-08-27T12:05:00Z", + "qualificationExpiresAfterDays": 90, + "adapters": [ + { + "adapterId": "opensearch-generation", + "displayName": "OpenSearch generation projection", + "implementation": "OpenSearchGenerationBoundRecordSearchProjection", + "implementationArtifacts": [ + { + "path": "src/Vyral.Aws/OpenSearchGenerationBoundRecordSearchProjection.cs", + "sha256": "sha256:" + "3" * 64, + } + ], + "provider": "opensearch", + "topology": "single-node-container", + "profileId": "vector-v1", + "strategyVersion": "opensearch-3.8", + "qualification": "local_conformant", + "status": "current", + "capabilities": ["complete-coverage", "vector"], + "generationEvidencePolicy": { + "descriptorSchema": "vyral.record-search-projection-generation.v1", + "requiresExactGenerationBinding": True, + "requiresCompleteRequestedCoverage": True, + "mutableHealthEstablishesCompleteness": False, + }, + "evidence": [ + evidence, + {**evidence, "kind": "local_gate", "reference": "tests/Vyral.Tests.Aws/OpenSearchRecordSearchProjectionLocalTests.cs"}, + ], + } + ], + } + + +def _rejects(artifact: dict[str, object], expected: str, *, allow_dirty: bool = True) -> None: + try: + MODULE.validate( + artifact, + as_of=datetime(2026, 8, 27, 12, 10, tzinfo=timezone.utc), + source=SOURCE, + allow_dirty=allow_dirty, + public_disclosure=True, + ) + except SystemExit as error: + if expected not in str(error): + raise SystemExit(f"Unexpected qualification rejection: {error}") from error + return + raise SystemExit(f"Qualification verifier accepted invalid evidence: {expected}") + + +def main() -> int: + artifact = _artifact() + MODULE.validate( + artifact, + as_of=datetime(2026, 8, 27, 12, 10, tzinfo=timezone.utc), + source=SOURCE, + allow_dirty=True, + public_disclosure=True, + ) + + clean_required = deepcopy(artifact) + _rejects(clean_required, "dirty source tree", allow_dirty=False) + + missing_local = deepcopy(artifact) + missing_local["adapters"][0]["evidence"] = missing_local["adapters"][0]["evidence"][:1] + _rejects(missing_local, "local_gate") + + substituted_source = deepcopy(artifact) + substituted_source["adapters"][0]["evidence"][0]["sourceTreeDigest"] = "sha256:" + "2" * 64 + _rejects(substituted_source, "does not match") + + duplicate_implementation = deepcopy(artifact) + duplicate_implementation["adapters"][0]["implementationArtifacts"].append( + duplicate_implementation["adapters"][0]["implementationArtifacts"][0] + ) + _rejects(duplicate_implementation, "duplicate implementation artifact") + + unpaired_generation = deepcopy(artifact) + unpaired_generation["adapters"][0]["evidence"][0]["generationIds"] = ["generation-a"] + _rejects(unpaired_generation, "must be paired") + + endpoint_reference = deepcopy(artifact) + endpoint_reference["adapters"][0]["evidence"][0]["reference"] = "https://provider.example.invalid/evidence" + _rejects(endpoint_reference, "exposes non-portable") + + missing_disclosure = deepcopy(artifact) + del missing_disclosure["adapters"][0]["evidence"][0]["disclosure"] + _rejects(missing_disclosure, "declare its disclosure boundary") + + consumer_validated = deepcopy(artifact) + adapter = consumer_validated["adapters"][0] + adapter["qualification"] = "consumer_validated" + base_evidence = adapter["evidence"][0] + adapter["evidence"].extend( + [ + { + **base_evidence, + "kind": "live_gate", + "reference": "qualification/evidence/retrieval-live.json", + }, + { + **base_evidence, + "kind": "cleanup", + "reference": "qualification/evidence/retrieval-cleanup.json", + }, + { + **base_evidence, + "kind": "consumer_validation", + "reference": "urn:vyral:private-consumer-evidence:sha256:" + "a" * 64, + "disclosure": "private_opaque", + }, + ] + ) + MODULE.validate( + consumer_validated, + as_of=datetime(2026, 8, 27, 12, 10, tzinfo=timezone.utc), + source=SOURCE, + allow_dirty=True, + public_disclosure=True, + ) + + transferred_private_source = deepcopy(consumer_validated) + transferred_private_source["adapters"][0]["evidence"][-1]["sourceTreeDigest"] = ( + "sha256:" + "9" * 64 + ) + MODULE.validate( + transferred_private_source, + as_of=datetime(2026, 8, 27, 12, 10, tzinfo=timezone.utc), + source=SOURCE, + allow_dirty=True, + public_disclosure=True, + ) + + named_consumer = deepcopy(consumer_validated) + named_consumer["adapters"][0]["evidence"][-1].update( + { + "reference": "private-consumer/qualification-receipt.json", + "disclosure": "public", + } + ) + _rejects(named_consumer, "must remain private_opaque") + + exposed_generation = deepcopy(consumer_validated) + private_evidence = exposed_generation["adapters"][0]["evidence"][-1] + private_evidence["generationIds"] = ["consumer-generation"] + private_evidence["descriptorDigests"] = ["sha256:" + "b" * 64] + _rejects(exposed_generation, "cannot expose consumer generation identifiers") + + print("retrieval-projection-qualification-test=ok") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-python-runtime-install.py b/scripts/verify-python-runtime-install.py index fd37076..4ed975e 100755 --- a/scripts/verify-python-runtime-install.py +++ b/scripts/verify-python-runtime-install.py @@ -24,6 +24,7 @@ run_bundled_canonical_scenario, run_bundled_external_worker_scenario, run_bundled_goldens, + run_bundled_projection_generation_scenario, run_bundled_native_execution_scenario, run_bundled_record_store_scenario, ) @@ -36,7 +37,8 @@ assert readiness.contract.operation_count == 129 assert readiness.contract.rest_operation_count == 133 assert readiness.contract.schema_count == 263 -assert len(run_bundled_goldens()) == 12 +assert len(run_bundled_goldens()) == 13 +assert len(run_bundled_projection_generation_scenario()) == 15 assert len(run_bundled_record_store_scenario()) == 17 assert len(run_bundled_external_worker_scenario()) == 3 assert len(run_bundled_canonical_scenario()) == 6 diff --git a/scripts/verify-python-runtime.sh b/scripts/verify-python-runtime.sh index af9c732..6e9c517 100755 --- a/scripts/verify-python-runtime.sh +++ b/scripts/verify-python-runtime.sh @@ -78,11 +78,13 @@ required_package_files = { "vyral_runtime/_conformance/runtime/v1/scenario.schema.json", "vyral_runtime/_conformance/runtime/v1/scenarios/goldens/primitives-hashing.json", "vyral_runtime/_conformance/runtime/v1/scenarios/goldens/collection-snapshot-hash.json", + "vyral_runtime/_conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json", "vyral_runtime/_conformance/runtime/v1/scenarios/goldens/embedding-vectors.json", "vyral_runtime/_conformance/runtime/v1/scenarios/goldens/rag-ingestion-plan.json", "vyral_runtime/_conformance/runtime/v1/scenarios/goldens/graph-record-mapping.json", "vyral_runtime/_conformance/runtime/v1/scenarios/records/core-crud.json", "vyral_runtime/_conformance/runtime/v1/scenarios/records/query-semantics.json", + "vyral_runtime/_conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json", "vyral_runtime/_conformance/runtime/v1/scenarios/external-workers/handler-lifecycle.json", "vyral_runtime/_conformance/runtime/v1/scenarios/canonical/strong-profile.json", "vyral_runtime/_conformance/runtime/v1/scenarios/execution/native-lifecycle.json", @@ -123,11 +125,13 @@ with tarfile.open(sdist_path, "r:gz") as archive: "src/vyral_runtime/_conformance/runtime/v1/manifest.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/primitives-hashing.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/collection-snapshot-hash.json", + "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/record-search-projection-generation.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/embedding-vectors.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/rag-ingestion-plan.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/goldens/graph-record-mapping.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/records/core-crud.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/records/query-semantics.json", + "src/vyral_runtime/_conformance/runtime/v1/scenarios/retrieval/generation-bound-lifecycle.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/external-workers/handler-lifecycle.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/canonical/strong-profile.json", "src/vyral_runtime/_conformance/runtime/v1/scenarios/execution/native-lifecycle.json", @@ -149,6 +153,7 @@ from vyral_runtime import ( run_bundled_native_execution_scenario, run_bundled_external_worker_scenario, run_bundled_goldens, + run_bundled_projection_generation_scenario, run_bundled_record_store_scenario, run_bundled_record_store_scenarios, ) @@ -161,7 +166,8 @@ assert readiness.contract is not None assert readiness.contract.operation_count == 129 assert readiness.contract.rest_operation_count == 133 assert readiness.contract.schema_count == 263 -assert len(run_bundled_goldens()) == 12 +assert len(run_bundled_goldens()) == 13 +assert len(run_bundled_projection_generation_scenario()) == 15 assert len(run_bundled_record_store_scenario()) == 17 assert len(run_bundled_record_store_scenarios()) == 30 assert len(run_bundled_external_worker_scenario()) == 3 @@ -253,6 +259,7 @@ from vyral_runtime import ( run_bundled_native_execution_scenario, run_bundled_external_worker_scenario, run_bundled_goldens, + run_bundled_projection_generation_scenario, run_bundled_record_store_scenario, ) @@ -261,7 +268,8 @@ readiness = runtime.readiness() assert readiness.status == "ok" assert readiness.contract is not None assert readiness.contract.operation_count == 129 -assert len(run_bundled_goldens()) == 12 +assert len(run_bundled_goldens()) == 13 +assert len(run_bundled_projection_generation_scenario()) == 15 assert len(run_bundled_record_store_scenario()) == 17 assert len(run_bundled_external_worker_scenario()) == 3 assert len(run_bundled_canonical_scenario()) == 6 diff --git a/scripts/verify-retrieval-projection-qualification.py b/scripts/verify-retrieval-projection-qualification.py new file mode 100644 index 0000000..35519b8 --- /dev/null +++ b/scripts/verify-retrieval-projection-qualification.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Validate retrieval-projection qualification evidence and its source binding.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import hashlib +import importlib.util +import json +from pathlib import Path +import re +from typing import Any + +from jsonschema import Draft202012Validator, FormatChecker + + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA = ROOT / "qualification/retrieval-projection-qualification.schema.json" +SOURCE_EVIDENCE_HELPER = ROOT / "scripts/compute-source-tree-evidence.py" +OPAQUE_CONSUMER_REFERENCE = re.compile( + r"^urn:vyral:private-(?:consumer-)?evidence:sha256:[0-9a-f]{64}$" +) + + +def _timestamp(value: str, label: str) -> datetime: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise SystemExit(f"{label} is not an ISO-8601 timestamp.") from error + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise SystemExit(f"{label} must include a UTC offset.") + return parsed.astimezone(timezone.utc) + + +def _source_evidence(root: Path, commit: str | None = None) -> dict[str, object]: + specification = importlib.util.spec_from_file_location( + "vyral_source_tree_evidence", + SOURCE_EVIDENCE_HELPER, + ) + if specification is None or specification.loader is None: + raise SystemExit("Could not load the source-tree evidence helper.") + module = importlib.util.module_from_spec(specification) + specification.loader.exec_module(module) + return module.compute_commit(root, commit) if commit else module.compute(root) + + +def _schema_errors(artifact: Any) -> list[str]: + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + validator = Draft202012Validator(schema, format_checker=FormatChecker()) + return [ + f"{'/'.join(str(part) for part in error.absolute_path) or ''}: {error.message}" + for error in sorted(validator.iter_errors(artifact), key=lambda item: list(item.absolute_path)) + ] + + +def validate( + artifact: dict[str, Any], + *, + as_of: datetime, + source: dict[str, object] | None, + allow_dirty: bool, + public_disclosure: bool = False, + source_root: Path | None = None, +) -> None: + errors = _schema_errors(artifact) + if errors: + raise SystemExit("Retrieval qualification schema validation failed:\n" + "\n".join(errors)) + + generated = _timestamp(artifact["generatedAtUtc"], "generatedAtUtc") + if generated > as_of + timedelta(minutes=5): + raise SystemExit("generatedAtUtc cannot be in the future.") + maximum_age = timedelta(days=artifact["qualificationExpiresAfterDays"]) + adapter_ids: set[str] = set() + historical_sources: dict[str, dict[str, object]] = {} + + required_kinds = { + "prototype": {"unit_gate"}, + "local_conformant": {"unit_gate", "local_gate"}, + "live_qualified": {"unit_gate", "local_gate", "live_gate", "cleanup"}, + "consumer_validated": { + "unit_gate", + "local_gate", + "live_gate", + "cleanup", + "consumer_validation", + }, + } + forbidden_reference = re.compile( + r"(?:https?://|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+|access[_-]?key|secret|token)", + re.IGNORECASE, + ) + + for adapter in artifact["adapters"]: + adapter_id = adapter["adapterId"] + if adapter_id in adapter_ids: + raise SystemExit(f"Duplicate retrieval adapter ID: {adapter_id}") + adapter_ids.add(adapter_id) + artifact_paths: set[str] = set() + for implementation_artifact in adapter["implementationArtifacts"]: + relative = implementation_artifact["path"] + parts = Path(relative).parts + if Path(relative).is_absolute() or ".." in parts or relative in artifact_paths: + raise SystemExit( + f"Adapter {adapter_id} has an invalid or duplicate implementation artifact path." + ) + artifact_paths.add(relative) + if source_root is not None: + path = source_root.resolve() / relative + if not path.is_file() or path.is_symlink(): + raise SystemExit( + f"Adapter {adapter_id} implementation artifact is unavailable: {relative}" + ) + observed_digest = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + if observed_digest != implementation_artifact["sha256"]: + raise SystemExit( + f"Adapter {adapter_id} implementation artifact digest does not match: {relative}" + ) + evidence_kinds = {evidence["kind"] for evidence in adapter["evidence"]} + missing_kinds = required_kinds[adapter["qualification"]] - evidence_kinds + if missing_kinds: + raise SystemExit( + f"Adapter {adapter_id} lacks evidence required for {adapter['qualification']}: " + + ", ".join(sorted(missing_kinds)) + ) + + stale_evidence = False + for evidence in adapter["evidence"]: + observed = _timestamp(evidence["observedAtUtc"], f"{adapter_id} observedAtUtc") + expires = _timestamp(evidence["expiresAtUtc"], f"{adapter_id} expiresAtUtc") + if observed > generated: + raise SystemExit(f"Adapter {adapter_id} evidence was observed after artifact generation.") + if expires <= observed or expires - observed > maximum_age: + raise SystemExit(f"Adapter {adapter_id} evidence has an invalid qualification lifetime.") + stale_evidence = stale_evidence or expires <= as_of + if len(evidence["generationIds"]) != len(evidence["descriptorDigests"]): + raise SystemExit( + f"Adapter {adapter_id} generation IDs and descriptor digests must be paired." + ) + if forbidden_reference.search(evidence["reference"]): + raise SystemExit(f"Adapter {adapter_id} evidence reference exposes non-portable or sensitive material.") + disclosure = evidence.get("disclosure") + if public_disclosure and disclosure not in {"public", "private_opaque"}: + raise SystemExit( + f"Adapter {adapter_id} public evidence must declare its disclosure boundary." + ) + if disclosure == "private_opaque": + if not OPAQUE_CONSUMER_REFERENCE.fullmatch(evidence["reference"]): + raise SystemExit( + f"Adapter {adapter_id} private evidence must use an opaque consumer-evidence reference." + ) + if evidence["generationIds"] or evidence["descriptorDigests"]: + raise SystemExit( + f"Adapter {adapter_id} private evidence cannot expose consumer generation identifiers." + ) + elif OPAQUE_CONSUMER_REFERENCE.fullmatch(evidence["reference"]): + raise SystemExit( + f"Adapter {adapter_id} opaque evidence must declare private_opaque disclosure." + ) + if evidence["kind"] == "consumer_validation" and disclosure != "private_opaque": + raise SystemExit( + f"Adapter {adapter_id} consumer validation must remain private_opaque." + ) + if evidence["sourceDirty"] and not allow_dirty: + raise SystemExit( + f"Adapter {adapter_id} evidence is bound to a dirty source tree; " + "use --allow-dirty only for private rehearsal." + ) + if source_root is not None and disclosure != "private_opaque": + commit = evidence["sourceCommit"] + if commit not in historical_sources: + historical_sources[commit] = _source_evidence(source_root, commit) + historical = historical_sources[commit] + for field in ("sourceCommit", "sourceTreeDigest", "sourceDirty"): + if evidence[field] != historical[field]: + raise SystemExit( + f"Adapter {adapter_id} evidence {field} does not match its source commit." + ) + elif source is not None and disclosure != "private_opaque": + for field in ("sourceCommit", "sourceTreeDigest", "sourceDirty"): + if evidence[field] != source[field]: + raise SystemExit( + f"Adapter {adapter_id} evidence {field} does not match the selected source tree." + ) + + expected_status = "stale" if stale_evidence else "current" + if adapter["status"] != expected_status: + raise SystemExit( + f"Adapter {adapter_id} status must be {expected_status} at the selected as-of time." + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact", type=Path) + parser.add_argument("--source-root", type=Path) + parser.add_argument("--as-of", help="ISO-8601 verification time; defaults to now") + parser.add_argument("--allow-dirty", action="store_true") + parser.add_argument( + "--public-disclosure", + action="store_true", + help="require explicit disclosure labels and opaque consumer-validation evidence", + ) + arguments = parser.parse_args() + + artifact = json.loads(arguments.artifact.read_text(encoding="utf-8")) + as_of = _timestamp(arguments.as_of, "--as-of") if arguments.as_of else datetime.now(timezone.utc) + validate( + artifact, + as_of=as_of, + source=None, + allow_dirty=arguments.allow_dirty, + public_disclosure=arguments.public_disclosure, + source_root=arguments.source_root, + ) + print( + "retrieval-projection-qualification=ok " + f"adapters={len(artifact['adapters'])} dirty-allowed={str(arguments.allow_dirty).lower()} " + f"public-disclosure={str(arguments.public_disclosure).lower()}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-runtime-conformance.py b/scripts/verify-runtime-conformance.py index 4918a75..1ae00d4 100755 --- a/scripts/verify-runtime-conformance.py +++ b/scripts/verify-runtime-conformance.py @@ -28,6 +28,7 @@ run_bundled_record_store_scenario, run_bundled_record_store_scenarios, run_bundled_goldens, + run_bundled_projection_generation_scenario, ) @@ -88,6 +89,9 @@ def main() -> int: native_results = run_bundled_native_execution_scenario( FIXTURE_ROOT ) + projection_generation_results = run_bundled_projection_generation_scenario( + FIXTURE_ROOT + ) print( f"runtime-conformance=ok fixture={FIXTURE_VERSION} contract={CONTRACT_VERSION} " f"runner={RUNTIME_VERSION} minimum-runner={manifest.runner_version} " @@ -96,7 +100,8 @@ def main() -> int: f"record-steps={len(record_scenario_results)} " f"external-worker-steps={len(worker_results)} " f"canonical-steps={len(canonical_results)} " - f"native-execution-steps={len(native_results)}" + f"native-execution-steps={len(native_results)} " + f"projection-generation-steps={len(projection_generation_results)}" ) return 0 diff --git a/scripts/verify-worker-r2-generation-projection.py b/scripts/verify-worker-r2-generation-projection.py new file mode 100755 index 0000000..9526317 --- /dev/null +++ b/scripts/verify-worker-r2-generation-projection.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +"""Build and exercise the consumer-neutral Worker/R2 generation projection proof.""" + +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +import hashlib +import json +import math +from pathlib import Path +import struct +import subprocess +from tempfile import TemporaryDirectory +from typing import Any, Iterable, Mapping + +import jsonschema + + +ROOT = Path(__file__).resolve().parent.parent +EXPERIMENT = ROOT / "experiments" / "worker-r2-generation-projection" +CONTRACT_SCHEMA = ( + ROOT + / "src" + / "Vyral.Abstractions" + / "contracts" + / "record-search-projection-generation.v1.schema.json" +) +PARTITION = "public" +BUNDLE_SCHEMA = "vyral.worker-r2-proof-bundle.v1" +REPORT_SCHEMA = "vyral.worker-r2-proof-report.v1" +MANIFEST_SCHEMA = "vyral.private.worker-r2-manifest.v1" +SHARD_SCHEMA = "vyral.private.worker-r2-shard.v1" +CATALOG_SCHEMA = "vyral.private.worker-r2-catalog.v1" +ACTIVE_SCHEMA = "vyral.private.worker-r2-active.v1" + + +class ProofError(RuntimeError): + """The Worker/R2 proof could not establish a required invariant.""" + + +def canonical_bytes(value: Any) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def digest_bytes(value: bytes) -> str: + return "sha256:" + hashlib.sha256(value).hexdigest() + + +def digest_json(value: Any) -> str: + return digest_bytes(canonical_bytes(value)) + + +def object_key(digest: str) -> str: + return "objects/sha256/" + digest.removeprefix("sha256:") + ".json" + + +def score_hex(value: float) -> str: + return struct.pack(">d", value).hex() + + +def route(identifier: str, shard_count: int) -> int: + prefix = hashlib.sha256(identifier.encode("utf-8")).digest()[:8] + return int.from_bytes(prefix, "big") % shard_count + + +def records(count: int) -> list[dict[str, Any]]: + categories = ("agents", "execution", "records", "retrieval") + return [ + { + "partitionKey": PARTITION, + "id": f"record-{index:03d}", + "revision": 1 + (index % 3), + "tokens": ["portable", categories[index % len(categories)], "evidence"], + "metadata": { + "category": categories[index % len(categories)], + "ordinal": index, + "published": True, + }, + } + for index in range(count) + ] + + +def request(query: str, *, limit: int, scan_limit: int) -> dict[str, Any]: + return { + "schema": "vyral.record-search-projection-request.v1", + "generationId": None, + "expectedDescriptorDigest": None, + "query": { + "partitionKeys": [PARTITION], + "filter": None, + "vector": None, + "lexical": { + "query": query, + "fields": None, + "top": limit, + "scanLimit": scan_limit, + "minScore": None, + "scoring": "bm25", + "matchMode": "any", + "fieldBoosts": None, + "bm25K1": 1.2, + "bm25B": 0.75, + "phraseBoost": 0.15, + "exactBoost": 0.25, + "metadataBoost": 0.1, + "prefixMatching": False, + "prefixMinChars": 3, + "requiredPhraseGroups": None, + }, + "orderBy": None, + "limit": limit, + "continuationToken": None, + }, + "deadlineUtc": None, + } + + +def expected_candidates( + documents: Iterable[Mapping[str, Any]], + query: str, + idf: Mapping[str, float], + *, + limit: int, +) -> list[dict[str, Any]]: + normalized = query.lower().strip() + direct = [document for document in documents if document["id"] == normalized] + if direct: + values = [(1000.0, document) for document in direct] + else: + terms = sorted(set(normalized.split())) + values = [] + for document in documents: + frequencies = Counter(document["tokens"]) + score = 0.0 + for term in terms: + frequency = float(frequencies.get(term, 0)) + if frequency == 0: + continue + denominator = frequency + 1.2 * ( + 1 - 0.75 + 0.75 * len(document["tokens"]) / 3.0 + ) + score += idf[term] * frequency * (1.2 + 1) / denominator + if score > 0: + values.append((score, document)) + values.sort(key=lambda item: (-item[0], item[1]["partitionKey"], item[1]["id"])) + return [ + { + "partitionKey": document["partitionKey"], + "id": document["id"], + "revision": document["revision"], + "scoreHex": score_hex(score), + } + for score, document in values[:limit] + ] + + +def descriptor_material(descriptor: Mapping[str, Any]) -> dict[str, Any]: + return { + key: descriptor[key] + for key in ( + "schema", + "collection", + "generationId", + "providerId", + "profileId", + "strategyVersion", + "sourceManifestDigest", + "recordRevisionSetDigest", + "projectionSchemaDigest", + "analyzerDigest", + "configurationDigest", + "expectedItemCount", + "expectedPartitions", + "capabilities", + "artifacts", + "createdAtUtc", + ) + } + + +def build_bundle(record_count: int) -> tuple[dict[str, Any], dict[str, Any]]: + documents = records(record_count) + shard_count = 4 + candidate_capacity = 50 + max_work_units = record_count * 16 + source_manifest_digest = digest_json( + [ + { + "partitionKey": value["partitionKey"], + "id": value["id"], + "revision": value["revision"], + } + for value in documents + ] + ) + revision_set_digest = digest_json( + sorted((value["partitionKey"], value["id"], value["revision"]) for value in documents) + ) + projection_schema_digest = digest_bytes(CONTRACT_SCHEMA.read_bytes()) + analyzer_digest = digest_json( + {"tokenPattern": "[a-z0-9]+", "stopWords": [], "queryAliases": []} + ) + generation_material = { + "provider": "cloudflare-worker-r2", + "sourceManifestDigest": source_manifest_digest, + "recordRevisionSetDigest": revision_set_digest, + "projectionSchemaDigest": projection_schema_digest, + "analyzerDigest": analyzer_digest, + "shardCount": shard_count, + "candidateCapacity": candidate_capacity, + "maxWorkUnits": max_work_units, + } + generation_id = "worker-r2-" + hashlib.sha256( + canonical_bytes(generation_material) + ).hexdigest()[:24] + + document_frequency = Counter( + term for document in documents for term in set(document["tokens"]) + ) + idf = { + term: math.log( + 1 + (record_count - frequency + 0.5) / (frequency + 0.5) + ) + for term, frequency in document_frequency.items() + } + per_shard: list[list[dict[str, Any]]] = [[] for _ in range(shard_count)] + for document in documents: + per_shard[route(document["id"], shard_count)].append(document) + + objects: dict[str, str] = {} + declarations: list[dict[str, Any]] = [] + shard_keys: list[str] = [] + for shard_index, shard_documents in enumerate(per_shard): + direct: dict[str, list[int]] = defaultdict(list) + terms: dict[str, list[Any]] = {} + postings: dict[str, list[list[Any]]] = defaultdict(list) + shard_records = [] + for ordinal, document in enumerate(shard_documents): + shard_records.append( + { + "partitionKey": document["partitionKey"], + "id": document["id"], + "revision": document["revision"], + "length": len(document["tokens"]), + "metadata": document["metadata"], + } + ) + direct[document["id"]].append(ordinal) + for term, frequency in Counter(document["tokens"]).items(): + postings[term].append([ordinal, float(frequency)]) + for term, values in postings.items(): + terms[term] = [idf[term], values] + shard = { + "schemaVersion": SHARD_SCHEMA, + "generationId": generation_id, + "sourceManifestDigest": source_manifest_digest, + "shardId": f"shard-{shard_index:02d}", + "partitions": [PARTITION], + "itemCount": len(shard_records), + "records": shard_records, + "directMap": dict(sorted(direct.items())), + "terms": dict(sorted(terms.items())), + } + encoded = canonical_bytes(shard) + content_hash = digest_bytes(encoded) + key = object_key(content_hash) + objects[key] = encoded.decode("utf-8") + shard_keys.append(key) + declarations.append( + { + "id": shard["shardId"], + "key": key, + "contentHash": content_hash, + "sizeBytes": len(encoded), + "itemCount": shard["itemCount"], + "partitions": [PARTITION], + } + ) + + manifest = { + "schemaVersion": MANIFEST_SCHEMA, + "generationId": generation_id, + "sourceManifestDigest": source_manifest_digest, + "recordRevisionSetDigest": revision_set_digest, + "projectionSchemaDigest": projection_schema_digest, + "analyzerDigest": analyzer_digest, + "scoringContract": "global-bm25-like-card-v1", + "tieBreak": "score-desc-partition-id-asc-v1", + "k1": 1.2, + "b": 0.75, + "tokenPattern": "[a-z0-9]+", + "stopWords": [], + "queryAliases": [], + "expectedItemCount": record_count, + "expectedPartitions": [PARTITION], + "averageDocumentLength": 3.0, + "candidateCapacity": candidate_capacity, + "maxWorkUnits": max_work_units, + "shards": declarations, + } + manifest_bytes = canonical_bytes(manifest) + manifest_digest = digest_bytes(manifest_bytes) + manifest_key = object_key(manifest_digest) + objects[manifest_key] = manifest_bytes.decode("utf-8") + + descriptor = { + "schema": "vyral.record-search-projection-generation.v1", + "collection": "portable-sample", + "generationId": generation_id, + "providerId": "cloudflare-worker-r2", + "profileId": "global-bm25-like-card-v1", + "strategyVersion": "private-json-shards-v1", + "sourceManifestDigest": source_manifest_digest, + "recordRevisionSetDigest": revision_set_digest, + "projectionSchemaDigest": projection_schema_digest, + "analyzerDigest": analyzer_digest, + "configurationDigest": digest_json(generation_material), + "expectedItemCount": record_count, + "expectedPartitions": [PARTITION], + "capabilities": ["completeCoverage", "generationPinnedContinuation", "lexical"], + "artifacts": [ + { + "id": "worker-r2-manifest", + "kind": "worker-r2-generation-manifest", + "contentHash": manifest_digest, + "sizeBytes": len(manifest_bytes), + "mediaType": "application/json", + } + ], + "createdAtUtc": "2026-08-27T00:00:00Z", + } + descriptor["descriptorDigest"] = digest_json(descriptor_material(descriptor)) + schema = json.loads(CONTRACT_SCHEMA.read_text(encoding="utf-8")) + jsonschema.validate(descriptor, schema) + + catalog = { + "schemaVersion": CATALOG_SCHEMA, + "collection": descriptor["collection"], + "generationId": generation_id, + "state": "active", + "descriptor": descriptor, + "manifestKey": manifest_key, + "availablePartitions": [PARTITION], + } + active = { + "schemaVersion": ACTIVE_SCHEMA, + "collection": descriptor["collection"], + "generationId": generation_id, + "descriptorDigest": descriptor["descriptorDigest"], + } + catalog_key = f"catalog/{descriptor['collection']}/{generation_id}.json" + active_key = f"active/{descriptor['collection']}.json" + objects[catalog_key] = canonical_bytes(catalog).decode("utf-8") + objects[active_key] = canonical_bytes(active).decode("utf-8") + + query_texts = ("portable", "retrieval", "portable execution", "record-000") + queries = [ + { + "id": f"query-{index + 1}", + "request": request(query, limit=10, scan_limit=max_work_units), + "expected": expected_candidates(documents, query, idf, limit=10), + } + for index, query in enumerate(query_texts) + ] + paged_query = { + "id": "paged-portable", + "request": request("portable", limit=15, scan_limit=max_work_units), + "expected": expected_candidates(documents, "portable", idf, limit=15), + } + bundle = { + "schemaVersion": BUNDLE_SCHEMA, + "collection": descriptor["collection"], + "generationId": generation_id, + "descriptor": descriptor, + "objects": objects, + "catalogKey": catalog_key, + "activeKey": active_key, + "manifestKey": manifest_key, + "shardKeys": shard_keys, + "queries": queries, + "pagedQuery": paged_query, + "workBoundQuery": queries[0], + } + evidence = { + "recordCount": record_count, + "queryCount": len(queries), + "generationId": generation_id, + "descriptorDigest": descriptor["descriptorDigest"], + "manifestDigest": manifest_digest, + "shardDigests": [value["contentHash"] for value in declarations], + "sourceManifestDigest": source_manifest_digest, + "recordRevisionSetDigest": revision_set_digest, + "shardCount": shard_count, + } + return bundle, evidence + + +def validate_report(report: Mapping[str, Any], query_count: int) -> None: + required_true = ( + "unauthenticatedRequestRejected", + "wrongAuthenticationRejected", + "nonJsonContentRejected", + "oversizedBodyRejected", + "malformedDescriptorDigestRejected", + "exactGenerationSelectionPassed", + "retainedGenerationContinuationPassed", + "continuationTamperRejected", + "continuationRequestSubstitutionRejected", + "retiredGenerationRejected", + "verifiedContentCacheHit", + "missingShardFailedClosed", + "corruptShardFailedClosed", + "incompleteCoverageFailedClosed", + "descriptorFenceRejected", + "expiredDeadlineRejected", + "workLimitFailedClosed", + "inspectionVerifiesArtifacts", + "readerBindingConfigurationFailedClosed", + ) + if report.get("schemaVersion") != REPORT_SCHEMA or report.get("status") != "complete": + raise ProofError("Worker/R2 harness did not return a complete public proof report") + if report.get("queryCount") != query_count: + raise ProofError("Worker/R2 harness did not run every deterministic query") + if report.get("exactCandidateAndScoreParityCount") != query_count: + raise ProofError("Worker/R2 candidates or scores diverged from the exhaustive oracle") + if any(report.get(field) is not True for field in required_true): + raise ProofError("Worker/R2 harness omitted a required lifecycle or failure proof") + if report.get("bindingMode") == "service-reader": + if report.get("serviceReaderGuardsPassed") is not True: + raise ProofError("Service-reader authorization or mutation guards did not pass") + elif report.get("bindingMode") == "direct-r2": + if report.get("serviceReaderGuardsPassed") is not None: + raise ProofError("Direct R2 proof unexpectedly reported service-reader evidence") + else: + raise ProofError("Worker/R2 harness reported an unsupported binding mode") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--records", type=int, default=100) + arguments = parser.parse_args() + if arguments.records < 16: + parser.error("--records must be at least 16 so pagination and work bounds are exercised") + + bundle, generation = build_bundle(arguments.records) + with TemporaryDirectory(prefix="vyral-worker-r2-proof-") as temporary: + bundle_path = Path(temporary) / "bundle.json" + bundle_path.write_bytes(canonical_bytes(bundle)) + reports: dict[str, Any] = {} + for mode in ("direct-r2", "service-reader"): + completed = subprocess.run( + [ + "node", + str(EXPERIMENT / "verify.mjs"), + "--bundle", + str(bundle_path), + "--mode", + mode, + ], + cwd=EXPERIMENT, + check=False, + stdout=subprocess.PIPE, + text=True, + timeout=180, + ) + if completed.returncode != 0: + raise ProofError( + f"Worker/R2 {mode} harness failed with exit code {completed.returncode}" + ) + report = json.loads(completed.stdout) + validate_report(report, len(bundle["queries"])) + reports[mode] = report + + receipt = { + "schemaVersion": "vyral.worker-r2-local-proof-receipt.v1", + "status": "complete", + "generation": generation, + "implementation": { + "workerSha256": digest_bytes((EXPERIMENT / "src" / "worker.mjs").read_bytes()), + "objectReaderSha256": digest_bytes( + (EXPERIMENT / "src" / "object-reader.mjs").read_bytes() + ), + }, + "verification": { + mode: { + key: value + for key, value in report.items() + if key not in {"schemaVersion", "status", "sampleResults"} + } + for mode, report in reports.items() + }, + "boundaries": { + "candidateOnly": True, + "consumerAuthorizationMovedIntoVyral": False, + "providerIndexFormatDeclaredPortable": False, + "networkHopRequired": False, + "liveCloudflareQualifiedByThisReceipt": False, + }, + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( + json.dumps( + { + "output": str(arguments.output), + "status": receipt["status"], + "queries": len(bundle["queries"]), + "records": arguments.records, + "modes": sorted(reports), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/Vyral.Abstractions/Interfaces/ContentAddressedObjectStoreExtensions.cs b/src/Vyral.Abstractions/Interfaces/ContentAddressedObjectStoreExtensions.cs new file mode 100644 index 0000000..dae9c07 --- /dev/null +++ b/src/Vyral.Abstractions/Interfaces/ContentAddressedObjectStoreExtensions.cs @@ -0,0 +1,176 @@ +using System.Security.Cryptography; +using Vyral.Abstractions.Models; + +namespace Vyral.Abstractions.Interfaces; + +public static class ContentAddressedObjectStoreExtensions +{ + /// + /// Stores immutable bytes under a SHA-256-derived key. An existing object is accepted only when + /// its provider-independent content hash and length match. The method never overwrites a + /// different object and uses a conditional create to make retries safe. + /// + public static async Task PutContentAddressedAsync( + this IObjectStore store, + string container, + string keyPrefix, + ReadOnlyMemory content, + string? contentType = null, + Dictionary? metadata = null, + CancellationToken ct = default) + { + var hash = "sha256:" + Convert.ToHexString(SHA256.HashData(content.Span)).ToLowerInvariant(); + using var stream = new MemoryStream(content.ToArray(), writable: false); + return await store.PutContentAddressedAsync( + container, + keyPrefix, + stream, + content.Length, + hash, + contentType, + metadata, + ct); + } + + /// + /// Streaming content-addressed write for large immutable artifacts. The caller supplies the + /// digest and length produced while building the artifact; the object store's independently + /// reported identity must match both before the receipt is accepted. + /// + public static async Task PutContentAddressedAsync( + this IObjectStore store, + string container, + string keyPrefix, + Stream content, + long contentLength, + string contentHash, + string? contentType = null, + Dictionary? metadata = null, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(content); + if (!content.CanRead) + { + throw new InvalidOperationException("A content-addressed object stream must be readable."); + } + if (contentLength < 0) + { + throw new InvalidOperationException("A content-addressed object length cannot be negative."); + } + RequireSha256(contentHash); + if (string.IsNullOrWhiteSpace(container)) + { + throw new InvalidOperationException("A content-addressed object container is required."); + } + if (string.IsNullOrWhiteSpace(keyPrefix)) + { + throw new InvalidOperationException("A content-addressed object key prefix is required."); + } + ObjectMetadataValidator.ValidateUserMetadata(metadata); + + var normalizedPrefix = keyPrefix.Trim().Trim('/'); + if (normalizedPrefix.Length == 0 || normalizedPrefix.Split('/').Any(part => part is "" or "." or "..")) + { + throw new InvalidOperationException("A content-addressed object key prefix must be a safe relative path."); + } + var key = normalizedPrefix + "/sha256/" + contentHash[7..]; + var read = new ObjectReadRequest { Container = container, Key = key }; + var existing = await store.GetObjectAsync(read, ct); + if (existing is not null) + { + await existing.Content.DisposeAsync(); + return Verify(existing, contentHash, contentLength, replayed: true); + } + + ObjectInfo written; + try + { + written = await store.PutObjectAsync(new ObjectWriteRequest + { + Container = container, + Key = key, + Content = content, + ContentType = contentType, + Metadata = metadata is null + ? null + : new Dictionary(metadata, StringComparer.Ordinal), + IfNoneMatch = "*" + }, ct); + } + catch (InvalidOperationException) + { + // A concurrent writer may have won the conditional create. Re-read and accept only the + // exact immutable object; otherwise preserve the original failure. + existing = await store.GetObjectAsync(read, ct); + if (existing is null) + { + throw; + } + await existing.Content.DisposeAsync(); + return Verify(existing, contentHash, contentLength, replayed: true); + } + + try + { + return Verify(written, contentHash, contentLength, replayed: false); + } + catch + { + try + { + await store.DeleteObjectAsync(new ObjectDeleteRequest + { + Container = written.Container, + Key = written.Key, + IfMatch = written.Etag + }, CancellationToken.None); + } + catch + { + // Preserve the integrity failure. Cleanup is best-effort and must not disguise it. + } + throw; + } + } + + private static ContentAddressedObjectReceipt Verify( + ObjectInfo info, + string expectedHash, + long expectedLength, + bool replayed) + { + if (!string.Equals(info.ContentHash, expectedHash, StringComparison.Ordinal) || + info.ContentLength != expectedLength) + { + throw new InvalidOperationException( + "The content-addressed object identity does not match its expected SHA-256 and length."); + } + return new ContentAddressedObjectReceipt + { + Object = info, + ContentHash = expectedHash, + Replayed = replayed + }; + } + + private static void RequireSha256(string value) + { + var valid = value is { Length: 71 } && value.StartsWith("sha256:", StringComparison.Ordinal); + if (valid) + { + foreach (var character in value.AsSpan(7)) + { + if (character is not (>= '0' and <= '9') and not (>= 'a' and <= 'f')) + { + valid = false; + break; + } + } + } + if (!valid) + { + throw new InvalidOperationException("A content-addressed object requires a lowercase SHA-256 digest."); + } + } +} diff --git a/src/Vyral.Abstractions/Interfaces/GenerationBoundRecordSearchProjectionExtensions.cs b/src/Vyral.Abstractions/Interfaces/GenerationBoundRecordSearchProjectionExtensions.cs new file mode 100644 index 0000000..72236c7 --- /dev/null +++ b/src/Vyral.Abstractions/Interfaces/GenerationBoundRecordSearchProjectionExtensions.cs @@ -0,0 +1,55 @@ +using Vyral.Abstractions.Models; + +namespace Vyral.Abstractions.Interfaces; + +public static class GenerationBoundRecordSearchProjectionExtensions +{ + /// + /// Hydrates successful generation-bound candidates from canonical storage. Projection evidence + /// is preserved verbatim; a missing or differing canonical revision is omitted and counted rather + /// than allowing the derived index to become authoritative. + /// + public static async Task SearchGenerationAndHydrateAsync( + this IGenerationBoundRecordSearchProjection projection, + IRecordCollectionStore canonicalStore, + RecordCollectionPolicy policy, + GenerationBoundRecordSearchProjectionRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(projection); + ArgumentNullException.ThrowIfNull(canonicalStore); + ArgumentNullException.ThrowIfNull(policy); + ArgumentNullException.ThrowIfNull(request); + + var candidates = await projection.SearchGenerationAsync(policy, request, ct); + RecordSearchProjectionGenerationContract.ValidateResult(candidates); + if (candidates.Status != RecordSearchProjectionResultStatuses.Succeeded) + { + return new HydratedGenerationBoundRecordSearchProjectionResult { Projection = candidates }; + } + + var items = new List(candidates.Items.Count); + var stale = 0; + foreach (var candidate in candidates.Items) + { + var record = await canonicalStore.GetRecordAsync( + policy.Name, + candidate.PartitionKey, + candidate.Id, + ct); + if (record is null || record.Revision != candidate.Revision) + { + stale++; + continue; + } + items.Add(new VyralRecordMatch { Record = record, Score = candidate.Score }); + } + + return new HydratedGenerationBoundRecordSearchProjectionResult + { + Projection = candidates, + Items = items, + StaleCandidatesDiscarded = stale + }; + } +} diff --git a/src/Vyral.Abstractions/Interfaces/IGenerationBoundRecordSearchProjection.cs b/src/Vyral.Abstractions/Interfaces/IGenerationBoundRecordSearchProjection.cs new file mode 100644 index 0000000..d6bdb02 --- /dev/null +++ b/src/Vyral.Abstractions/Interfaces/IGenerationBoundRecordSearchProjection.cs @@ -0,0 +1,30 @@ +using Vyral.Abstractions.Models; + +namespace Vyral.Abstractions.Interfaces; + +/// +/// Optional candidate-only search projection for providers that can prove immutable generation +/// identity and complete requested-partition coverage. It does not replace +/// for simple eventual projections and does not grant access, +/// select application policy, or return canonical records. +/// +public interface IGenerationBoundRecordSearchProjection +{ + /// + /// Inspects immutable evidence plus current eligibility observations. A null generation ID + /// selects the active generation. Mutable health never substitutes for descriptor completeness. + /// + Task InspectGenerationAsync( + RecordCollectionPolicy policy, + string? generationId = null, + CancellationToken ct = default); + + /// + /// Returns candidates only after every requested partition is covered by one retained immutable + /// generation. Incomplete and unavailable outcomes fail closed with no candidates. + /// + Task SearchGenerationAsync( + RecordCollectionPolicy policy, + GenerationBoundRecordSearchProjectionRequest request, + CancellationToken ct = default); +} diff --git a/src/Vyral.Abstractions/Interfaces/IRecordSearchProjectionGenerationBuilder.cs b/src/Vyral.Abstractions/Interfaces/IRecordSearchProjectionGenerationBuilder.cs new file mode 100644 index 0000000..fdece95 --- /dev/null +++ b/src/Vyral.Abstractions/Interfaces/IRecordSearchProjectionGenerationBuilder.cs @@ -0,0 +1,20 @@ +using Vyral.Abstractions.Models; + +namespace Vyral.Abstractions.Interfaces; + +/// +/// Experimental host-composition seam for constructing and verifying immutable projection +/// generations. It deliberately excludes activation, rollback, retirement, authorization, and +/// application adoption. Implementations store provider-native bytes through the supplied portable +/// object store and return only a compact verified descriptor receipt. +/// +public interface IRecordSearchProjectionGenerationBuilder +{ + string BuilderId { get; } + + Task BuildAndVerifyAsync( + RecordSearchProjectionGenerationBuildRequest request, + IObjectStore artifactStore, + Func? reportProgress = null, + CancellationToken ct = default); +} diff --git a/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationService.cs b/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationService.cs index 538c3f9..7a4705e 100644 --- a/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationService.cs +++ b/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationService.cs @@ -66,6 +66,17 @@ public class RetrievalEvaluationVariant [JsonPropertyName("label")] public string? Label { get; set; } + /// + /// Optional host-resolved evaluation target. This is deliberately separate from the variant + /// id: a variant names one comparison arm, while the target selects an exact registered + /// retrieval implementation and, when supplied, fences an immutable generation. + /// + // This remains an in-process experiment until the versioned wire schemas and peer SDKs are + // deliberately expanded. JsonIgnore prevents the .NET DTO from silently creating an + // undocumented server contract in the meantime. + [JsonIgnore] + public RetrievalEvaluationTargetReference? Target { get; set; } + [JsonPropertyName("profile")] public string? Profile { get; set; } @@ -429,6 +440,9 @@ public class RetrievalEvaluationVariantResult [JsonPropertyName("label")] public string? Label { get; set; } + [JsonIgnore] + public RetrievalEvaluationTargetEvidence? Target { get; set; } + [JsonPropertyName("status")] public string Status { get; set; } = string.Empty; @@ -448,6 +462,36 @@ public class RetrievalEvaluationVariantResult public List Cases { get; set; } = new(); } +/// +/// Host-resolved target selection for evaluation. The identifier refers to configured code, not a +/// caller-controlled URL or provider SDK. Generation fields are optional caller fences; the +/// resolved evidence always reports the exact target actually used. +/// +public sealed class RetrievalEvaluationTargetReference +{ + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + + [JsonPropertyName("generationId")] + public string? GenerationId { get; set; } + + [JsonPropertyName("expectedGenerationDescriptorDigest")] + public string? ExpectedGenerationDescriptorDigest { get; set; } +} + +/// Immutable identity reported for the resolved evaluation target. +public sealed class RetrievalEvaluationTargetEvidence +{ + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + + [JsonPropertyName("generationId")] + public string? GenerationId { get; set; } + + [JsonPropertyName("generationDescriptorDigest")] + public string? GenerationDescriptorDigest { get; set; } +} + public class RetrievalEvaluationMetrics { [JsonPropertyName("requested")] diff --git a/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationTargetResolver.cs b/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationTargetResolver.cs new file mode 100644 index 0000000..936fcdd --- /dev/null +++ b/src/Vyral.Abstractions/Interfaces/IRetrievalEvaluationTargetResolver.cs @@ -0,0 +1,17 @@ +namespace Vyral.Abstractions.Interfaces; + +/// +/// Resolves an evaluation target from host-owned registrations. Implementations must not interpret +/// the target id as a URL, assembly name, provider type, or other caller-selected code location. +/// +public interface IRetrievalEvaluationTargetResolver +{ + RetrievalEvaluationResolvedTarget Resolve(RetrievalEvaluationTargetReference target); +} + +/// One resolved retrieval service and the immutable identity it will exercise. +public sealed class RetrievalEvaluationResolvedTarget +{ + public required IRetrievalService Service { get; init; } + public required RetrievalEvaluationTargetEvidence Evidence { get; init; } +} diff --git a/src/Vyral.Abstractions/Models/GenerationBoundRecordSearchProjectionModels.cs b/src/Vyral.Abstractions/Models/GenerationBoundRecordSearchProjectionModels.cs new file mode 100644 index 0000000..7ec60f2 --- /dev/null +++ b/src/Vyral.Abstractions/Models/GenerationBoundRecordSearchProjectionModels.cs @@ -0,0 +1,987 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Vyral.Abstractions.Models; + +public static class RecordSearchProjectionGenerationSchemas +{ + public const string DescriptorV1 = "vyral.record-search-projection-generation.v1"; + public const string SearchRequestV1 = "vyral.record-search-projection-request.v1"; + public const string SearchResultV1 = "vyral.record-search-projection-result.v1"; + public const string InspectionV1 = "vyral.record-search-projection-inspection.v1"; + public const string BuildRequestV1 = "vyral.record-search-projection-build-request.v1"; + public const string BuildReceiptV1 = "vyral.record-search-projection-build-receipt.v1"; +} + +public static class RecordSearchProjectionBuildStatuses +{ + public const string Verified = "verified"; +} + +public static class RecordSearchProjectionGenerationStates +{ + public const string Active = "active"; + public const string Retained = "retained"; + public const string Retired = "retired"; +} + +public static class RecordSearchProjectionCoverageStatuses +{ + public const string Complete = "complete"; + public const string Incomplete = "incomplete"; + public const string Unavailable = "unavailable"; +} + +public static class RecordSearchProjectionResultStatuses +{ + public const string Succeeded = "succeeded"; + public const string Failed = "failed"; +} + +public static class RecordSearchProjectionFailureCodes +{ + public const string GenerationUnavailable = "generationUnavailable"; + public const string GenerationRetired = "generationRetired"; + public const string GenerationDescriptorMismatch = "generationDescriptorMismatch"; + public const string CoverageIncomplete = "coverageIncomplete"; + public const string InvalidContinuation = "invalidContinuation"; + public const string ExpiredContinuation = "expiredContinuation"; + public const string DeadlineExceeded = "deadlineExceeded"; + public const string WorkLimitExceeded = "workLimitExceeded"; + public const string UnsupportedQuery = "unsupportedQuery"; +} + +public static class RecordSearchProjectionCacheStatuses +{ + public const string Hit = "hit"; + public const string Miss = "miss"; + public const string Bypass = "bypass"; + public const string NotApplicable = "notApplicable"; +} + +public static class RecordSearchProjectionGenerationCapabilities +{ + public const string CompleteCoverage = "completeCoverage"; + /// + /// Safety property: every continuation the adapter emits is authenticated and bound to its + /// immutable generation. This does not itself claim that a particular query shape is pageable; + /// a bounded first-page-only implementation can satisfy it by never emitting a continuation. + /// + public const string GenerationPinnedContinuation = "generationPinnedContinuation"; + public const string Lexical = "lexical"; + public const string Vector = "vector"; +} + +/// +/// Portable identity and completeness evidence for one immutable derived search generation. +/// Provider-native index files and routing structures remain opaque; this descriptor binds only +/// the evidence a consumer needs to select and verify a generation safely. +/// +public sealed class RecordSearchProjectionGenerationDescriptor +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } = RecordSearchProjectionGenerationSchemas.DescriptorV1; + + [JsonPropertyName("collection")] + public string Collection { get; set; } = string.Empty; + + [JsonPropertyName("generationId")] + public string GenerationId { get; set; } = string.Empty; + + [JsonPropertyName("providerId")] + public string ProviderId { get; set; } = string.Empty; + + [JsonPropertyName("profileId")] + public string ProfileId { get; set; } = string.Empty; + + [JsonPropertyName("strategyVersion")] + public string StrategyVersion { get; set; } = string.Empty; + + [JsonPropertyName("sourceManifestDigest")] + public string SourceManifestDigest { get; set; } = string.Empty; + + [JsonPropertyName("recordRevisionSetDigest")] + public string RecordRevisionSetDigest { get; set; } = string.Empty; + + [JsonPropertyName("projectionSchemaDigest")] + public string ProjectionSchemaDigest { get; set; } = string.Empty; + + [JsonPropertyName("analyzerDigest")] + public string? AnalyzerDigest { get; set; } + + [JsonPropertyName("configurationDigest")] + public string ConfigurationDigest { get; set; } = string.Empty; + + [JsonPropertyName("expectedItemCount")] + public long ExpectedItemCount { get; set; } + + [JsonPropertyName("expectedPartitions")] + public List ExpectedPartitions { get; set; } = new(); + + [JsonPropertyName("capabilities")] + public List Capabilities { get; set; } = new(); + + [JsonPropertyName("artifacts")] + public List Artifacts { get; set; } = new(); + + [JsonPropertyName("createdAtUtc")] + public DateTime CreatedAtUtc { get; set; } + + [JsonPropertyName("descriptorDigest")] + public string DescriptorDigest { get; set; } = string.Empty; +} + +public sealed class RecordSearchProjectionGenerationArtifact +{ + [JsonPropertyName("id")] + public string Id { get; set; } = string.Empty; + + [JsonPropertyName("kind")] + public string Kind { get; set; } = string.Empty; + + [JsonPropertyName("contentHash")] + public string ContentHash { get; set; } = string.Empty; + + [JsonPropertyName("sizeBytes")] + public long SizeBytes { get; set; } + + [JsonPropertyName("mediaType")] + public string? MediaType { get; set; } +} + +public sealed class GenerationBoundRecordSearchProjectionRequest +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } = RecordSearchProjectionGenerationSchemas.SearchRequestV1; + + /// + /// Exact generation requested. Null selects the active generation for a first page. A + /// continuation always selects its retained generation and cannot be remapped to the active one. + /// + [JsonPropertyName("generationId")] + public string? GenerationId { get; set; } + + /// Optional caller fence against descriptor substitution. + [JsonPropertyName("expectedDescriptorDigest")] + public string? ExpectedDescriptorDigest { get; set; } + + [JsonPropertyName("query")] + public QueryEnvelope Query { get; set; } = new(); + + [JsonPropertyName("deadlineUtc")] + public DateTime? DeadlineUtc { get; set; } +} + +public sealed class RecordSearchProjectionCoverage +{ + [JsonPropertyName("status")] + public string Status { get; set; } = RecordSearchProjectionCoverageStatuses.Unavailable; + + [JsonPropertyName("requestedPartitions")] + public List RequestedPartitions { get; set; } = new(); + + [JsonPropertyName("coveredPartitions")] + public List CoveredPartitions { get; set; } = new(); + + [JsonPropertyName("missingPartitions")] + public List MissingPartitions { get; set; } = new(); +} + +public sealed class RecordSearchProjectionWorkDiagnostics +{ + [JsonPropertyName("workLimit")] + public long? WorkLimit { get; set; } + + [JsonPropertyName("workUnits")] + public long? WorkUnits { get; set; } + + [JsonPropertyName("candidateBound")] + public long? CandidateBound { get; set; } + + [JsonPropertyName("candidateCount")] + public long? CandidateCount { get; set; } + + [JsonPropertyName("returnedCount")] + public long ReturnedCount { get; set; } + + [JsonPropertyName("cacheStatus")] + public string CacheStatus { get; set; } = RecordSearchProjectionCacheStatuses.NotApplicable; + + /// + /// Bounded, privacy-safe adapter diagnostics. Provider SDK objects, query text, credentials, + /// authorization decisions, and raw logs do not belong here. + /// + [JsonPropertyName("details")] + public JsonObject? Details { get; set; } +} + +public sealed class RecordSearchProjectionFailure +{ + [JsonPropertyName("code")] + public string Code { get; set; } = string.Empty; + + [JsonPropertyName("message")] + public string Message { get; set; } = string.Empty; + + [JsonPropertyName("retryable")] + public bool Retryable { get; set; } +} + +/// +/// Candidate-only result bound to one immutable generation. A failed or incomplete result must +/// never contain candidates or a continuation token. +/// +public sealed class GenerationBoundRecordSearchProjectionResult +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } = RecordSearchProjectionGenerationSchemas.SearchResultV1; + + [JsonPropertyName("status")] + public string Status { get; set; } = RecordSearchProjectionResultStatuses.Failed; + + [JsonPropertyName("generationId")] + public string? GenerationId { get; set; } + + [JsonPropertyName("generationDescriptorDigest")] + public string? GenerationDescriptorDigest { get; set; } + + [JsonPropertyName("items")] + public List Items { get; set; } = new(); + + [JsonPropertyName("continuationToken")] + public string? ContinuationToken { get; set; } + + [JsonPropertyName("consistency")] + public string Consistency { get; set; } = "immutableGeneration"; + + [JsonPropertyName("coverage")] + public RecordSearchProjectionCoverage Coverage { get; set; } = new(); + + [JsonPropertyName("diagnostics")] + public RecordSearchProjectionWorkDiagnostics Diagnostics { get; set; } = new(); + + [JsonPropertyName("failure")] + public RecordSearchProjectionFailure? Failure { get; set; } +} + +public sealed class RecordSearchProjectionGenerationInspection +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } = RecordSearchProjectionGenerationSchemas.InspectionV1; + + [JsonPropertyName("descriptor")] + public RecordSearchProjectionGenerationDescriptor Descriptor { get; set; } = new(); + + [JsonPropertyName("state")] + public string State { get; set; } = RecordSearchProjectionGenerationStates.Retained; + + [JsonPropertyName("availablePartitions")] + public List AvailablePartitions { get; set; } = new(); + + [JsonPropertyName("coverageStatus")] + public string CoverageStatus { get; set; } = RecordSearchProjectionCoverageStatuses.Unavailable; + + [JsonPropertyName("observedAtUtc")] + public DateTime ObservedAtUtc { get; set; } +} + +/// +/// Portable payload for durable construction and verification. Idempotency belongs to the +/// execution admission request so it is hashed and enforced by IExecutionRuntime rather than +/// duplicated inside this payload. +/// +public sealed class RecordSearchProjectionGenerationBuildRequest +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } = RecordSearchProjectionGenerationSchemas.BuildRequestV1; + + [JsonPropertyName("collection")] + public string Collection { get; set; } = string.Empty; + + [JsonPropertyName("generationId")] + public string GenerationId { get; set; } = string.Empty; + + [JsonPropertyName("builderId")] + public string BuilderId { get; set; } = string.Empty; + + [JsonPropertyName("providerId")] + public string ProviderId { get; set; } = string.Empty; + + [JsonPropertyName("profileId")] + public string ProfileId { get; set; } = string.Empty; + + [JsonPropertyName("strategyVersion")] + public string StrategyVersion { get; set; } = string.Empty; + + [JsonPropertyName("sourceManifestRef")] + public string SourceManifestRef { get; set; } = string.Empty; + + [JsonPropertyName("sourceManifestDigest")] + public string SourceManifestDigest { get; set; } = string.Empty; + + [JsonPropertyName("expectedRecordRevisionSetDigest")] + public string? ExpectedRecordRevisionSetDigest { get; set; } + + [JsonPropertyName("projectionSchemaDigest")] + public string ProjectionSchemaDigest { get; set; } = string.Empty; + + [JsonPropertyName("analyzerDigest")] + public string? AnalyzerDigest { get; set; } + + [JsonPropertyName("configurationDigest")] + public string ConfigurationDigest { get; set; } = string.Empty; + + [JsonPropertyName("expectedItemCount")] + public long ExpectedItemCount { get; set; } + + [JsonPropertyName("expectedPartitions")] + public List ExpectedPartitions { get; set; } = new(); + + [JsonPropertyName("deadlineUtc")] + public DateTime? DeadlineUtc { get; set; } +} + +public sealed class RecordSearchProjectionGenerationBuildProgress +{ + [JsonPropertyName("stage")] + public string Stage { get; set; } = string.Empty; + + [JsonPropertyName("completed")] + public long Completed { get; set; } + + [JsonPropertyName("total")] + public long? Total { get; set; } + + [JsonPropertyName("checkpoint")] + public JsonObject? Checkpoint { get; set; } +} + +/// +/// Compact terminal receipt. Full provider logs and index bytes remain in immutable artifacts; +/// this receipt binds their digests to the verified portable descriptor. +/// +public sealed class RecordSearchProjectionGenerationBuildReceipt +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } = RecordSearchProjectionGenerationSchemas.BuildReceiptV1; + + [JsonPropertyName("status")] + public string Status { get; set; } = RecordSearchProjectionBuildStatuses.Verified; + + [JsonPropertyName("builderId")] + public string BuilderId { get; set; } = string.Empty; + + [JsonPropertyName("descriptor")] + public RecordSearchProjectionGenerationDescriptor Descriptor { get; set; } = new(); + + [JsonPropertyName("evaluationReceiptDigest")] + public string? EvaluationReceiptDigest { get; set; } + + [JsonPropertyName("builtAtUtc")] + public DateTime BuiltAtUtc { get; set; } +} + +public sealed class HydratedGenerationBoundRecordSearchProjectionResult +{ + [JsonPropertyName("projection")] + public GenerationBoundRecordSearchProjectionResult Projection { get; set; } = new(); + + [JsonPropertyName("items")] + public List Items { get; set; } = new(); + + [JsonPropertyName("staleCandidatesDiscarded")] + public int StaleCandidatesDiscarded { get; set; } +} + +/// Deterministic validation and hashing shared by projection implementations. +public static class RecordSearchProjectionGenerationContract +{ + private static readonly JsonSerializerOptions JsonOptions = + new(JsonSerializerDefaults.Web) { WriteIndented = false }; + + public static string SealDescriptor(RecordSearchProjectionGenerationDescriptor descriptor) + { + ArgumentNullException.ThrowIfNull(descriptor); + ValidateDescriptor(descriptor, requireDescriptorDigest: false); + descriptor.DescriptorDigest = ComputeDescriptorDigest(descriptor); + return descriptor.DescriptorDigest; + } + + public static string ComputeDescriptorDigest(RecordSearchProjectionGenerationDescriptor descriptor) + { + ArgumentNullException.ThrowIfNull(descriptor); + var material = new + { + schema = descriptor.Schema, + collection = descriptor.Collection, + generationId = descriptor.GenerationId, + providerId = descriptor.ProviderId, + profileId = descriptor.ProfileId, + strategyVersion = descriptor.StrategyVersion, + sourceManifestDigest = descriptor.SourceManifestDigest, + recordRevisionSetDigest = descriptor.RecordRevisionSetDigest, + projectionSchemaDigest = descriptor.ProjectionSchemaDigest, + analyzerDigest = descriptor.AnalyzerDigest, + configurationDigest = descriptor.ConfigurationDigest, + expectedItemCount = descriptor.ExpectedItemCount, + expectedPartitions = descriptor.ExpectedPartitions.OrderBy(value => value, StringComparer.Ordinal), + capabilities = descriptor.Capabilities.OrderBy(value => value, StringComparer.Ordinal), + artifacts = descriptor.Artifacts + .OrderBy(value => value.Id, StringComparer.Ordinal) + .Select(value => new + { + id = value.Id, + kind = value.Kind, + contentHash = value.ContentHash, + sizeBytes = value.SizeBytes, + mediaType = value.MediaType + }), + createdAtUtc = descriptor.CreatedAtUtc.ToUniversalTime() + }; + return Hash(CanonicalJson.SerializeUtf8(material, JsonOptions)); + } + + public static string ComputeRequestFingerprint( + GenerationBoundRecordSearchProjectionRequest request, + string generationId, + string descriptorDigest, + IReadOnlyCollection requestedPartitions) + { + ArgumentNullException.ThrowIfNull(request); + var query = request.Query ?? throw new InvalidOperationException("A generation-bound projection request requires a query."); + var material = new + { + schema = request.Schema, + generationId, + descriptorDigest, + requestedPartitions = requestedPartitions.OrderBy(value => value, StringComparer.Ordinal), + query = new + { + partitionKeys = query.PartitionKeys?.OrderBy(value => value, StringComparer.Ordinal), + filter = query.Filter, + vector = query.Vector, + lexical = query.Lexical, + orderBy = query.OrderBy, + limit = query.Limit + } + }; + return Hash(CanonicalJson.SerializeUtf8(material, JsonOptions)); + } + + public static void ValidateDescriptor( + RecordSearchProjectionGenerationDescriptor descriptor, + bool requireDescriptorDigest = true) + { + ArgumentNullException.ThrowIfNull(descriptor); + RequireExact(descriptor.Schema, RecordSearchProjectionGenerationSchemas.DescriptorV1, "descriptor schema"); + RequireIdentifier(descriptor.Collection, "collection"); + RequireIdentifier(descriptor.GenerationId, "generationId"); + RequireIdentifier(descriptor.ProviderId, "providerId"); + RequireIdentifier(descriptor.ProfileId, "profileId"); + RequireIdentifier(descriptor.StrategyVersion, "strategyVersion"); + RequireSha256(descriptor.SourceManifestDigest, "sourceManifestDigest"); + RequireSha256(descriptor.RecordRevisionSetDigest, "recordRevisionSetDigest"); + RequireSha256(descriptor.ProjectionSchemaDigest, "projectionSchemaDigest"); + if (descriptor.AnalyzerDigest is not null) + { + RequireSha256(descriptor.AnalyzerDigest, "analyzerDigest"); + } + RequireSha256(descriptor.ConfigurationDigest, "configurationDigest"); + + if (descriptor.ExpectedItemCount < 0) + { + throw new InvalidOperationException("expectedItemCount cannot be negative."); + } + RequireCanonicalSet(descriptor.ExpectedPartitions, "expectedPartitions", requireNonEmpty: true); + RequireCanonicalSet(descriptor.Capabilities, "capabilities", requireNonEmpty: true); + if (!descriptor.Capabilities.Contains(RecordSearchProjectionGenerationCapabilities.CompleteCoverage, StringComparer.Ordinal) || + !descriptor.Capabilities.Contains(RecordSearchProjectionGenerationCapabilities.GenerationPinnedContinuation, StringComparer.Ordinal)) + { + throw new InvalidOperationException( + "A generation-bound descriptor requires complete-coverage and generation-pinned-continuation capabilities."); + } + if (descriptor.CreatedAtUtc == default || descriptor.CreatedAtUtc.Kind != DateTimeKind.Utc) + { + throw new InvalidOperationException("createdAtUtc must be a non-default UTC timestamp."); + } + + var artifactIds = new List(descriptor.Artifacts.Count); + foreach (var artifact in descriptor.Artifacts) + { + RequireIdentifier(artifact.Id, "artifact id"); + RequireIdentifier(artifact.Kind, "artifact kind"); + RequireSha256(artifact.ContentHash, $"artifact '{artifact.Id}' contentHash"); + if (artifact.SizeBytes < 0) + { + throw new InvalidOperationException($"Artifact '{artifact.Id}' sizeBytes cannot be negative."); + } + if (artifact.MediaType is { } mediaType && + (string.IsNullOrWhiteSpace(mediaType) || mediaType.Length > 200 || + !string.Equals(mediaType, mediaType.Trim(), StringComparison.Ordinal) || + mediaType.Any(char.IsControl))) + { + throw new InvalidOperationException($"Artifact '{artifact.Id}' mediaType is invalid."); + } + artifactIds.Add(artifact.Id); + } + RequireCanonicalSet(artifactIds, "artifact ids", requireNonEmpty: false); + + if (!requireDescriptorDigest) + { + return; + } + RequireSha256(descriptor.DescriptorDigest, "descriptorDigest"); + var actual = ComputeDescriptorDigest(descriptor); + if (!string.Equals(actual, descriptor.DescriptorDigest, StringComparison.Ordinal)) + { + throw new InvalidOperationException("descriptorDigest does not match the canonical descriptor material."); + } + } + + public static void ValidateRequest(GenerationBoundRecordSearchProjectionRequest request) + { + ArgumentNullException.ThrowIfNull(request); + RequireExact(request.Schema, RecordSearchProjectionGenerationSchemas.SearchRequestV1, "request schema"); + if (request.GenerationId is not null) + { + RequireIdentifier(request.GenerationId, "generationId"); + } + if (request.ExpectedDescriptorDigest is not null) + { + RequireSha256(request.ExpectedDescriptorDigest, "expectedDescriptorDigest"); + } + if (request.Query is null) + { + throw new InvalidOperationException("A generation-bound projection request requires a query."); + } + if (request.DeadlineUtc is { } deadline && deadline.Kind != DateTimeKind.Utc) + { + throw new InvalidOperationException("deadlineUtc must be UTC when supplied."); + } + } + + public static void ValidateResult(GenerationBoundRecordSearchProjectionResult result) + { + ArgumentNullException.ThrowIfNull(result); + RequireExact(result.Schema, RecordSearchProjectionGenerationSchemas.SearchResultV1, "result schema"); + if (result.Consistency != "immutableGeneration") + { + throw new InvalidOperationException("A generation-bound result must use immutableGeneration consistency."); + } + ValidateCoverage(result.Coverage); + ValidateDiagnostics(result.Diagnostics); + if (result.GenerationId is not null) + { + RequireIdentifier(result.GenerationId, "generationId"); + } + if (result.GenerationDescriptorDigest is not null) + { + RequireSha256(result.GenerationDescriptorDigest, "generationDescriptorDigest"); + if (result.GenerationId is null) + { + throw new InvalidOperationException("A generation descriptor digest requires its generation ID."); + } + } + if (result.ContinuationToken is { Length: > 8192 }) + { + throw new InvalidOperationException("A generation-bound continuation cannot exceed 8192 characters."); + } + + if (result.Status == RecordSearchProjectionResultStatuses.Succeeded) + { + RequireIdentifier(result.GenerationId, "generationId"); + RequireSha256(result.GenerationDescriptorDigest, "generationDescriptorDigest"); + if (result.Failure is not null) + { + throw new InvalidOperationException("A successful generation-bound result cannot include a failure."); + } + if (result.Coverage.Status != RecordSearchProjectionCoverageStatuses.Complete) + { + throw new InvalidOperationException("A successful generation-bound result requires complete coverage."); + } + if (result.Diagnostics.ReturnedCount != result.Items.Count) + { + throw new InvalidOperationException("returnedCount must equal the number of returned candidates."); + } + ValidateCandidates(result.Items); + return; + } + + if (result.Status != RecordSearchProjectionResultStatuses.Failed) + { + throw new InvalidOperationException("A generation-bound result status must be succeeded or failed."); + } + if (result.Failure is null || string.IsNullOrWhiteSpace(result.Failure.Message)) + { + throw new InvalidOperationException("A failed generation-bound result requires a structured failure."); + } + RequireIdentifier(result.Failure.Code, "failure code"); + if (result.Failure.Message.Length > 1000) + { + throw new InvalidOperationException("A generation-bound failure message cannot exceed 1000 characters."); + } + if (result.Failure.Message.Any(char.IsControl)) + { + throw new InvalidOperationException("A generation-bound failure message cannot contain control characters."); + } + if (result.Items.Count != 0 || result.ContinuationToken is not null || result.Diagnostics.ReturnedCount != 0) + { + throw new InvalidOperationException("A failed generation-bound result cannot expose candidates or a continuation token."); + } + } + + public static void ValidateInspection(RecordSearchProjectionGenerationInspection inspection) + { + ArgumentNullException.ThrowIfNull(inspection); + RequireExact(inspection.Schema, RecordSearchProjectionGenerationSchemas.InspectionV1, "inspection schema"); + ValidateDescriptor(inspection.Descriptor); + if (inspection.State is not RecordSearchProjectionGenerationStates.Active and + not RecordSearchProjectionGenerationStates.Retained and + not RecordSearchProjectionGenerationStates.Retired) + { + throw new InvalidOperationException("A generation inspection state must be active, retained, or retired."); + } + RequireCanonicalSet(inspection.AvailablePartitions, "availablePartitions", requireNonEmpty: false); + if (inspection.AvailablePartitions.Except(inspection.Descriptor.ExpectedPartitions, StringComparer.Ordinal).Any()) + { + throw new InvalidOperationException("availablePartitions must be a subset of expectedPartitions."); + } + if (inspection.State == RecordSearchProjectionGenerationStates.Retired && inspection.AvailablePartitions.Count != 0) + { + throw new InvalidOperationException("A retired generation cannot report available partitions."); + } + var expectedCoverage = inspection.State == RecordSearchProjectionGenerationStates.Retired + ? RecordSearchProjectionCoverageStatuses.Unavailable + : inspection.AvailablePartitions.SequenceEqual(inspection.Descriptor.ExpectedPartitions, StringComparer.Ordinal) + ? RecordSearchProjectionCoverageStatuses.Complete + : RecordSearchProjectionCoverageStatuses.Incomplete; + if (!string.Equals(expectedCoverage, inspection.CoverageStatus, StringComparison.Ordinal)) + { + throw new InvalidOperationException("coverageStatus does not match the inspected partition evidence and lifecycle state."); + } + if (inspection.ObservedAtUtc == default || inspection.ObservedAtUtc.Kind != DateTimeKind.Utc) + { + throw new InvalidOperationException("observedAtUtc must be a non-default UTC timestamp."); + } + } + + public static void ValidateBuildRequest(RecordSearchProjectionGenerationBuildRequest request) + { + ArgumentNullException.ThrowIfNull(request); + RequireExact(request.Schema, RecordSearchProjectionGenerationSchemas.BuildRequestV1, "build request schema"); + RequireIdentifier(request.Collection, "collection"); + RequireIdentifier(request.GenerationId, "generationId"); + RequireIdentifier(request.BuilderId, "builderId"); + RequireIdentifier(request.ProviderId, "providerId"); + RequireIdentifier(request.ProfileId, "profileId"); + RequireIdentifier(request.StrategyVersion, "strategyVersion"); + RequireIdentifier(request.SourceManifestRef, "sourceManifestRef"); + if (request.SourceManifestRef.Contains('?') || request.SourceManifestRef.Contains('#')) + { + throw new InvalidOperationException("sourceManifestRef must be a stable reference without query parameters or fragments."); + } + if (Uri.TryCreate(request.SourceManifestRef, UriKind.Absolute, out var sourceUri) && + !string.IsNullOrEmpty(sourceUri.UserInfo)) + { + throw new InvalidOperationException("sourceManifestRef cannot contain URI credentials."); + } + RequireSha256(request.SourceManifestDigest, "sourceManifestDigest"); + if (request.ExpectedRecordRevisionSetDigest is not null) + { + RequireSha256(request.ExpectedRecordRevisionSetDigest, "expectedRecordRevisionSetDigest"); + } + RequireSha256(request.ProjectionSchemaDigest, "projectionSchemaDigest"); + if (request.AnalyzerDigest is not null) + { + RequireSha256(request.AnalyzerDigest, "analyzerDigest"); + } + RequireSha256(request.ConfigurationDigest, "configurationDigest"); + if (request.ExpectedItemCount < 0) + { + throw new InvalidOperationException("expectedItemCount cannot be negative."); + } + RequireCanonicalSet(request.ExpectedPartitions, "expectedPartitions", requireNonEmpty: true); + if (request.DeadlineUtc is { } deadline && deadline.Kind != DateTimeKind.Utc) + { + throw new InvalidOperationException("deadlineUtc must be UTC when supplied."); + } + } + + public static void ValidateBuildReceipt( + RecordSearchProjectionGenerationBuildRequest request, + RecordSearchProjectionGenerationBuildReceipt receipt) + { + ValidateBuildRequest(request); + ArgumentNullException.ThrowIfNull(receipt); + RequireExact(receipt.Schema, RecordSearchProjectionGenerationSchemas.BuildReceiptV1, "build receipt schema"); + RequireExact(receipt.Status, RecordSearchProjectionBuildStatuses.Verified, "build receipt status"); + RequireExact(receipt.BuilderId, request.BuilderId, "build receipt builderId"); + ValidateDescriptor(receipt.Descriptor); + if (!string.Equals(receipt.Descriptor.Collection, request.Collection, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.GenerationId, request.GenerationId, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.ProviderId, request.ProviderId, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.ProfileId, request.ProfileId, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.StrategyVersion, request.StrategyVersion, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.SourceManifestDigest, request.SourceManifestDigest, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.ProjectionSchemaDigest, request.ProjectionSchemaDigest, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.AnalyzerDigest, request.AnalyzerDigest, StringComparison.Ordinal) || + !string.Equals(receipt.Descriptor.ConfigurationDigest, request.ConfigurationDigest, StringComparison.Ordinal) || + receipt.Descriptor.ExpectedItemCount != request.ExpectedItemCount || + !receipt.Descriptor.ExpectedPartitions.SequenceEqual(request.ExpectedPartitions, StringComparer.Ordinal)) + { + throw new InvalidOperationException("The verified descriptor does not match its admitted build request."); + } + if (request.ExpectedRecordRevisionSetDigest is not null && + !string.Equals(receipt.Descriptor.RecordRevisionSetDigest, request.ExpectedRecordRevisionSetDigest, StringComparison.Ordinal)) + { + throw new InvalidOperationException("The verified record/revision-set digest does not match the admitted expectation."); + } + if (receipt.EvaluationReceiptDigest is not null) + { + RequireSha256(receipt.EvaluationReceiptDigest, "evaluationReceiptDigest"); + } + if (receipt.BuiltAtUtc == default || receipt.BuiltAtUtc.Kind != DateTimeKind.Utc) + { + throw new InvalidOperationException("builtAtUtc must be a non-default UTC timestamp."); + } + if (receipt.Descriptor.CreatedAtUtc > receipt.BuiltAtUtc) + { + throw new InvalidOperationException("A descriptor cannot be created after its build receipt."); + } + if (request.DeadlineUtc is { } deadline && receipt.BuiltAtUtc > deadline) + { + throw new InvalidOperationException("A build receipt completed after its admitted deadline."); + } + } + + public static void ValidateBuildProgress(RecordSearchProjectionGenerationBuildProgress progress) + { + ArgumentNullException.ThrowIfNull(progress); + RequireIdentifier(progress.Stage, "build progress stage"); + if (progress.Completed < 0 || progress.Total < 0) + { + throw new InvalidOperationException("Projection generation progress counts cannot be negative."); + } + if (progress.Total is { } total && progress.Completed > total) + { + throw new InvalidOperationException("Projection generation completed work cannot exceed total work."); + } + if (progress.Checkpoint is not null) + { + ValidateBoundedSafeDetails(progress.Checkpoint, "Projection generation checkpoint"); + } + } + + private static void ValidateCoverage(RecordSearchProjectionCoverage coverage) + { + if (coverage is null) + { + throw new InvalidOperationException("A generation-bound result requires coverage evidence."); + } + if (coverage.Status is not RecordSearchProjectionCoverageStatuses.Complete and + not RecordSearchProjectionCoverageStatuses.Incomplete and + not RecordSearchProjectionCoverageStatuses.Unavailable) + { + throw new InvalidOperationException("Coverage status must be complete, incomplete, or unavailable."); + } + RequireCanonicalSet(coverage.RequestedPartitions, "requestedPartitions", requireNonEmpty: false); + RequireCanonicalSet(coverage.CoveredPartitions, "coveredPartitions", requireNonEmpty: false); + RequireCanonicalSet(coverage.MissingPartitions, "missingPartitions", requireNonEmpty: false); + if (coverage.CoveredPartitions.Except(coverage.RequestedPartitions, StringComparer.Ordinal).Any() || + coverage.MissingPartitions.Except(coverage.RequestedPartitions, StringComparer.Ordinal).Any() || + coverage.CoveredPartitions.Intersect(coverage.MissingPartitions, StringComparer.Ordinal).Any()) + { + throw new InvalidOperationException("Coverage partitions must be disjoint subsets of requestedPartitions."); + } + var union = coverage.CoveredPartitions + .Concat(coverage.MissingPartitions) + .OrderBy(value => value, StringComparer.Ordinal); + if (!union.SequenceEqual(coverage.RequestedPartitions, StringComparer.Ordinal)) + { + throw new InvalidOperationException("Covered and missing partitions must account for every requested partition."); + } + if (coverage.Status == RecordSearchProjectionCoverageStatuses.Complete && coverage.MissingPartitions.Count != 0) + { + throw new InvalidOperationException("Complete coverage cannot include missing partitions."); + } + if (coverage.Status == RecordSearchProjectionCoverageStatuses.Incomplete && coverage.MissingPartitions.Count == 0) + { + throw new InvalidOperationException("Incomplete coverage requires at least one missing partition."); + } + if (coverage.Status == RecordSearchProjectionCoverageStatuses.Unavailable && coverage.CoveredPartitions.Count != 0) + { + throw new InvalidOperationException("Unavailable coverage cannot claim covered partitions."); + } + } + + private static void ValidateDiagnostics(RecordSearchProjectionWorkDiagnostics diagnostics) + { + if (diagnostics is null) + { + throw new InvalidOperationException("A generation-bound result requires bounded diagnostics."); + } + foreach (var (name, value) in new[] + { + ("workLimit", diagnostics.WorkLimit), + ("workUnits", diagnostics.WorkUnits), + ("candidateBound", diagnostics.CandidateBound), + ("candidateCount", diagnostics.CandidateCount), + ("returnedCount", (long?)diagnostics.ReturnedCount) + }) + { + if (value < 0) + { + throw new InvalidOperationException($"{name} cannot be negative."); + } + } + if (diagnostics.CacheStatus is not RecordSearchProjectionCacheStatuses.Hit and + not RecordSearchProjectionCacheStatuses.Miss and + not RecordSearchProjectionCacheStatuses.Bypass and + not RecordSearchProjectionCacheStatuses.NotApplicable) + { + throw new InvalidOperationException("cacheStatus is not recognized."); + } + if (diagnostics.CandidateCount is { } candidateCount && candidateCount < diagnostics.ReturnedCount) + { + throw new InvalidOperationException("candidateCount cannot be less than returnedCount."); + } + if (diagnostics.CandidateBound is { } candidateBound && + diagnostics.CandidateCount is { } boundedCandidateCount && + candidateBound < boundedCandidateCount) + { + throw new InvalidOperationException("candidateBound cannot be less than candidateCount."); + } + if (diagnostics.Details is not null) + { + ValidateBoundedSafeDetails(diagnostics.Details, "Projection diagnostics details"); + } + } + + private static void ValidateBoundedSafeDetails(JsonObject details, string label) + { + var bytes = JsonSerializer.SerializeToUtf8Bytes(details, JsonOptions); + if (bytes.Length > 16_384) + { + throw new InvalidOperationException($"{label} cannot exceed 16384 UTF-8 bytes."); + } + ValidateSafePropertyNames(details, label); + } + + private static void ValidateSafePropertyNames(JsonNode node, string label) + { + if (node is JsonObject objectNode) + { + foreach (var property in objectNode) + { + if (IsSensitiveName(property.Key)) + { + throw new InvalidOperationException($"{label} cannot include sensitive field '{property.Key}'."); + } + if (property.Value is not null) + { + ValidateSafePropertyNames(property.Value, label); + } + } + return; + } + if (node is JsonArray arrayNode) + { + foreach (var item in arrayNode) + { + if (item is not null) + { + ValidateSafePropertyNames(item, label); + } + } + } + } + + private static void ValidateCandidates(IEnumerable candidates) + { + var identities = new HashSet(StringComparer.Ordinal); + foreach (var candidate in candidates) + { + RequireIdentifier(candidate.PartitionKey, "candidate partitionKey"); + RequireIdentifier(candidate.Id, "candidate id"); + if (candidate.Revision <= 0 || !float.IsFinite(candidate.Score)) + { + throw new InvalidOperationException("Projection candidates require a positive revision and finite score."); + } + if (!identities.Add(candidate.PartitionKey + "\n" + candidate.Id)) + { + throw new InvalidOperationException("Projection candidates must have unique partition/id identities."); + } + } + } + + private static void RequireCanonicalSet(IReadOnlyCollection values, string name, bool requireNonEmpty) + { + if (requireNonEmpty && values.Count == 0) + { + throw new InvalidOperationException($"{name} must not be empty."); + } + var canonical = values.OrderBy(value => value, StringComparer.Ordinal).ToList(); + foreach (var value in canonical) + { + RequireIdentifier(value, name + " item"); + } + if (canonical.Distinct(StringComparer.Ordinal).Count() != canonical.Count) + { + throw new InvalidOperationException($"{name} must contain unique non-empty values."); + } + if (!canonical.SequenceEqual(values, StringComparer.Ordinal)) + { + throw new InvalidOperationException($"{name} must use canonical ordinal ordering."); + } + } + + private static void RequireIdentifier(string? value, string name) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > 200 || + !string.Equals(value, value.Trim(), StringComparison.Ordinal) || value.Any(char.IsControl)) + { + throw new InvalidOperationException($"{name} must be a trimmed non-empty value no longer than 200 characters."); + } + } + + private static bool IsSensitiveName(string value) => + value.Contains("secret", StringComparison.OrdinalIgnoreCase) || + value.Contains("token", StringComparison.OrdinalIgnoreCase) || + value.Contains("password", StringComparison.OrdinalIgnoreCase) || + value.Contains("credential", StringComparison.OrdinalIgnoreCase) || + value.Contains("authorization", StringComparison.OrdinalIgnoreCase) || + value.Contains("cookie", StringComparison.OrdinalIgnoreCase); + + private static void RequireExact(string? actual, string expected, string name) + { + if (!string.Equals(actual, expected, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"{name} must be '{expected}'."); + } + } + + private static void RequireSha256(string? value, string name) + { + var valid = value is { Length: 71 } && value.StartsWith("sha256:", StringComparison.Ordinal); + if (valid) + { + foreach (var character in value!.AsSpan(7)) + { + if (character is not (>= '0' and <= '9') and not (>= 'a' and <= 'f')) + { + valid = false; + break; + } + } + } + if (!valid) + { + throw new InvalidOperationException($"{name} must be a lowercase sha256 digest."); + } + } + + private static string Hash(byte[] bytes) => + "sha256:" + Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); +} diff --git a/src/Vyral.Abstractions/Models/ObjectStoreModels.cs b/src/Vyral.Abstractions/Models/ObjectStoreModels.cs index 0d38b52..27f1c69 100644 --- a/src/Vyral.Abstractions/Models/ObjectStoreModels.cs +++ b/src/Vyral.Abstractions/Models/ObjectStoreModels.cs @@ -106,3 +106,15 @@ public class ObjectListResult [JsonPropertyName("continuationToken")] public string? ContinuationToken { get; set; } } + +public sealed class ContentAddressedObjectReceipt +{ + [JsonPropertyName("object")] + public ObjectInfo Object { get; set; } = new(); + + [JsonPropertyName("contentHash")] + public string ContentHash { get; set; } = string.Empty; + + [JsonPropertyName("replayed")] + public bool Replayed { get; set; } +} diff --git a/src/Vyral.Abstractions/contracts/fixtures/record-search-projection-generation.v1.valid.json b/src/Vyral.Abstractions/contracts/fixtures/record-search-projection-generation.v1.valid.json new file mode 100644 index 0000000..af4efb8 --- /dev/null +++ b/src/Vyral.Abstractions/contracts/fixtures/record-search-projection-generation.v1.valid.json @@ -0,0 +1,41 @@ +{ + "schema": "vyral.record-search-projection-generation.v1", + "collection": "library", + "generationId": "gen-2026-08-27-a", + "providerId": "local-exhaustive", + "profileId": "lexical-exhaustive-v1", + "strategyVersion": "exhaustive-token-v1", + "sourceManifestDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "recordRevisionSetDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "projectionSchemaDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "analyzerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "configurationDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "expectedItemCount": 3, + "expectedPartitions": [ + "public-a", + "public-b" + ], + "capabilities": [ + "completeCoverage", + "generationPinnedContinuation", + "lexical" + ], + "artifacts": [ + { + "id": "partition-public-a", + "kind": "index-part", + "contentHash": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "sizeBytes": 128, + "mediaType": "application/octet-stream" + }, + { + "id": "partition-public-b", + "kind": "index-part", + "contentHash": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "sizeBytes": 96, + "mediaType": "application/octet-stream" + } + ], + "createdAtUtc": "2026-08-27T12:00:00Z", + "descriptorDigest": "sha256:3b92c8927a2a93f12d99e64e03d118e29ac57284663943fa257568ae178d31d5" +} diff --git a/src/Vyral.Abstractions/contracts/record-search-projection-generation.v1.schema.json b/src/Vyral.Abstractions/contracts/record-search-projection-generation.v1.schema.json new file mode 100644 index 0000000..b64d117 --- /dev/null +++ b/src/Vyral.Abstractions/contracts/record-search-projection-generation.v1.schema.json @@ -0,0 +1,387 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openvyral.com/contracts/record-search-projection-generation.v1.schema.json", + "title": "Vyral immutable record-search projection generation", + "description": "Provider-neutral identity and completeness evidence for one immutable candidate-only record-search projection generation.", + "$ref": "#/$defs/generationDescriptor", + "$defs": { + "sha256": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "identifier": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "pattern": "^\\S(?:.*\\S)?$" + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "contentHash", "sizeBytes", "mediaType"], + "properties": { + "id": { "$ref": "#/$defs/identifier" }, + "kind": { "$ref": "#/$defs/identifier" }, + "contentHash": { "$ref": "#/$defs/sha256" }, + "sizeBytes": { "type": "integer", "minimum": 0 }, + "mediaType": { "type": ["string", "null"], "maxLength": 200 } + } + }, + "generationDescriptor": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "collection", + "generationId", + "providerId", + "profileId", + "strategyVersion", + "sourceManifestDigest", + "recordRevisionSetDigest", + "projectionSchemaDigest", + "analyzerDigest", + "configurationDigest", + "expectedItemCount", + "expectedPartitions", + "capabilities", + "artifacts", + "createdAtUtc", + "descriptorDigest" + ], + "properties": { + "schema": { "const": "vyral.record-search-projection-generation.v1" }, + "collection": { "$ref": "#/$defs/identifier" }, + "generationId": { "$ref": "#/$defs/identifier" }, + "providerId": { "$ref": "#/$defs/identifier" }, + "profileId": { "$ref": "#/$defs/identifier" }, + "strategyVersion": { "$ref": "#/$defs/identifier" }, + "sourceManifestDigest": { "$ref": "#/$defs/sha256" }, + "recordRevisionSetDigest": { "$ref": "#/$defs/sha256" }, + "projectionSchemaDigest": { "$ref": "#/$defs/sha256" }, + "analyzerDigest": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "configurationDigest": { "$ref": "#/$defs/sha256" }, + "expectedItemCount": { "type": "integer", "minimum": 0 }, + "expectedPartitions": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "capabilities": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" }, + "allOf": [ + { "contains": { "const": "completeCoverage" } }, + { "contains": { "const": "generationPinnedContinuation" } } + ] + }, + "artifacts": { + "type": "array", + "items": { "$ref": "#/$defs/artifact" } + }, + "createdAtUtc": { "type": "string", "format": "date-time" }, + "descriptorDigest": { "$ref": "#/$defs/sha256" } + } + }, + "queryEnvelope": { + "type": "object", + "additionalProperties": false, + "properties": { + "partitionKeys": { + "type": ["array", "null"], + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "filter": { "type": ["object", "null"] }, + "vector": { "type": ["object", "null"] }, + "lexical": { "type": ["object", "null"] }, + "orderBy": { "type": ["array", "null"] }, + "limit": { "type": ["integer", "null"], "minimum": 1 }, + "continuationToken": { "type": ["string", "null"], "maxLength": 8192 } + } + }, + "searchRequest": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "generationId", "expectedDescriptorDigest", "query", "deadlineUtc"], + "properties": { + "schema": { "const": "vyral.record-search-projection-request.v1" }, + "generationId": { + "oneOf": [ + { "$ref": "#/$defs/identifier" }, + { "type": "null" } + ] + }, + "expectedDescriptorDigest": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "query": { "$ref": "#/$defs/queryEnvelope" }, + "deadlineUtc": { "type": ["string", "null"], "format": "date-time" } + } + }, + "candidate": { + "type": "object", + "additionalProperties": false, + "required": ["partitionKey", "id", "revision", "score"], + "properties": { + "partitionKey": { "$ref": "#/$defs/identifier" }, + "id": { "$ref": "#/$defs/identifier" }, + "revision": { "type": "integer", "minimum": 1 }, + "score": { "type": "number" } + } + }, + "coverage": { + "type": "object", + "additionalProperties": false, + "required": ["status", "requestedPartitions", "coveredPartitions", "missingPartitions"], + "properties": { + "status": { "enum": ["complete", "incomplete", "unavailable"] }, + "requestedPartitions": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "coveredPartitions": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "missingPartitions": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + } + } + }, + "diagnostics": { + "type": "object", + "additionalProperties": false, + "required": [ + "workLimit", + "workUnits", + "candidateBound", + "candidateCount", + "returnedCount", + "cacheStatus", + "details" + ], + "properties": { + "workLimit": { "type": ["integer", "null"], "minimum": 0 }, + "workUnits": { "type": ["integer", "null"], "minimum": 0 }, + "candidateBound": { "type": ["integer", "null"], "minimum": 0 }, + "candidateCount": { "type": ["integer", "null"], "minimum": 0 }, + "returnedCount": { "type": "integer", "minimum": 0 }, + "cacheStatus": { "enum": ["hit", "miss", "bypass", "notApplicable"] }, + "details": { "type": ["object", "null"] } + } + }, + "failure": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message", "retryable"], + "properties": { + "code": { "$ref": "#/$defs/identifier" }, + "message": { "type": "string", "minLength": 1, "maxLength": 1000 }, + "retryable": { "type": "boolean" } + } + }, + "searchResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "status", + "generationId", + "generationDescriptorDigest", + "items", + "continuationToken", + "consistency", + "coverage", + "diagnostics", + "failure" + ], + "properties": { + "schema": { "const": "vyral.record-search-projection-result.v1" }, + "status": { "enum": ["succeeded", "failed"] }, + "generationId": { + "oneOf": [ + { "$ref": "#/$defs/identifier" }, + { "type": "null" } + ] + }, + "generationDescriptorDigest": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "items": { "type": "array", "items": { "$ref": "#/$defs/candidate" } }, + "continuationToken": { "type": ["string", "null"], "maxLength": 8192 }, + "consistency": { "const": "immutableGeneration" }, + "coverage": { "$ref": "#/$defs/coverage" }, + "diagnostics": { "$ref": "#/$defs/diagnostics" }, + "failure": { + "oneOf": [ + { "$ref": "#/$defs/failure" }, + { "type": "null" } + ] + } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "succeeded" } }, "required": ["status"] }, + "then": { + "properties": { + "generationId": { "$ref": "#/$defs/identifier" }, + "generationDescriptorDigest": { "$ref": "#/$defs/sha256" }, + "coverage": { + "allOf": [ + { "$ref": "#/$defs/coverage" }, + { "properties": { "status": { "const": "complete" }, "missingPartitions": { "maxItems": 0 } } } + ] + }, + "failure": { "type": "null" } + } + }, + "else": { + "properties": { + "items": { "maxItems": 0 }, + "continuationToken": { "type": "null" }, + "failure": { "$ref": "#/$defs/failure" } + } + } + } + ] + }, + "buildRequest": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "collection", + "generationId", + "builderId", + "providerId", + "profileId", + "strategyVersion", + "sourceManifestRef", + "sourceManifestDigest", + "expectedRecordRevisionSetDigest", + "projectionSchemaDigest", + "analyzerDigest", + "configurationDigest", + "expectedItemCount", + "expectedPartitions", + "deadlineUtc" + ], + "properties": { + "schema": { "const": "vyral.record-search-projection-build-request.v1" }, + "collection": { "$ref": "#/$defs/identifier" }, + "generationId": { "$ref": "#/$defs/identifier" }, + "builderId": { "$ref": "#/$defs/identifier" }, + "providerId": { "$ref": "#/$defs/identifier" }, + "profileId": { "$ref": "#/$defs/identifier" }, + "strategyVersion": { "$ref": "#/$defs/identifier" }, + "sourceManifestRef": { "$ref": "#/$defs/identifier" }, + "sourceManifestDigest": { "$ref": "#/$defs/sha256" }, + "expectedRecordRevisionSetDigest": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "projectionSchemaDigest": { "$ref": "#/$defs/sha256" }, + "analyzerDigest": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "configurationDigest": { "$ref": "#/$defs/sha256" }, + "expectedItemCount": { "type": "integer", "minimum": 0 }, + "expectedPartitions": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "deadlineUtc": { "type": ["string", "null"], "format": "date-time" } + } + }, + "buildProgress": { + "type": "object", + "additionalProperties": false, + "required": ["stage", "completed", "total", "checkpoint"], + "properties": { + "stage": { "$ref": "#/$defs/identifier" }, + "completed": { "type": "integer", "minimum": 0 }, + "total": { "type": ["integer", "null"], "minimum": 0 }, + "checkpoint": { "type": ["object", "null"] } + } + }, + "buildReceipt": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "status", + "builderId", + "descriptor", + "evaluationReceiptDigest", + "builtAtUtc" + ], + "properties": { + "schema": { "const": "vyral.record-search-projection-build-receipt.v1" }, + "status": { "const": "verified" }, + "builderId": { "$ref": "#/$defs/identifier" }, + "descriptor": { "$ref": "#/$defs/generationDescriptor" }, + "evaluationReceiptDigest": { + "oneOf": [ + { "$ref": "#/$defs/sha256" }, + { "type": "null" } + ] + }, + "builtAtUtc": { "type": "string", "format": "date-time" } + } + }, + "inspection": { + "type": "object", + "additionalProperties": false, + "required": ["schema", "descriptor", "state", "availablePartitions", "coverageStatus", "observedAtUtc"], + "properties": { + "schema": { "const": "vyral.record-search-projection-inspection.v1" }, + "descriptor": { "$ref": "#/$defs/generationDescriptor" }, + "state": { "enum": ["active", "retained", "retired"] }, + "availablePartitions": { + "type": "array", + "uniqueItems": true, + "items": { "$ref": "#/$defs/identifier" } + }, + "coverageStatus": { "enum": ["complete", "incomplete", "unavailable"] }, + "observedAtUtc": { "type": "string", "format": "date-time" } + }, + "allOf": [ + { + "if": { "properties": { "state": { "const": "retired" } }, "required": ["state"] }, + "then": { + "properties": { + "availablePartitions": { "maxItems": 0 }, + "coverageStatus": { "const": "unavailable" } + } + } + } + ] + } + } +} diff --git a/src/Vyral.Aws/OpenSearchGenerationBoundRecordSearchProjection.cs b/src/Vyral.Aws/OpenSearchGenerationBoundRecordSearchProjection.cs new file mode 100644 index 0000000..12a6475 --- /dev/null +++ b/src/Vyral.Aws/OpenSearchGenerationBoundRecordSearchProjection.cs @@ -0,0 +1,724 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; + +namespace Vyral.Aws; + +public static class OpenSearchProjectionGenerationBinding +{ + public const string ArtifactId = "provider-generation"; + public const string ArtifactKind = "opensearchIndexBinding"; + private const string BindingVersion = "vyral.aws.opensearch-index-binding.v1"; + + /// + /// Binds an opaque provider index coordinate to a portable descriptor without disclosing the + /// coordinate in the descriptor. The adapter independently verifies the same material before + /// it accepts candidates. + /// + public static string ComputeContentHash(string indexName, string indexUuid) + { + indexName = OpenSearchRecordSearchProjectionOptions.ValidateIndexName(indexName); + ValidateIndexUuid(indexUuid); + var material = $"{BindingVersion}\n{indexName}\n{indexUuid}"; + return "sha256:" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(material))); + } + + /// + /// Computes the exact policy material that controls the OpenSearch projection mapping and + /// provider query field names. Retained generations must be queried with the policy they were + /// built from, not whichever policy is current when the request arrives. + /// + public static string ComputeProjectionSchemaDigest(RecordCollectionPolicy policy) + { + ArgumentNullException.ThrowIfNull(policy); + RecordIdentityValidator.ValidateCollectionName(policy.Name); + var material = new + { + schema = "vyral.aws.opensearch-projection-schema.v1", + collection = policy.Name, + indexedMetadata = policy.IndexedMetadata.OrderBy(value => value, StringComparer.Ordinal), + vectors = policy.VectorPolicies + .OrderBy(value => value.Name, StringComparer.Ordinal) + .ThenBy(value => value.Path, StringComparer.Ordinal) + .Select(value => new + { + name = value.Name, + path = value.Path, + dimensions = value.Dimensions, + datatype = value.Datatype, + distanceFunction = value.DistanceFunction, + indexType = value.IndexType + }) + }; + return "sha256:" + Convert.ToHexStringLower( + SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(material))); + } + + internal static void ValidateIndexUuid(string? indexUuid) + { + if (string.IsNullOrWhiteSpace(indexUuid) || indexUuid.Length > 200 || + !string.Equals(indexUuid, indexUuid.Trim(), StringComparison.Ordinal) || + indexUuid.Any(char.IsControl)) + { + throw new InvalidOperationException("An OpenSearch generation requires a bounded index UUID."); + } + } +} + +public sealed class OpenSearchRecordSearchProjectionGeneration +{ + public RecordSearchProjectionGenerationDescriptor Descriptor { get; set; } = new(); + public string IndexName { get; set; } = string.Empty; + public string IndexUuid { get; set; } = string.Empty; + public string State { get; set; } = RecordSearchProjectionGenerationStates.Retained; + public List? AvailablePartitions { get; set; } +} + +public sealed class OpenSearchGenerationBoundRecordSearchProjectionOptions +{ + public const string DefaultProviderId = "aws-opensearch"; + public string ProviderId { get; set; } = DefaultProviderId; + public TimeProvider TimeProvider { get; set; } = TimeProvider.System; +} + +/// +/// Exact-generation, candidate-only OpenSearch adapter. Hosts register immutable provider indexes +/// from verified build evidence; callers can select only those allowlisted registrations, never an +/// arbitrary index or endpoint. Every search verifies the index UUID, read-only block, non-timeout +/// response, complete shard participation, and exact hit index before exposing candidates. +/// +public sealed class OpenSearchGenerationBoundRecordSearchProjection : IGenerationBoundRecordSearchProjection +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private readonly IOpenSearchTransport _transport; + private readonly OpenSearchRecordSearchProjection _searchProjection; + private readonly OpenSearchGenerationBoundRecordSearchProjectionOptions _options; + private readonly int _maximumCandidates; + private readonly Dictionary> _collections = new(StringComparer.Ordinal); + private readonly Dictionary _activeGenerations = new(StringComparer.Ordinal); + + public OpenSearchGenerationBoundRecordSearchProjection( + IOpenSearchTransport transport, + OpenSearchRecordSearchProjectionOptions searchOptions, + IEnumerable generations, + OpenSearchGenerationBoundRecordSearchProjectionOptions? options = null) + { + _transport = transport ?? throw new ArgumentNullException(nameof(transport)); + ArgumentNullException.ThrowIfNull(searchOptions); + ArgumentNullException.ThrowIfNull(generations); + var configuredOptions = options ?? new OpenSearchGenerationBoundRecordSearchProjectionOptions(); + if (string.IsNullOrWhiteSpace(configuredOptions.ProviderId) || configuredOptions.ProviderId.Length > 200 || + !string.Equals(configuredOptions.ProviderId, configuredOptions.ProviderId.Trim(), StringComparison.Ordinal) || + configuredOptions.ProviderId.Any(char.IsControl) || configuredOptions.TimeProvider is null) + { + throw new InvalidOperationException("The OpenSearch generation options are invalid."); + } + _options = new OpenSearchGenerationBoundRecordSearchProjectionOptions + { + ProviderId = configuredOptions.ProviderId, + TimeProvider = configuredOptions.TimeProvider + }; + _searchProjection = new OpenSearchRecordSearchProjection( + transport, + new OpenSearchRecordSearchProjectionOptions { MaximumCandidates = searchOptions.MaximumCandidates }); + _maximumCandidates = searchOptions.MaximumCandidates; + + foreach (var generation in generations) + { + Register(generation); + } + } + + public async Task InspectGenerationAsync( + RecordCollectionPolicy policy, + string? generationId = null, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(policy); + var registration = Resolve(policy.Name, generationId); + if (registration is null) + { + return null; + } + + var available = registration.State == RecordSearchProjectionGenerationStates.Retired + ? new List() + : PolicyMatchesDescriptor(policy, registration.Descriptor) && + await VerifyRemoteBindingAsync(registration, ct) + ? registration.AvailablePartitions.ToList() + : new List(); + var inspection = new RecordSearchProjectionGenerationInspection + { + Descriptor = CloneDescriptor(registration.Descriptor), + State = registration.State, + AvailablePartitions = available, + CoverageStatus = registration.State == RecordSearchProjectionGenerationStates.Retired + ? RecordSearchProjectionCoverageStatuses.Unavailable + : available.SequenceEqual(registration.Descriptor.ExpectedPartitions, StringComparer.Ordinal) + ? RecordSearchProjectionCoverageStatuses.Complete + : RecordSearchProjectionCoverageStatuses.Incomplete, + ObservedAtUtc = UtcNow() + }; + RecordSearchProjectionGenerationContract.ValidateInspection(inspection); + return inspection; + } + + public async Task SearchGenerationAsync( + RecordCollectionPolicy policy, + GenerationBoundRecordSearchProjectionRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(policy); + RecordSearchProjectionGenerationContract.ValidateRequest(request); + ct.ThrowIfCancellationRequested(); + + var requestedWithoutDescriptor = CanonicalPartitions(request.Query.PartitionKeys ?? new List()); + if (!string.IsNullOrWhiteSpace(request.Query.ContinuationToken)) + { + return Failure( + request, + null, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.InvalidContinuation, + "This OpenSearch generation path does not issue pageable vector continuations.", + retryable: false, + requestedWithoutDescriptor); + } + + var registration = Resolve(policy.Name, request.GenerationId); + if (registration is null) + { + return Failure( + request, + null, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.GenerationUnavailable, + "The requested generation is unavailable.", + retryable: true, + requestedWithoutDescriptor, + generationId: request.GenerationId); + } + + var requested = request.Query.PartitionKeys is { Count: > 0 } + ? CanonicalPartitions(request.Query.PartitionKeys) + : registration.Descriptor.ExpectedPartitions.ToList(); + if (registration.State == RecordSearchProjectionGenerationStates.Retired) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.GenerationRetired, + "The requested generation has been retired.", + retryable: false, + requested); + } + if (!PolicyMatchesDescriptor(policy, registration.Descriptor)) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.GenerationDescriptorMismatch, + "The collection policy does not match the selected generation's projection schema.", + retryable: false, + requested, + requested); + } + if (request.ExpectedDescriptorDigest is not null && + !string.Equals(request.ExpectedDescriptorDigest, registration.Descriptor.DescriptorDigest, StringComparison.Ordinal)) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.GenerationDescriptorMismatch, + "The selected generation descriptor does not match the caller's expected digest.", + retryable: false, + requested, + requested); + } + + var covered = requested + .Intersect(registration.AvailablePartitions, StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + if (covered.Count != requested.Count) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "One or more requested partitions are not covered by the selected generation.", + retryable: true, + requested, + covered); + } + if (request.DeadlineUtc is { } deadline && deadline <= UtcNow()) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.DeadlineExceeded, + "The generation-bound search deadline has elapsed.", + retryable: true, + requested, + covered); + } + if (request.Query.Vector is null || request.Query.Lexical is not null || request.Query.OrderBy is { Count: > 0 }) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.UnsupportedQuery, + "This OpenSearch generation path supports bounded vector candidate retrieval only.", + retryable: false, + requested, + covered); + } + try + { + FilterValueNormalizer.ValidateFilter(request.Query.Filter); + RecordVectorValidator.ValidateSearchVector(policy.Name, policy, request.Query.Vector); + var requestedCandidates = request.Query.Limit ?? request.Query.Vector.Top; + if (request.Query.Vector.MinScore.HasValue || request.Query.Vector.Top <= 0 || requestedCandidates <= 0 || + request.Query.Vector.Top > _maximumCandidates || requestedCandidates > _maximumCandidates) + { + throw new NotSupportedException(); + } + } + catch (Exception exception) when (exception is InvalidOperationException or NotSupportedException) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.UnsupportedQuery, + "The request uses query behavior that is not supported by this OpenSearch generation.", + retryable: false, + requested, + covered); + } + + RecordSearchProjectionResult candidates; + using var deadlineSource = CreateDeadlineSource(request.DeadlineUtc, ct); + var remoteCancellation = deadlineSource?.Token ?? ct; + try + { + if (!await VerifyRemoteBindingAsync(registration, remoteCancellation)) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch did not prove the selected immutable generation binding.", + retryable: true, + requested); + } + candidates = await _searchProjection.SearchIndexAsync( + policy, + request.Query, + registration.IndexName, + requireCompleteResponse: true, + remoteCancellation); + if (!CandidatesAreBounded(candidates.Items, requested) || + !await VerifyRemoteBindingAsync(registration, remoteCancellation)) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch did not preserve the bounded immutable generation through search completion.", + retryable: true, + requested); + } + if (request.DeadlineUtc is { } completedDeadline && completedDeadline <= UtcNow()) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.DeadlineExceeded, + "The generation-bound search deadline elapsed before OpenSearch completed.", + retryable: true, + requested, + covered); + } + } + catch (NotSupportedException) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.UnsupportedQuery, + "The request uses query behavior that is not supported by this OpenSearch generation.", + retryable: false, + requested, + covered); + } + catch (InvalidOperationException) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch did not prove a complete generation-bound search.", + retryable: true, + requested); + } + catch (HttpRequestException) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch did not prove a complete generation-bound search.", + retryable: true, + requested); + } + catch (Exception exception) when (exception is JsonException or KeyNotFoundException) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch returned a malformed generation-bound search response.", + retryable: true, + requested); + } + catch (OperationCanceledException) when ( + request.DeadlineUtc.HasValue && !ct.IsCancellationRequested && + deadlineSource?.IsCancellationRequested == true) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.DeadlineExceeded, + "The generation-bound search deadline elapsed before OpenSearch completed.", + retryable: true, + requested, + covered); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch did not complete the generation-bound search.", + retryable: true, + requested); + } + + var candidateBound = Math.Max(request.Query.Vector.Top, request.Query.Limit ?? request.Query.Vector.Top); + var result = new GenerationBoundRecordSearchProjectionResult + { + Status = RecordSearchProjectionResultStatuses.Succeeded, + GenerationId = registration.Descriptor.GenerationId, + GenerationDescriptorDigest = registration.Descriptor.DescriptorDigest, + Items = candidates.Items, + Coverage = new RecordSearchProjectionCoverage + { + Status = RecordSearchProjectionCoverageStatuses.Complete, + RequestedPartitions = requested, + CoveredPartitions = covered + }, + Diagnostics = new RecordSearchProjectionWorkDiagnostics + { + CandidateBound = candidateBound, + CandidateCount = candidates.Items.Count, + ReturnedCount = candidates.Items.Count, + CacheStatus = RecordSearchProjectionCacheStatuses.NotApplicable + } + }; + try + { + RecordSearchProjectionGenerationContract.ValidateResult(result); + return result; + } + catch (InvalidOperationException) + { + return Failure( + request, + registration, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "OpenSearch returned candidates that do not satisfy the portable projection contract.", + retryable: true, + requested); + } + } + + private void Register(OpenSearchRecordSearchProjectionGeneration generation) + { + ArgumentNullException.ThrowIfNull(generation); + RecordSearchProjectionGenerationContract.ValidateDescriptor(generation.Descriptor); + if (!string.Equals(generation.Descriptor.ProviderId, _options.ProviderId, StringComparison.Ordinal)) + { + throw new InvalidOperationException("An OpenSearch generation descriptor has an unexpected provider ID."); + } + var requiredCapabilities = new[] + { + RecordSearchProjectionGenerationCapabilities.CompleteCoverage, + RecordSearchProjectionGenerationCapabilities.GenerationPinnedContinuation, + RecordSearchProjectionGenerationCapabilities.Vector + }; + if (requiredCapabilities.Except(generation.Descriptor.Capabilities, StringComparer.Ordinal).Any()) + { + throw new InvalidOperationException( + "An OpenSearch generation descriptor must declare complete coverage, generation-pinned continuation safety, and vector capability."); + } + if (generation.State is not RecordSearchProjectionGenerationStates.Active and + not RecordSearchProjectionGenerationStates.Retained and + not RecordSearchProjectionGenerationStates.Retired) + { + throw new InvalidOperationException("An OpenSearch generation state must be active, retained, or retired."); + } + var indexName = OpenSearchRecordSearchProjectionOptions.ValidateIndexName(generation.IndexName); + OpenSearchProjectionGenerationBinding.ValidateIndexUuid(generation.IndexUuid); + var binding = generation.Descriptor.Artifacts.SingleOrDefault(artifact => + string.Equals(artifact.Id, OpenSearchProjectionGenerationBinding.ArtifactId, StringComparison.Ordinal) && + string.Equals(artifact.Kind, OpenSearchProjectionGenerationBinding.ArtifactKind, StringComparison.Ordinal)); + var expectedHash = OpenSearchProjectionGenerationBinding.ComputeContentHash(indexName, generation.IndexUuid); + if (binding is null || !string.Equals(binding.ContentHash, expectedHash, StringComparison.Ordinal)) + { + throw new InvalidOperationException("The OpenSearch index coordinate is not bound to the generation descriptor."); + } + + IEnumerable availableSource = generation.State == RecordSearchProjectionGenerationStates.Retired + ? Array.Empty() + : generation.AvailablePartitions ?? generation.Descriptor.ExpectedPartitions; + var available = availableSource + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + if (available.Distinct(StringComparer.Ordinal).Count() != available.Count || + available.Except(generation.Descriptor.ExpectedPartitions, StringComparer.Ordinal).Any()) + { + throw new InvalidOperationException("OpenSearch available partitions must be a unique descriptor subset."); + } + if (generation.State == RecordSearchProjectionGenerationStates.Active && + !available.SequenceEqual(generation.Descriptor.ExpectedPartitions, StringComparer.Ordinal)) + { + throw new InvalidOperationException("An incomplete OpenSearch generation cannot be active."); + } + + if (!_collections.TryGetValue(generation.Descriptor.Collection, out var collection)) + { + collection = new Dictionary(StringComparer.Ordinal); + _collections.Add(generation.Descriptor.Collection, collection); + } + if (!collection.TryAdd(generation.Descriptor.GenerationId, new Registration( + CloneDescriptor(generation.Descriptor), indexName, generation.IndexUuid, generation.State, available))) + { + throw new InvalidOperationException("An OpenSearch generation ID can be registered only once."); + } + if (generation.State == RecordSearchProjectionGenerationStates.Active && + !_activeGenerations.TryAdd(generation.Descriptor.Collection, generation.Descriptor.GenerationId)) + { + throw new InvalidOperationException("A collection can register only one active OpenSearch generation."); + } + } + + private Registration? Resolve(string collection, string? generationId) + { + if (!_collections.TryGetValue(collection, out var generations)) + { + return null; + } + if (string.IsNullOrWhiteSpace(generationId) && + !_activeGenerations.TryGetValue(collection, out generationId)) + { + return null; + } + return generations.GetValueOrDefault(generationId!); + } + + private async Task VerifyRemoteBindingAsync(Registration registration, CancellationToken ct) + { + if (registration.State == RecordSearchProjectionGenerationStates.Retired) + { + return false; + } + try + { + var response = await _transport.SendAsync( + HttpMethod.Get, + $"/{registration.IndexName}/_settings?flat_settings=true", + null, + ct); + if (response.StatusCode is < 200 or >= 300) + { + return false; + } + using var document = JsonDocument.Parse(response.Body); + if (!document.RootElement.TryGetProperty(registration.IndexName, out var index) || + !index.TryGetProperty("settings", out var settings)) + { + return false; + } + var uuid = ReadSetting(settings, "index.uuid", "uuid"); + var readOnlyBlock = ReadSetting(settings, "index.blocks.read_only", "blocks", "read_only"); + return uuid is not null && + string.Equals(uuid, registration.IndexUuid, StringComparison.Ordinal) && + string.Equals(readOnlyBlock, "true", StringComparison.OrdinalIgnoreCase) && + string.Equals( + OpenSearchProjectionGenerationBinding.ComputeContentHash(registration.IndexName, uuid), + registration.BindingContentHash, + StringComparison.Ordinal); + } + catch (JsonException) + { + return false; + } + catch (HttpRequestException) + { + return false; + } + catch (InvalidOperationException) + { + return false; + } + } + + private static string? ReadSetting(JsonElement settings, string flatName, params string[] nestedPath) + { + if (settings.TryGetProperty(flatName, out var flat) && flat.ValueKind == JsonValueKind.String) + { + return flat.GetString(); + } + var current = settings; + if (current.TryGetProperty("index", out var index)) + { + current = index; + } + foreach (var segment in nestedPath) + { + if (!current.TryGetProperty(segment, out current)) + { + return null; + } + } + return current.ValueKind == JsonValueKind.String ? current.GetString() : null; + } + + private GenerationBoundRecordSearchProjectionResult Failure( + GenerationBoundRecordSearchProjectionRequest request, + Registration? registration, + string coverageStatus, + string code, + string message, + bool retryable, + List requested, + List? covered = null, + string? generationId = null) + { + covered = covered?.ToList() ?? new List(); + if (coverageStatus == RecordSearchProjectionCoverageStatuses.Unavailable) + { + covered.Clear(); + } + var result = new GenerationBoundRecordSearchProjectionResult + { + Status = RecordSearchProjectionResultStatuses.Failed, + GenerationId = registration?.Descriptor.GenerationId ?? generationId, + GenerationDescriptorDigest = registration?.Descriptor.DescriptorDigest, + Coverage = new RecordSearchProjectionCoverage + { + Status = coverageStatus, + RequestedPartitions = requested, + CoveredPartitions = covered, + MissingPartitions = requested.Except(covered, StringComparer.Ordinal).OrderBy(value => value, StringComparer.Ordinal).ToList() + }, + Diagnostics = new RecordSearchProjectionWorkDiagnostics(), + Failure = new RecordSearchProjectionFailure { Code = code, Message = message, Retryable = retryable } + }; + RecordSearchProjectionGenerationContract.ValidateResult(result); + return result; + } + + private static List CanonicalPartitions(IEnumerable partitions) => + partitions.Distinct(StringComparer.Ordinal).OrderBy(value => value, StringComparer.Ordinal).ToList(); + + private static bool PolicyMatchesDescriptor( + RecordCollectionPolicy policy, + RecordSearchProjectionGenerationDescriptor descriptor) => + string.Equals( + OpenSearchProjectionGenerationBinding.ComputeProjectionSchemaDigest(policy), + descriptor.ProjectionSchemaDigest, + StringComparison.Ordinal); + + private static bool CandidatesAreBounded( + IEnumerable candidates, + IReadOnlyCollection requestedPartitions) + { + var identities = new HashSet(StringComparer.Ordinal); + foreach (var candidate in candidates) + { + if (!requestedPartitions.Contains(candidate.PartitionKey, StringComparer.Ordinal) || + candidate.Revision <= 0 || !float.IsFinite(candidate.Score) || + !identities.Add(candidate.PartitionKey + "\n" + candidate.Id)) + { + return false; + } + } + return true; + } + + private CancellationTokenSource? CreateDeadlineSource(DateTime? deadlineUtc, CancellationToken ct) + { + if (deadlineUtc is null) + { + return null; + } + var remaining = deadlineUtc.Value - UtcNow(); + var source = CancellationTokenSource.CreateLinkedTokenSource(ct); + source.CancelAfter(remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero); + return source; + } + + private DateTime UtcNow() => _options.TimeProvider.GetUtcNow().UtcDateTime; + + private static RecordSearchProjectionGenerationDescriptor CloneDescriptor( + RecordSearchProjectionGenerationDescriptor descriptor) => + JsonSerializer.Deserialize( + JsonSerializer.Serialize(descriptor, JsonOptions), + JsonOptions) ?? throw new InvalidOperationException("Could not clone an OpenSearch generation descriptor."); + + private sealed class Registration + { + public Registration( + RecordSearchProjectionGenerationDescriptor descriptor, + string indexName, + string indexUuid, + string state, + List availablePartitions) + { + Descriptor = descriptor; + IndexName = indexName; + IndexUuid = indexUuid; + State = state; + AvailablePartitions = availablePartitions; + BindingContentHash = OpenSearchProjectionGenerationBinding.ComputeContentHash(indexName, indexUuid); + } + + public RecordSearchProjectionGenerationDescriptor Descriptor { get; } + public string IndexName { get; } + public string IndexUuid { get; } + public string State { get; } + public List AvailablePartitions { get; } + public string BindingContentHash { get; } + } +} diff --git a/src/Vyral.Aws/OpenSearchRecordSearchProjection.cs b/src/Vyral.Aws/OpenSearchRecordSearchProjection.cs index 72a1bc5..f507181 100644 --- a/src/Vyral.Aws/OpenSearchRecordSearchProjection.cs +++ b/src/Vyral.Aws/OpenSearchRecordSearchProjection.cs @@ -105,13 +105,22 @@ public async Task ProjectAsync(RecordSearchProjectionChange change, Cancellation ThrowForFailure("project the canonical record into OpenSearch", response); } - public async Task SearchAsync( + public Task SearchAsync( RecordCollectionPolicy policy, QueryEnvelope query, + CancellationToken ct = default) => + SearchIndexAsync(policy, query, _options.GetIndexName(policy), requireCompleteResponse: false, ct); + + internal async Task SearchIndexAsync( + RecordCollectionPolicy policy, + QueryEnvelope query, + string index, + bool requireCompleteResponse, CancellationToken ct = default) { ValidatePolicy(policy); ArgumentNullException.ThrowIfNull(query); + index = OpenSearchRecordSearchProjectionOptions.ValidateIndexName(index); if (query.Vector is null) throw new NotSupportedException("The OpenSearch projection currently supports vector candidate retrieval only."); if (query.Lexical is not null || query.OrderBy is { Count: > 0 }) @@ -160,7 +169,9 @@ public async Task SearchAsync( } }; var response = await _transport.SendAsync(HttpMethod.Post, - $"/{_options.GetIndexName(policy)}/_search", + requireCompleteResponse + ? $"/{index}/_search?allow_partial_search_results=false" + : $"/{index}/_search", request.ToJsonString(JsonOptions), ct); if (response.StatusCode is < 200 or >= 300) ThrowForFailure("search the OpenSearch projection", response); @@ -168,10 +179,22 @@ public async Task SearchAsync( try { using var document = JsonDocument.Parse(response.Body); - var hits = document.RootElement.GetProperty("hits").GetProperty("hits"); + var root = document.RootElement; + if (requireCompleteResponse) + { + ValidateCompleteSearchResponse(root, index); + } + var hits = root.GetProperty("hits").GetProperty("hits"); var results = new List(); foreach (var hit in hits.EnumerateArray()) { + if (requireCompleteResponse && + (!hit.TryGetProperty("_index", out var hitIndex) || + hitIndex.ValueKind != JsonValueKind.String || + !string.Equals(hitIndex.GetString(), index, StringComparison.Ordinal))) + { + throw new InvalidOperationException("OpenSearch returned a candidate from outside the selected immutable generation index."); + } var source = hit.GetProperty("_source"); if (!source.TryGetProperty("partitionKey", out var partitionKey) || !source.TryGetProperty("id", out var id) || @@ -206,6 +229,28 @@ public async Task SearchAsync( } } + private static void ValidateCompleteSearchResponse(JsonElement root, string expectedIndex) + { + if (!root.TryGetProperty("timed_out", out var timedOut) || + timedOut.ValueKind is not JsonValueKind.True and not JsonValueKind.False || + timedOut.GetBoolean()) + { + throw new InvalidOperationException("OpenSearch did not prove a complete non-timeout search response."); + } + if (!root.TryGetProperty("_shards", out var shards) || + !shards.TryGetProperty("total", out var total) || + !shards.TryGetProperty("successful", out var successful) || + !shards.TryGetProperty("failed", out var failed) || + !total.TryGetInt32(out var totalValue) || totalValue <= 0 || + !successful.TryGetInt32(out var successfulValue) || + !failed.TryGetInt32(out var failedValue) || + failedValue != 0 || successfulValue != totalValue) + { + throw new InvalidOperationException( + $"OpenSearch did not prove complete shard coverage for immutable index '{expectedIndex}'."); + } + } + private static JsonObject BuildCreateIndexPayload(RecordCollectionPolicy policy) { var vectorProperties = new JsonObject(); diff --git a/src/Vyral.Aws/OpenSearchRecordSearchProjectionOptions.cs b/src/Vyral.Aws/OpenSearchRecordSearchProjectionOptions.cs index cb22426..61ada46 100644 --- a/src/Vyral.Aws/OpenSearchRecordSearchProjectionOptions.cs +++ b/src/Vyral.Aws/OpenSearchRecordSearchProjectionOptions.cs @@ -58,7 +58,7 @@ internal string GetIndexName(string collection) return ValidateIndexName(IndexNameFactory(collection)); } - private static string ValidateIndexName(string? index) + internal static string ValidateIndexName(string? index) { if (string.IsNullOrWhiteSpace(index) || index.Length > MaximumIndexNameLength || !char.IsLetterOrDigit(index[0]) || diff --git a/src/Vyral.Aws/README.md b/src/Vyral.Aws/README.md index 52ec6b3..14d28d4 100644 --- a/src/Vyral.Aws/README.md +++ b/src/Vyral.Aws/README.md @@ -38,3 +38,22 @@ scripts/validate-opensearch-local.sh This local gate deliberately does not validate a managed endpoint's SigV4 authorization, IAM policy, VPC/networking, encryption, or stream delivery. + +## Generation-bound OpenSearch + +`OpenSearchGenerationBoundRecordSearchProjection` implements the optional +`IGenerationBoundRecordSearchProjection` contract for exact, immutable provider indexes. Hosts +register an allowlisted index name and provider-issued UUID together with a portable generation +descriptor. Inspection and search verify that binding, require the index to remain read-only, +disable partial search results, reject timeouts or incomplete shard participation, and accept hits +only from the selected index. + +The adapter returns identities, canonical revisions, and scores only. Use +`SearchGenerationAndHydrateAsync` to retrieve matching revisions from the canonical record store; +stale candidates are discarded rather than promoted to authoritative data. This implementation is +first-page-only and rejects continuations rather than claiming unproven paging behavior. + +The local OpenSearch gate exercises the exact generation binding and fail-closed writable-index +transition. It remains local conformance evidence: managed AWS identity separation, SigV4, VPC +policy, service scaling, availability, and administrator mutation risks require their own live +qualification. diff --git a/src/Vyral.Local/GenerationBoundRetrievalEvaluationTargetResolver.cs b/src/Vyral.Local/GenerationBoundRetrievalEvaluationTargetResolver.cs new file mode 100644 index 0000000..cc457b9 --- /dev/null +++ b/src/Vyral.Local/GenerationBoundRetrievalEvaluationTargetResolver.cs @@ -0,0 +1,263 @@ +using System.Text.Json.Nodes; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; + +namespace Vyral.Local; + +/// +/// Host-owned registration for one exact generation-bound evaluation target. Registrations carry +/// code references directly; target ids are dictionary keys and are never interpreted as dynamic +/// endpoints, assemblies, or provider types. +/// +public sealed class GenerationBoundRetrievalEvaluationTargetRegistration +{ + public required string Id { get; init; } + public required IGenerationBoundRecordSearchProjection Projection { get; init; } + public required IRecordCollectionStore CanonicalStore { get; init; } + public required RecordCollectionPolicy Policy { get; init; } + public required string GenerationId { get; init; } + public required string GenerationDescriptorDigest { get; init; } +} + +/// +/// Resolves explicitly registered candidate-only projection generations for the existing retrieval +/// evaluation metrics. This keeps target selection out of query profiles and makes descriptor +/// substitution an evaluation failure rather than a silent comparison change. +/// +public sealed class GenerationBoundRetrievalEvaluationTargetResolver : IRetrievalEvaluationTargetResolver +{ + private readonly IReadOnlyDictionary _targets; + + public GenerationBoundRetrievalEvaluationTargetResolver( + IEnumerable targets) + { + ArgumentNullException.ThrowIfNull(targets); + var registrations = targets.ToList(); + if (registrations.Count == 0) + { + throw new InvalidOperationException("At least one generation-bound evaluation target is required."); + } + var duplicate = registrations + .GroupBy(target => target.Id, StringComparer.Ordinal) + .FirstOrDefault(group => group.Count() > 1); + if (duplicate is not null) + { + throw new InvalidOperationException( + $"Generation-bound evaluation target id '{duplicate.Key}' is duplicated."); + } + foreach (var registration in registrations) + { + ValidateRegistration(registration); + } + _targets = registrations.ToDictionary(target => target.Id, StringComparer.Ordinal); + } + + public RetrievalEvaluationResolvedTarget Resolve(RetrievalEvaluationTargetReference target) + { + ArgumentNullException.ThrowIfNull(target); + if (!_targets.TryGetValue(target.Id, out var registration)) + { + throw new InvalidOperationException( + $"Retrieval evaluation target '{target.Id}' is not registered."); + } + if (target.GenerationId is not null && + !string.Equals(target.GenerationId, registration.GenerationId, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Retrieval evaluation target '{target.Id}' does not match the requested generation."); + } + if (target.ExpectedGenerationDescriptorDigest is not null && + !string.Equals( + target.ExpectedGenerationDescriptorDigest, + registration.GenerationDescriptorDigest, + StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Retrieval evaluation target '{target.Id}' does not match the expected generation descriptor."); + } + return new RetrievalEvaluationResolvedTarget + { + Service = new GenerationBoundRetrievalServiceAdapter(registration), + Evidence = new RetrievalEvaluationTargetEvidence + { + Id = registration.Id, + GenerationId = registration.GenerationId, + GenerationDescriptorDigest = registration.GenerationDescriptorDigest + } + }; + } + + private static void ValidateRegistration( + GenerationBoundRetrievalEvaluationTargetRegistration registration) + { + ArgumentNullException.ThrowIfNull(registration); + ArgumentNullException.ThrowIfNull(registration.Projection); + ArgumentNullException.ThrowIfNull(registration.CanonicalStore); + ArgumentNullException.ThrowIfNull(registration.Policy); + if (string.IsNullOrWhiteSpace(registration.Id) || registration.Id.Length > 200 || + !string.Equals(registration.Id, registration.Id.Trim(), StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "A generation-bound evaluation target requires a bounded canonical id."); + } + RecordSearchProjectionGenerationContract.ValidateRequest( + new GenerationBoundRecordSearchProjectionRequest + { + GenerationId = registration.GenerationId, + ExpectedDescriptorDigest = registration.GenerationDescriptorDigest, + Query = new QueryEnvelope() + }); + if (string.IsNullOrWhiteSpace(registration.Policy.Name)) + { + throw new InvalidOperationException( + "A generation-bound evaluation target requires a collection policy."); + } + } +} + +/// +/// Projects an exact candidate-only generation into the existing evaluation service's retrieval +/// input shape. Canonical hydration remains mandatory; stale candidate revisions fail the target +/// rather than silently changing its measured result set. +/// +public sealed class GenerationBoundRetrievalServiceAdapter : IRetrievalService +{ + private readonly GenerationBoundRetrievalEvaluationTargetRegistration _registration; + + public GenerationBoundRetrievalServiceAdapter( + GenerationBoundRetrievalEvaluationTargetRegistration registration) + { + _registration = registration ?? throw new ArgumentNullException(nameof(registration)); + } + + public async Task SearchAsync( + RetrievalRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(request); + if (request.Collections is null || request.Collections.Count != 1 || + !string.Equals(request.Collections[0], _registration.Policy.Name, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "A generation-bound evaluation target requires its exact registered collection."); + } + if (request.Limit <= 0) + { + throw new InvalidOperationException( + "A generation-bound evaluation target requires a positive result limit."); + } + if (!string.IsNullOrWhiteSpace(request.Profile) || request.MinScore.HasValue || + request.Embedding is not null || request.VectorFields is { Count: > 0 } || + request.Hybrid is not null || request.Rerank?.Enabled == true || + !string.IsNullOrWhiteSpace(request.SearchMode) && + !string.Equals(request.SearchMode, SearchModes.Lexical, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "This generation-bound evaluation target supports unprofiled lexical candidate comparison without a score threshold or reranking."); + } + + var lexical = CloneLexical(request.Lexical); + if (string.IsNullOrWhiteSpace(lexical.Query)) + { + lexical.Query = request.Query; + } + lexical.Top = request.Limit; + var generationRequest = new GenerationBoundRecordSearchProjectionRequest + { + GenerationId = _registration.GenerationId, + ExpectedDescriptorDigest = _registration.GenerationDescriptorDigest, + Query = new QueryEnvelope + { + PartitionKeys = request.PartitionKeys?.ToList(), + Filter = request.Filter, + Lexical = lexical, + Limit = request.Limit + } + }; + var hydrated = await _registration.Projection.SearchGenerationAndHydrateAsync( + _registration.CanonicalStore, + _registration.Policy, + generationRequest, + ct); + var projection = hydrated.Projection; + if (projection.Status != RecordSearchProjectionResultStatuses.Succeeded) + { + throw new InvalidOperationException( + $"Generation-bound evaluation target failed with '{projection.Failure?.Code ?? "unknown"}'."); + } + if (hydrated.StaleCandidatesDiscarded != 0) + { + throw new InvalidOperationException( + "Generation-bound evaluation target encountered stale canonical candidate revisions."); + } + + var results = hydrated.Items + .Select((item, index) => new RetrievalMatch + { + Rank = index + 1, + Score = item.Score, + Collection = _registration.Policy.Name, + Record = item.Record, + Diagnostics = new RetrievalDiagnostics + { + CandidateSources = ["projection.generation-bound"], + CandidateCounts = new Dictionary + { + ["projection"] = projection.Diagnostics.CandidateCount is { } count + ? checked((int)Math.Min(count, int.MaxValue)) + : projection.Items.Count + }, + ReasonCodes = ["coverage.complete", "generation.pinned"], + Details = new Dictionary + { + ["projectionGenerationId"] = projection.GenerationId, + ["projectionGenerationDescriptorDigest"] = projection.GenerationDescriptorDigest, + ["projectionCoverageStatus"] = projection.Coverage.Status + } + } + }) + .ToList(); + return new RetrievalResultEnvelope + { + Query = request.Query, + Results = results, + Trace = new JsonObject + { + ["projectionGenerationId"] = projection.GenerationId, + ["projectionGenerationDescriptorDigest"] = projection.GenerationDescriptorDigest, + ["projectionCoverageStatus"] = projection.Coverage.Status, + ["projectionCandidateCount"] = projection.Diagnostics.CandidateCount, + ["projectionReturnedCount"] = projection.Diagnostics.ReturnedCount, + ["projectionStaleCandidatesDiscarded"] = hydrated.StaleCandidatesDiscarded + } + }; + } + + private static LexicalSearchOptions CloneLexical(LexicalSearchOptions? source) + { + source ??= LexicalSearchOptions.Default; + return new LexicalSearchOptions + { + Query = source.Query, + Fields = source.Fields?.ToList(), + Top = source.Top, + ScanLimit = source.ScanLimit, + MinScore = source.MinScore, + Scoring = source.Scoring, + MatchMode = source.MatchMode, + FieldBoosts = source.FieldBoosts is null + ? null + : new Dictionary(source.FieldBoosts, StringComparer.Ordinal), + Bm25K1 = source.Bm25K1, + Bm25B = source.Bm25B, + PhraseBoost = source.PhraseBoost, + ExactBoost = source.ExactBoost, + MetadataBoost = source.MetadataBoost, + PrefixMatching = source.PrefixMatching, + PrefixMinChars = source.PrefixMinChars, + RequiredPhraseGroups = source.RequiredPhraseGroups? + .Select(group => group.ToList()) + .ToList() + }; + } +} diff --git a/src/Vyral.Local/LocalGenerationBoundRecordSearchProjection.cs b/src/Vyral.Local/LocalGenerationBoundRecordSearchProjection.cs new file mode 100644 index 0000000..22be41e --- /dev/null +++ b/src/Vyral.Local/LocalGenerationBoundRecordSearchProjection.cs @@ -0,0 +1,835 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; + +namespace Vyral.Local; + +public sealed class LocalGenerationBoundRecordSearchProjectionOptions +{ + public byte[] ContinuationSigningKey { get; set; } = Array.Empty(); + public TimeSpan ContinuationLifetime { get; set; } = TimeSpan.FromMinutes(15); + public int DefaultResultLimit { get; set; } = 50; + public int MaxResultLimit { get; set; } = 1000; + public int DefaultWorkLimit { get; set; } = 100_000; + public int MaxWorkLimit { get; set; } = 1_000_000; + public int MaxContinuationTokenChars { get; set; } = 8192; + public TimeProvider TimeProvider { get; set; } = TimeProvider.System; +} + +public sealed class LocalRecordSearchProjectionDocument +{ + public RecordSearchProjectionCandidate Candidate { get; set; } = new(); + public string SearchText { get; set; } = string.Empty; +} + +public sealed class LocalRecordSearchProjectionGeneration +{ + public RecordSearchProjectionGenerationDescriptor Descriptor { get; set; } = new(); + public List Documents { get; set; } = new(); + public List? AvailablePartitions { get; set; } +} + +/// +/// Deterministic exhaustive reference for the optional generation-bound projection contract. Its +/// local publication controls are deliberately not part of the portable interface: they exist to +/// exercise generation switching, retention, retirement, and coverage faults before a provider +/// lifecycle is standardized. +/// +public sealed class LocalGenerationBoundRecordSearchProjection : IGenerationBoundRecordSearchProjection +{ + private const string ContinuationVersion = "vyral.local-record-projection-continuation.v1"; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + private readonly object _gate = new(); + private readonly Dictionary> _collections = new(StringComparer.Ordinal); + private readonly Dictionary _activeGenerations = new(StringComparer.Ordinal); + private readonly byte[] _signingKey; + private readonly LocalGenerationBoundRecordSearchProjectionOptions _options; + + public LocalGenerationBoundRecordSearchProjection(LocalGenerationBoundRecordSearchProjectionOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.ContinuationSigningKey is not { Length: >= 32 }) + { + throw new InvalidOperationException("A local generation-bound projection requires a continuation signing key of at least 32 bytes."); + } + if (options.ContinuationLifetime <= TimeSpan.Zero || options.ContinuationLifetime > TimeSpan.FromDays(1)) + { + throw new InvalidOperationException("ContinuationLifetime must be greater than zero and no longer than one day."); + } + if (options.DefaultResultLimit <= 0 || options.MaxResultLimit < options.DefaultResultLimit || + options.MaxResultLimit > 10_000) + { + throw new InvalidOperationException("Result limits are invalid."); + } + if (options.DefaultWorkLimit <= 0 || options.MaxWorkLimit < options.DefaultWorkLimit || + options.MaxWorkLimit > 10_000_000) + { + throw new InvalidOperationException("Work limits are invalid."); + } + if (options.MaxContinuationTokenChars is < 256 or > 8192) + { + throw new InvalidOperationException("MaxContinuationTokenChars must be between 256 and 8192."); + } + if (options.TimeProvider is null) + { + throw new InvalidOperationException("A local generation-bound projection requires a time provider."); + } + _options = new LocalGenerationBoundRecordSearchProjectionOptions + { + ContinuationLifetime = options.ContinuationLifetime, + DefaultResultLimit = options.DefaultResultLimit, + MaxResultLimit = options.MaxResultLimit, + DefaultWorkLimit = options.DefaultWorkLimit, + MaxWorkLimit = options.MaxWorkLimit, + MaxContinuationTokenChars = options.MaxContinuationTokenChars, + TimeProvider = options.TimeProvider + }; + _signingKey = options.ContinuationSigningKey.ToArray(); + } + + public void PublishGeneration(LocalRecordSearchProjectionGeneration generation) + { + ArgumentNullException.ThrowIfNull(generation); + RecordSearchProjectionGenerationContract.ValidateDescriptor(generation.Descriptor); + ValidateDocuments(generation); + + var available = (generation.AvailablePartitions ?? generation.Descriptor.ExpectedPartitions) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + if (available.Distinct(StringComparer.Ordinal).Count() != available.Count || + available.Except(generation.Descriptor.ExpectedPartitions, StringComparer.Ordinal).Any()) + { + throw new InvalidOperationException("Available partitions must be a unique subset of descriptor expectedPartitions."); + } + + var entry = new GenerationEntry( + CloneDescriptor(generation.Descriptor), + generation.Documents.Select(CloneDocument).ToList(), + available, + RecordSearchProjectionGenerationStates.Retained); + lock (_gate) + { + if (!_collections.TryGetValue(generation.Descriptor.Collection, out var generations)) + { + generations = new Dictionary(StringComparer.Ordinal); + _collections.Add(generation.Descriptor.Collection, generations); + } + if (generations.TryGetValue(generation.Descriptor.GenerationId, out var existing)) + { + if (!string.Equals(existing.Descriptor.DescriptorDigest, generation.Descriptor.DescriptorDigest, StringComparison.Ordinal)) + { + throw new InvalidOperationException("An immutable generation ID cannot be republished with different descriptor evidence."); + } + if (!GenerationContentEquals(existing, entry)) + { + throw new InvalidOperationException("An immutable generation ID cannot be republished with different documents or coverage evidence."); + } + return; + } + generations.Add(generation.Descriptor.GenerationId, entry); + } + } + + public void ActivateGeneration(string collection, string generationId) + { + lock (_gate) + { + var entry = GetEntry(collection, generationId) + ?? throw new InvalidOperationException("The generation is unavailable."); + if (entry.State == RecordSearchProjectionGenerationStates.Retired) + { + throw new InvalidOperationException("A retired generation cannot be activated."); + } + if (!entry.AvailablePartitions.SequenceEqual(entry.Descriptor.ExpectedPartitions, StringComparer.Ordinal)) + { + throw new InvalidOperationException("An incomplete generation cannot be activated."); + } + if (_activeGenerations.TryGetValue(collection, out var priorId) && + GetEntry(collection, priorId) is { } prior) + { + prior.State = RecordSearchProjectionGenerationStates.Retained; + } + entry.State = RecordSearchProjectionGenerationStates.Active; + _activeGenerations[collection] = generationId; + } + } + + public void RetireGeneration(string collection, string generationId) + { + lock (_gate) + { + var entry = GetEntry(collection, generationId) + ?? throw new InvalidOperationException("The generation is unavailable."); + if (_activeGenerations.TryGetValue(collection, out var active) && + string.Equals(active, generationId, StringComparison.Ordinal)) + { + throw new InvalidOperationException("The active generation cannot be retired before another generation is activated."); + } + entry.State = RecordSearchProjectionGenerationStates.Retired; + entry.AvailablePartitions.Clear(); + } + } + + public void SetAvailablePartitions(string collection, string generationId, IEnumerable partitions) + { + ArgumentNullException.ThrowIfNull(partitions); + var available = partitions.OrderBy(value => value, StringComparer.Ordinal).ToList(); + lock (_gate) + { + var entry = GetEntry(collection, generationId) + ?? throw new InvalidOperationException("The generation is unavailable."); + if (entry.State == RecordSearchProjectionGenerationStates.Retired) + { + throw new InvalidOperationException("A retired generation cannot become available."); + } + if (available.Distinct(StringComparer.Ordinal).Count() != available.Count || + available.Except(entry.Descriptor.ExpectedPartitions, StringComparer.Ordinal).Any()) + { + throw new InvalidOperationException("Available partitions must be a unique subset of descriptor expectedPartitions."); + } + entry.AvailablePartitions = available; + } + } + + public Task InspectGenerationAsync( + RecordCollectionPolicy policy, + string? generationId = null, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(policy); + ct.ThrowIfCancellationRequested(); + GenerationEntry? snapshot; + lock (_gate) + { + var resolvedId = generationId; + if (string.IsNullOrWhiteSpace(resolvedId) && !_activeGenerations.TryGetValue(policy.Name, out resolvedId)) + { + return Task.FromResult(null); + } + snapshot = GetEntry(policy.Name, resolvedId!); + if (snapshot is null) + { + return Task.FromResult(null); + } + snapshot = snapshot.Clone(); + } + + var coverage = snapshot.State == RecordSearchProjectionGenerationStates.Retired + ? RecordSearchProjectionCoverageStatuses.Unavailable + : snapshot.AvailablePartitions.SequenceEqual(snapshot.Descriptor.ExpectedPartitions, StringComparer.Ordinal) + ? RecordSearchProjectionCoverageStatuses.Complete + : RecordSearchProjectionCoverageStatuses.Incomplete; + var inspection = new RecordSearchProjectionGenerationInspection + { + Descriptor = snapshot.Descriptor, + State = snapshot.State, + AvailablePartitions = snapshot.AvailablePartitions, + CoverageStatus = coverage, + ObservedAtUtc = UtcNow() + }; + RecordSearchProjectionGenerationContract.ValidateInspection(inspection); + return Task.FromResult(inspection); + } + + public Task SearchGenerationAsync( + RecordCollectionPolicy policy, + GenerationBoundRecordSearchProjectionRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(policy); + RecordSearchProjectionGenerationContract.ValidateRequest(request); + ct.ThrowIfCancellationRequested(); + var now = UtcNow(); + + ContinuationPayload? continuation = null; + if (!string.IsNullOrWhiteSpace(request.Query.ContinuationToken)) + { + var continuationFailure = ReadContinuation(request.Query.ContinuationToken!, now, out continuation); + if (continuationFailure is not null) + { + return Task.FromResult(Failure( + request, + null, + RecordSearchProjectionCoverageStatuses.Unavailable, + continuationFailure.Value.Code, + continuationFailure.Value.Message, + retryable: false)); + } + if (request.GenerationId is not null && !string.Equals(request.GenerationId, continuation!.GenerationId, StringComparison.Ordinal)) + { + return Task.FromResult(Failure( + request, + null, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.InvalidContinuation, + "The requested generation does not match the continuation generation.", + retryable: false)); + } + } + + var generationId = continuation?.GenerationId ?? request.GenerationId; + GenerationEntry? entry; + lock (_gate) + { + if (generationId is null && !_activeGenerations.TryGetValue(policy.Name, out generationId)) + { + return Task.FromResult(Failure( + request, + null, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.GenerationUnavailable, + "No active generation is available.", + retryable: true)); + } + entry = GetEntry(policy.Name, generationId!)?.Clone(); + } + + if (entry is null) + { + return Task.FromResult(Failure( + request, + null, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.GenerationUnavailable, + "The requested generation is unavailable.", + retryable: true, + generationId: generationId)); + } + + var requestedPartitions = ResolveRequestedPartitions(request.Query, entry.Descriptor); + if (entry.State == RecordSearchProjectionGenerationStates.Retired) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Unavailable, + RecordSearchProjectionFailureCodes.GenerationRetired, + "The requested generation has been retired.", + retryable: false, + requestedPartitions: requestedPartitions)); + } + + var coveredPartitions = requestedPartitions + .Intersect(entry.AvailablePartitions, StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + var missingPartitions = requestedPartitions + .Except(coveredPartitions, StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + if (missingPartitions.Count != 0) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Incomplete, + RecordSearchProjectionFailureCodes.CoverageIncomplete, + "One or more requested partitions are not covered by the selected generation.", + retryable: true, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + + if (request.DeadlineUtc is { } deadline && deadline <= now) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.DeadlineExceeded, + "The generation-bound search deadline has elapsed.", + retryable: true, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + if (request.ExpectedDescriptorDigest is not null && + !string.Equals(request.ExpectedDescriptorDigest, entry.Descriptor.DescriptorDigest, StringComparison.Ordinal)) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.GenerationDescriptorMismatch, + "The selected generation descriptor does not match the caller's expected digest.", + retryable: false, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + + var unsupported = ValidateSupportedQuery(request.Query); + if (unsupported is not null) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.UnsupportedQuery, + unsupported, + retryable: false, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + + var requestFingerprint = RecordSearchProjectionGenerationContract.ComputeRequestFingerprint( + request, + entry.Descriptor.GenerationId, + entry.Descriptor.DescriptorDigest, + requestedPartitions); + if (continuation is not null && + (!string.Equals(continuation.DescriptorDigest, entry.Descriptor.DescriptorDigest, StringComparison.Ordinal) || + !string.Equals(continuation.RequestFingerprint, requestFingerprint, StringComparison.Ordinal))) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.InvalidContinuation, + "The continuation does not match the immutable generation or request.", + retryable: false, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + + var documents = entry.Documents + .Where(document => requestedPartitions.Contains(document.Candidate.PartitionKey, StringComparer.Ordinal)) + .ToList(); + var requestedWorkLimit = request.Query.Lexical?.ScanLimit ?? _options.DefaultWorkLimit; + var workLimit = Math.Min(requestedWorkLimit, _options.MaxWorkLimit); + if (documents.Count > workLimit) + { + var failed = Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.WorkLimitExceeded, + "The exhaustive reference work bound was exceeded; no partial candidate page was returned.", + retryable: false, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions); + failed.Diagnostics.WorkLimit = workLimit; + failed.Diagnostics.WorkUnits = documents.Count; + failed.Diagnostics.CandidateBound = workLimit; + RecordSearchProjectionGenerationContract.ValidateResult(failed); + return Task.FromResult(failed); + } + + ct.ThrowIfCancellationRequested(); + var matches = Search(documents, request.Query.Lexical, ct); + if (request.DeadlineUtc is { } completedDeadline && completedDeadline <= UtcNow()) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.DeadlineExceeded, + "The generation-bound search deadline elapsed before the bounded scan completed.", + retryable: true, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + var limit = request.Query.Limit ?? request.Query.Lexical?.Top ?? _options.DefaultResultLimit; + if (limit <= 0 || limit > _options.MaxResultLimit || + request.Query.Lexical is { Top: > 0 } lexical && lexical.Top > _options.MaxResultLimit) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.UnsupportedQuery, + $"The result limit must be between 1 and {_options.MaxResultLimit}.", + retryable: false, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + + var offset = continuation?.Offset ?? 0; + if (offset < 0 || offset > matches.Count) + { + return Task.FromResult(Failure( + request, + entry, + RecordSearchProjectionCoverageStatuses.Complete, + RecordSearchProjectionFailureCodes.InvalidContinuation, + "The continuation offset is outside the retained result set.", + retryable: false, + requestedPartitions: requestedPartitions, + coveredPartitions: coveredPartitions)); + } + var page = matches.Skip(offset).Take(limit).ToList(); + var nextOffset = offset + page.Count; + var next = nextOffset < matches.Count + ? WriteContinuation(new ContinuationPayload + { + Version = ContinuationVersion, + GenerationId = entry.Descriptor.GenerationId, + DescriptorDigest = entry.Descriptor.DescriptorDigest, + RequestFingerprint = requestFingerprint, + Offset = nextOffset, + ExpiresAtUtc = now.Add(_options.ContinuationLifetime) + }) + : null; + var result = new GenerationBoundRecordSearchProjectionResult + { + Status = RecordSearchProjectionResultStatuses.Succeeded, + GenerationId = entry.Descriptor.GenerationId, + GenerationDescriptorDigest = entry.Descriptor.DescriptorDigest, + Items = page, + ContinuationToken = next, + Coverage = new RecordSearchProjectionCoverage + { + Status = RecordSearchProjectionCoverageStatuses.Complete, + RequestedPartitions = requestedPartitions, + CoveredPartitions = coveredPartitions + }, + Diagnostics = new RecordSearchProjectionWorkDiagnostics + { + WorkLimit = workLimit, + WorkUnits = documents.Count, + CandidateBound = workLimit, + CandidateCount = matches.Count, + ReturnedCount = page.Count, + CacheStatus = RecordSearchProjectionCacheStatuses.NotApplicable + } + }; + RecordSearchProjectionGenerationContract.ValidateResult(result); + return Task.FromResult(result); + } + + private GenerationBoundRecordSearchProjectionResult Failure( + GenerationBoundRecordSearchProjectionRequest request, + GenerationEntry? entry, + string coverageStatus, + string failureCode, + string message, + bool retryable, + string? generationId = null, + List? requestedPartitions = null, + List? coveredPartitions = null) + { + requestedPartitions ??= request.Query.PartitionKeys? + .Distinct(StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList() ?? new List(); + requestedPartitions = requestedPartitions.ToList(); + coveredPartitions = coveredPartitions?.ToList() ?? new List(); + if (coverageStatus == RecordSearchProjectionCoverageStatuses.Unavailable) + { + coveredPartitions.Clear(); + } + var missing = requestedPartitions + .Except(coveredPartitions, StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + var result = new GenerationBoundRecordSearchProjectionResult + { + Status = RecordSearchProjectionResultStatuses.Failed, + GenerationId = entry?.Descriptor.GenerationId ?? generationId, + GenerationDescriptorDigest = entry?.Descriptor.DescriptorDigest, + Coverage = new RecordSearchProjectionCoverage + { + Status = coverageStatus, + RequestedPartitions = requestedPartitions, + CoveredPartitions = coveredPartitions, + MissingPartitions = missing + }, + Diagnostics = new RecordSearchProjectionWorkDiagnostics(), + Failure = new RecordSearchProjectionFailure + { + Code = failureCode, + Message = message, + Retryable = retryable + } + }; + RecordSearchProjectionGenerationContract.ValidateResult(result); + return result; + } + + private static List Search( + IReadOnlyCollection documents, + LexicalSearchOptions? lexical, + CancellationToken ct) + { + var queryTokens = Tokenize(lexical?.Query ?? string.Empty); + var requireAll = lexical?.MatchMode == LexicalMatchModes.All; + var matches = new List(); + foreach (var document in documents) + { + ct.ThrowIfCancellationRequested(); + var textTokens = Tokenize(document.SearchText); + var counts = textTokens + .GroupBy(value => value, StringComparer.Ordinal) + .ToDictionary(group => group.Key, group => group.Count(), StringComparer.Ordinal); + var matched = queryTokens.Count == 0 + ? 1 + : queryTokens.Sum(token => counts.GetValueOrDefault(token)); + var accepted = queryTokens.Count == 0 || + (requireAll + ? queryTokens.All(token => counts.ContainsKey(token)) + : matched > 0); + if (!accepted) + { + continue; + } + matches.Add(new RecordSearchProjectionCandidate + { + PartitionKey = document.Candidate.PartitionKey, + Id = document.Candidate.Id, + Revision = document.Candidate.Revision, + Score = matched + }); + } + return matches + .OrderByDescending(candidate => candidate.Score) + .ThenBy(candidate => candidate.PartitionKey, StringComparer.Ordinal) + .ThenBy(candidate => candidate.Id, StringComparer.Ordinal) + .ToList(); + } + + private static List Tokenize(string value) + { + var tokens = new List(); + var current = new StringBuilder(); + foreach (var character in value.Normalize(NormalizationForm.FormC)) + { + if (char.IsLetterOrDigit(character) || character == '_') + { + current.Append(char.ToLowerInvariant(character)); + continue; + } + FlushToken(current, tokens); + } + FlushToken(current, tokens); + return tokens; + } + + private static void FlushToken(StringBuilder current, List tokens) + { + if (current.Length == 0) + { + return; + } + tokens.Add(current.ToString()); + current.Clear(); + } + + private static string? ValidateSupportedQuery(QueryEnvelope query) + { + if (query.Filter is not null || query.Vector is not null || query.OrderBy is { Count: > 0 }) + { + return "The exhaustive local reference currently supports partition and lexical selection only."; + } + if (query.Lexical is { } lexical) + { + if (lexical.ScanLimit <= 0) + { + return "lexical.scanLimit must be greater than zero."; + } + if (lexical.MatchMode is not LexicalMatchModes.Any and not LexicalMatchModes.All) + { + return "The exhaustive local reference supports lexical matchMode any or all."; + } + if (lexical.Fields is { Count: > 0 } || lexical.RequiredPhraseGroups is { Count: > 0 } || + lexical.PrefixMatching || lexical.FieldBoosts is { Count: > 0 } || lexical.MinScore.HasValue) + { + return "Fielded, phrase, prefix, boosted, and score-threshold lexical behavior is capability-gated outside this reference selector."; + } + if (!string.Equals(lexical.Scoring, LexicalScorings.Bm25, StringComparison.Ordinal) || + lexical.Bm25K1 != 1.2f || lexical.Bm25B != 0.75f || lexical.PhraseBoost != 0.15f || + lexical.ExactBoost != 0.25f || lexical.MetadataBoost != 0.10f) + { + return "The exhaustive local reference does not accept caller-selected scoring parameters; scores are defined by the generation strategy."; + } + if (lexical.Top <= 0) + { + return "lexical.top must be greater than zero."; + } + } + return null; + } + + private static List ResolveRequestedPartitions( + QueryEnvelope query, + RecordSearchProjectionGenerationDescriptor descriptor) => + (query.PartitionKeys is { Count: > 0 } ? query.PartitionKeys : descriptor.ExpectedPartitions) + .Distinct(StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToList(); + + private (string Code, string Message)? ReadContinuation( + string token, + DateTime now, + out ContinuationPayload? payload) + { + payload = null; + if (token.Length > _options.MaxContinuationTokenChars) + { + return (RecordSearchProjectionFailureCodes.InvalidContinuation, "The continuation exceeds the configured size bound."); + } + var parts = token.Split('.', StringSplitOptions.None); + if (parts.Length != 2 || !TryDecode(parts[0], out var body) || !TryDecode(parts[1], out var signature)) + { + return (RecordSearchProjectionFailureCodes.InvalidContinuation, "The continuation is malformed."); + } + var expected = HMACSHA256.HashData(_signingKey, body); + if (!CryptographicOperations.FixedTimeEquals(expected, signature)) + { + return (RecordSearchProjectionFailureCodes.InvalidContinuation, "The continuation signature is invalid."); + } + try + { + payload = JsonSerializer.Deserialize(body, JsonOptions); + } + catch (JsonException) + { + return (RecordSearchProjectionFailureCodes.InvalidContinuation, "The continuation payload is invalid."); + } + if (payload is null || payload.Version != ContinuationVersion || payload.Offset < 0 || + string.IsNullOrWhiteSpace(payload.GenerationId) || string.IsNullOrWhiteSpace(payload.DescriptorDigest) || + string.IsNullOrWhiteSpace(payload.RequestFingerprint)) + { + payload = null; + return (RecordSearchProjectionFailureCodes.InvalidContinuation, "The continuation payload is invalid."); + } + if (payload.ExpiresAtUtc <= now) + { + return (RecordSearchProjectionFailureCodes.ExpiredContinuation, "The continuation has expired."); + } + return null; + } + + private string WriteContinuation(ContinuationPayload payload) + { + var body = JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptions); + var signature = HMACSHA256.HashData(_signingKey, body); + return Encode(body) + "." + Encode(signature); + } + + private static string Encode(byte[] value) => + Convert.ToBase64String(value).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + private static bool TryDecode(string value, out byte[] bytes) + { + try + { + var normalized = value.Replace('-', '+').Replace('_', '/'); + normalized += new string('=', (4 - normalized.Length % 4) % 4); + bytes = Convert.FromBase64String(normalized); + return true; + } + catch (FormatException) + { + bytes = Array.Empty(); + return false; + } + } + + private GenerationEntry? GetEntry(string collection, string generationId) => + _collections.TryGetValue(collection, out var generations) && generations.TryGetValue(generationId, out var entry) + ? entry + : null; + + private DateTime UtcNow() => _options.TimeProvider.GetUtcNow().UtcDateTime; + + private static void ValidateDocuments(LocalRecordSearchProjectionGeneration generation) + { + if (generation.Documents.Count != generation.Descriptor.ExpectedItemCount) + { + throw new InvalidOperationException("Generation document count does not match descriptor expectedItemCount."); + } + var identities = new HashSet(StringComparer.Ordinal); + foreach (var document in generation.Documents) + { + var candidate = document.Candidate ?? throw new InvalidOperationException("A local generation document requires a candidate identity."); + if (!generation.Descriptor.ExpectedPartitions.Contains(candidate.PartitionKey, StringComparer.Ordinal)) + { + throw new InvalidOperationException("A generation document partition is absent from descriptor expectedPartitions."); + } + if (string.IsNullOrWhiteSpace(candidate.Id) || candidate.Id.Length > 200 || + !string.Equals(candidate.Id, candidate.Id.Trim(), StringComparison.Ordinal) || + candidate.Id.Any(char.IsControl) || candidate.Revision <= 0 || !float.IsFinite(candidate.Score)) + { + throw new InvalidOperationException("A generation document requires a non-empty ID, positive revision, and finite score."); + } + if (!identities.Add(candidate.PartitionKey + "\n" + candidate.Id)) + { + throw new InvalidOperationException("Generation documents must have unique partition/id identities."); + } + if (document.SearchText is null) + { + throw new InvalidOperationException("A local generation document requires search text."); + } + } + } + + private static RecordSearchProjectionGenerationDescriptor CloneDescriptor( + RecordSearchProjectionGenerationDescriptor descriptor) => + JsonSerializer.Deserialize( + JsonSerializer.Serialize(descriptor, JsonOptions), + JsonOptions) ?? throw new InvalidOperationException("The generation descriptor could not be cloned."); + + private static LocalRecordSearchProjectionDocument CloneDocument(LocalRecordSearchProjectionDocument document) => new() + { + Candidate = new RecordSearchProjectionCandidate + { + PartitionKey = document.Candidate.PartitionKey, + Id = document.Candidate.Id, + Revision = document.Candidate.Revision, + Score = document.Candidate.Score + }, + SearchText = document.SearchText + }; + + private static bool GenerationContentEquals(GenerationEntry left, GenerationEntry right) => + left.AvailablePartitions.SequenceEqual(right.AvailablePartitions, StringComparer.Ordinal) && + left.Documents.Count == right.Documents.Count && + OrderDocuments(left.Documents).Zip(OrderDocuments(right.Documents)).All(pair => + string.Equals(pair.First.Candidate.PartitionKey, pair.Second.Candidate.PartitionKey, StringComparison.Ordinal) && + string.Equals(pair.First.Candidate.Id, pair.Second.Candidate.Id, StringComparison.Ordinal) && + pair.First.Candidate.Revision == pair.Second.Candidate.Revision && + pair.First.Candidate.Score.Equals(pair.Second.Candidate.Score) && + string.Equals(pair.First.SearchText, pair.Second.SearchText, StringComparison.Ordinal)); + + private static IEnumerable OrderDocuments( + IEnumerable documents) => + documents + .OrderBy(document => document.Candidate.PartitionKey, StringComparer.Ordinal) + .ThenBy(document => document.Candidate.Id, StringComparer.Ordinal); + + private sealed class GenerationEntry + { + public GenerationEntry( + RecordSearchProjectionGenerationDescriptor descriptor, + List documents, + List availablePartitions, + string state) + { + Descriptor = descriptor; + Documents = documents; + AvailablePartitions = availablePartitions; + State = state; + } + + public RecordSearchProjectionGenerationDescriptor Descriptor { get; } + public List Documents { get; } + public List AvailablePartitions { get; set; } + public string State { get; set; } + + public GenerationEntry Clone() => new( + CloneDescriptor(Descriptor), + Documents.Select(CloneDocument).ToList(), + AvailablePartitions.ToList(), + State); + } + + private sealed class ContinuationPayload + { + public string Version { get; set; } = string.Empty; + public string GenerationId { get; set; } = string.Empty; + public string DescriptorDigest { get; set; } = string.Empty; + public string RequestFingerprint { get; set; } = string.Empty; + public int Offset { get; set; } + public DateTime ExpiresAtUtc { get; set; } + } +} diff --git a/src/Vyral.Local/LocalRetrievalEvaluationService.cs b/src/Vyral.Local/LocalRetrievalEvaluationService.cs index cd4fb19..97874a3 100644 --- a/src/Vyral.Local/LocalRetrievalEvaluationService.cs +++ b/src/Vyral.Local/LocalRetrievalEvaluationService.cs @@ -19,16 +19,29 @@ public class LocalRetrievalEvaluationService : IRetrievalEvaluationService private const int MaxComparisonVariants = 12; private const int MaxK = 100; private readonly IRetrievalService _retrievalService; + private readonly IRetrievalEvaluationTargetResolver? _targetResolver; - public LocalRetrievalEvaluationService(IRetrievalService retrievalService) + public LocalRetrievalEvaluationService( + IRetrievalService retrievalService, + IRetrievalEvaluationTargetResolver? targetResolver = null) { _retrievalService = retrievalService; + _targetResolver = targetResolver; } public async Task EvaluateAsync( RetrievalEvaluationRequest request, CancellationToken ct = default, IProgress? progress = null) + { + return await EvaluateTargetAsync(_retrievalService, request, ct, progress); + } + + private static async Task EvaluateTargetAsync( + IRetrievalService retrievalService, + RetrievalEvaluationRequest request, + CancellationToken ct, + IProgress? progress = null) { ValidateRequest(request); ct.ThrowIfCancellationRequested(); @@ -43,7 +56,7 @@ public async Task EvaluateAsync( var start = DateTime.UtcNow; try { - var caseResult = await EvaluateCaseAsync(request, testCase, i, ct); + var caseResult = await EvaluateCaseAsync(retrievalService, request, testCase, i, ct); caseResult.DurationMs = (DateTime.UtcNow - start).TotalMilliseconds; result.Cases.Add(caseResult); result.Succeeded++; @@ -109,13 +122,19 @@ public async Task CompareAsync( var start = DateTime.UtcNow; try { - var evaluation = await EvaluateAsync(new RetrievalEvaluationRequest + var resolvedTarget = ResolveTarget(variant); + var evaluation = await EvaluateTargetAsync(resolvedTarget.Service, new RetrievalEvaluationRequest { Cases = ApplyVariant(request.Cases, variant, ct), ContinueOnError = request.ContinueOnError, DefaultK = request.DefaultK, IncludeTopResults = request.IncludeTopResults }, ct); + if (variant.Target is not null && evaluation.Failed != 0) + { + throw new InvalidOperationException( + $"Retrieval evaluation target '{variant.Target.Id}' failed {evaluation.Failed} case(s)."); + } var metrics = ToMetrics(evaluation); if (i == 0) { @@ -126,6 +145,7 @@ public async Task CompareAsync( { Id = variant.Id, Label = variant.Label, + Target = variant.Target is null ? null : resolvedTarget.Evidence, Status = EvaluationVariantStatuses.Succeeded, DurationMs = (DateTime.UtcNow - start).TotalMilliseconds, Metrics = metrics, @@ -142,6 +162,7 @@ public async Task CompareAsync( { Id = variant.Id, Label = variant.Label, + Target = null, Status = EvaluationVariantStatuses.Failed, Error = ex.Message, DurationMs = (DateTime.UtcNow - start).TotalMilliseconds, @@ -164,6 +185,24 @@ public async Task CompareAsync( return result; } + private RetrievalEvaluationResolvedTarget ResolveTarget(RetrievalEvaluationVariant variant) + { + if (variant.Target is null) + { + return new RetrievalEvaluationResolvedTarget + { + Service = _retrievalService, + Evidence = new RetrievalEvaluationTargetEvidence { Id = "default" } + }; + } + if (_targetResolver is null) + { + throw new InvalidOperationException( + $"Retrieval evaluation target '{variant.Target.Id}' is not configured."); + } + return _targetResolver.Resolve(variant.Target); + } + private static void ReportComparisonProgress( IProgress? progress, RetrievalEvaluationComparisonResult result, @@ -213,7 +252,8 @@ private static RetrievalEvaluationComparisonResult CloneComparisonResult(Retriev return JsonSerializer.Deserialize(json, JsonOptions)!; } - private async Task EvaluateCaseAsync( + private static async Task EvaluateCaseAsync( + IRetrievalService retrievalService, RetrievalEvaluationRequest request, RetrievalEvaluationCase testCase, int index, @@ -224,7 +264,7 @@ private async Task EvaluateCaseAsync( var k = ResolveK(request, testCase); var retrievalRequest = CloneRetrievalRequest(testCase.Request, k, request.IncludeTopResults); - var retrieval = await _retrievalService.SearchAsync(retrievalRequest, ct); + var retrieval = await retrievalService.SearchAsync(retrievalRequest, ct); ct.ThrowIfCancellationRequested(); var topResults = retrieval.Results.OrderBy(match => match.Rank).Take(k).ToList(); var rerankEnabled = GetTraceBool(retrieval.Trace, "rerankEnabled") ?? (retrievalRequest.Rerank?.Enabled == true); @@ -929,6 +969,39 @@ private static void ValidateComparisonRequest(RetrievalEvaluationComparisonReque { throw new InvalidOperationException("Retrieval evaluation comparison variant limit must be greater than zero when provided."); } + + ValidateTargetReference(variant.Target); + } + } + + private static void ValidateTargetReference(RetrievalEvaluationTargetReference? target) + { + if (target is null) + { + return; + } + ValidateTargetIdentifier(target.Id, "target id"); + if (target.GenerationId is not null) + { + ValidateTargetIdentifier(target.GenerationId, "target generationId"); + } + if (target.ExpectedGenerationDescriptorDigest is { } digest && + (digest.Length != 71 || !digest.StartsWith("sha256:", StringComparison.Ordinal) || + digest[7..].Any(character => + !((character >= '0' && character <= '9') || (character >= 'a' && character <= 'f'))))) + { + throw new InvalidOperationException( + "Retrieval evaluation target expectedGenerationDescriptorDigest must be a lowercase SHA-256 digest."); + } + } + + private static void ValidateTargetIdentifier(string value, string label) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > 200 || + !string.Equals(value, value.Trim(), StringComparison.Ordinal) || value.Any(char.IsControl)) + { + throw new InvalidOperationException( + $"Retrieval evaluation {label} must be a bounded non-whitespace identifier."); } } diff --git a/src/Vyral.Server/ExecutionRuntimeProjectionGenerationAdapter.cs b/src/Vyral.Server/ExecutionRuntimeProjectionGenerationAdapter.cs new file mode 100644 index 0000000..b359d62 --- /dev/null +++ b/src/Vyral.Server/ExecutionRuntimeProjectionGenerationAdapter.cs @@ -0,0 +1,275 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; +using Vyral.Execution; + +namespace Vyral.Server; + +/// +/// Experimental management-plane composition for durable projection generation. Search adapters +/// do not depend on this class and no per-query network hop is introduced. +/// +public sealed class ExecutionRuntimeProjectionGenerationAdapter +{ + public const string PluginId = "vyral.retrieval.projection-generation"; + public const string BuildHandlerId = "vyral.retrieval.projection-generation.build"; + private readonly IExecutionRuntime _runtime; + private readonly IRecordSearchProjectionGenerationBuilder _builder; + + public ExecutionRuntimeProjectionGenerationAdapter( + IExecutionRuntime runtime, + IRecordSearchProjectionGenerationBuilder builder, + IObjectStore artifactStore) + { + _runtime = runtime ?? throw new ArgumentNullException(nameof(runtime)); + _builder = builder ?? throw new ArgumentNullException(nameof(builder)); + ArgumentNullException.ThrowIfNull(artifactStore); + if (string.IsNullOrWhiteSpace(builder.BuilderId)) + { + throw new InvalidOperationException("A projection generation builder requires a stable builder ID."); + } + _runtime.RegisterPlugin(new ProjectionGenerationPlugin(builder, artifactStore)); + } + + public Task StartBuildAsync( + RecordSearchProjectionGenerationBuildRequest request, + string idempotencyKey, + ExecutionScope? scope = null, + CancellationToken ct = default) + { + RecordSearchProjectionGenerationContract.ValidateBuildRequest(request); + if (!string.Equals(request.BuilderId, _builder.BuilderId, StringComparison.Ordinal)) + { + throw new InvalidOperationException("The build request does not target the configured builder."); + } + if (string.IsNullOrWhiteSpace(idempotencyKey)) + { + throw new InvalidOperationException("A durable projection generation build requires an idempotency key."); + } + return _runtime.StartRunAsync(new ExecutionRunRequest + { + HandlerId = BuildHandlerId, + PluginId = PluginId, + Payload = JsonSerializer.SerializeToNode(request, ExecutionJson.Options), + IdempotencyKey = idempotencyKey, + Scope = scope, + RetryPolicy = new ExecutionRetryPolicy + { + MaxAttempts = 3, + InitialDelaySeconds = 1, + MaxDelaySeconds = 30, + BackoffMultiplier = 2 + }, + Tags = + { + ["vyral.job"] = "retrieval-projection-generation", + ["vyral.retrieval.builder"] = request.BuilderId, + ["vyral.retrieval.generation"] = request.GenerationId, + ["vyral.retrieval.profile"] = request.ProfileId + } + }, ct); + } + + public async Task GetAsync( + string runId, + bool includeResult = true, + CancellationToken ct = default) + { + var run = await _runtime.GetRunAsync(runId, includeResult, ct); + return IsProjectionGenerationRun(run) ? run : null; + } + + public async Task CancelAsync(string runId, CancellationToken ct = default) + { + var existing = await _runtime.GetRunAsync(runId, includeResult: false, ct); + return IsProjectionGenerationRun(existing) + ? await _runtime.CancelRunAsync(runId, ct) + : null; + } + + private static bool IsProjectionGenerationRun(ExecutionRun? run) => + run is not null && + string.Equals(run.HandlerId, BuildHandlerId, StringComparison.Ordinal) && + string.Equals(run.PluginId, PluginId, StringComparison.Ordinal); + + private sealed class ProjectionGenerationPlugin : IExecutionPlugin + { + private readonly IReadOnlyList _handlers; + + public ProjectionGenerationPlugin( + IRecordSearchProjectionGenerationBuilder builder, + IObjectStore artifactStore) + { + _handlers = [new BuildHandler(builder, artifactStore)]; + Descriptor = new ExecutionPluginDescriptor + { + PluginId = PluginId, + Name = "Vyral retrieval projection generation", + Version = "0.1.0", + Handlers = [BuildHandler.DescriptorValue] + }; + } + + public ExecutionPluginDescriptor Descriptor { get; } + public IReadOnlyList Handlers => _handlers; + } + + private sealed class BuildHandler : IExecutionHandler + { + public static readonly ExecutionHandlerDescriptor DescriptorValue = new() + { + HandlerId = BuildHandlerId, + PluginId = PluginId, + DisplayName = "Build and verify an immutable retrieval projection generation", + Description = "Builds provider-native artifacts and returns a compact generation-bound verification receipt.", + MaxAttempts = 3, + Tags = + { + ["vyral.job"] = "retrieval-projection-generation" + } + }; + + private readonly IRecordSearchProjectionGenerationBuilder _builder; + private readonly IObjectStore _artifactStore; + + public BuildHandler(IRecordSearchProjectionGenerationBuilder builder, IObjectStore artifactStore) + { + _builder = builder; + _artifactStore = artifactStore; + } + + public ExecutionHandlerDescriptor Descriptor => DescriptorValue; + + public async Task ExecuteAsync( + IExecutionRunContext context, + CancellationToken ct = default) + { + var request = context.Run.Payload?.Deserialize(ExecutionJson.Options) + ?? throw new InvalidOperationException("A projection generation build payload is required."); + RecordSearchProjectionGenerationContract.ValidateBuildRequest(request); + if (!string.Equals(request.BuilderId, _builder.BuilderId, StringComparison.Ordinal)) + { + return ExecutionRunResult.Failed( + ExecutionFailureClasses.Validation, + "The durable build targets a different projection generation builder."); + } + if (request.DeadlineUtc is { } deadline && deadline <= DateTime.UtcNow) + { + return ExecutionRunResult.Failed( + ExecutionFailureClasses.Timeout, + "The projection generation build deadline elapsed before execution began."); + } + + var buildDeadline = request.DeadlineUtc; + using var deadlineSource = buildDeadline.HasValue + ? CancellationTokenSource.CreateLinkedTokenSource(ct) + : null; + if (deadlineSource is not null) + { + var remaining = buildDeadline!.Value - DateTime.UtcNow; + deadlineSource.CancelAfter(remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero); + } + + RecordSearchProjectionGenerationBuildReceipt receipt; + try + { + var buildCancellation = deadlineSource?.Token ?? ct; + receipt = await _builder.BuildAndVerifyAsync( + request, + _artifactStore, + (update, _) => ReportProgressAsync(context, update, buildCancellation), + buildCancellation); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested && deadlineSource?.IsCancellationRequested == true) + { + return ExecutionRunResult.Failed( + ExecutionFailureClasses.Timeout, + "The projection generation build deadline elapsed during execution."); + } + if (request.DeadlineUtc is { } completedDeadline && completedDeadline <= DateTime.UtcNow) + { + return ExecutionRunResult.Failed( + ExecutionFailureClasses.Timeout, + "The projection generation build deadline elapsed before verification completed."); + } + try + { + RecordSearchProjectionGenerationContract.ValidateBuildReceipt(request, receipt); + } + catch (InvalidOperationException exception) + { + return ExecutionRunResult.Failed( + ExecutionFailureClasses.Validation, + "Projection generation verification receipt was rejected: " + exception.Message); + } + + var result = JsonSerializer.SerializeToNode(receipt, ExecutionJson.Options); + await context.PutArtifactAsync(new ExecutionArtifactWrite + { + Name = "projection-generation-receipt", + Kind = ExecutionArtifactKinds.Json, + MediaType = "application/json", + Content = result?.DeepClone(), + Metadata = + { + ["generationId"] = receipt.Descriptor.GenerationId, + ["descriptorDigest"] = receipt.Descriptor.DescriptorDigest, + ["builderId"] = receipt.BuilderId + } + }, ct); + var status = new JsonObject + { + ["stage"] = "verified", + ["generationId"] = receipt.Descriptor.GenerationId, + ["descriptorDigest"] = receipt.Descriptor.DescriptorDigest, + ["artifactCount"] = receipt.Descriptor.Artifacts.Count + }; + await context.ReportAsync(new ExecutionRunUpdate + { + Requested = checked((int)Math.Min(receipt.Descriptor.ExpectedItemCount, int.MaxValue)), + Attempted = checked((int)Math.Min(receipt.Descriptor.ExpectedItemCount, int.MaxValue)), + Succeeded = checked((int)Math.Min(receipt.Descriptor.ExpectedItemCount, int.MaxValue)), + Failed = 0, + Progress = 1, + CurrentStep = "verified", + Result = result?.DeepClone(), + StatusDetails = status + }, ct); + return ExecutionRunResult.Succeeded(result, status); + } + + private static async Task ReportProgressAsync( + IExecutionRunContext context, + RecordSearchProjectionGenerationBuildProgress progress, + CancellationToken ct) + { + RecordSearchProjectionGenerationContract.ValidateBuildProgress(progress); + var ratio = progress.Total is > 0 + ? Math.Clamp(progress.Completed / (double)progress.Total.Value, 0, 1) + : 0; + var checkpoint = new JsonObject + { + ["stage"] = progress.Stage, + ["completed"] = progress.Completed, + ["total"] = progress.Total + }; + if (progress.Checkpoint is not null) + { + checkpoint["builder"] = progress.Checkpoint.DeepClone(); + } + await context.PutCheckpointAsync(new ExecutionCheckpointWrite + { + Key = "projection-generation-progress", + Content = checkpoint.DeepClone() + }, ct); + await context.ReportAsync(new ExecutionRunUpdate + { + Progress = ratio, + CurrentStep = progress.Stage, + StatusDetails = checkpoint + }, ct); + } + } + +} diff --git a/tests/Vyral.Tests.Aws/OpenSearchGenerationBoundRecordSearchProjectionTests.cs b/tests/Vyral.Tests.Aws/OpenSearchGenerationBoundRecordSearchProjectionTests.cs new file mode 100644 index 0000000..c408715 --- /dev/null +++ b/tests/Vyral.Tests.Aws/OpenSearchGenerationBoundRecordSearchProjectionTests.cs @@ -0,0 +1,478 @@ +using System.Security.Cryptography; +using System.Text; +using Vyral.Abstractions.Models; +using Vyral.Aws; + +namespace Vyral.Tests.Aws; + +public sealed class OpenSearchGenerationBoundRecordSearchProjectionTests +{ + private const string IndexName = "vyral-evidence-generation-a"; + private const string IndexUuid = "uuid-generation-a"; + + [Fact] + public async Task Search_UsesTheDescriptorBoundExactReadOnlyIndexAndReturnsCompleteCandidates() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSearch(IndexName, totalShards: 2, successfulShards: 2, failedShards: 0); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + var projection = CreateProjection(transport); + var request = CreateRequest(); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), request); + + Assert.Equal(RecordSearchProjectionResultStatuses.Succeeded, result.Status); + Assert.Equal("generation-a", result.GenerationId); + Assert.Single(result.Items); + Assert.Equal("record-a", result.Items[0].Id); + Assert.Equal(RecordSearchProjectionCoverageStatuses.Complete, result.Coverage.Status); + Assert.Collection( + transport.Requests, + settings => Assert.Equal($"/{IndexName}/_settings?flat_settings=true", settings.PathAndQuery), + search => + { + Assert.Equal($"/{IndexName}/_search?allow_partial_search_results=false", search.PathAndQuery); + Assert.DoesNotContain("alias", search.PathAndQuery, StringComparison.OrdinalIgnoreCase); + }, + finalSettings => Assert.Equal($"/{IndexName}/_settings?flat_settings=true", finalSettings.PathAndQuery)); + } + + [Theory] + [InlineData(2, 1, 1, false)] + [InlineData(2, 2, 0, true)] + public async Task Search_FailsClosedOnPartialOrTimedOutResponses( + int totalShards, + int successfulShards, + int failedShards, + bool timedOut) + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSearch(IndexName, totalShards, successfulShards, failedShards, timedOut); + var projection = CreateProjection(transport); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Null(result.ContinuationToken); + Assert.Equal(new[] { "tenant-a" }, result.Coverage.MissingPartitions); + } + + [Fact] + public async Task Search_FailsClosedWhenTheRemoteIndexUuidDoesNotMatchTheDescriptorBinding() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, "substituted-index-uuid", readOnly: true); + var projection = CreateProjection(transport); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Single(transport.Requests); + } + + [Fact] + public async Task Search_FailsClosedWhenTheGenerationIndexIsStillWritable() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: false); + var projection = CreateProjection(transport); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Single(transport.Requests); + } + + [Fact] + public async Task Search_ConvertsProviderTransportFailureIntoAnEmptyRetryableResult() + { + var projection = CreateProjection(new ThrowingTransport()); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.True(result.Failure.Retryable); + Assert.Empty(result.Items); + } + + [Fact] + public async Task Search_FailsClosedWhenAHitComesFromOutsideTheSelectedIndex() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSearch("vyral-evidence-other", totalShards: 1, successfulShards: 1, failedShards: 0); + var projection = CreateProjection(transport); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Empty(result.Items); + } + + [Fact] + public async Task Search_FailsClosedWhenAHitEscapesTheRequestedLogicalPartition() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSearch( + IndexName, + totalShards: 1, + successfulShards: 1, + failedShards: 0, + hitPartition: "tenant-b"); + var projection = CreateProjection(transport); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Empty(result.Items); + } + + [Fact] + public async Task Search_FailsClosedIfTheIndexBecomesWritableBeforeSearchCompletion() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSearch(IndexName, totalShards: 1, successfulShards: 1, failedShards: 0); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: false); + var projection = CreateProjection(transport); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.True(result.Failure.Retryable); + Assert.Empty(result.Items); + Assert.Null(result.ContinuationToken); + } + + [Fact] + public async Task Search_FailsWhenTheDeadlineElapsesEvenIfTheTransportIgnoresCancellation() + { + var clock = new MutableTimeProvider(new DateTimeOffset(2026, 8, 27, 12, 0, 0, TimeSpan.Zero)); + var transport = new QueueTransport + { + OnRequest = requestNumber => + { + if (requestNumber == 2) + { + clock.UtcNow = clock.UtcNow.AddMinutes(1); + } + } + }; + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSearch(IndexName, totalShards: 1, successfulShards: 1, failedShards: 0); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + var projection = CreateProjection(transport, timeProvider: clock); + var request = CreateRequest(); + request.DeadlineUtc = clock.UtcNow.AddSeconds(30).UtcDateTime; + + var result = await projection.SearchGenerationAsync(CreatePolicy(), request); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.DeadlineExceeded, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Null(result.ContinuationToken); + } + + [Fact] + public void RegistrationRequiresVectorCapabilityInAdditionToPortableSafetyCapabilities() + { + var descriptor = CreateDescriptor(); + descriptor.Capabilities.Remove(RecordSearchProjectionGenerationCapabilities.Vector); + RecordSearchProjectionGenerationContract.SealDescriptor(descriptor); + + var exception = Assert.Throws(() => + new OpenSearchGenerationBoundRecordSearchProjection( + new QueueTransport(), + new OpenSearchRecordSearchProjectionOptions { MaximumCandidates = 100 }, + [ + new OpenSearchRecordSearchProjectionGeneration + { + Descriptor = descriptor, + IndexName = IndexName, + IndexUuid = IndexUuid, + State = RecordSearchProjectionGenerationStates.Active, + AvailablePartitions = ["tenant-a"] + } + ])); + + Assert.Contains("vector capability", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Search_RejectsDescriptorSubstitutionBeforeCallingOpenSearch() + { + var transport = new QueueTransport(); + var projection = CreateProjection(transport); + var request = CreateRequest(); + request.ExpectedDescriptorDigest = Hash("substituted-descriptor"); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), request); + + Assert.Equal(RecordSearchProjectionFailureCodes.GenerationDescriptorMismatch, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Empty(transport.Requests); + } + + [Fact] + public async Task Search_RejectsCollectionPolicyDriftBeforeCallingOpenSearch() + { + var transport = new QueueTransport(); + var projection = CreateProjection(transport); + var changedPolicy = CreatePolicy(); + changedPolicy.IndexedMetadata.Add("/metadata/new-field"); + + var result = await projection.SearchGenerationAsync(changedPolicy, CreateRequest()); + + Assert.Equal(RecordSearchProjectionFailureCodes.GenerationDescriptorMismatch, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Empty(transport.Requests); + } + + [Fact] + public async Task Search_RejectsIncompleteLogicalCoverageBeforeCallingOpenSearch() + { + var transport = new QueueTransport(); + var projection = CreateProjection(transport, availablePartitions: Array.Empty(), state: RecordSearchProjectionGenerationStates.Retained); + + var result = await projection.SearchGenerationAsync(CreatePolicy(), CreateRequest()); + + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Equal(new[] { "tenant-a" }, result.Coverage.MissingPartitions); + Assert.Empty(transport.Requests); + } + + [Fact] + public async Task Search_DoesNotAcceptAContinuationForTheNonPageableVectorShape() + { + var transport = new QueueTransport(); + var projection = CreateProjection(transport); + var request = CreateRequest(); + request.Query.ContinuationToken = "opaque-provider-token"; + + var result = await projection.SearchGenerationAsync(CreatePolicy(), request); + + Assert.Equal(RecordSearchProjectionFailureCodes.InvalidContinuation, result.Failure!.Code); + Assert.Empty(result.Items); + Assert.Empty(transport.Requests); + } + + [Fact] + public async Task Inspection_RequiresTheSameImmutableRemoteBinding() + { + var transport = new QueueTransport(); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: true); + transport.EnqueueSettings(IndexName, IndexUuid, readOnly: false); + var projection = CreateProjection(transport); + + var complete = await projection.InspectGenerationAsync(CreatePolicy(), "generation-a"); + var incomplete = await projection.InspectGenerationAsync(CreatePolicy(), "generation-a"); + + Assert.Equal(RecordSearchProjectionCoverageStatuses.Complete, complete!.CoverageStatus); + Assert.Equal(new[] { "tenant-a" }, complete.AvailablePartitions); + Assert.Equal(RecordSearchProjectionCoverageStatuses.Incomplete, incomplete!.CoverageStatus); + Assert.Empty(incomplete.AvailablePartitions); + } + + private static OpenSearchGenerationBoundRecordSearchProjection CreateProjection( + IOpenSearchTransport transport, + IEnumerable? availablePartitions = null, + string state = RecordSearchProjectionGenerationStates.Active, + TimeProvider? timeProvider = null) + { + var descriptor = CreateDescriptor(); + return new OpenSearchGenerationBoundRecordSearchProjection( + transport, + new OpenSearchRecordSearchProjectionOptions { MaximumCandidates = 100 }, + new[] + { + new OpenSearchRecordSearchProjectionGeneration + { + Descriptor = descriptor, + IndexName = IndexName, + IndexUuid = IndexUuid, + State = state, + AvailablePartitions = availablePartitions?.ToList() + } + }, + new OpenSearchGenerationBoundRecordSearchProjectionOptions + { + TimeProvider = timeProvider ?? TimeProvider.System + }); + } + + private static RecordSearchProjectionGenerationDescriptor CreateDescriptor() + { + var descriptor = new RecordSearchProjectionGenerationDescriptor + { + Collection = "evidence", + GenerationId = "generation-a", + ProviderId = OpenSearchGenerationBoundRecordSearchProjectionOptions.DefaultProviderId, + ProfileId = "vector-v1", + StrategyVersion = "1", + SourceManifestDigest = Hash("source"), + RecordRevisionSetDigest = Hash("revisions"), + ProjectionSchemaDigest = OpenSearchProjectionGenerationBinding.ComputeProjectionSchemaDigest(CreatePolicy()), + AnalyzerDigest = Hash("analyzer"), + ConfigurationDigest = Hash("configuration"), + ExpectedItemCount = 1, + ExpectedPartitions = new List { "tenant-a" }, + Capabilities = new List + { + RecordSearchProjectionGenerationCapabilities.CompleteCoverage, + RecordSearchProjectionGenerationCapabilities.GenerationPinnedContinuation, + RecordSearchProjectionGenerationCapabilities.Vector + }, + Artifacts = new List + { + new() + { + Id = OpenSearchProjectionGenerationBinding.ArtifactId, + Kind = OpenSearchProjectionGenerationBinding.ArtifactKind, + ContentHash = OpenSearchProjectionGenerationBinding.ComputeContentHash(IndexName, IndexUuid), + SizeBytes = 0 + } + }, + CreatedAtUtc = new DateTime(2026, 8, 27, 12, 0, 0, DateTimeKind.Utc) + }; + RecordSearchProjectionGenerationContract.SealDescriptor(descriptor); + return descriptor; + } + + private static GenerationBoundRecordSearchProjectionRequest CreateRequest() + { + var descriptor = CreateDescriptor(); + return new GenerationBoundRecordSearchProjectionRequest + { + GenerationId = descriptor.GenerationId, + ExpectedDescriptorDigest = descriptor.DescriptorDigest, + Query = new QueryEnvelope + { + PartitionKeys = new List { "tenant-a" }, + Vector = new VectorSearchOptions + { + Field = "embedding", + Value = new[] { 0.1f, 0.2f, 0.3f }, + Top = 10 + }, + Limit = 10 + } + }; + } + + private static RecordCollectionPolicy CreatePolicy() => new() + { + Name = "evidence", + VectorPolicies = new List + { + new() + { + Name = "embedding", + Path = "/vectors/embedding/values", + Dimensions = 3, + DistanceFunction = DistanceFunctions.Cosine, + IndexType = IndexTypes.DiskAnn + } + } + }; + + private static string Hash(string value) => + "sha256:" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private sealed class QueueTransport : IOpenSearchTransport + { + private readonly Queue _responses = new(); + public List<(HttpMethod Method, string PathAndQuery, string JsonBody)> Requests { get; } = new(); + public Action? OnRequest { get; init; } + + public void EnqueueSettings(string indexName, string indexUuid, bool readOnly) => + _responses.Enqueue(new OpenSearchTransportResponse + { + StatusCode = 200, + Body = $$""" + { + "{{indexName}}": { + "settings": { + "index.uuid": "{{indexUuid}}", + "index.blocks.read_only": "{{readOnly.ToString().ToLowerInvariant()}}" + } + } + } + """ + }); + + public void EnqueueSearch( + string hitIndex, + int totalShards, + int successfulShards, + int failedShards, + bool timedOut = false, + string hitPartition = "tenant-a") => + _responses.Enqueue(new OpenSearchTransportResponse + { + StatusCode = 200, + Body = $$""" + { + "timed_out": {{timedOut.ToString().ToLowerInvariant()}}, + "_shards": { + "total": {{totalShards}}, + "successful": {{successfulShards}}, + "skipped": 0, + "failed": {{failedShards}} + }, + "hits": { + "hits": [ + { + "_index": "{{hitIndex}}", + "_score": 0.9, + "_source": { + "partitionKey": "{{hitPartition}}", + "id": "record-a", + "revision": 4 + } + } + ] + } + } + """ + }); + + public Task SendAsync( + HttpMethod method, + string pathAndQuery, + string? jsonBody, + CancellationToken ct = default) + { + Requests.Add((method, pathAndQuery, jsonBody ?? string.Empty)); + OnRequest?.Invoke(Requests.Count); + return Task.FromResult(_responses.Dequeue()); + } + } + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + public DateTimeOffset UtcNow { get; set; } = utcNow; + public override DateTimeOffset GetUtcNow() => UtcNow; + } + + private sealed class ThrowingTransport : IOpenSearchTransport + { + public Task SendAsync( + HttpMethod method, + string pathAndQuery, + string? jsonBody, + CancellationToken ct = default) => + throw new HttpRequestException("provider endpoint unavailable"); + } +} diff --git a/tests/Vyral.Tests.Aws/OpenSearchRecordSearchProjectionLocalTests.cs b/tests/Vyral.Tests.Aws/OpenSearchRecordSearchProjectionLocalTests.cs index d6debad..8cfb3c5 100644 --- a/tests/Vyral.Tests.Aws/OpenSearchRecordSearchProjectionLocalTests.cs +++ b/tests/Vyral.Tests.Aws/OpenSearchRecordSearchProjectionLocalTests.cs @@ -1,6 +1,12 @@ using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading.Tasks; using Amazon.Runtime; +using Vyral.Abstractions.Models; using Vyral.Aws; namespace Vyral.Tests.Aws; @@ -29,4 +35,205 @@ public async Task Projection_IndexesSearchesRevisionFencesAndDeletesAgainstLocal await OpenSearchRecordSearchProjectionQualification.RunAsync(projection, policy); } + + [OpenSearchLocalFact] + public async Task GenerationBoundProjection_BindsAnImmutableIndexAndFailsClosedWhenItBecomesWritable() + { + var endpoint = OpenSearchLocalSettings.GetEndpoint(); + var policy = OpenSearchRecordSearchProjectionQualification.CreatePolicy( + AwsLiveSettings.UniquePrefix("vyral-it-opensearch-generation")); + var indexName = policy.Name; + var projectionOptions = new OpenSearchRecordSearchProjectionOptions + { + PolicyIndexNameFactory = _ => indexName, + MaximumCandidates = 10 + }; + + using var transport = new AwsSigV4OpenSearchTransport( + endpoint, + region: "us-east-1", + credentials: new BasicAWSCredentials("local", "local")); + var writer = new OpenSearchRecordSearchProjection(transport, projectionOptions); + var created = false; + + try + { + await writer.EnsureCollectionAsync(policy); + created = true; + await writer.ProjectAsync(RecordSearchProjectionChange.Upsert(policy.Name, CreateRecord())); + RequireSuccess(await transport.SendAsync(HttpMethod.Post, $"/{indexName}/_refresh", null)); + + var indexUuid = await ReadIndexUuidAsync(transport, indexName); + await SetReadOnlyAsync(transport, indexName, readOnly: true); + var descriptor = CreateDescriptor(policy, indexName, indexUuid); + var projection = new OpenSearchGenerationBoundRecordSearchProjection( + transport, + projectionOptions, + [ + new OpenSearchRecordSearchProjectionGeneration + { + Descriptor = descriptor, + IndexName = indexName, + IndexUuid = indexUuid, + State = RecordSearchProjectionGenerationStates.Active, + AvailablePartitions = ["tenant-a"] + } + ]); + + var inspection = await projection.InspectGenerationAsync(policy, descriptor.GenerationId); + Assert.NotNull(inspection); + Assert.Equal(RecordSearchProjectionCoverageStatuses.Complete, inspection!.CoverageStatus); + Assert.Equal(["tenant-a"], inspection.AvailablePartitions); + + var result = await projection.SearchGenerationAsync(policy, CreateRequest(descriptor)); + Assert.Equal(RecordSearchProjectionResultStatuses.Succeeded, result.Status); + Assert.Equal(descriptor.GenerationId, result.GenerationId); + Assert.Equal(descriptor.DescriptorDigest, result.GenerationDescriptorDigest); + var candidate = Assert.Single(result.Items); + Assert.Equal("tenant-a", candidate.PartitionKey); + Assert.Equal("record-a", candidate.Id); + Assert.Equal(1, candidate.Revision); + + var blockedRecord = CreateRecord(); + blockedRecord.Revision = 2; + blockedRecord.Etag = "rev:2"; + await Assert.ThrowsAsync(() => + writer.ProjectAsync(RecordSearchProjectionChange.Upsert(policy.Name, blockedRecord))); + + await SetReadOnlyAsync(transport, indexName, readOnly: false); + var writable = await projection.SearchGenerationAsync(policy, CreateRequest(descriptor)); + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, writable.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, writable.Failure!.Code); + Assert.Empty(writable.Items); + Assert.Null(writable.ContinuationToken); + } + finally + { + if (created) + { + await SetReadOnlyAsync(transport, indexName, readOnly: false, allowMissing: true); + var deleted = await transport.SendAsync(HttpMethod.Delete, $"/{indexName}", null); + Assert.True(deleted.StatusCode is >= 200 and < 300 or 404); + } + } + } + + private static VyralRecord CreateRecord() => new() + { + Id = "record-a", + PartitionKey = "tenant-a", + Type = "record", + Revision = 1, + Etag = "rev:1", + Metadata = new JsonObject { ["status"] = "current" }, + Vectors = new Dictionary + { + ["embedding"] = new() { Values = [0.1f, 0.2f, 0.3f], Dimensions = 3 } + } + }; + + private static RecordSearchProjectionGenerationDescriptor CreateDescriptor( + RecordCollectionPolicy policy, + string indexName, + string indexUuid) + { + var descriptor = new RecordSearchProjectionGenerationDescriptor + { + Collection = policy.Name, + GenerationId = "generation-a", + ProviderId = OpenSearchGenerationBoundRecordSearchProjectionOptions.DefaultProviderId, + ProfileId = "vector-v1", + StrategyVersion = "opensearch-3.8", + SourceManifestDigest = Hash("source-manifest"), + RecordRevisionSetDigest = Hash("record-revisions"), + ProjectionSchemaDigest = OpenSearchProjectionGenerationBinding.ComputeProjectionSchemaDigest(policy), + AnalyzerDigest = Hash("analyzer"), + ConfigurationDigest = Hash("configuration"), + ExpectedItemCount = 1, + ExpectedPartitions = ["tenant-a"], + Capabilities = + [ + RecordSearchProjectionGenerationCapabilities.CompleteCoverage, + RecordSearchProjectionGenerationCapabilities.GenerationPinnedContinuation, + RecordSearchProjectionGenerationCapabilities.Vector + ], + Artifacts = + [ + new RecordSearchProjectionGenerationArtifact + { + Id = OpenSearchProjectionGenerationBinding.ArtifactId, + Kind = OpenSearchProjectionGenerationBinding.ArtifactKind, + ContentHash = OpenSearchProjectionGenerationBinding.ComputeContentHash(indexName, indexUuid), + SizeBytes = 0 + } + ], + CreatedAtUtc = DateTime.UtcNow + }; + RecordSearchProjectionGenerationContract.SealDescriptor(descriptor); + return descriptor; + } + + private static GenerationBoundRecordSearchProjectionRequest CreateRequest( + RecordSearchProjectionGenerationDescriptor descriptor) => new() + { + GenerationId = descriptor.GenerationId, + ExpectedDescriptorDigest = descriptor.DescriptorDigest, + Query = new QueryEnvelope + { + PartitionKeys = ["tenant-a"], + Vector = new VectorSearchOptions + { + Field = "embedding", + Value = [0.1f, 0.2f, 0.3f], + Top = 1 + }, + Limit = 1 + } + }; + + private static async Task ReadIndexUuidAsync(IOpenSearchTransport transport, string indexName) + { + var response = await transport.SendAsync( + HttpMethod.Get, + $"/{indexName}/_settings?flat_settings=true", + null); + RequireSuccess(response); + using var document = JsonDocument.Parse(response.Body); + return document.RootElement.GetProperty(indexName) + .GetProperty("settings") + .GetProperty("index.uuid") + .GetString() + ?? throw new InvalidOperationException("The local OpenSearch index did not report its UUID."); + } + + private static async Task SetReadOnlyAsync( + IOpenSearchTransport transport, + string indexName, + bool readOnly, + bool allowMissing = false) + { + var response = await transport.SendAsync( + HttpMethod.Put, + $"/{indexName}/_settings", + new JsonObject + { + ["index"] = new JsonObject + { + ["blocks"] = new JsonObject { ["read_only"] = readOnly } + } + }.ToJsonString()); + if (allowMissing && response.StatusCode == 404) + { + return; + } + RequireSuccess(response); + } + + private static void RequireSuccess(OpenSearchTransportResponse response) => + Assert.True( + response.StatusCode is >= 200 and < 300, + $"The local OpenSearch operation failed with HTTP {response.StatusCode}."); + + private static string Hash(string value) => + "sha256:" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value))); } diff --git a/tests/Vyral.Tests.Conformance/GenerationBoundProjectionLifecycleFixtureTests.cs b/tests/Vyral.Tests.Conformance/GenerationBoundProjectionLifecycleFixtureTests.cs new file mode 100644 index 0000000..9d49f4e --- /dev/null +++ b/tests/Vyral.Tests.Conformance/GenerationBoundProjectionLifecycleFixtureTests.cs @@ -0,0 +1,192 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Abstractions.Models; +using Vyral.Local; + +namespace Vyral.Tests.Conformance; + +public sealed class GenerationBoundProjectionLifecycleFixtureTests +{ + private const string ScenarioId = "records.projection-generation-lifecycle.v1"; + private const string ScenarioResource = "Vyral.Tests.Conformance.runtime-v1-generation-bound-lifecycle.json"; + private const string ManifestResource = "Vyral.Tests.Conformance.runtime-v1-manifest.json"; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + [Fact] + public async Task LocalReferenceExecutesPortableLifecycleScenario() + { + var scenarioBytes = ReadResource(ScenarioResource); + using var manifest = JsonDocument.Parse(ReadResource(ManifestResource)); + var descriptor = manifest.RootElement + .GetProperty("scenarios") + .EnumerateArray() + .Single(item => item.GetProperty("id").GetString() == ScenarioId); + Assert.Equal( + descriptor.GetProperty("sha256").GetString(), + "sha256:" + Convert.ToHexStringLower(SHA256.HashData(scenarioBytes))); + + using var scenario = JsonDocument.Parse(scenarioBytes); + var projection = new LocalGenerationBoundRecordSearchProjection( + new LocalGenerationBoundRecordSearchProjectionOptions + { + ContinuationSigningKey = SHA256.HashData("portable-generation-fixture-key"u8.ToArray()) + }); + var continuations = new Dictionary(StringComparer.Ordinal); + + foreach (var step in scenario.RootElement.GetProperty("steps").EnumerateArray()) + { + var actual = await ExecuteAsync(projection, continuations, step); + Assert.True(JsonNode.DeepEquals( + JsonNode.Parse(step.GetProperty("expect").GetProperty("value").GetRawText()), + JsonSerializer.SerializeToNode(actual, JsonOptions)), + $"Lifecycle step '{step.GetProperty("id").GetString()}' did not match its portable expectation."); + } + } + + private static async Task ExecuteAsync( + LocalGenerationBoundRecordSearchProjection projection, + IDictionary continuations, + JsonElement step) + { + var operation = step.GetProperty("operation").GetString(); + var arguments = step.GetProperty("arguments"); + switch (operation) + { + case "records.projection-generation-publish": + var descriptor = arguments.GetProperty("descriptor") + .Deserialize(JsonOptions)!; + var documents = arguments.GetProperty("documents") + .EnumerateArray() + .Select(item => new LocalRecordSearchProjectionDocument + { + Candidate = new RecordSearchProjectionCandidate + { + PartitionKey = item.GetProperty("partitionKey").GetString()!, + Id = item.GetProperty("id").GetString()!, + Revision = item.GetProperty("revision").GetInt32() + }, + SearchText = item.GetProperty("searchText").GetString()! + }) + .ToList(); + projection.PublishGeneration(new LocalRecordSearchProjectionGeneration + { + Descriptor = descriptor, + Documents = documents + }); + return new { status = "ok" }; + + case "records.projection-generation-activate": + projection.ActivateGeneration(Collection(arguments), Generation(arguments)); + return new { status = "ok" }; + + case "records.projection-generation-retire": + projection.RetireGeneration(Collection(arguments), Generation(arguments)); + return new { status = "ok" }; + + case "records.projection-generation-set-available": + projection.SetAvailablePartitions( + Collection(arguments), + Generation(arguments), + arguments.GetProperty("availablePartitions").EnumerateArray().Select(item => item.GetString()!)); + return new { status = "ok" }; + + case "records.projection-generation-inspect": + var inspection = await projection.InspectGenerationAsync( + new RecordCollectionPolicy { Name = Collection(arguments) }); + Assert.NotNull(inspection); + return new + { + generationId = inspection.Descriptor.GenerationId, + state = inspection.State, + coverageStatus = inspection.CoverageStatus, + availablePartitions = inspection.AvailablePartitions + }; + + case "records.projection-generation-search": + return await SearchAsync(projection, continuations, arguments); + + default: + throw new InvalidOperationException($"Unsupported lifecycle fixture operation '{operation}'."); + } + } + + private static async Task SearchAsync( + LocalGenerationBoundRecordSearchProjection projection, + IDictionary continuations, + JsonElement arguments) + { + string? continuation = null; + if (arguments.TryGetProperty("continuationRef", out var continuationRef)) + { + continuation = continuations[continuationRef.GetString()!]; + } + if (arguments.TryGetProperty("tamperContinuationRef", out var tamperRef)) + { + continuation = Tamper(continuations[tamperRef.GetString()!]); + } + + var request = new GenerationBoundRecordSearchProjectionRequest + { + ExpectedDescriptorDigest = arguments.TryGetProperty("expectedDescriptorDigest", out var digest) + ? digest.GetString() + : null, + Query = new QueryEnvelope + { + PartitionKeys = arguments.GetProperty("partitionKeys") + .EnumerateArray() + .Select(item => item.GetString()!) + .ToList(), + Lexical = new LexicalSearchOptions + { + Query = arguments.GetProperty("query").GetString()!, + ScanLimit = 100 + }, + Limit = arguments.GetProperty("limit").GetInt32(), + ContinuationToken = continuation + } + }; + var result = await projection.SearchGenerationAsync( + new RecordCollectionPolicy { Name = Collection(arguments) }, + request); + if (arguments.TryGetProperty("saveContinuationAs", out var saveAs)) + { + Assert.False(string.IsNullOrWhiteSpace(result.ContinuationToken)); + continuations[saveAs.GetString()!] = result.ContinuationToken!; + } + return new + { + status = result.Status, + generationId = result.GenerationId, + ids = result.Items.Select(item => item.Id).ToList(), + continuation = result.ContinuationToken is null ? "absent" : "present", + coverageStatus = result.Coverage.Status, + coveredPartitions = result.Coverage.CoveredPartitions, + missingPartitions = result.Coverage.MissingPartitions, + failureCode = result.Failure?.Code + }; + } + + private static string Collection(JsonElement arguments) => + arguments.GetProperty("collection").GetString()!; + + private static string Generation(JsonElement arguments) => + arguments.GetProperty("generationId").GetString()!; + + private static string Tamper(string token) + { + var signatureStart = token.IndexOf('.', StringComparison.Ordinal) + 1; + var replacement = token[signatureStart] == 'A' ? 'B' : 'A'; + return token[..signatureStart] + replacement + token[(signatureStart + 1)..]; + } + + private static byte[] ReadResource(string name) + { + using var stream = typeof(GenerationBoundProjectionLifecycleFixtureTests).Assembly + .GetManifestResourceStream(name) + ?? throw new InvalidOperationException($"Embedded conformance resource '{name}' is unavailable."); + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + return buffer.ToArray(); + } +} diff --git a/tests/Vyral.Tests.Conformance/PortableGenerationBoundProjectionFixtureTests.cs b/tests/Vyral.Tests.Conformance/PortableGenerationBoundProjectionFixtureTests.cs new file mode 100644 index 0000000..e7e53b7 --- /dev/null +++ b/tests/Vyral.Tests.Conformance/PortableGenerationBoundProjectionFixtureTests.cs @@ -0,0 +1,79 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Abstractions.Models; + +namespace Vyral.Tests.Conformance; + +public sealed class PortableGenerationBoundProjectionFixtureTests +{ + private const string ScenarioId = "records.projection-generation.v1"; + private const string ManifestResource = "Vyral.Tests.Conformance.runtime-v1-manifest.json"; + private const string ScenarioResource = "Vyral.Tests.Conformance.runtime-v1-record-search-projection-generation.json"; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + [Fact] + public void DescriptorDigestMatchesPortableGoldenAndPackagedFixture() + { + var scenarioBytes = ReadResource(ScenarioResource); + using var manifest = JsonDocument.Parse(ReadResource(ManifestResource)); + var descriptor = manifest.RootElement + .GetProperty("scenarios") + .EnumerateArray() + .Single(item => item.GetProperty("id").GetString() == ScenarioId); + Assert.Equal( + descriptor.GetProperty("sha256").GetString(), + "sha256:" + Convert.ToHexStringLower(SHA256.HashData(scenarioBytes))); + Assert.Equal("vyral.runtime.retrieval-generation.v1", descriptor.GetProperty("profile").GetString()); + + using var scenario = JsonDocument.Parse(scenarioBytes); + var step = Assert.Single(scenario.RootElement.GetProperty("steps").EnumerateArray()); + var golden = step.GetProperty("arguments").GetProperty("descriptor") + .Deserialize(JsonOptions)!; + RecordSearchProjectionGenerationContract.ValidateDescriptor(golden); + Assert.Equal( + step.GetProperty("expect").GetProperty("value").GetString(), + RecordSearchProjectionGenerationContract.ComputeDescriptorDigest(golden)); + + var fixturePath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "record-search-projection-generation.v1.valid.json"); + var fixtureNode = JsonNode.Parse(File.ReadAllText(fixturePath)); + var goldenNode = JsonNode.Parse(step.GetProperty("arguments").GetProperty("descriptor").GetRawText()); + Assert.True(JsonNode.DeepEquals(goldenNode, fixtureNode)); + + var schemaPath = Path.Combine(AppContext.BaseDirectory, "contracts", "record-search-projection-generation.v1.schema.json"); + var schema = JsonNode.Parse(File.ReadAllText(schemaPath))!.AsObject(); + Assert.Equal(RecordSearchProjectionGenerationSchemas.DescriptorV1, schema["$defs"]!["generationDescriptor"]!["properties"]!["schema"]!["const"]!.GetValue()); + Assert.NotNull(schema["$defs"]!["searchRequest"]); + Assert.NotNull(schema["$defs"]!["searchResult"]); + Assert.NotNull(schema["$defs"]!["buildRequest"]); + Assert.NotNull(schema["$defs"]!["buildReceipt"]); + Assert.NotNull(schema["$defs"]!["inspection"]); + } + + [Fact] + public void DescriptorDigestIgnoresInputObjectAndSetOrderingButValidationRequiresCanonicalWireOrder() + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "record-search-projection-generation.v1.valid.json"); + var canonical = JsonSerializer.Deserialize(File.ReadAllText(path), JsonOptions)!; + var reordered = JsonSerializer.Deserialize(File.ReadAllText(path), JsonOptions)!; + reordered.ExpectedPartitions.Reverse(); + reordered.Capabilities.Reverse(); + reordered.Artifacts.Reverse(); + + Assert.Equal( + RecordSearchProjectionGenerationContract.ComputeDescriptorDigest(canonical), + RecordSearchProjectionGenerationContract.ComputeDescriptorDigest(reordered)); + Assert.Throws(() => + RecordSearchProjectionGenerationContract.ValidateDescriptor(reordered)); + } + + private static byte[] ReadResource(string name) + { + using var stream = typeof(PortableGenerationBoundProjectionFixtureTests).Assembly + .GetManifestResourceStream(name) + ?? throw new InvalidOperationException($"Embedded conformance resource '{name}' is unavailable."); + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + return buffer.ToArray(); + } +} diff --git a/tests/Vyral.Tests.Conformance/RetrievalProjectionQualificationSchemaTests.cs b/tests/Vyral.Tests.Conformance/RetrievalProjectionQualificationSchemaTests.cs new file mode 100644 index 0000000..eb6f63d --- /dev/null +++ b/tests/Vyral.Tests.Conformance/RetrievalProjectionQualificationSchemaTests.cs @@ -0,0 +1,40 @@ +using System.Text.Json.Nodes; + +namespace Vyral.Tests.Conformance; + +public sealed class RetrievalProjectionQualificationSchemaTests +{ + [Fact] + public void RetrievalQualificationIsSeparateFromGenerationAndExecutionEvidence() + { + var path = Path.Combine( + AppContext.BaseDirectory, + "qualification", + "retrieval-projection-qualification.schema.json"); + var schema = JsonNode.Parse(File.ReadAllText(path))!.AsObject(); + Assert.Equal( + "Vyral retrieval projection adapter qualification artifact", + schema["title"]!.GetValue()); + var policy = schema["$defs"]!["generationEvidencePolicy"]!["properties"]!; + Assert.True(policy["requiresExactGenerationBinding"]!["const"]!.GetValue()); + Assert.True(policy["requiresCompleteRequestedCoverage"]!["const"]!.GetValue()); + Assert.False(policy["mutableHealthEstablishesCompleteness"]!["const"]!.GetValue()); + Assert.Equal( + "vyral.record-search-projection-generation.v1", + policy["descriptorSchema"]!["const"]!.GetValue()); + var evidence = schema["$defs"]!["evidence"]!; + var evidenceKinds = evidence["properties"]!["kind"]!["enum"]!.AsArray() + .Select(value => value!.GetValue()) + .ToList(); + Assert.Contains("local_gate", evidenceKinds); + var requiredEvidence = evidence["required"]!.AsArray() + .Select(value => value!.GetValue()) + .ToList(); + Assert.Contains("sourceTreeDigest", requiredEvidence); + Assert.Contains("sourceDirty", requiredEvidence); + var adapterRequired = schema["$defs"]!["adapter"]!["required"]!.AsArray() + .Select(value => value!.GetValue()) + .ToList(); + Assert.Contains("implementationArtifacts", adapterRequired); + } +} diff --git a/tests/Vyral.Tests.Conformance/Vyral.Tests.Conformance.csproj b/tests/Vyral.Tests.Conformance/Vyral.Tests.Conformance.csproj index b983312..6cb7d3e 100644 --- a/tests/Vyral.Tests.Conformance/Vyral.Tests.Conformance.csproj +++ b/tests/Vyral.Tests.Conformance/Vyral.Tests.Conformance.csproj @@ -42,6 +42,10 @@ LogicalName="Vyral.Tests.Conformance.runtime-v1-admission-receipts.json" /> + + + + + + + + diff --git a/tests/Vyral.Tests.Local/ExecutionRuntimeProjectionGenerationAdapterTests.cs b/tests/Vyral.Tests.Local/ExecutionRuntimeProjectionGenerationAdapterTests.cs new file mode 100644 index 0000000..375a04b --- /dev/null +++ b/tests/Vyral.Tests.Local/ExecutionRuntimeProjectionGenerationAdapterTests.cs @@ -0,0 +1,201 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; +using Vyral.Execution; +using Vyral.Execution.Local; +using Vyral.Local; +using Vyral.Server; + +namespace Vyral.Tests.Local; + +public sealed class ExecutionRuntimeProjectionGenerationAdapterTests +{ + [Fact] + public async Task BuildRunIsDurableIdempotentCheckpointedAndArtifactBound() + { + var root = Path.Combine(Path.GetTempPath(), $"vyral-projection-generation-{Guid.NewGuid():N}"); + var runtime = new LocalExecutionRuntime(new LocalExecutionRuntimeOptions + { + DatabasePath = Path.Combine(root, "execution.sqlite"), + ArtifactDirectory = Path.Combine(root, "execution-artifacts") + }); + var objects = new FileObjectStore(Path.Combine(root, "objects")); + var builder = new FixtureGenerationBuilder(); + var adapter = new ExecutionRuntimeProjectionGenerationAdapter(runtime, builder, objects); + var request = BuildRequest(); + + var admitted = await adapter.StartBuildAsync(request, "library:generation-a"); + var replayed = await adapter.StartBuildAsync(request, "library:generation-a"); + var completed = await WaitForRunAsync(runtime, admitted.Id, ExecutionRunStatuses.Succeeded); + + Assert.Equal(admitted.Id, replayed.Id); + Assert.True(replayed.AdmissionReplayed); + Assert.Equal(1, builder.Calls); + var receipt = completed.Result!.Deserialize(ExecutionJson.Options)!; + RecordSearchProjectionGenerationContract.ValidateBuildReceipt(request, receipt); + Assert.Equal("generation-a", receipt.Descriptor.GenerationId); + Assert.Single(receipt.Descriptor.Artifacts); + + var checkpoint = await runtime.GetCheckpointAsync(admitted.Id, "projection-generation-progress"); + Assert.NotNull(checkpoint); + Assert.Equal("verify", checkpoint!.Content!["stage"]!.GetValue()); + var runArtifacts = await runtime.ListArtifactsAsync(admitted.Id); + var terminal = Assert.Single(runArtifacts); + Assert.Equal("projection-generation-receipt", terminal.Name); + Assert.Equal(receipt.Descriptor.DescriptorDigest, terminal.Metadata["descriptorDigest"]); + + var objectReceipt = Assert.Single(receipt.Descriptor.Artifacts); + var stored = await objects.GetObjectAsync(new ObjectReadRequest + { + Container = "retrieval", + Key = "generations/generation-a/sha256/" + objectReceipt.ContentHash[7..] + }); + Assert.NotNull(stored); + await stored!.Content.DisposeAsync(); + Assert.Equal(objectReceipt.ContentHash, stored.ContentHash); + } + + [Fact] + public async Task AdapterRejectsMissingIdempotencyAndMismatchedBuilderBeforeAdmission() + { + var root = Path.Combine(Path.GetTempPath(), $"vyral-projection-generation-{Guid.NewGuid():N}"); + var runtime = new LocalExecutionRuntime(new LocalExecutionRuntimeOptions + { + DatabasePath = Path.Combine(root, "execution.sqlite") + }); + var adapter = new ExecutionRuntimeProjectionGenerationAdapter( + runtime, + new FixtureGenerationBuilder(), + new FileObjectStore(Path.Combine(root, "objects"))); + var request = BuildRequest(); + + await Assert.ThrowsAsync(() => adapter.StartBuildAsync(request, "")); + request.BuilderId = "another-builder"; + await Assert.ThrowsAsync(() => adapter.StartBuildAsync(request, "key")); + request.BuilderId = FixtureGenerationBuilder.Id; + request.SourceManifestRef = "https://objects.example/manifest?token=not-stable"; + await Assert.ThrowsAsync(() => adapter.StartBuildAsync(request, "key")); + Assert.Empty(await runtime.ListRunsAsync()); + } + + private static RecordSearchProjectionGenerationBuildRequest BuildRequest() => new() + { + Collection = "library", + GenerationId = "generation-a", + BuilderId = FixtureGenerationBuilder.Id, + ProviderId = "local-exhaustive", + ProfileId = "lexical-exhaustive-v1", + StrategyVersion = "exhaustive-token-v1", + SourceManifestRef = "manifests/library-generation-a.json", + SourceManifestDigest = Hash("manifest"), + ExpectedRecordRevisionSetDigest = Hash("record-revisions"), + ProjectionSchemaDigest = Hash("projection-schema"), + AnalyzerDigest = Hash("analyzer"), + ConfigurationDigest = Hash("configuration"), + ExpectedItemCount = 2, + ExpectedPartitions = ["public-a", "public-b"], + DeadlineUtc = DateTime.UtcNow.AddMinutes(5) + }; + + private static async Task WaitForRunAsync(IExecutionRuntime runtime, string id, string status) + { + ExecutionRun? run = null; + for (var i = 0; i < 200; i++) + { + run = await runtime.GetRunAsync(id); + if (run?.Status == status) + { + return run; + } + await Task.Delay(25); + } + throw new InvalidOperationException($"Run {id} did not reach {status}; last status was {run?.Status} and error was {run?.Error}."); + } + + private static string Hash(string value) => + "sha256:" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private sealed class FixtureGenerationBuilder : IRecordSearchProjectionGenerationBuilder + { + public const string Id = "fixture-local-generation-builder"; + private int _calls; + public string BuilderId => Id; + public int Calls => Volatile.Read(ref _calls); + + public async Task BuildAndVerifyAsync( + RecordSearchProjectionGenerationBuildRequest request, + IObjectStore artifactStore, + Func? reportProgress = null, + CancellationToken ct = default) + { + Interlocked.Increment(ref _calls); + if (reportProgress is not null) + { + await reportProgress(new RecordSearchProjectionGenerationBuildProgress + { + Stage = "build", + Completed = 1, + Total = 2 + }, ct); + } + var bytes = Encoding.UTF8.GetBytes("immutable local projection generation"); + var artifact = await artifactStore.PutContentAddressedAsync( + "retrieval", + "generations/" + request.GenerationId, + bytes, + "application/octet-stream", + ct: ct); + if (reportProgress is not null) + { + await reportProgress(new RecordSearchProjectionGenerationBuildProgress + { + Stage = "verify", + Completed = 2, + Total = 2, + Checkpoint = new System.Text.Json.Nodes.JsonObject + { + ["artifactHash"] = artifact.ContentHash + } + }, ct); + } + var descriptor = new RecordSearchProjectionGenerationDescriptor + { + Collection = request.Collection, + GenerationId = request.GenerationId, + ProviderId = request.ProviderId, + ProfileId = request.ProfileId, + StrategyVersion = request.StrategyVersion, + SourceManifestDigest = request.SourceManifestDigest, + RecordRevisionSetDigest = request.ExpectedRecordRevisionSetDigest!, + ProjectionSchemaDigest = request.ProjectionSchemaDigest, + AnalyzerDigest = request.AnalyzerDigest, + ConfigurationDigest = request.ConfigurationDigest, + ExpectedItemCount = request.ExpectedItemCount, + ExpectedPartitions = request.ExpectedPartitions.ToList(), + Capabilities = ["completeCoverage", "generationPinnedContinuation", "lexical"], + Artifacts = + [ + new RecordSearchProjectionGenerationArtifact + { + Id = "index", + Kind = "index-part", + ContentHash = artifact.ContentHash, + SizeBytes = artifact.Object.ContentLength, + MediaType = artifact.Object.ContentType + } + ], + CreatedAtUtc = DateTime.UtcNow + }; + RecordSearchProjectionGenerationContract.SealDescriptor(descriptor); + return new RecordSearchProjectionGenerationBuildReceipt + { + BuilderId = BuilderId, + Descriptor = descriptor, + EvaluationReceiptDigest = Hash("evaluation"), + BuiltAtUtc = DateTime.UtcNow + }; + } + } +} diff --git a/tests/Vyral.Tests.Local/FileObjectStoreTests.cs b/tests/Vyral.Tests.Local/FileObjectStoreTests.cs index 2488af6..b3337f3 100644 --- a/tests/Vyral.Tests.Local/FileObjectStoreTests.cs +++ b/tests/Vyral.Tests.Local/FileObjectStoreTests.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Vyral.Abstractions.Interfaces; using Vyral.Abstractions.Models; using Vyral.Local; using Xunit; @@ -11,6 +12,56 @@ namespace Vyral.Tests.Local; public class FileObjectStoreTests { + [Fact] + public async Task ContentAddressedWrite_IsImmutableAndReplaySafe() + { + var root = Path.Combine(Path.GetTempPath(), $"vyral-objects-{Guid.NewGuid():N}"); + var store = new FileObjectStore(root); + var content = Encoding.UTF8.GetBytes("immutable generation part"); + + var first = await store.PutContentAddressedAsync( + "objects", + "retrieval/generations/generation-a", + content, + "application/octet-stream"); + var second = await store.PutContentAddressedAsync( + "objects", + "retrieval/generations/generation-a", + content, + "application/octet-stream"); + + Assert.False(first.Replayed); + Assert.True(second.Replayed); + Assert.Equal(first.ContentHash, second.ContentHash); + Assert.Equal(first.Object.Key, second.Object.Key); + Assert.Contains("/sha256/", first.Object.Key, StringComparison.Ordinal); + Assert.EndsWith(first.ContentHash[7..], first.Object.Key, StringComparison.Ordinal); + } + + [Fact] + public async Task StreamingContentAddressedWrite_RejectsAndRemovesMismatchedContent() + { + var root = Path.Combine(Path.GetTempPath(), $"vyral-objects-{Guid.NewGuid():N}"); + var store = new FileObjectStore(root); + var actual = Encoding.UTF8.GetBytes("actual"); + var expected = Encoding.UTF8.GetBytes("expected"); + var expectedHash = "sha256:" + Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData(expected)); + await using var stream = new MemoryStream(actual, writable: false); + + await Assert.ThrowsAsync(() => store.PutContentAddressedAsync( + "objects", + "retrieval/generations/generation-a", + stream, + actual.Length, + expectedHash)); + + Assert.Null(await store.GetObjectAsync(new ObjectReadRequest + { + Container = "objects", + Key = "retrieval/generations/generation-a/sha256/" + expectedHash[7..] + })); + } + [Fact] public async Task PutGetListDelete_RoundTripsObjectDeterministically() { diff --git a/tests/Vyral.Tests.Local/GenerationBoundRetrievalEvaluationTargetTests.cs b/tests/Vyral.Tests.Local/GenerationBoundRetrievalEvaluationTargetTests.cs new file mode 100644 index 0000000..77cf3f6 --- /dev/null +++ b/tests/Vyral.Tests.Local/GenerationBoundRetrievalEvaluationTargetTests.cs @@ -0,0 +1,304 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; +using Vyral.Local; + +namespace Vyral.Tests.Local; + +public sealed class GenerationBoundRetrievalEvaluationTargetTests : IAsyncLifetime +{ + private readonly string _path = Path.Combine( + Path.GetTempPath(), + $"vyral-generation-evaluation-{Guid.NewGuid():N}.sqlite"); + private readonly RecordCollectionPolicy _policy = new() { Name = "library" }; + private SqliteRecordCollectionStore _store = null!; + + public async Task InitializeAsync() + { + _store = new SqliteRecordCollectionStore(_path); + await _store.InitializeAsync(); + await _store.CreateCollectionAsync(_policy); + await _store.UpsertRecordAsync(_policy.Name, Record("expected", "portable contracts")); + await _store.UpsertRecordAsync(_policy.Name, Record("hard-negative", "unrelated provider")); + } + + public Task DisposeAsync() + { + if (File.Exists(_path)) + { + File.Delete(_path); + } + return Task.CompletedTask; + } + + [Fact] + public async Task ComparisonResolvesAndReportsExactGenerationTargets() + { + var projection = CreateProjection(); + var exhaustive = CreateGeneration("generation-exhaustive"); + var indexed = CreateGeneration("generation-indexed"); + projection.PublishGeneration(exhaustive); + projection.PublishGeneration(indexed); + var resolver = Resolver( + Target("exhaustive", projection, exhaustive), + Target("indexed", projection, indexed)); + var defaultService = new RejectingRetrievalService(); + var evaluation = new LocalRetrievalEvaluationService(defaultService, resolver); + + var result = await evaluation.CompareAsync(Comparison( + Variant("exhaustive-arm", "exhaustive", exhaustive), + Variant("indexed-arm", "indexed", indexed))); + + Assert.Equal(2, result.VariantsSucceeded); + Assert.Equal(0, result.VariantsFailed); + Assert.Equal(2, result.Variants.Count); + Assert.Equal(0, defaultService.Calls); + foreach (var variant in result.Variants) + { + Assert.Equal(EvaluationVariantStatuses.Succeeded, variant.Status); + Assert.NotNull(variant.Target); + Assert.Equal(1, variant.Metrics.HitRate); + Assert.Equal(0, variant.Metrics.HardNegativeHitRate); + Assert.Equal(1, variant.Metrics.Succeeded); + Assert.Equal(0, variant.Metrics.Failed); + var testCase = Assert.Single(variant.Cases); + Assert.Equal("expected", Assert.Single(testCase.TopResults).Id); + } + Assert.Equal("generation-exhaustive", result.Variants[0].Target!.GenerationId); + Assert.Equal(exhaustive.Descriptor.DescriptorDigest, result.Variants[0].Target!.GenerationDescriptorDigest); + Assert.Equal("generation-indexed", result.Variants[1].Target!.GenerationId); + Assert.Equal(indexed.Descriptor.DescriptorDigest, result.Variants[1].Target!.GenerationDescriptorDigest); + } + + [Fact] + public async Task DescriptorSubstitutionFailsTheVariantWithoutUsingDefaultTarget() + { + var projection = CreateProjection(); + var generation = CreateGeneration("generation-a"); + projection.PublishGeneration(generation); + var defaultService = new RejectingRetrievalService(); + var evaluation = new LocalRetrievalEvaluationService( + defaultService, + Resolver(Target("registered", projection, generation))); + var variant = Variant("candidate", "registered", generation); + variant.Target!.ExpectedGenerationDescriptorDigest = Hash("substituted-descriptor"); + + var result = await evaluation.CompareAsync(Comparison(variant)); + + var failed = Assert.Single(result.Variants); + Assert.Equal(EvaluationVariantStatuses.Failed, failed.Status); + Assert.Contains("does not match the expected generation descriptor", failed.Error, StringComparison.Ordinal); + Assert.Null(failed.Target); + Assert.Equal(0, defaultService.Calls); + } + + [Fact] + public async Task StaleCandidateRevisionFailsTheTargetInsteadOfChangingMetrics() + { + var projection = CreateProjection(); + var generation = CreateGeneration("generation-stale", expectedRevision: 2); + projection.PublishGeneration(generation); + var evaluation = new LocalRetrievalEvaluationService( + new RejectingRetrievalService(), + Resolver(Target("stale", projection, generation))); + + var result = await evaluation.CompareAsync(Comparison( + Variant("stale-arm", "stale", generation))); + + var failed = Assert.Single(result.Variants); + Assert.Equal(EvaluationVariantStatuses.Failed, failed.Status); + Assert.Contains("failed 1 case", failed.Error, StringComparison.Ordinal); + Assert.Equal(0, failed.Metrics.Succeeded); + Assert.Null(failed.Target); + } + + [Fact] + public async Task MissingLogicalCoverageFailsTheTargetWithoutPartialCandidates() + { + var projection = CreateProjection(); + var generation = CreateGeneration("generation-incomplete"); + projection.PublishGeneration(generation); + projection.SetAvailablePartitions(_policy.Name, generation.Descriptor.GenerationId, []); + var evaluation = new LocalRetrievalEvaluationService( + new RejectingRetrievalService(), + Resolver(Target("incomplete", projection, generation))); + + var result = await evaluation.CompareAsync(Comparison( + Variant("incomplete-arm", "incomplete", generation))); + + var failed = Assert.Single(result.Variants); + Assert.Equal(EvaluationVariantStatuses.Failed, failed.Status); + Assert.Contains("failed 1 case", failed.Error, StringComparison.Ordinal); + Assert.Null(failed.Target); + } + + [Fact] + public void ExperimentalTargetsDoNotSilentlyExpandThePublicJsonContract() + { + var variant = new RetrievalEvaluationVariant + { + Id = "candidate", + Target = new RetrievalEvaluationTargetReference { Id = "registered" } + }; + var result = new RetrievalEvaluationVariantResult + { + Id = "candidate", + Target = new RetrievalEvaluationTargetEvidence { Id = "registered" } + }; + + Assert.DoesNotContain("target", JsonSerializer.Serialize(variant), StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("target", JsonSerializer.Serialize(result), StringComparison.OrdinalIgnoreCase); + } + + private GenerationBoundRetrievalEvaluationTargetResolver Resolver( + params GenerationBoundRetrievalEvaluationTargetRegistration[] targets) => new(targets); + + private GenerationBoundRetrievalEvaluationTargetRegistration Target( + string id, + IGenerationBoundRecordSearchProjection projection, + LocalRecordSearchProjectionGeneration generation) => new() + { + Id = id, + Projection = projection, + CanonicalStore = _store, + Policy = _policy, + GenerationId = generation.Descriptor.GenerationId, + GenerationDescriptorDigest = generation.Descriptor.DescriptorDigest + }; + + private static RetrievalEvaluationComparisonRequest Comparison( + params RetrievalEvaluationVariant[] variants) => new() + { + Cases = + [ + new RetrievalEvaluationCase + { + Name = "portable-contracts", + Request = new RetrievalRequest + { + Query = "portable", + Collections = ["library"], + SearchMode = SearchModes.Lexical, + Limit = 5 + }, + Expected = [new RetrievalEvaluationExpectedMatch { Id = "expected" }], + HardNegatives = + [ + new RetrievalEvaluationHardNegativeMatch + { + Id = "hard-negative", + Reason = "must not match a portable-contract query" + } + ], + K = 5 + } + ], + Variants = variants.ToList(), + IncludeTopResults = true, + IncludeCaseResults = true + }; + + private static RetrievalEvaluationVariant Variant( + string variantId, + string targetId, + LocalRecordSearchProjectionGeneration generation) => new() + { + Id = variantId, + Target = new RetrievalEvaluationTargetReference + { + Id = targetId, + GenerationId = generation.Descriptor.GenerationId, + ExpectedGenerationDescriptorDigest = generation.Descriptor.DescriptorDigest + } + }; + + private LocalGenerationBoundRecordSearchProjection CreateProjection() => new( + new LocalGenerationBoundRecordSearchProjectionOptions + { + ContinuationSigningKey = SHA256.HashData( + Encoding.UTF8.GetBytes("generation-evaluation-test-key")), + DefaultWorkLimit = 100, + MaxWorkLimit = 100 + }); + + private LocalRecordSearchProjectionGeneration CreateGeneration( + string generationId, + int expectedRevision = 1) + { + var descriptor = new RecordSearchProjectionGenerationDescriptor + { + Collection = _policy.Name, + GenerationId = generationId, + ProviderId = "local-test", + ProfileId = "lexical-test-v1", + StrategyVersion = "token-test-v1", + SourceManifestDigest = Hash(generationId + ":manifest"), + RecordRevisionSetDigest = Hash(generationId + ":records"), + ProjectionSchemaDigest = Hash("projection-schema"), + AnalyzerDigest = Hash("analyzer"), + ConfigurationDigest = Hash("configuration"), + ExpectedItemCount = 2, + ExpectedPartitions = ["public"], + Capabilities = ["completeCoverage", "generationPinnedContinuation", "lexical"], + CreatedAtUtc = new DateTime(2026, 8, 27, 12, 0, 0, DateTimeKind.Utc) + }; + RecordSearchProjectionGenerationContract.SealDescriptor(descriptor); + return new LocalRecordSearchProjectionGeneration + { + Descriptor = descriptor, + Documents = + [ + new LocalRecordSearchProjectionDocument + { + Candidate = new RecordSearchProjectionCandidate + { + PartitionKey = "public", + Id = "expected", + Revision = expectedRevision, + Score = 1 + }, + SearchText = "portable portable contracts" + }, + new LocalRecordSearchProjectionDocument + { + Candidate = new RecordSearchProjectionCandidate + { + PartitionKey = "public", + Id = "hard-negative", + Revision = 1, + Score = 1 + }, + SearchText = "unrelated provider" + } + ] + }; + } + + private static VyralRecord Record(string id, string text) => new() + { + Id = id, + PartitionKey = "public", + Type = "note", + Content = new JsonObject { ["text"] = text } + }; + + private static string Hash(string value) => + "sha256:" + Convert.ToHexStringLower( + SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private sealed class RejectingRetrievalService : IRetrievalService + { + public int Calls { get; private set; } + + public Task SearchAsync( + RetrievalRequest request, + CancellationToken ct = default) + { + Calls++; + throw new InvalidOperationException("The default retrieval service must not be used."); + } + } +} diff --git a/tests/Vyral.Tests.Local/LocalGenerationBoundRecordSearchProjectionTests.cs b/tests/Vyral.Tests.Local/LocalGenerationBoundRecordSearchProjectionTests.cs new file mode 100644 index 0000000..83ec456 --- /dev/null +++ b/tests/Vyral.Tests.Local/LocalGenerationBoundRecordSearchProjectionTests.cs @@ -0,0 +1,374 @@ +using System.Security.Cryptography; +using System.Text; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; +using Vyral.Local; + +namespace Vyral.Tests.Local; + +public sealed class LocalGenerationBoundRecordSearchProjectionTests +{ + private static readonly RecordCollectionPolicy Policy = new() { Name = "library" }; + + [Fact] + public async Task ActiveSwitchKeepsExistingContinuationPinnedToRetainedGeneration() + { + var projection = CreateProjection(); + var firstGeneration = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + var secondGeneration = CreateGeneration("generation-b", "portable execution", "portable evidence"); + projection.PublishGeneration(firstGeneration); + projection.ActivateGeneration(Policy.Name, firstGeneration.Descriptor.GenerationId); + + var request = SearchRequest(limit: 1); + var firstPage = await projection.SearchGenerationAsync(Policy, request); + Assert.Equal(RecordSearchProjectionResultStatuses.Succeeded, firstPage.Status); + Assert.Equal("generation-a", firstPage.GenerationId); + Assert.Equal("a-1", Assert.Single(firstPage.Items).Id); + Assert.NotNull(firstPage.ContinuationToken); + + projection.PublishGeneration(secondGeneration); + projection.ActivateGeneration(Policy.Name, secondGeneration.Descriptor.GenerationId); + + request.Query.ContinuationToken = firstPage.ContinuationToken; + var retainedPage = await projection.SearchGenerationAsync(Policy, request); + Assert.Equal(RecordSearchProjectionResultStatuses.Succeeded, retainedPage.Status); + Assert.Equal("generation-a", retainedPage.GenerationId); + Assert.Equal("b-1", Assert.Single(retainedPage.Items).Id); + + var fresh = await projection.SearchGenerationAsync(Policy, SearchRequest(limit: 1)); + Assert.Equal("generation-b", fresh.GenerationId); + Assert.Equal("a-1", Assert.Single(fresh.Items).Id); + } + + [Fact] + public async Task RetiredContinuationFailsInsteadOfRemappingToActiveGeneration() + { + var projection = CreateProjection(); + var firstGeneration = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + var secondGeneration = CreateGeneration("generation-b", "portable execution", "portable evidence"); + projection.PublishGeneration(firstGeneration); + projection.ActivateGeneration(Policy.Name, "generation-a"); + var firstPage = await projection.SearchGenerationAsync(Policy, SearchRequest(limit: 1)); + + projection.PublishGeneration(secondGeneration); + projection.ActivateGeneration(Policy.Name, "generation-b"); + projection.RetireGeneration(Policy.Name, "generation-a"); + + var request = SearchRequest(limit: 1); + request.Query.ContinuationToken = firstPage.ContinuationToken; + var result = await projection.SearchGenerationAsync(Policy, request); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.GenerationRetired, result.Failure!.Code); + Assert.Equal("generation-a", result.GenerationId); + Assert.Empty(result.Items); + Assert.Null(result.ContinuationToken); + Assert.Equal(RecordSearchProjectionCoverageStatuses.Unavailable, result.Coverage.Status); + } + + [Fact] + public async Task MissingPartitionFailsClosedWithoutPartialCandidates() + { + var projection = CreateProjection(); + var generation = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + projection.PublishGeneration(generation); + projection.ActivateGeneration(Policy.Name, "generation-a"); + projection.SetAvailablePartitions(Policy.Name, "generation-a", ["public-a"]); + + var result = await projection.SearchGenerationAsync(Policy, SearchRequest(limit: 10)); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.CoverageIncomplete, result.Failure!.Code); + Assert.Equal(RecordSearchProjectionCoverageStatuses.Incomplete, result.Coverage.Status); + Assert.Equal(["public-a"], result.Coverage.CoveredPartitions); + Assert.Equal(["public-b"], result.Coverage.MissingPartitions); + Assert.Empty(result.Items); + Assert.Null(result.ContinuationToken); + + var inspection = await projection.InspectGenerationAsync(Policy); + Assert.NotNull(inspection); + Assert.Equal(RecordSearchProjectionCoverageStatuses.Incomplete, inspection!.CoverageStatus); + } + + [Fact] + public async Task TamperedOrCrossRequestContinuationFailsExplicitly() + { + var projection = CreateProjection(); + var generation = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + projection.PublishGeneration(generation); + projection.ActivateGeneration(Policy.Name, "generation-a"); + var first = await projection.SearchGenerationAsync(Policy, SearchRequest(limit: 1)); + + var tamperedRequest = SearchRequest(limit: 1); + var token = first.ContinuationToken!; + var signatureStart = token.IndexOf('.', StringComparison.Ordinal) + 1; + var replacement = token[signatureStart] == 'A' ? 'B' : 'A'; + tamperedRequest.Query.ContinuationToken = + token[..signatureStart] + replacement + token[(signatureStart + 1)..]; + var tampered = await projection.SearchGenerationAsync(Policy, tamperedRequest); + Assert.Equal(RecordSearchProjectionFailureCodes.InvalidContinuation, tampered.Failure!.Code); + + var changedRequest = SearchRequest(limit: 1, query: "retrieval"); + changedRequest.Query.ContinuationToken = token; + var changed = await projection.SearchGenerationAsync(Policy, changedRequest); + Assert.Equal(RecordSearchProjectionFailureCodes.InvalidContinuation, changed.Failure!.Code); + + var oversizedRequest = SearchRequest(limit: 1); + oversizedRequest.Query.ContinuationToken = new string('x', 8193); + var oversized = await projection.SearchGenerationAsync(Policy, oversizedRequest); + Assert.Equal(RecordSearchProjectionFailureCodes.InvalidContinuation, oversized.Failure!.Code); + } + + [Fact] + public async Task ExpiredContinuationFailsExplicitly() + { + var clock = new MutableTimeProvider(new DateTimeOffset(2026, 8, 27, 12, 0, 0, TimeSpan.Zero)); + var projection = CreateProjection(clock, TimeSpan.FromMinutes(5)); + projection.PublishGeneration(CreateGeneration("generation-a", "portable contracts", "portable retrieval")); + projection.ActivateGeneration(Policy.Name, "generation-a"); + var first = await projection.SearchGenerationAsync(Policy, SearchRequest(limit: 1)); + clock.UtcNow = clock.UtcNow.AddMinutes(6); + + var request = SearchRequest(limit: 1); + request.Query.ContinuationToken = first.ContinuationToken; + var expired = await projection.SearchGenerationAsync(Policy, request); + + Assert.Equal(RecordSearchProjectionFailureCodes.ExpiredContinuation, expired.Failure!.Code); + Assert.Empty(expired.Items); + } + + [Fact] + public async Task DescriptorFenceDeadlineAndWorkBoundFailWithoutCandidates() + { + var clock = new MutableTimeProvider(new DateTimeOffset(2026, 8, 27, 12, 0, 0, TimeSpan.Zero)); + var projection = CreateProjection(clock); + var generation = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + projection.PublishGeneration(generation); + projection.ActivateGeneration(Policy.Name, "generation-a"); + + var fencedRequest = SearchRequest(limit: 10); + fencedRequest.ExpectedDescriptorDigest = Hash("wrong"); + var fenced = await projection.SearchGenerationAsync(Policy, fencedRequest); + Assert.Equal(RecordSearchProjectionFailureCodes.GenerationDescriptorMismatch, fenced.Failure!.Code); + + var expiredRequest = SearchRequest(limit: 10); + expiredRequest.DeadlineUtc = clock.UtcNow.AddSeconds(-1).UtcDateTime; + var expired = await projection.SearchGenerationAsync(Policy, expiredRequest); + Assert.Equal(RecordSearchProjectionFailureCodes.DeadlineExceeded, expired.Failure!.Code); + + var boundedRequest = SearchRequest(limit: 10); + boundedRequest.Query.Lexical!.ScanLimit = 1; + var bounded = await projection.SearchGenerationAsync(Policy, boundedRequest); + Assert.Equal(RecordSearchProjectionFailureCodes.WorkLimitExceeded, bounded.Failure!.Code); + Assert.Equal(3, bounded.Diagnostics.WorkUnits); + Assert.Empty(bounded.Items); + } + + [Fact] + public async Task CallerSelectedScoringParametersFailInsteadOfBeingSilentlyIgnored() + { + var projection = CreateProjection(); + projection.PublishGeneration(CreateGeneration("generation-a", "portable contracts", "portable retrieval")); + projection.ActivateGeneration(Policy.Name, "generation-a"); + var request = SearchRequest(limit: 10); + request.Query.Lexical!.Bm25K1 = 2; + + var result = await projection.SearchGenerationAsync(Policy, request); + + Assert.Equal(RecordSearchProjectionResultStatuses.Failed, result.Status); + Assert.Equal(RecordSearchProjectionFailureCodes.UnsupportedQuery, result.Failure!.Code); + Assert.Contains("generation strategy", result.Failure.Message, StringComparison.Ordinal); + Assert.Empty(result.Items); + } + + [Fact] + public async Task HydrationPreservesProjectionEvidenceAndRejectsStaleCandidates() + { + var path = Path.Combine(Path.GetTempPath(), $"vyral-generation-hydration-{Guid.NewGuid():N}.sqlite"); + var store = new SqliteRecordCollectionStore(path); + await store.InitializeAsync(); + await store.CreateCollectionAsync(Policy); + await store.UpsertRecordAsync(Policy.Name, new VyralRecord + { + Id = "a-1", + PartitionKey = "public-a", + Type = "note", + Content = new System.Text.Json.Nodes.JsonObject { ["text"] = "portable contracts" } + }); + + var projection = CreateProjection(); + var generation = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + projection.PublishGeneration(generation); + projection.ActivateGeneration(Policy.Name, "generation-a"); + + var hydrated = await projection.SearchGenerationAndHydrateAsync( + store, + Policy, + SearchRequest(limit: 10)); + + Assert.Equal("generation-a", hydrated.Projection.GenerationId); + Assert.Equal(2, hydrated.Projection.Items.Count); + Assert.Equal("a-1", Assert.Single(hydrated.Items).Record.Id); + Assert.Equal(1, hydrated.StaleCandidatesDiscarded); + } + + [Fact] + public async Task CancellationAndInvalidPartialFailureAreRejected() + { + var projection = CreateProjection(); + projection.PublishGeneration(CreateGeneration("generation-a", "portable contracts", "portable retrieval")); + projection.ActivateGeneration(Policy.Name, "generation-a"); + using var source = new CancellationTokenSource(); + source.Cancel(); + await Assert.ThrowsAnyAsync(() => + projection.SearchGenerationAsync(Policy, SearchRequest(limit: 10), source.Token)); + + var invalid = new GenerationBoundRecordSearchProjectionResult + { + Status = RecordSearchProjectionResultStatuses.Failed, + Items = [new RecordSearchProjectionCandidate { PartitionKey = "public-a", Id = "a-1", Revision = 1, Score = 1 }], + Coverage = new RecordSearchProjectionCoverage(), + Diagnostics = new RecordSearchProjectionWorkDiagnostics { ReturnedCount = 1 }, + Failure = new RecordSearchProjectionFailure { Code = "failed", Message = "failed" } + }; + Assert.Throws(() => + RecordSearchProjectionGenerationContract.ValidateResult(invalid)); + + var sensitive = new GenerationBoundRecordSearchProjectionResult + { + Status = RecordSearchProjectionResultStatuses.Failed, + Coverage = new RecordSearchProjectionCoverage(), + Diagnostics = new RecordSearchProjectionWorkDiagnostics + { + Details = new System.Text.Json.Nodes.JsonObject { ["authorizationToken"] = "not-allowed" } + }, + Failure = new RecordSearchProjectionFailure { Code = "failed", Message = "failed" } + }; + Assert.Throws(() => + RecordSearchProjectionGenerationContract.ValidateResult(sensitive)); + + Assert.Throws(() => + RecordSearchProjectionGenerationContract.ValidateBuildProgress( + new RecordSearchProjectionGenerationBuildProgress + { + Stage = "build", + Completed = 2, + Total = 1 + })); + Assert.Throws(() => + RecordSearchProjectionGenerationContract.ValidateBuildProgress( + new RecordSearchProjectionGenerationBuildProgress + { + Stage = "build", + Completed = 1, + Total = 1, + Checkpoint = new System.Text.Json.Nodes.JsonObject + { + ["provider"] = new System.Text.Json.Nodes.JsonObject + { + ["accessToken"] = "not-allowed" + } + } + })); + } + + [Fact] + public void RepublishingAnImmutableGenerationRequiresIdenticalContentAndCoverage() + { + var projection = CreateProjection(); + var original = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + projection.PublishGeneration(original); + + var reordered = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + reordered.Documents.Reverse(); + projection.PublishGeneration(reordered); + + var substituted = CreateGeneration("generation-a", "substituted content", "portable retrieval"); + Assert.Throws(() => projection.PublishGeneration(substituted)); + + var changedCoverage = CreateGeneration("generation-a", "portable contracts", "portable retrieval"); + changedCoverage.AvailablePartitions = ["public-a"]; + Assert.Throws(() => projection.PublishGeneration(changedCoverage)); + } + + private static LocalGenerationBoundRecordSearchProjection CreateProjection( + TimeProvider? timeProvider = null, + TimeSpan? continuationLifetime = null) => new(new LocalGenerationBoundRecordSearchProjectionOptions + { + ContinuationSigningKey = SHA256.HashData(Encoding.UTF8.GetBytes("local-generation-bound-test-key")), + TimeProvider = timeProvider ?? TimeProvider.System, + ContinuationLifetime = continuationLifetime ?? TimeSpan.FromMinutes(15) + }); + + private static GenerationBoundRecordSearchProjectionRequest SearchRequest(int limit, string query = "portable") => new() + { + Query = new QueryEnvelope + { + PartitionKeys = ["public-a", "public-b"], + Limit = limit, + Lexical = new LexicalSearchOptions + { + Query = query, + Top = limit, + ScanLimit = 100, + MatchMode = LexicalMatchModes.Any + } + } + }; + + private static LocalRecordSearchProjectionGeneration CreateGeneration( + string generationId, + string firstText, + string secondText) + { + var descriptor = new RecordSearchProjectionGenerationDescriptor + { + Collection = Policy.Name, + GenerationId = generationId, + ProviderId = "local-exhaustive", + ProfileId = "lexical-exhaustive-v1", + StrategyVersion = "exhaustive-token-v1", + SourceManifestDigest = Hash(generationId + ":manifest"), + RecordRevisionSetDigest = Hash(generationId + ":records"), + ProjectionSchemaDigest = Hash("projection-schema-v1"), + AnalyzerDigest = Hash("analyzer-v1"), + ConfigurationDigest = Hash("configuration-v1"), + ExpectedItemCount = 3, + ExpectedPartitions = ["public-a", "public-b"], + Capabilities = ["completeCoverage", "generationPinnedContinuation", "lexical"], + CreatedAtUtc = new DateTime(2026, 8, 27, 12, 0, 0, DateTimeKind.Utc) + }; + RecordSearchProjectionGenerationContract.SealDescriptor(descriptor); + return new LocalRecordSearchProjectionGeneration + { + Descriptor = descriptor, + Documents = + [ + Document("public-a", "a-1", firstText), + Document("public-a", "a-2", "provider native semantics"), + Document("public-b", "b-1", secondText) + ] + }; + } + + private static LocalRecordSearchProjectionDocument Document(string partition, string id, string text) => new() + { + Candidate = new RecordSearchProjectionCandidate + { + PartitionKey = partition, + Id = id, + Revision = 1, + Score = 1 + }, + SearchText = text + }; + + private static string Hash(string value) => + "sha256:" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(value))); + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + public DateTimeOffset UtcNow { get; set; } = utcNow; + public override DateTimeOffset GetUtcNow() => UtcNow; + } +}