From 63330c0250a3357ec3b2cb9dd1d98f65bc7ebbd3 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Wed, 19 Aug 2026 00:07:44 -0400 Subject: [PATCH 1/3] Add hosted execution worker --- Dockerfile | 12 +- README.md | 12 +- Vyral.sln | 15 + deploy/README.md | 43 +- deploy/google-cloud-run.env.example | 23 + deploy/google-hosted-worker.env.example | 37 + deploy/preflight-google-execution.sh | 10 + docs/guides/consumer-handoff.md | 10 +- .../ExecutionWorkerClient.cs | 26 + src/Vyral.HostedWorker/HostedWorkerOptions.cs | 132 ++++ src/Vyral.HostedWorker/Program.cs | 118 +++ .../Vyral.HostedWorker.csproj | 15 + src/Vyral.HostedWorker/packages.lock.json | 688 ++++++++++++++++++ .../ArtifactRecordIngestionHostedPlugin.cs | 105 +++ .../ArtifactRecordIngestionService.cs | 14 + ...onRuntimeArtifactRecordIngestionAdapter.cs | 126 +--- src/Vyral.Server/Program.cs | 109 +-- src/Vyral.Server/ServerStorageFactory.cs | 90 +++ src/Vyral.Server/VyralExecutionAccess.cs | 12 +- .../ExecutionWorkerClientTests.cs | 10 +- ...oogleCloudExecutionDispatchOptionsTests.cs | 16 + .../ExecutionRuntimeFactoryTests.cs | 13 + .../ExecutionRuntimeTests.cs | 72 ++ .../Vyral.Tests.Local.csproj | 1 + 24 files changed, 1498 insertions(+), 211 deletions(-) create mode 100644 deploy/google-hosted-worker.env.example create mode 100644 src/Vyral.HostedWorker/HostedWorkerOptions.cs create mode 100644 src/Vyral.HostedWorker/Program.cs create mode 100644 src/Vyral.HostedWorker/Vyral.HostedWorker.csproj create mode 100644 src/Vyral.HostedWorker/packages.lock.json create mode 100644 src/Vyral.Server/ArtifactRecordIngestionHostedPlugin.cs create mode 100644 src/Vyral.Server/ServerStorageFactory.cs diff --git a/Dockerfile b/Dockerfile index 563cce2..6d59079 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,15 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0@sha256:e1fc6e423f543119c406d24e2e687d67c5 WORKDIR /src COPY . . RUN dotnet restore src/Vyral.Server/Vyral.Server.csproj --locked-mode --disable-parallel \ + && dotnet restore src/Vyral.HostedWorker/Vyral.HostedWorker.csproj --locked-mode --disable-parallel \ && dotnet publish src/Vyral.Server/Vyral.Server.csproj \ -c Release \ - -o /app/publish \ + -o /app/publish/server \ + --no-restore \ + /p:UseAppHost=false \ + && dotnet publish src/Vyral.HostedWorker/Vyral.HostedWorker.csproj \ + -c Release \ + -o /app/publish/worker \ --no-restore \ /p:UseAppHost=false \ && mkdir -p /app/publish/.vyral @@ -31,4 +37,6 @@ ENV ASPNETCORE_URLS=http://0.0.0.0:8080 \ COPY --from=build --chown=1654:1654 /app/publish . USER 1654 EXPOSE 8080 -ENTRYPOINT ["dotnet", "Vyral.Server.dll"] +# The default is the public API server. Deploy the same pinned image as the +# least-privilege generic worker with: dotnet worker/Vyral.HostedWorker.dll +ENTRYPOINT ["dotnet", "server/Vyral.Server.dll"] diff --git a/README.md b/README.md index f2f61da..c9ab4a0 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,11 @@ Vyral is most useful when an application needs to cross one or more provider seams without giving those providers ownership of its data and execution semantics. +Vyral grows by portable capability, not by reproducing provider APIs. A shared +contract earns its place when independent provider shapes—or an established +application model—show a durable common core; provider-native behavior remains +explicit at the edge. + Vyral does not assume vector search is always the answer. For current, structured sources that an application can safely access, direct source search is often the simplest first path. Indexed lexical, vector, hybrid, and reranked @@ -353,6 +358,11 @@ Vyral uses two canonical mutation shapes: - receipt-bound asynchronous operations durably admit work and return a run or job receipt whose status can be queried independently. +Vyral-owned generic handlers can also run through a separately deployed, +same-version hosted worker (preview), so consumers retain the public admission +contract without reimplementing Vyral storage or lease behavior. The initial +deployment shape is documented for [Google Cloud Run](deploy/README.md#vyral-hosted-generic-handlers). + The local SQLite runtime is the reference implementation. Azure Durable, AWS, Google Cloud Tasks, and Temporal adapters implement different subsets and carry separate qualification evidence. External workers use leases, @@ -392,7 +402,7 @@ Adapter authors should begin with the ## Repository map ```text -src/ .NET contracts, runtimes, server, and provider adapters +src/ .NET contracts, runtimes, API/hosted-worker services, and provider adapters clients/ Python and JavaScript HTTP SDKs; Go external-worker client runtimes/ Peer runtime implementations, currently Python contracts/ OpenAPI-derived public SDK catalog and JSON schemas diff --git a/Vyral.sln b/Vyral.sln index 5c5ac6f..08d5a25 100644 --- a/Vyral.sln +++ b/Vyral.sln @@ -17,6 +17,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.Local", "src\Vyral.Lo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.Server", "src\Vyral.Server\Vyral.Server.csproj", "{8EC7614A-6D41-4752-BC17-6858811C2B48}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.HostedWorker", "src\Vyral.HostedWorker\Vyral.HostedWorker.csproj", "{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{AAF8E491-F193-4C07-8669-9F3F823ED378}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.Tests.Azure", "tests\Vyral.Tests.Azure\Vyral.Tests.Azure.csproj", "{425659EC-721E-463D-85E4-764273695F21}" @@ -183,6 +185,18 @@ Global {8EC7614A-6D41-4752-BC17-6858811C2B48}.Release|x64.Build.0 = Release|Any CPU {8EC7614A-6D41-4752-BC17-6858811C2B48}.Release|x86.ActiveCfg = Release|Any CPU {8EC7614A-6D41-4752-BC17-6858811C2B48}.Release|x86.Build.0 = Release|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x64.ActiveCfg = Debug|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x64.Build.0 = Debug|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x86.ActiveCfg = Debug|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x86.Build.0 = Debug|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|Any CPU.Build.0 = Release|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x64.ActiveCfg = Release|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x64.Build.0 = Release|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x86.ActiveCfg = Release|Any CPU + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x86.Build.0 = Release|Any CPU {425659EC-721E-463D-85E4-764273695F21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {425659EC-721E-463D-85E4-764273695F21}.Debug|Any CPU.Build.0 = Debug|Any CPU {425659EC-721E-463D-85E4-764273695F21}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -662,6 +676,7 @@ Global {2D62965C-C36B-4EB8-BFE9-20EB6B23AEF5} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44} {C9131552-2AC8-44F6-BE8D-17F223D0F5E1} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44} {8EC7614A-6D41-4752-BC17-6858811C2B48} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44} + {A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44} {425659EC-721E-463D-85E4-764273695F21} = {AAF8E491-F193-4C07-8669-9F3F823ED378} {23B12913-7D75-46A6-A36B-F3755A1102BD} = {AAF8E491-F193-4C07-8669-9F3F823ED378} {F248B3EE-1C18-48CF-89C4-72D3F93F2C57} = {AAF8E491-F193-4C07-8669-9F3F823ED378} diff --git a/deploy/README.md b/deploy/README.md index ac1c2bd..f406e7a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -196,6 +196,43 @@ email. It rejects unscoped starts, worker-id impersonation, un-routed handlers, and maintenance calls without a dedicated policy. Keep test-only public worker fixtures out of the shared project. +### Vyral-hosted generic handlers + +Consumer-neutral Vyral handlers may use the same external-worker protocol as a +consumer worker. The initial handler, `vyral.artifacts.record-ingest`, preserves +the public `POST /ingest/record-artifact` admission contract while a separately +deployed Vyral worker performs the staged-object read, object publish, record +upsert, and best-effort staging cleanup. + +Set `VYRAL_INGEST_STAGING_CONTAINER` identically on the API and worker. With +GCS it is normally the Vyral artifact bucket, using a private +`record-artifact/` object prefix; it is not a product-owned bucket. A manifest's +published artifact container must likewise be one the Vyral worker is allowed to +write. + +Deploy the same pinned Vyral image as a separate Cloud Run service with command +`dotnet` and argument `worker/Vyral.HostedWorker.dll`. Configure the API route +and external handler descriptor as shown in +[`google-cloud-run.env.example`](google-cloud-run.env.example), and configure +the worker from [`google-hosted-worker.env.example`](google-hosted-worker.env.example). +The worker must have a distinct service account. The Cloud Tasks dispatch +service account gets `run.invoker` only on the worker. The worker gets +`run.invoker` only on the Vyral API, plus Firestore and object permissions for +the Vyral storage plane; it does not need Cloud Tasks enqueue, execution-state, +or consumer-deployment permissions. If the API retains API-key defense in +depth, mount only that Vyral API-key secret into the worker. Each product policy +that allows artifact admission must also allow `vyral.artifacts.record-ingest` +and the hosted worker id. This keeps the generic worker from crossing a product +scope merely because it can claim the handler. + +The queue message is only a run id and dispatch reason. The worker validates the +Cloud Tasks OIDC callback, leases the run through Vyral, and returns success only +after duplicate-safe lease completion. The original `202 Accepted` response and +its `Location` execution-run receipt remain the consumer's status surface; no +consumer implementation is required to complete the generic work. Do not route a +generic Vyral handler to a consumer worker or grant consumers access to the +Vyral admission-staging prefix. + ### Execution deployment preflight Run the read-only preflight after deploying candidate Vyral and worker Cloud Run services, but @@ -208,9 +245,9 @@ deletes a resource. ```bash VYRAL_EXECUTION_PROJECT_ID=your-gcp-project-id \ VYRAL_EXECUTION_SERVER_SERVICE=vyral-server \ -VYRAL_EXECUTION_WORKER_SERVICE=product-example-worker \ -VYRAL_EXECUTION_WORKER_ID=product-example-worker \ -VYRAL_EXECUTION_HANDLER_IDS=product.example.job \ +VYRAL_EXECUTION_WORKER_SERVICE=vyral-hosted-worker \ +VYRAL_EXECUTION_WORKER_ID=vyral-hosted-artifact-worker \ +VYRAL_EXECUTION_HANDLER_IDS=vyral.artifacts.record-ingest \ VYRAL_EXECUTION_CONFIG_FILE=deploy/google-cloud-run.env \ deploy/preflight-google-execution.sh ``` diff --git a/deploy/google-cloud-run.env.example b/deploy/google-cloud-run.env.example index b6454a3..06e065d 100644 --- a/deploy/google-cloud-run.env.example +++ b/deploy/google-cloud-run.env.example @@ -19,6 +19,9 @@ VYRAL_TRACE_STORE=google-firestore VYRAL_OBJECT_STORE=google-cloud-storage VYRAL_GCS_BUCKET=your-gcp-project-id-vyral-artifacts VYRAL_OBJECT_PROBE_CONTAINER=your-gcp-project-id-vyral-artifacts +# Both the API and the hosted worker use this private staging prefix in the shared Vyral bucket. +# It is not a consumer artifact container or a second bucket to grant to product services. +VYRAL_INGEST_STAGING_CONTAINER=your-gcp-project-id-vyral-artifacts VYRAL_FIRESTORE_ROOT_COLLECTION=vyral # Optional durable execution plane. Leave this unset to retain the default local SQLite @@ -46,11 +49,23 @@ ExecutionRuntime__MaxRetainedTerminalRuns=500 ExecutionRuntime__ExternalHandlers__0__HandlerId=product.example.job ExecutionRuntime__ExternalHandlers__0__PluginId=product.example ExecutionRuntime__ExternalHandlers__0__DisplayName=Product example job +# Vyral-owned generic handlers are external only when an explicitly configured Vyral hosted +# worker is deployed. Do not give consumers staging-object or generic storage access. +ExecutionRuntime__Google__WorkerRoutes__1__HandlerId=vyral.artifacts.record-ingest +ExecutionRuntime__Google__WorkerRoutes__1__WorkerUrl=https://replace-with-vyral-hosted-worker/tasks/execution +ExecutionRuntime__Google__WorkerRoutes__1__OidcAudience=https://replace-with-vyral-hosted-worker +ExecutionRuntime__ExternalHandlers__1__HandlerId=vyral.artifacts.record-ingest +ExecutionRuntime__ExternalHandlers__1__PluginId=vyral.artifacts +ExecutionRuntime__ExternalHandlers__1__DisplayName=Vyral artifact and record ingest # Product policy is the durable scope boundary that binds the verified worker identity below. ExecutionRuntime__ProductPolicies__0__ProductId=product-example ExecutionRuntime__ProductPolicies__0__AllowedTenantIds__0=tenant-example ExecutionRuntime__ProductPolicies__0__AllowedHandlerIds__0=product.example.job ExecutionRuntime__ProductPolicies__0__AllowedServiceIdentities__0=product-example-worker +# A product that uses the public artifact admission route grants the generic handler and hosted +# Vyral worker explicitly. Do not infer either permission from generic storage access. +ExecutionRuntime__ProductPolicies__0__AllowedHandlerIds__1=vyral.artifacts.record-ingest +ExecutionRuntime__ProductPolicies__0__AllowedServiceIdentities__1=vyral-hosted-artifact-worker # Shared execution requires a verified Cloud Run OIDC workload identity for every execution API # request. Bind each service account to exactly the product, tenants, handlers, and operations it @@ -70,6 +85,14 @@ Server__ExecutionAccess__IdentityPolicies__1__WorkerId=product-example-worker Server__ExecutionAccess__IdentityPolicies__1__AllowedTenantIds__0=tenant-example Server__ExecutionAccess__IdentityPolicies__1__AllowedHandlerIds__0=product.example.job Server__ExecutionAccess__IdentityPolicies__1__AllowedOperations__0=worker +# The generic Vyral worker is independently authenticated. Repeat this policy for every product +# scope it may process, with that product's allowed tenants. +Server__ExecutionAccess__IdentityPolicies__2__Principal=vyral-hosted-worker@your-gcp-project-id.iam.gserviceaccount.com +Server__ExecutionAccess__IdentityPolicies__2__ProductId=product-example +Server__ExecutionAccess__IdentityPolicies__2__WorkerId=vyral-hosted-artifact-worker +Server__ExecutionAccess__IdentityPolicies__2__AllowedTenantIds__0=tenant-example +Server__ExecutionAccess__IdentityPolicies__2__AllowedHandlerIds__0=vyral.artifacts.record-ingest +Server__ExecutionAccess__IdentityPolicies__2__AllowedOperations__0=worker VYRAL_API_KEY_HEADER=X-Vyral-Api-Key diff --git a/deploy/google-hosted-worker.env.example b/deploy/google-hosted-worker.env.example new file mode 100644 index 0000000..12daf6d --- /dev/null +++ b/deploy/google-hosted-worker.env.example @@ -0,0 +1,37 @@ +# Vyral hosted generic worker for Cloud Tasks -> Cloud Run. Run the same pinned Vyral image with +# command `dotnet` and argument `worker/Vyral.HostedWorker.dll`. +# +# This worker owns only Vyral's generic staged artifact/record publish behavior. Consumer code, +# collection policy, provenance, encryption, and domain retry choices remain outside this service. + +ASPNETCORE_ENVIRONMENT=Production +GOOGLE_CLOUD_PROJECT=your-gcp-project-id +VYRAL_GCP_PROJECT_ID=your-gcp-project-id + +# Match the Vyral API's generic storage plane. Grant this worker's service account only the +# Firestore record and GCS object permissions required by this generic handler. +VYRAL_RECORD_STORE=google-firestore +VYRAL_OBJECT_STORE=google-cloud-storage +VYRAL_GCS_BUCKET=your-gcp-project-id-vyral-artifacts +VYRAL_FIRESTORE_ROOT_COLLECTION=vyral +# Must exactly match the Vyral API's private generic staging container. +VYRAL_INGEST_STAGING_CONTAINER=your-gcp-project-id-vyral-artifacts + +# The Vyral API worker protocol endpoint and this worker's stable identity. Both values must +# match an API ExecutionAccess worker policy. Every product policy that allows artifact admission +# must also allow this handler and worker id for its own tenant scope. +HostedWorker__VyralUrl=https://replace-with-vyral-server +HostedWorker__WorkerId=vyral-hosted-artifact-worker +HostedWorker__HandlerIds__0=vyral.artifacts.record-ingest +HostedWorker__LeaseTtlSeconds=60 +HostedWorker__HeartbeatSeconds=20 +# The worker always presents its Cloud Run identity to Vyral. If the Vyral API also requires an +# API key, mount one Vyral API-key secret into this exact value; do not place its value in this file. +# HostedWorker__ApiKey=mounted-secret-value +# HostedWorker__ApiKeyHeader=X-Vyral-Api-Key + +# Cloud Tasks calls the worker with an OIDC token minted for the Cloud Run URL. Cloud Run IAM must +# also grant that task service account run.invoker on this worker service. +HostedWorker__TaskAuthentication__Mode=google-oidc +HostedWorker__TaskAuthentication__AllowedAudiences__0=https://replace-with-vyral-hosted-worker +HostedWorker__TaskAuthentication__AllowedPrincipals__0=vyral-cloud-tasks@your-gcp-project-id.iam.gserviceaccount.com diff --git a/deploy/preflight-google-execution.sh b/deploy/preflight-google-execution.sh index 6eadcd4..712fea0 100755 --- a/deploy/preflight-google-execution.sh +++ b/deploy/preflight-google-execution.sh @@ -300,6 +300,7 @@ DEFAULT_OIDC_AUDIENCE="$(config_value "ExecutionRuntime__Google__OidcAudience")" ARTIFACT_OBJECT_CONTAINER="$(config_value "ExecutionRuntime__Google__ArtifactObjectContainer")" OBJECT_STORE="$(config_value "VYRAL_OBJECT_STORE")" GCS_BUCKET="$(config_value "VYRAL_GCS_BUCKET")" +INGEST_STAGING_CONTAINER="$(config_value "VYRAL_INGEST_STAGING_CONTAINER")" AUTH_MODE="$(config_value "Server__ExecutionAccess__AuthenticationMode")" RECORD_ROOT="$(config_value "VYRAL_FIRESTORE_ROOT_COLLECTION")" RUNTIME_ADAPTER="$(config_value "ExecutionRuntime__Adapter")" @@ -367,6 +368,15 @@ fi if has_run_invoker "$SERVER_SERVICE" "$WORKER_SERVICE_ACCOUNT"; then pass "worker service account can invoke Vyral server"; else fail "worker service account lacks roles/run.invoker on Vyral server"; fi if has_project_role "$SERVER_SERVICE_ACCOUNT" "roles/datastore.user"; then pass "Vyral runtime can use Firestore"; else fail "Vyral runtime lacks roles/datastore.user"; fi if has_project_role "$SERVER_SERVICE_ACCOUNT" "roles/cloudtasks.enqueuer"; then pass "Vyral runtime can enqueue Cloud Tasks"; else fail "Vyral runtime lacks roles/cloudtasks.enqueuer"; fi +for handler_id in "${HANDLER_IDS[@]}"; do + if [[ "$handler_id" != "vyral.artifacts.record-ingest" ]]; then + continue + fi + config_value_is_real "VYRAL_INGEST_STAGING_CONTAINER" "$INGEST_STAGING_CONTAINER" || true + if [[ "$INGEST_STAGING_CONTAINER" == "$GCS_BUCKET" ]]; then pass "generic ingestion staging uses the Vyral object bucket"; else fail "VYRAL_INGEST_STAGING_CONTAINER must equal VYRAL_GCS_BUCKET for least-privilege hosted ingestion"; fi + if has_project_role "$WORKER_SERVICE_ACCOUNT" "roles/datastore.user"; then pass "Vyral hosted artifact worker can use Firestore records"; else fail "Vyral hosted artifact worker lacks roles/datastore.user"; fi + if [[ -n "$GCS_BUCKET" ]] && has_bucket_object_access "$GCS_BUCKET" "$WORKER_SERVICE_ACCOUNT"; then pass "Vyral hosted artifact worker can read and write generic objects"; else fail "Vyral hosted artifact worker lacks object access on $GCS_BUCKET"; fi +done PROJECT_NUMBER="$(gcloud projects describe "$PROJECT_ID" --format='value(projectNumber)' 2>/dev/null || true)" TASKS_AGENT="service-${PROJECT_NUMBER}@gcp-sa-cloudtasks.iam.gserviceaccount.com" for tasks_service_account in $(printf '%s\n' "${ROUTE_TASKS_SERVICE_ACCOUNTS[@]}" | sort -u); do diff --git a/docs/guides/consumer-handoff.md b/docs/guides/consumer-handoff.md index 8bde3fb..6df7814 100644 --- a/docs/guides/consumer-handoff.md +++ b/docs/guides/consumer-handoff.md @@ -87,9 +87,13 @@ Consumers decide whether and how to derive pseudonymous identifiers. Vyral stores only the metadata and content that their manifest supplies. For Cloud Run, keep specialized work request-bound and let Vyral own generic -durable acceptance of the supplied artifact and record. A producer may select -inline or queue-backed processing without changing the contract. Its integration -owns the queue, callback identity, retry policy, and route. +durable acceptance of the supplied artifact and record. The public operation +returns a `202` admission receipt; a Vyral-hosted generic worker completes the +configured external execution route. Consumer-owned workers remain for +consumer-defined handlers, not Vyral's generic staging, storage publication, or +lease-completion behavior. The Vyral deployment owns the generic queue, callback +identity, and reconciliation. A producer owns a queue, callback identity, retry +policy, and route only for its own consumer-defined handler. An upstream may attach a signed external context to its manifest. The host configures a public-key verifier and expected issuer, audience, and key id. diff --git a/src/Vyral.Execution.WorkerClient/ExecutionWorkerClient.cs b/src/Vyral.Execution.WorkerClient/ExecutionWorkerClient.cs index 87bae74..8c6ec58 100644 --- a/src/Vyral.Execution.WorkerClient/ExecutionWorkerClient.cs +++ b/src/Vyral.Execution.WorkerClient/ExecutionWorkerClient.cs @@ -101,6 +101,12 @@ public sealed class ExecutionWorkerClientOptions public required string WorkerId { get; init; } public required IReadOnlyList HandlerIds { get; init; } public IExecutionWorkerTokenSource? TokenSource { get; init; } + /// + /// Optional API key supplied in addition to the worker identity. This is useful when the + /// Vyral API retains API-key defense in depth behind an identity-aware gateway. + /// + public string? ApiKey { get; init; } + public string ApiKeyHeader { get; init; } = "X-Vyral-Api-Key"; public Action? Observe { get; init; } } @@ -133,6 +139,8 @@ public sealed class ExecutionWorkerClient : IExecutionWorkerTransport private readonly string _workerId; private readonly IReadOnlyList _handlerIds; private readonly IExecutionWorkerTokenSource? _tokenSource; + private readonly string? _apiKey; + private readonly string? _apiKeyHeader; private readonly Action? _observe; public ExecutionWorkerClient(HttpClient client, ExecutionWorkerClientOptions options) @@ -146,6 +154,8 @@ public ExecutionWorkerClient(HttpClient client, ExecutionWorkerClientOptions opt _handlerIds = options.HandlerIds.Where(id => !string.IsNullOrWhiteSpace(id)).Select(id => id.Trim()).Distinct(StringComparer.Ordinal).ToList(); if (_handlerIds.Count == 0) throw new InvalidOperationException("At least one Vyral worker handler id is required."); _tokenSource = options.TokenSource; + _apiKey = string.IsNullOrWhiteSpace(options.ApiKey) ? null : options.ApiKey.Trim(); + _apiKeyHeader = _apiKey is null ? null : RequireHeaderName(options.ApiKeyHeader); _observe = options.Observe; } @@ -247,6 +257,10 @@ await SendAsync(operation, path, runId, payload, allowNoContent: false, ct: c if (string.IsNullOrWhiteSpace(token)) throw new InvalidOperationException("Vyral worker token source returned an empty token."); request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); } + if (_apiKey is not null) + { + request.Headers.Add(_apiKeyHeader!, _apiKey); + } using var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct); statusCode = response.StatusCode; @@ -283,4 +297,16 @@ private void Observe(string operation, string path, string? runId, HttpStatusCod } }); private static string Require(string value, string description) => string.IsNullOrWhiteSpace(value) ? throw new InvalidOperationException($"{description} is required.") : value.Trim(); + + private static string RequireHeaderName(string? value) + { + var candidate = value?.Trim(); + if (string.IsNullOrWhiteSpace(candidate) || !candidate.All(character => + char.IsAsciiLetterOrDigit(character) || character == '-')) + { + throw new InvalidOperationException("Vyral worker API-key header must contain only letters, digits, and hyphens."); + } + + return candidate; + } } diff --git a/src/Vyral.HostedWorker/HostedWorkerOptions.cs b/src/Vyral.HostedWorker/HostedWorkerOptions.cs new file mode 100644 index 0000000..b4ecd55 --- /dev/null +++ b/src/Vyral.HostedWorker/HostedWorkerOptions.cs @@ -0,0 +1,132 @@ +using Microsoft.Extensions.Configuration; +using Vyral.Server; + +namespace Vyral.HostedWorker; + +/// +/// Deployment-owned configuration for the Vyral generic-handler worker. It deliberately has no +/// consumer, queue, collection, or provenance defaults. +/// +public sealed class HostedWorkerOptions +{ + public string VyralUrl { get; init; } = string.Empty; + public string WorkerId { get; init; } = string.Empty; + public IReadOnlyList HandlerIds { get; init; } = Array.Empty(); + public string? ApiKey { get; init; } + public string ApiKeyHeader { get; init; } = "X-Vyral-Api-Key"; + public string CallbackPath { get; init; } = "/tasks/execution"; + public double LeaseTtlSeconds { get; init; } = 60; + public int HeartbeatSeconds { get; init; } = 20; + public HostedWorkerTaskAuthenticationOptions TaskAuthentication { get; init; } = new(); + + public static HostedWorkerOptions FromConfiguration(IConfiguration configuration) + { + var section = configuration.GetSection("HostedWorker"); + return new HostedWorkerOptions + { + VyralUrl = FirstNonEmpty(section["VyralUrl"], section["vyralUrl"], configuration["VYRAL_HOSTED_WORKER_VYRAL_URL"]) ?? string.Empty, + WorkerId = FirstNonEmpty(section["WorkerId"], section["workerId"], configuration["VYRAL_HOSTED_WORKER_ID"]) ?? string.Empty, + HandlerIds = ReadValues(section, "HandlerIds", "handlerIds", configuration["VYRAL_HOSTED_WORKER_HANDLER_IDS"]), + ApiKey = FirstNonEmpty(section["ApiKey"], section["apiKey"], configuration["VYRAL_HOSTED_WORKER_API_KEY"]), + ApiKeyHeader = FirstNonEmpty(section["ApiKeyHeader"], section["apiKeyHeader"], configuration["VYRAL_HOSTED_WORKER_API_KEY_HEADER"], "X-Vyral-Api-Key")!, + CallbackPath = FirstNonEmpty(section["CallbackPath"], section["callbackPath"], configuration["VYRAL_HOSTED_WORKER_CALLBACK_PATH"], "/tasks/execution")!, + LeaseTtlSeconds = ParsePositiveDouble(FirstNonEmpty(section["LeaseTtlSeconds"], section["leaseTtlSeconds"], configuration["VYRAL_HOSTED_WORKER_LEASE_TTL_SECONDS"]), 60), + HeartbeatSeconds = ParsePositiveInt(FirstNonEmpty(section["HeartbeatSeconds"], section["heartbeatSeconds"], configuration["VYRAL_HOSTED_WORKER_HEARTBEAT_SECONDS"]), 20), + TaskAuthentication = HostedWorkerTaskAuthenticationOptions.FromConfiguration(configuration) + }; + } + + public void Validate() + { + if (!Uri.TryCreate(VyralUrl, UriKind.Absolute, out var uri) || !string.IsNullOrEmpty(uri.UserInfo)) + { + throw new InvalidOperationException("HostedWorker:VyralUrl must be an absolute URL without user credentials."); + } + if (string.IsNullOrWhiteSpace(WorkerId)) throw new InvalidOperationException("HostedWorker:WorkerId is required."); + if (HandlerIds.Count == 0) throw new InvalidOperationException("HostedWorker:HandlerIds requires at least one Vyral hosted handler."); + if (HandlerIds.Any(id => !string.Equals(id, ArtifactRecordIngestionHostedPlugin.HandlerId, StringComparison.Ordinal))) + { + throw new InvalidOperationException("HostedWorker:HandlerIds contains an unsupported Vyral hosted handler."); + } + if (!CallbackPath.StartsWith("/", StringComparison.Ordinal) || CallbackPath.StartsWith("//", StringComparison.Ordinal)) + { + throw new InvalidOperationException("HostedWorker:CallbackPath must be an absolute path."); + } + if (!double.IsFinite(LeaseTtlSeconds) || LeaseTtlSeconds <= 0 || LeaseTtlSeconds > 900) + { + throw new InvalidOperationException("HostedWorker:LeaseTtlSeconds must be greater than zero and no more than 900."); + } + if (HeartbeatSeconds <= 0 || HeartbeatSeconds >= LeaseTtlSeconds) + { + throw new InvalidOperationException("HostedWorker:HeartbeatSeconds must be positive and shorter than HostedWorker:LeaseTtlSeconds."); + } + TaskAuthentication.Validate(); + } + + private static IReadOnlyList ReadValues(IConfigurationSection section, string name, string alternateName, string? environmentValue) + { + var values = section.GetSection(name).GetChildren().Select(item => item.Value) + .Concat(section.GetSection(alternateName).GetChildren().Select(item => item.Value)) + .Append(section[name]) + .Append(section[alternateName]) + .Append(environmentValue) + .Where(item => !string.IsNullOrWhiteSpace(item)) + .SelectMany(item => item!.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + .Distinct(StringComparer.Ordinal) + .OrderBy(item => item, StringComparer.Ordinal) + .ToList(); + return values; + } + + private static string? FirstNonEmpty(params string?[] values) => values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value))?.Trim(); + private static int ParsePositiveInt(string? value, int fallback) => int.TryParse(value, out var parsed) && parsed > 0 ? parsed : fallback; + private static double ParsePositiveDouble(string? value, double fallback) => double.TryParse(value, out var parsed) && double.IsFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +public sealed class HostedWorkerTaskAuthenticationOptions +{ + public string Mode { get; init; } = "google-oidc"; + public IReadOnlySet AllowedAudiences { get; init; } = new HashSet(StringComparer.Ordinal); + public IReadOnlySet AllowedPrincipals { get; init; } = new HashSet(StringComparer.Ordinal); + public string DevelopmentIdentityHeader { get; init; } = "X-Vyral-Development-Identity"; + + public static HostedWorkerTaskAuthenticationOptions FromConfiguration(IConfiguration configuration) + { + var section = configuration.GetSection("HostedWorker:TaskAuthentication"); + return new HostedWorkerTaskAuthenticationOptions + { + Mode = section["Mode"] ?? section["mode"] ?? configuration["VYRAL_HOSTED_WORKER_TASK_AUTH_MODE"] ?? "google-oidc", + AllowedAudiences = ReadSet(section, "AllowedAudiences", "allowedAudiences", configuration["VYRAL_HOSTED_WORKER_TASK_ALLOWED_AUDIENCES"]), + AllowedPrincipals = ReadSet(section, "AllowedPrincipals", "allowedPrincipals", configuration["VYRAL_HOSTED_WORKER_TASK_ALLOWED_PRINCIPALS"]), + DevelopmentIdentityHeader = section["DevelopmentIdentityHeader"] ?? section["developmentIdentityHeader"] ?? "X-Vyral-Development-Identity" + }; + } + + public void Validate() + { + if (!string.Equals(Mode, "google-oidc", StringComparison.Ordinal) && + !string.Equals(Mode, "development-header", StringComparison.Ordinal)) + { + throw new InvalidOperationException("HostedWorker:TaskAuthentication:Mode must be google-oidc or development-header."); + } + if (AllowedPrincipals.Count == 0) + { + throw new InvalidOperationException("HostedWorker:TaskAuthentication:AllowedPrincipals requires at least one callback identity."); + } + if (string.Equals(Mode, "google-oidc", StringComparison.Ordinal) && AllowedAudiences.Count == 0) + { + throw new InvalidOperationException("HostedWorker:TaskAuthentication:AllowedAudiences is required for Google OIDC."); + } + } + + private static IReadOnlySet ReadSet(IConfigurationSection section, string name, string alternateName, string? environmentValue) => + new HashSet( + section.GetSection(name).GetChildren().Select(item => item.Value) + .Concat(section.GetSection(alternateName).GetChildren().Select(item => item.Value)) + .Append(section[name]) + .Append(section[alternateName]) + .Append(environmentValue) + .Where(item => !string.IsNullOrWhiteSpace(item)) + .SelectMany(item => item!.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)), + StringComparer.Ordinal); +} diff --git a/src/Vyral.HostedWorker/Program.cs b/src/Vyral.HostedWorker/Program.cs new file mode 100644 index 0000000..9456a4f --- /dev/null +++ b/src/Vyral.HostedWorker/Program.cs @@ -0,0 +1,118 @@ +using System.Text.Json; +using Vyral.Execution; +using Vyral.Execution.WorkerClient; +using Vyral.Google; +using Vyral.HostedWorker; +using Vyral.Server; + +var builder = WebApplication.CreateBuilder(args); +var options = HostedWorkerOptions.FromConfiguration(builder.Configuration); +options.Validate(); + +var storageOptions = ServerStorageOptions.FromConfiguration(builder.Configuration); +var records = await ServerStorageFactory.CreateRecordStoreAsync(storageOptions); +var objects = ServerStorageFactory.CreateObjectStore(storageOptions); +var ingestionOptions = ArtifactRecordIngestionOptions.FromConfiguration(builder.Configuration); +var plugin = new ArtifactRecordIngestionHostedPlugin( + objects, + new ArtifactRecordIngestionService(records, objects, ingestionOptions)); +var authentication = new HostedWorkerTaskAuthenticator(options.TaskAuthentication, builder.Environment); +var transport = new ExecutionWorkerClient( + new HttpClient { Timeout = TimeSpan.FromSeconds(30) }, + new ExecutionWorkerClientOptions + { + BaseUri = new Uri(options.VyralUrl, UriKind.Absolute), + WorkerId = options.WorkerId, + HandlerIds = options.HandlerIds, + TokenSource = new GoogleMetadataOidcTokenSource(options.VyralUrl), + ApiKey = options.ApiKey, + ApiKeyHeader = options.ApiKeyHeader + }); +var worker = new ExecutionPluginWorker( + transport, + [plugin], + new ExecutionPluginWorkerOptions + { + LeaseTtlSeconds = options.LeaseTtlSeconds, + HeartbeatInterval = TimeSpan.FromSeconds(options.HeartbeatSeconds) + }); + +var app = builder.Build(); +app.MapGet("/health", () => Results.Ok(new { status = "ok", workerId = options.WorkerId, handlers = options.HandlerIds })); +app.MapPost(options.CallbackPath, async (HttpContext context, CancellationToken ct) => +{ + if (!string.Equals(context.Request.Headers["X-Vyral-Execution-Dispatch"], "1", StringComparison.Ordinal)) + { + return Results.BadRequest(new { error = "Vyral execution dispatch header is required." }); + } + + if (!await authentication.IsAuthorizedAsync(context, ct)) + { + return Results.Unauthorized(); + } + + var message = await JsonSerializer.DeserializeAsync( + context.Request.Body, + ExecutionJson.Options, + ct); + if (message is null || string.IsNullOrWhiteSpace(message.RunId)) + { + return Results.BadRequest(new { error = "Vyral execution dispatch requires a run id." }); + } + + _ = await worker.RunOnceAsync(message.RunId, ct); + return Results.NoContent(); +}); + +await app.RunAsync(); + +internal sealed class HostedWorkerTaskAuthenticator +{ + private readonly HostedWorkerTaskAuthenticationOptions _options; + private readonly IHostEnvironment _environment; + private readonly GoogleExecutionTokenValidator _google = new(); + + public HostedWorkerTaskAuthenticator(HostedWorkerTaskAuthenticationOptions options, IHostEnvironment environment) + { + _options = options; + _environment = environment; + } + + public async Task IsAuthorizedAsync(HttpContext context, CancellationToken ct) + { + try + { + var principal = string.Equals(_options.Mode, "google-oidc", StringComparison.Ordinal) + ? await _google.ValidateAsync(GetBearerToken(context.Request) ?? string.Empty, _options.AllowedAudiences, ct) + : GetDevelopmentPrincipal(context.Request); + return _options.AllowedPrincipals.Contains(principal); + } + catch (ExecutionAccessDeniedException) + { + return false; + } + catch (InvalidOperationException) + { + return false; + } + } + + private string GetDevelopmentPrincipal(HttpRequest request) + { + if (!_environment.IsDevelopment()) + { + throw new ExecutionAccessDeniedException("Development-header worker authentication is disabled outside Development."); + } + var principal = request.Headers[_options.DevelopmentIdentityHeader].ToString().Trim(); + if (string.IsNullOrWhiteSpace(principal)) throw new ExecutionAccessDeniedException("A development worker identity header is required."); + return principal; + } + + private static string? GetBearerToken(HttpRequest request) + { + var value = request.Headers["X-Serverless-Authorization"].ToString(); + if (string.IsNullOrWhiteSpace(value)) value = request.Headers.Authorization.ToString(); + const string prefix = "Bearer "; + return value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) ? value[prefix.Length..].Trim() : null; + } +} diff --git a/src/Vyral.HostedWorker/Vyral.HostedWorker.csproj b/src/Vyral.HostedWorker/Vyral.HostedWorker.csproj new file mode 100644 index 0000000..a18562c --- /dev/null +++ b/src/Vyral.HostedWorker/Vyral.HostedWorker.csproj @@ -0,0 +1,15 @@ + + + + + + + + + net10.0 + enable + enable + false + + + diff --git a/src/Vyral.HostedWorker/packages.lock.json b/src/Vyral.HostedWorker/packages.lock.json new file mode 100644 index 0000000..bec517a --- /dev/null +++ b/src/Vyral.HostedWorker/packages.lock.json @@ -0,0 +1,688 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "4.0.101", + "contentHash": "7Q0z0fLpy+w6h4G5z/qZ6PYGzZ0DgYjYkVdGMnhtjkgQaKX+Sjm0YxfFxFILRewxC4o6wwnrbvwaSj41LgRRfg==" + }, + "AWSSDK.DynamoDBv2": { + "type": "Transitive", + "resolved": "4.0.103.1", + "contentHash": "lWjDWid4uPdgi1b5dARsvvsFDUmgJf+uAnInD3yGSdNXZoEPd2nDZfVTKFxi9ny4BdlmV4znmVFf2uQoqPPIfg==", + "dependencies": { + "AWSSDK.Core": "[4.0.101, 5.0.0)" + } + }, + "AWSSDK.S3": { + "type": "Transitive", + "resolved": "4.0.102.1", + "contentHash": "zGP3FW4D0wfbmYxoOWgNoPcwoA8E+kWH6KezOBPiLpAOi3l4cygFF15jiWdma6R7D+M9qu3DzdpWq/VlyIJX+Q==", + "dependencies": { + "AWSSDK.Core": "[4.0.101, 5.0.0)" + } + }, + "AWSSDK.Signin": { + "type": "Transitive", + "resolved": "4.0.101.5", + "contentHash": "+XVnd+4efxEkdC74RTdHmxwJ2VhyPLT5TeGBoDTQ8LSQXYTfsN4Q2JWcS/ZAw151xjS5W89+i8FyvCM8Z7zGQg==", + "dependencies": { + "BouncyCastle.Cryptography": "2.6.2" + } + }, + "AWSSDK.SQS": { + "type": "Transitive", + "resolved": "4.0.100.8", + "contentHash": "6WZzJc+1T6VJuFDorQmGbhLabJhkFIp2zTK9y6fbGprCqUCtnzF/FufmpjKcAJZ17xxDpxpgo/wQ0leZHHTQ8g==", + "dependencies": { + "AWSSDK.Core": "[4.0.101, 5.0.0)" + } + }, + "Azure.Core": { + "type": "Transitive", + "resolved": "1.55.0", + "contentHash": "c7femvYS/xEUrLP1sNZN+DoQZmx3X+G3ZXtOOz0RL/7cGMCM3JVqepVmRd3AwM4IK8AtTGS4GOigCO+d/zaSsQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.3", + "Microsoft.Identity.Client": "4.83.1", + "Microsoft.Identity.Client.Extensions.Msal": "4.83.1", + "System.ClientModel": "1.11.0", + "System.Memory.Data": "10.0.3" + } + }, + "Azure.Storage.Blobs": { + "type": "Transitive", + "resolved": "12.29.1", + "contentHash": "pWh7xZBto8YcwiO853AB3uijTfVqs9PDte0Bu9/Zd8O9nwKiitWm0Jej1vsNkmYUzeG1552f8vJPjmtW30pBUQ==", + "dependencies": { + "Azure.Core": "1.55.0", + "Azure.Storage.Common": "12.28.0" + } + }, + "Azure.Storage.Common": { + "type": "Transitive", + "resolved": "12.28.0", + "contentHash": "5l8YhNrks38zKLGFW2BBFLwieyamH/xauABSASsdpZv79G0f+n2n22IjkRmUqCmALSupkwRHh2LOhbW3yj5Qgw==", + "dependencies": { + "Azure.Core": "1.55.0", + "System.IO.Hashing": "10.0.3" + } + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.6.2", + "contentHash": "7oWOcvnntmMKNzDLsdxAYqApt+AjpRpP2CShjMfIa3umZ42UQMvH0tl1qAliYPNYO6vTdcGMqnRrCPmsfzTI1w==" + }, + "Google.Api.CommonProtos": { + "type": "Transitive", + "resolved": "2.17.0", + "contentHash": "elfQPknFr495hm7vdy6ZlgyQh6yzZq9TU7sS35L/Fj/fqjM/mUGau9gVJLhvQEtUlPjtR80hpn/m9HvBMyCXIw==", + "dependencies": { + "Google.Protobuf": "[3.31.1, 4.0.0]" + } + }, + "Google.Api.Gax": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "ujDpZk7O2VqAEIqcSlhH0FKzVpGZWly/qcNjHxpNUrL445Tei9PHnu4V1pwW2WDiMDvo3TpL1Bi4DeU525Afrg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Api.Gax.Grpc": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "QgYF0bd8z6xWSVLGkGAMPB70seJSa9J02lFn8LT0Rb7PjSxZYpX+wU6LbVmghxDi6Wd64taulpX37pFq20J61g==", + "dependencies": { + "Google.Api.CommonProtos": "2.17.0", + "Google.Api.Gax": "4.13.1", + "Google.Apis.Auth": "1.73.0", + "Grpc.Auth": "[2.71.0, 3.0.0)", + "Grpc.Core.Api": "[2.71.0, 3.0.0)", + "Grpc.Net.Client": "[2.71.0, 3.0.0)" + } + }, + "Google.Api.Gax.Rest": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "wDfFZXKoHC6sy+Yfub/4cLQOb5ozBVPqDOrxCUAeTfku4d96ILqgzulJSdkPHbiT3oWB3hlYTdJgUwNSJjyo+A==", + "dependencies": { + "Google.Api.Gax": "4.13.1", + "Google.Apis.Auth": "1.73.0" + } + }, + "Google.Apis": { + "type": "Transitive", + "resolved": "1.75.0", + "contentHash": "ZqODi2IvyTBezeGztemXv6U/+VinyqxxPiyoW2CZbzIrUp+a35Rt5tzUjXHPXK9nA1YQi/w8ABpYQpBm31ditw==", + "dependencies": { + "Google.Apis.Core": "1.75.0" + } + }, + "Google.Apis.Auth": { + "type": "Transitive", + "resolved": "1.75.0", + "contentHash": "hzuGwUBIQYdFkChXm62E5Suxe+q5PHt2uE5EunGBco2j01uQJGlUgzNujZvGHMlAIEHaytzhdn3v3v52ZPgv2Q==", + "dependencies": { + "Google.Apis": "1.75.0", + "Google.Apis.Core": "1.75.0", + "System.Management": "7.0.2" + } + }, + "Google.Apis.Core": { + "type": "Transitive", + "resolved": "1.75.0", + "contentHash": "7AuI44XP4LzMFiOjdk4GCtCxJTIWZcjrXLeGjLYYSpTHHbiPkvm76XNym7zPOnD90sIg+zdTulg+I6D5W5spTQ==", + "dependencies": { + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Apis.Storage.v1": { + "type": "Transitive", + "resolved": "1.74.0.4115", + "contentHash": "9J3YPPSG53xwQLzdQ304dI+DMtVxJDpEazA2QQ2LA8IJoM4VUu4DpIxyhTGxpnAK4Db7s4LwzeNi6NkPKM3HzQ==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Auth": "1.74.0" + } + }, + "Google.Cloud.Firestore": { + "type": "Transitive", + "resolved": "4.4.0", + "contentHash": "A2gzk6VcPtew5mEiOlngNgqzdy3LEPyak8TQa8kvYNRu8x0bADFIbFNOT1s9DHdpplyI7FFW2WZjfXt3Olg9ww==", + "dependencies": { + "Google.Cloud.Firestore.V1": "4.4.0", + "Microsoft.Bcl.AsyncInterfaces": "10.0.10" + } + }, + "Google.Cloud.Firestore.V1": { + "type": "Transitive", + "resolved": "4.4.0", + "contentHash": "ZQpldY8YA5piE+d8tMbJWfLkMD5oTbJ9Mr4MWTtQP2ePg91UrqVEQ8f7BI/xYiporiJNamC9JsljgIVcuiZ0yw==", + "dependencies": { + "Google.Api.Gax.Grpc": "[4.13.1, 5.0.0)", + "Google.Cloud.Location": "[2.4.0, 3.0.0)", + "Google.LongRunning": "[3.5.0, 4.0.0)" + } + }, + "Google.Cloud.Iam.V1": { + "type": "Transitive", + "resolved": "3.5.0", + "contentHash": "CTLBAQ7WxsLoA4lovpvhF4MRUJH64smrIjuD/0jDrLs6zpbpf+f8Ur8ekRKv67kwlpN6Ug0q7EuT2csWDEczlQ==", + "dependencies": { + "Google.Api.Gax.Grpc": "[4.12.1, 5.0.0)" + } + }, + "Google.Cloud.Location": { + "type": "Transitive", + "resolved": "2.4.0", + "contentHash": "ZFChsp6OYVPBAzpoEWFMf/W9lfOgPj5BY+75/VjGV6EoirhA3YNlSKta3fX2MPaWH6RqJ9ZtNJT4vmdxhxbkbQ==", + "dependencies": { + "Google.Api.Gax.Grpc": "[4.12.1, 5.0.0)" + } + }, + "Google.Cloud.Storage.V1": { + "type": "Transitive", + "resolved": "4.15.0", + "contentHash": "TVlJsaIkMDwHbZPCcLFFKduFcFD0yJL1jsZmV/V/FMjv5iok/eoC0oy8/VHjxfmWmNGkJLPLMAe5MrFLtY7hzg==", + "dependencies": { + "Google.Api.Gax.Rest": "[4.13.1, 5.0.0)", + "Google.Apis.Storage.v1": "[1.74.0.4115, 2.0.0)" + } + }, + "Google.Cloud.Tasks.V2": { + "type": "Transitive", + "resolved": "3.6.0", + "contentHash": "EMmvqMDkivYjd+gVSoc4UgT8AsYdUZ2aDLoy7EjSNIySGuAefD79hNvD0kQ/LkaiMEn/+a4Lg90vikoPyR3nYw==", + "dependencies": { + "Google.Api.Gax.Grpc": "[4.12.1, 5.0.0)", + "Google.Cloud.Iam.V1": "[3.5.0, 4.0.0)", + "Google.Cloud.Location": "[2.4.0, 3.0.0)" + } + }, + "Google.LongRunning": { + "type": "Transitive", + "resolved": "3.5.0", + "contentHash": "W8xO6FA+rG8WjKOsyIjTKjeKLcyCrjBBYeEdZ4QBkKQcxmRczbrfKhKQmdorb2V35CqXeeTbue5Na6Zkgyv8ow==", + "dependencies": { + "Google.Api.Gax.Grpc": "[4.12.1, 5.0.0)" + } + }, + "Google.Protobuf": { + "type": "Transitive", + "resolved": "3.31.1", + "contentHash": "gSnJbUmGiOTdWddPhqzrEscHq9Ls6sqRDPB9WptckyjTUyx70JOOAaDLkFff8gManZNN3hllQ4aQInnQyq/Z/A==" + }, + "Grpc.Auth": { + "type": "Transitive", + "resolved": "2.71.0", + "contentHash": "t2aGh/pMgqmc3GimtYfC7VcgVY/VSbk6SLH+61wewsgK45tzxxD9nYYItT5bpLn7fbebirmHXfgJcVKIArd0cg==", + "dependencies": { + "Google.Apis.Auth": "1.69.0", + "Grpc.Core.Api": "2.71.0" + } + }, + "Grpc.Core.Api": { + "type": "Transitive", + "resolved": "2.71.0", + "contentHash": "QquqUC37yxsDzd1QaDRsH2+uuznWPTS8CVE2Yzwl3CvU4geTNkolQXoVN812M2IwT6zpv3jsZRc9ExJFNFslTg==" + }, + "Grpc.Net.Client": { + "type": "Transitive", + "resolved": "2.71.0", + "contentHash": "U1vr20r5ngoT9nlb7wejF28EKN+taMhJsV9XtK9MkiepTZwnKxxiarriiMfCHuDAfPUm9XUjFMn/RIuJ4YY61w==", + "dependencies": { + "Grpc.Net.Common": "2.71.0" + } + }, + "Grpc.Net.Common": { + "type": "Transitive", + "resolved": "2.71.0", + "contentHash": "v0c8R97TwRYwNXlC8GyRXwYTCNufpDfUtj9la+wUrZFzVWkFJuNAltU+c0yI3zu0jl54k7en6u2WKgZgd57r2Q==", + "dependencies": { + "Grpc.Core.Api": "2.71.0" + } + }, + "Microsoft.Azure.Cosmos": { + "type": "Transitive", + "resolved": "3.62.1", + "contentHash": "1jNOGwdtwHjk40zfxPv3dhwpqX7Ilex1PUuCPYDEMBAa3Y7HZS5GxoXbzGR+uxVJc4XKwFH9ZWkSLgswCgXlYQ==", + "dependencies": { + "Azure.Core": "1.44.1", + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Microsoft.Bcl.HashCode": "1.1.0", + "System.Configuration.ConfigurationManager": "6.0.0" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.10", + "contentHash": "TFI6OKYE1XZz4SGuTSH70c6SBdPpFktXsoa1gCxTr3mKrhmXirnvaS0tKz+J3ZWICEAmMpEGn59nO4ICtUpQXA==" + }, + "Microsoft.Bcl.HashCode": { + "type": "Transitive", + "resolved": "1.1.0", + "contentHash": "J2G1k+u5unBV+aYcwxo94ip16Rkp65pgWFb0R6zwJipzWNMgvqlWeuI7/+R+e8bob66LnSG+llLJ+z8wI94cHg==" + }, + "Microsoft.Data.Sqlite": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "7je7UELzm131GiLYc4PpZvfKXIgIyzPM+v+tjcd/nbnuWRfgcONYKzDTqJlURxwVCFsVnlpmq6y6yn4qvR8QXQ==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Data.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "hubA20AGenQ4Sx0ElWaPpB8DISjXpdx463+1zOGRslsT0e/t/06ITv+pHsop8CcJ0d8PZLfgnT7juCDVD79Dkw==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Extensions.AI.Abstractions": { + "type": "Transitive", + "resolved": "10.8.3", + "contentHash": "K0B05oApxmviWalNHPMBBcRC7erKiDATz3ENNR/jqTR9JwIwLRefgDhj2jCRwL1aca99pXUe0qyQC73/xIuZig==" + }, + "Microsoft.Identity.Client": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "jOLIrZ3cynoqHLLO1cXplFFabrhrMEYs/EuKHvmCyrOm1axqiVFT6nCSnHxk7w5+d2BeQfCdM12Yf/0X7OeS1g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.14.0" + } + }, + "Microsoft.Identity.Client.Extensions.Msal": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "I3k4J4Hj4KbLEFanjeUzzDOVecukETaTgEkJ7h2pP/Yazs6SLp6TVUTo/Eo+ptPXMwvc+iX7rBFtMSUrA7R+Mg==", + "dependencies": { + "Microsoft.Identity.Client": "4.83.1", + "System.Security.Cryptography.ProtectedData": "4.5.0" + } + }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "8.14.0", + "contentHash": "iwbCpSjD3ehfTwBhtSNEtKPK0ICun6ov7Ibx6ISNA9bfwIyzI2Siwyi9eJFCJBwxowK9xcA1mj+jBWiigeqgcQ==" + }, + "Microsoft.ML.OnnxRuntime": { + "type": "Transitive", + "resolved": "1.29.0", + "contentHash": "bUO3Ryc42UVkDGByBIYU9/ogRURGHeMOEq96LMWnDBVMQGZpIahBhPRCeu+Qq6+EJh/dfxzFJBKZa0/+yMmpWQ==", + "dependencies": { + "Microsoft.ML.OnnxRuntime.Managed": "1.29.0" + } + }, + "Microsoft.ML.OnnxRuntime.Managed": { + "type": "Transitive", + "resolved": "1.29.0", + "contentHash": "jLy5PtzvRQCBFOczzfRY7zpuMVUyLwjO0Jsdd8MBZdmnPIqx5vW9cYDNyCqkrYpwkof8GlqBVaEUMWrMWRtvuw==", + "dependencies": { + "System.Numerics.Tensors": "9.0.0" + } + }, + "Microsoft.Win32.SystemEvents": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "hqTM5628jSsQiv+HGpiq3WKBl2c8v1KZfby2J6Pr7pEPlK9waPdgEO6b8A/+/xn/yZ9ulv8HuqK71ONy2tg67A==" + }, + "ModelContextProtocol": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "4Pb9u02Nwsp0poueDsqNdyGRojFxOYpljB7zDBsq+aHL+Afou3OgxlBc3GWFVnsRMRJrUtWqDh3s6k2JgPzmrQ==", + "dependencies": { + "ModelContextProtocol.Core": "[2.2.0]" + } + }, + "ModelContextProtocol.AspNetCore": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "3JelDMuFIwFzXybsh6K30G6wXu5gmqKnuRnieqgBMuR0FkqqXRv4B+NYZuPgthSyNq5UwRyvv2U2VJBE3RD3PQ==", + "dependencies": { + "ModelContextProtocol": "[2.2.0]" + } + }, + "ModelContextProtocol.Core": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "FeBfXU6T8k+jw4afg4sfxdEX2rL/e5oKOk9ROOGztu9k47+7Bz08sdaToYt2XvMY1opNbwxYQOFMj6wH9TInhA==", + "dependencies": { + "Microsoft.Extensions.AI.Abstractions": "10.8.3" + } + }, + "ModelContextProtocol.Extensions.Tasks": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "cIGGEIL/KVbPBxflSF6TRKmjNTGk668e8x/R+Jx9l8VPHH70BisCmjdtHdkDHbbBr9gGMbjBjJdiNMJecweCGw==", + "dependencies": { + "ModelContextProtocol": "[2.2.0]" + } + }, + "MySqlConnector": { + "type": "Transitive", + "resolved": "2.6.2", + "contentHash": "dlTc/tsBa42szdvpj4nrHvQVUb0gsMwR9HT6J0Zz2OSqfP0CdYlGw+qQPqCpbnO97idfvclSjKYtM6y5QaBQaQ==" + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.4", + "contentHash": "pdgNNMai3zv51W5aq268sujXUyx7SNdE2bj1wZcWjAQrKMFZV260lbqYop1d2GM67JI1huLRwxo9ZqnfF/lC6A==" + }, + "Npgsql": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "7nb5YzXuvWWJxB0J8DiyL3we+X4FOctZrt0fIBnucOIaIevFEEwGQVZKtiu9olXdlNAK1eNgqSral6r/jlhI4w==" + }, + "Pgvector": { + "type": "Transitive", + "resolved": "0.3.2", + "contentHash": "n7M5LuNejHUmtWky3zCbNO+tP1Gnjiuv9Qtu4LyvB1602dD8RiBxxCQp9jEjM0ZFDxAZF1oOWkNIkXw46KT00Q==", + "dependencies": { + "Npgsql": "8.0.5" + } + }, + "SQLite": { + "type": "Transitive", + "resolved": "3.53.4", + "contentHash": "KN7jeWqgUPeBRe1FlcpZURzxomuKKEKHmBBQfg+Nx7NkY1LjKhzHvH+3ASkNvhayESE34nMBinL9CV21JfPRJw==" + }, + "SQLitePCLRaw.bundle_e_sqlite3": { + "type": "Transitive", + "resolved": "3.0.5", + "contentHash": "SW8iASIyWMrLzqabUHYQRvALhvD4ylSBsj4PgEVGwc36kQjc9xT5kSV/XQ9rU7nIpBWD+LPyX3Hlw5FzyUzGeQ==", + "dependencies": { + "SQLite": "3.53.4", + "SQLitePCLRaw.config.e_sqlite3": "3.0.5" + } + }, + "SQLitePCLRaw.config.e_sqlite3": { + "type": "Transitive", + "resolved": "3.0.5", + "contentHash": "aSk8WE5tF2MybESMgtZAEyMVCAWA0nBqOMc48HFoa9UoSdtm3goDXVzNRnefeKIwE6bV9NaNXptn1F9ReMQI0Q==", + "dependencies": { + "SQLitePCLRaw.provider.e_sqlite3": "3.0.5" + } + }, + "SQLitePCLRaw.core": { + "type": "Transitive", + "resolved": "3.0.5", + "contentHash": "k81AYXXRCw3Zj8rOhyBoCsx/U97KYDFI2CSKr/ijl5BwpsW/hX/4kBiDmerFaoust8nxBwa0IHQFw8MmSHRtnQ==" + }, + "SQLitePCLRaw.provider.e_sqlite3": { + "type": "Transitive", + "resolved": "3.0.5", + "contentHash": "um8YSWduhhuskTG2bHfZrBqMNQOydfU8pcseT+cu5RivOGyoUbCrGXxgoRNTmRYw2VbMWnEZVVFcneZT/5dsBg==", + "dependencies": { + "SQLitePCLRaw.core": "3.0.5" + } + }, + "System.ClientModel": { + "type": "Transitive", + "resolved": "1.11.0", + "contentHash": "1Wl32zh7TbvN+HAO8NqDuaN0Ao2Qu/0j0NJSrXGDtpUQsTXQYJ3C6hd9/Ds2IrgP4agMQYFoXB35GZfC21ByHw==", + "dependencies": { + "System.Memory.Data": "10.0.3" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "7.0.0", + "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" + }, + "System.Configuration.ConfigurationManager": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "7T+m0kDSlIPTHIkPMIu6m6tV6qsMqJpvQWW2jIc2qi7sn40qxFo0q+7mEQAhMPXZHMKnWrnv47ntGlM/ejvw3g==", + "dependencies": { + "System.Security.Cryptography.ProtectedData": "6.0.0", + "System.Security.Permissions": "6.0.0" + } + }, + "System.Drawing.Common": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "NfuoKUiP2nUWwKZN6twGqXioIe1zVD0RIj2t976A+czLHr2nY454RwwXs6JU9Htc6mwqL6Dn/nEL3dpVf2jOhg==", + "dependencies": { + "Microsoft.Win32.SystemEvents": "6.0.0" + } + }, + "System.IO.Hashing": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "La6ICwsdTKhVX+LKN+pvFjQRR3LhLwq3uKdi2knjLzRyPYBSydF4cjXidYxIiTcDD6XVYdsBWQEI8ZxiZ/OdIg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "7.0.2", + "contentHash": "/qEUN91mP/MUQmJnM5y5BdT7ZoPuVrtxnFlbJ8a3kBJGhe2wCzBfnPFtK2wTtEEcf3DMGR9J00GZZfg6HRI6yA==", + "dependencies": { + "System.CodeDom": "7.0.0" + } + }, + "System.Memory.Data": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "MaGhRfGunmrj/nHjtsi9XkhlYJ/ERGWrbA+BiSKNtGnAjc9XlG5EhAvak6VRcX5LYzPF6pBO8nJ613dTgzabig==" + }, + "System.Numerics.Tensors": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "4jNNt67NhCqf3bf2FN0mrsC+EH60L7Sny6poqVeiviB27Kw7TQzMmgn8HIaPc8E9EcuGusUyfeu21gLfzcBpDA==" + }, + "System.Security.Cryptography.ProtectedData": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "rp1gMNEZpvx9vP0JW0oHLxlf8oSiQgtno77Y4PLUBjSiDYoD77Y8uXHr1Ea5XG4/pIKhqAdxZ8v8OTUtqo9PeQ==" + }, + "System.Security.Permissions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "T/uuc7AklkDoxmcJ7LGkyX1CcSviZuLCa4jg3PekfJ7SU0niF0IVTXwUiNVP9DSpzou2PpxJ+eNY2IfDM90ZCg==", + "dependencies": { + "System.Windows.Extensions": "6.0.0" + } + }, + "System.Windows.Extensions": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "IXoJOXIqc39AIe+CIR7koBtRGMiCt/LPM3lI+PELtDIy9XdyeSrwXFdWV9dzJ2Awl0paLWUaknLxFQ5HpHZUog==", + "dependencies": { + "System.Drawing.Common": "6.0.0" + } + }, + "vyral.abstractions": { + "type": "Project", + "dependencies": { + "Vyral.Primitives": "[0.2.0, )" + } + }, + "vyral.aws": { + "type": "Project", + "dependencies": { + "AWSSDK.DynamoDBv2": "[4.0.103.1, )", + "AWSSDK.S3": "[4.0.102.1, )", + "AWSSDK.Signin": "[4.0.101.5, )", + "System.Numerics.Tensors": "[10.0.11, )", + "Vyral.Abstractions": "[0.3.0, )" + } + }, + "vyral.azure": { + "type": "Project", + "dependencies": { + "Azure.Storage.Blobs": "[12.29.1, )", + "Microsoft.Azure.Cosmos": "[3.62.1, )", + "Newtonsoft.Json": "[13.0.4, )", + "Vyral.Abstractions": "[0.3.0, )", + "Vyral.Execution.AzureDurable": "[0.2.0, )" + } + }, + "vyral.cloudflare": { + "type": "Project", + "dependencies": { + "AWSSDK.S3": "[4.0.102.1, )", + "Vyral.Abstractions": "[0.3.0, )", + "Vyral.Aws": "[0.3.0, )" + } + }, + "vyral.embeddings.onnx": { + "type": "Project", + "dependencies": { + "Microsoft.ML.OnnxRuntime": "[1.29.0, )", + "System.Numerics.Tensors": "[10.0.11, )", + "Vyral.Abstractions": "[0.3.0, )" + } + }, + "vyral.execution": { + "type": "Project", + "dependencies": { + "Vyral.Primitives": "[0.2.0, )" + } + }, + "vyral.execution.aws": { + "type": "Project", + "dependencies": { + "AWSSDK.DynamoDBv2": "[4.0.103.1, )", + "AWSSDK.SQS": "[4.0.100.8, )", + "Vyral.Abstractions": "[0.3.0, )", + "Vyral.Execution": "[0.2.0, )" + } + }, + "vyral.execution.azuredurable": { + "type": "Project", + "dependencies": { + "Vyral.Execution": "[0.2.0, )" + } + }, + "vyral.execution.local": { + "type": "Project", + "dependencies": { + "Microsoft.Data.Sqlite": "[10.0.11, )", + "SQLitePCLRaw.bundle_e_sqlite3": "[3.0.5, )", + "Vyral.Execution": "[0.2.0, )", + "Vyral.Primitives": "[0.2.0, )" + } + }, + "vyral.execution.workerclient": { + "type": "Project", + "dependencies": { + "Vyral.Execution": "[0.2.0, )" + } + }, + "vyral.google": { + "type": "Project", + "dependencies": { + "Google.Apis.Auth": "[1.75.0, )", + "Google.Apis.Core": "[1.75.0, )", + "Google.Cloud.Firestore": "[4.4.0, )", + "Google.Cloud.Storage.V1": "[4.15.0, )", + "Google.Cloud.Tasks.V2": "[3.6.0, )", + "Npgsql": "[10.0.3, )", + "Vyral.Abstractions": "[0.3.0, )", + "Vyral.Execution": "[0.2.0, )", + "Vyral.Pgvector": "[0.3.0, )" + } + }, + "vyral.local": { + "type": "Project", + "dependencies": { + "Microsoft.Data.Sqlite": "[10.0.11, )", + "SQLitePCLRaw.bundle_e_sqlite3": "[3.0.5, )", + "SQLitePCLRaw.core": "[3.0.5, )", + "System.Numerics.Tensors": "[10.0.11, )", + "Vyral.Abstractions": "[0.3.0, )" + } + }, + "vyral.mcp": { + "type": "Project", + "dependencies": { + "ModelContextProtocol.AspNetCore": "[2.2.0, )", + "ModelContextProtocol.Extensions.Tasks": "[2.2.0, )", + "Vyral.Abstractions": "[0.3.0, )", + "Vyral.Execution": "[0.2.0, )", + "Vyral.Primitives": "[0.2.0, )", + "Vyral.Providers.Abstractions": "[0.3.0, )" + } + }, + "vyral.mysql": { + "type": "Project", + "dependencies": { + "MySqlConnector": "[2.6.2, )", + "Vyral.Abstractions": "[0.3.0, )" + } + }, + "vyral.pgvector": { + "type": "Project", + "dependencies": { + "Npgsql": "[10.0.3, )", + "Pgvector": "[0.3.2, )", + "Vyral.Abstractions": "[0.3.0, )" + } + }, + "vyral.primitives": { + "type": "Project" + }, + "vyral.providers.abstractions": { + "type": "Project", + "dependencies": { + "Vyral.Primitives": "[0.2.0, )" + } + }, + "vyral.providers.cli": { + "type": "Project", + "dependencies": { + "Vyral.Providers.Abstractions": "[0.3.0, )" + } + }, + "vyral.providers.jules": { + "type": "Project", + "dependencies": { + "Vyral.Providers.Abstractions": "[0.3.0, )" + } + }, + "vyral.providers.local": { + "type": "Project", + "dependencies": { + "Vyral.Providers.Abstractions": "[0.3.0, )" + } + }, + "vyral.providers.onnx": { + "type": "Project", + "dependencies": { + "Vyral.Embeddings.Onnx": "[0.3.0, )", + "Vyral.Providers.Abstractions": "[0.3.0, )" + } + }, + "vyral.server": { + "type": "Project", + "dependencies": { + "AWSSDK.DynamoDBv2": "[4.0.103.1, )", + "AWSSDK.SQS": "[4.0.100.8, )", + "Google.Apis.Auth": "[1.75.0, )", + "Vyral.Abstractions": "[0.3.0, )", + "Vyral.Azure": "[0.3.0, )", + "Vyral.Cloudflare": "[0.3.0, )", + "Vyral.Embeddings.Onnx": "[0.3.0, )", + "Vyral.Execution": "[0.2.0, )", + "Vyral.Execution.Aws": "[0.2.0, )", + "Vyral.Execution.Local": "[0.2.0, )", + "Vyral.Google": "[0.3.0, )", + "Vyral.Local": "[0.3.0, )", + "Vyral.Mcp": "[0.3.0, )", + "Vyral.MySql": "[0.3.0, )", + "Vyral.Pgvector": "[0.3.0, )", + "Vyral.Providers.Abstractions": "[0.3.0, )", + "Vyral.Providers.Cli": "[0.3.0, )", + "Vyral.Providers.Jules": "[0.3.0, )", + "Vyral.Providers.Local": "[0.3.0, )", + "Vyral.Providers.Onnx": "[0.3.0, )" + } + } + } + } +} \ No newline at end of file diff --git a/src/Vyral.Server/ArtifactRecordIngestionHostedPlugin.cs b/src/Vyral.Server/ArtifactRecordIngestionHostedPlugin.cs new file mode 100644 index 0000000..150df54 --- /dev/null +++ b/src/Vyral.Server/ArtifactRecordIngestionHostedPlugin.cs @@ -0,0 +1,105 @@ +using System.Text.Json; +using Vyral.Abstractions.Interfaces; +using Vyral.Abstractions.Models; +using Vyral.Execution; + +namespace Vyral.Server; + +/// +/// Consumer-neutral implementation of the durable artifact/record publish workflow. +/// The same plugin may run in-process or through a Vyral hosted external worker. +/// +public sealed class ArtifactRecordIngestionHostedPlugin : IExecutionPlugin +{ + public const string PluginId = "vyral.artifacts"; + public const string HandlerId = "vyral.artifacts.record-ingest"; + public const string DefaultStagingContainer = "vyral-admission-staging"; + + private readonly IReadOnlyList _handlers; + + public ArtifactRecordIngestionHostedPlugin( + IObjectStore objects, + ArtifactRecordIngestionService ingestion) + { + _handlers = [new ArtifactRecordIngestionHandler(objects, ingestion)]; + } + + public ExecutionPluginDescriptor Descriptor { get; } = new() + { + PluginId = PluginId, + Name = "Vyral artifact/record ingestion", + Version = "1.0.0", + Handlers = { CreateHandlerDescriptor() } + }; + + public IReadOnlyList Handlers => _handlers; + + public static ExecutionHandlerDescriptor CreateHandlerDescriptor() => new() + { + HandlerId = HandlerId, + PluginId = PluginId, + DisplayName = "Ingest an artifact and its record", + Description = "Completes a staged cross-store artifact/record ingestion.", + MaxAttempts = 3, + ConcurrencyKey = HandlerId + }; + + private sealed class ArtifactRecordIngestionHandler : IExecutionHandler + { + private readonly IObjectStore _objects; + private readonly ArtifactRecordIngestionService _ingestion; + + public ArtifactRecordIngestionHandler(IObjectStore objects, ArtifactRecordIngestionService ingestion) + { + _objects = objects; + _ingestion = ingestion; + } + + public ExecutionHandlerDescriptor Descriptor { get; } = CreateHandlerDescriptor(); + + public async Task ExecuteAsync( + IExecutionRunContext context, + CancellationToken ct = default) + { + var payload = context.Run.Payload?.Deserialize(ExecutionJson.Options) + ?? throw new InvalidOperationException("Artifact record ingestion payload is required."); + var staged = await _objects.GetObjectAsync(new ObjectReadRequest + { + Container = payload.StagingContainer, + Key = payload.StagingKey + }, ct) ?? throw new InvalidOperationException("Staged artifact content is missing."); + + ArtifactRecordIngestReceipt receipt; + await using (staged.Content) + { + if (!string.Equals(staged.ContentHash, payload.ContentHash, StringComparison.Ordinal)) + throw new InvalidOperationException("Staged artifact content hash does not match its admission payload."); + receipt = await _ingestion.IngestAsync(payload.Manifest, staged.Content, ct); + } + + try + { + await _objects.DeleteObjectAsync(new ObjectDeleteRequest + { + Container = payload.StagingContainer, + Key = payload.StagingKey, + IfMatch = staged.Etag + }, ct); + } + catch (InvalidOperationException) + { + // Successful published work is authoritative; staging cleanup is best-effort. + } + + return ExecutionRunResult.Succeeded(JsonSerializer.SerializeToNode(receipt, ExecutionJson.Options)); + } + } +} + +internal sealed class ArtifactRecordIngestionPayload +{ + public ArtifactRecordIngestManifest Manifest { get; set; } = new(); + public string StagingContainer { get; set; } = string.Empty; + public string StagingKey { get; set; } = string.Empty; + public string ContentHash { get; set; } = string.Empty; +} diff --git a/src/Vyral.Server/ArtifactRecordIngestionService.cs b/src/Vyral.Server/ArtifactRecordIngestionService.cs index 47817d2..f8af755 100644 --- a/src/Vyral.Server/ArtifactRecordIngestionService.cs +++ b/src/Vyral.Server/ArtifactRecordIngestionService.cs @@ -158,6 +158,11 @@ internal static string ComputeSha256(byte[] bytes) => public sealed class ArtifactRecordIngestionOptions { public long MaxArtifactBytes { get; init; } = 16 * 1024 * 1024; + /// + /// Private, non-published object container used between request-bound admission and durable + /// completion. Hosted API and worker deployments must use the same value. + /// + public string StagingContainer { get; init; } = ArtifactRecordIngestionHostedPlugin.DefaultStagingContainer; public ExternalContextVerificationOptions ExternalContext { get; init; } = new(); public static ArtifactRecordIngestionOptions FromConfiguration(IConfiguration configuration) @@ -166,9 +171,15 @@ public static ArtifactRecordIngestionOptions FromConfiguration(IConfiguration co configuration["Ingest:MaxArtifactBytes"], configuration["VYRAL_INGEST_MAX_ARTIFACT_BYTES"], 16 * 1024 * 1024); + var stagingContainer = FirstNonEmpty( + configuration["Ingest:StagingContainer"], + configuration["VYRAL_INGEST_STAGING_CONTAINER"], + ArtifactRecordIngestionHostedPlugin.DefaultStagingContainer)!; + ObjectNameValidator.ValidateContainer(stagingContainer); return new ArtifactRecordIngestionOptions { MaxArtifactBytes = max, + StagingContainer = stagingContainer, ExternalContext = ExternalContextVerificationOptions.FromConfiguration(configuration) }; } @@ -184,6 +195,9 @@ private static long FirstPositiveLong(string? first, string? second, long fallba } return fallback; } + + private static string? FirstNonEmpty(params string?[] values) => + values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value))?.Trim(); } /// diff --git a/src/Vyral.Server/ExecutionRuntimeArtifactRecordIngestionAdapter.cs b/src/Vyral.Server/ExecutionRuntimeArtifactRecordIngestionAdapter.cs index 566cce3..42e12af 100644 --- a/src/Vyral.Server/ExecutionRuntimeArtifactRecordIngestionAdapter.cs +++ b/src/Vyral.Server/ExecutionRuntimeArtifactRecordIngestionAdapter.cs @@ -14,9 +14,9 @@ namespace Vyral.Server; /// public sealed class ExecutionRuntimeArtifactRecordIngestionAdapter { - public const string PluginId = "vyral.artifacts"; - public const string HandlerId = "vyral.artifacts.record-ingest"; - private const string StagingContainer = "vyral-admission-staging"; + public const string PluginId = ArtifactRecordIngestionHostedPlugin.PluginId; + public const string HandlerId = ArtifactRecordIngestionHostedPlugin.HandlerId; + public const string StagingContainer = ArtifactRecordIngestionHostedPlugin.DefaultStagingContainer; private readonly IExecutionRuntime _runtime; private readonly IObjectStore _objects; @@ -26,12 +26,16 @@ public ExecutionRuntimeArtifactRecordIngestionAdapter( IExecutionRuntime runtime, IObjectStore objects, ArtifactRecordIngestionService ingestion, - ArtifactRecordIngestionOptions options) + ArtifactRecordIngestionOptions options, + bool registerInProcessHandler = true) { _runtime = runtime; _objects = objects; _options = options; - _runtime.RegisterPlugin(new ArtifactRecordIngestionPlugin(objects, ingestion)); + if (registerInProcessHandler) + { + _runtime.RegisterPlugin(new ArtifactRecordIngestionHostedPlugin(objects, ingestion)); + } } public async Task StartAsync( @@ -62,9 +66,8 @@ public async Task StartAsync( Payload = JsonSerializer.SerializeToNode(new ArtifactRecordIngestionPayload { Manifest = manifest, - StagingContainer = StagingContainer, + StagingContainer = _options.StagingContainer, StagingKey = stagingKey, - StagingEtag = staged.Info.Etag, ContentHash = contentHash }, ExecutionJson.Options), IdempotencyKey = idempotencyKey, @@ -106,7 +109,7 @@ private async Task PutStagingObjectAsync( await using var content = new MemoryStream(bytes, writable: false); var info = await _objects.PutObjectAsync(new ObjectWriteRequest { - Container = StagingContainer, + Container = _options.StagingContainer, Key = key, Content = content, ContentType = "application/octet-stream", @@ -124,7 +127,7 @@ private async Task PutStagingObjectAsync( { var existing = await _objects.GetObjectAsync(new ObjectReadRequest { - Container = StagingContainer, + Container = _options.StagingContainer, Key = key }, ct); if (existing is null || @@ -159,7 +162,7 @@ private async Task TryDeleteStagingObjectAsync(string key, string? etag, Cancell { await _objects.DeleteObjectAsync(new ObjectDeleteRequest { - Container = StagingContainer, + Container = _options.StagingContainer, Key = key, IfMatch = etag }, ct); @@ -179,107 +182,4 @@ private static string BuildStagingKey(string? idempotencyKey) } private sealed record StagingWrite(ObjectInfo Info, bool Created); - - private sealed class ArtifactRecordIngestionPlugin : IExecutionPlugin - { - private readonly IReadOnlyList _handlers; - - public ArtifactRecordIngestionPlugin( - IObjectStore objects, - ArtifactRecordIngestionService ingestion) - { - _handlers = new[] { new ArtifactRecordIngestionHandler(objects, ingestion) }; - } - - public ExecutionPluginDescriptor Descriptor { get; } = new() - { - PluginId = PluginId, - Name = "Vyral artifact/record ingestion", - Version = "1.0.0", - Handlers = - { - new ExecutionHandlerDescriptor - { - HandlerId = HandlerId, - PluginId = PluginId, - DisplayName = "Ingest an artifact and its record", - Description = "Completes a staged cross-store artifact/record ingestion.", - MaxAttempts = 3, - ConcurrencyKey = "vyral.artifacts.record-ingest" - } - } - }; - - public IReadOnlyList Handlers => _handlers; - } - - private sealed class ArtifactRecordIngestionHandler : IExecutionHandler - { - private readonly IObjectStore _objects; - private readonly ArtifactRecordIngestionService _ingestion; - - public ArtifactRecordIngestionHandler( - IObjectStore objects, - ArtifactRecordIngestionService ingestion) - { - _objects = objects; - _ingestion = ingestion; - } - - public ExecutionHandlerDescriptor Descriptor { get; } = new() - { - HandlerId = HandlerId, - PluginId = PluginId, - DisplayName = "Ingest an artifact and its record", - MaxAttempts = 3, - ConcurrencyKey = "vyral.artifacts.record-ingest" - }; - - public async Task ExecuteAsync( - IExecutionRunContext context, - CancellationToken ct = default) - { - var payload = context.Run.Payload?.Deserialize(ExecutionJson.Options) - ?? throw new InvalidOperationException("Artifact record ingestion payload is required."); - var staged = await _objects.GetObjectAsync(new ObjectReadRequest - { - Container = payload.StagingContainer, - Key = payload.StagingKey - }, ct) ?? throw new InvalidOperationException("Staged artifact content is missing."); - - ArtifactRecordIngestReceipt receipt; - await using (staged.Content) - { - if (!string.Equals(staged.ContentHash, payload.ContentHash, StringComparison.Ordinal)) - throw new InvalidOperationException("Staged artifact content hash does not match its admission payload."); - receipt = await _ingestion.IngestAsync(payload.Manifest, staged.Content, ct); - } - - try - { - await _objects.DeleteObjectAsync(new ObjectDeleteRequest - { - Container = payload.StagingContainer, - Key = payload.StagingKey, - IfMatch = staged.Etag - }, ct); - } - catch (InvalidOperationException) - { - // Successful published work is authoritative; staging cleanup is best-effort. - } - - return ExecutionRunResult.Succeeded( - JsonSerializer.SerializeToNode(receipt, ExecutionJson.Options)); - } - } - - private sealed class ArtifactRecordIngestionPayload - { - public ArtifactRecordIngestManifest Manifest { get; set; } = new(); - public string StagingContainer { get; set; } = string.Empty; - public string StagingKey { get; set; } = string.Empty; - public string? StagingEtag { get; set; } - public string ContentHash { get; set; } = string.Empty; - } } diff --git a/src/Vyral.Server/Program.cs b/src/Vyral.Server/Program.cs index 4a1648a..2af6b73 100644 --- a/src/Vyral.Server/Program.cs +++ b/src/Vyral.Server/Program.cs @@ -50,15 +50,15 @@ LogStartupPhaseCompleted("embedding.provider", startupPhase, $"provider={embeddingProvider.ProviderId}; model={embeddingProvider.ModelId}; dimensions={embeddingProvider.Dimensions}"); startupPhase = LogStartupPhaseStarting("record_store", $"backend={storageOptions.RecordStore}"); -var store = await CreateRecordStoreAsync(storageOptions); +var store = await ServerStorageFactory.CreateRecordStoreAsync(storageOptions); LogStartupPhaseCompleted("record_store", startupPhase, $"backend={storageOptions.RecordStore}; store={store.GetType().Name}"); startupPhase = LogStartupPhaseStarting("trace_store", $"backend={storageOptions.TraceStore}"); -var traceStore = await CreateTraceStoreAsync(storageOptions); +var traceStore = await ServerStorageFactory.CreateTraceStoreAsync(storageOptions); LogStartupPhaseCompleted("trace_store", startupPhase, $"backend={storageOptions.TraceStore}; store={traceStore.GetType().Name}"); startupPhase = LogStartupPhaseStarting("object_store", $"backend={storageOptions.ObjectStore}"); -var objectStore = CreateObjectStore(storageOptions); +var objectStore = ServerStorageFactory.CreateObjectStore(storageOptions); LogStartupPhaseCompleted("object_store", startupPhase, $"backend={storageOptions.ObjectStore}; store={objectStore.GetType().Name}"); startupPhase = LogStartupPhaseStarting("canonical_store"); @@ -98,6 +98,18 @@ var executionRuntime = CreateExecutionRuntime(builder.Configuration, storageOptions, dbPath, objectStore); var externalExecutionRuntime = executionRuntime as IExternalExecutionWorkerRuntime; var configuredExternalHandlers = GetExternalExecutionHandlers(builder.Configuration); +var artifactIngestionIsExternal = configuredExternalHandlers.Any(handler => + string.Equals(handler.HandlerId, ExecutionRuntimeArtifactRecordIngestionAdapter.HandlerId, StringComparison.Ordinal)); +foreach (var handler in configuredExternalHandlers.Where(handler => + string.Equals(handler.HandlerId, ExecutionRuntimeArtifactRecordIngestionAdapter.HandlerId, StringComparison.Ordinal))) +{ + if (!string.IsNullOrWhiteSpace(handler.PluginId) && + !string.Equals(handler.PluginId, ExecutionRuntimeArtifactRecordIngestionAdapter.PluginId, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"External handler '{handler.HandlerId}' must use plugin '{ExecutionRuntimeArtifactRecordIngestionAdapter.PluginId}'."); + } +} if (externalExecutionRuntime is null && configuredExternalHandlers.Count > 0) { throw new InvalidOperationException("ExecutionRuntime:ExternalHandlers requires an adapter that implements IExternalExecutionWorkerRuntime."); @@ -193,7 +205,13 @@ builder.Services.AddSingleton(localExecutionRuntime); } builder.Services.AddSingleton(embeddingJobAdapter); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(services => + new ExecutionRuntimeArtifactRecordIngestionAdapter( + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService(), + services.GetRequiredService(), + registerInProcessHandler: !artifactIngestionIsExternal)); builder.Services.AddSingleton(); builder.Services.AddSingleton(services => new ExecutionRuntimeRagIngestionJobAdapter( @@ -2530,68 +2548,6 @@ static List GetTraceList(IReadOnlyDictionary values, st return new List { value.ToString()! }; } -static async Task CreateRecordStoreAsync(ServerStorageOptions options) -{ - switch (options.RecordStore) - { - case ServerStorageBackendIds.Sqlite: - { - var store = new SqliteRecordCollectionStore(options.DatabasePath); - await store.InitializeAsync(); - return store; - } - case ServerStorageBackendIds.GoogleFirestore: - return new FirestoreRecordCollectionStore(CreateFirestoreDb(options), options.GoogleFirestoreRootCollection); - case ServerStorageBackendIds.GoogleAlloyDb: - { - if (string.IsNullOrWhiteSpace(options.GoogleAlloyDbConnectionString)) - { - throw new InvalidOperationException("Google AlloyDB record store requires Google:AlloyDb:ConnectionString or VYRAL_ALLOYDB_CONNECTION_STRING."); - } - - var store = new AlloyDbRecordCollectionStore(options.GoogleAlloyDbConnectionString); - await store.InitializeAsync(); - return store; - } - default: - throw new InvalidOperationException($"Record store backend '{options.RecordStore}' is not supported by this server."); - } -} - -static async Task CreateTraceStoreAsync(ServerStorageOptions options) -{ - switch (options.TraceStore) - { - case ServerStorageBackendIds.Sqlite: - { - var traceStore = new SqliteTraceStore(options.DatabasePath); - await traceStore.InitializeAsync(); - return traceStore; - } - case ServerStorageBackendIds.GoogleFirestore: - return new FirestoreTraceStore(CreateFirestoreDb(options), options.GoogleFirestoreRootCollection); - default: - throw new InvalidOperationException($"Trace store backend '{options.TraceStore}' is not supported by this server."); - } -} - -static IObjectStore CreateObjectStore(ServerStorageOptions options) -{ - return options.ObjectStore switch - { - ServerStorageBackendIds.File => new FileObjectStore(options.ObjectsPath), - ServerStorageBackendIds.GoogleCloudStorage => new CloudStorageObjectStore(StorageClient.Create()), - ServerStorageBackendIds.CloudflareR2 => R2ObjectStore.Create(new CloudflareR2Options - { - AccountId = options.CloudflareAccountId, - AccessKeyId = options.CloudflareR2AccessKeyId, - SecretAccessKey = options.CloudflareR2SecretAccessKey, - ServiceUrl = options.CloudflareR2ServiceUrl - }), - _ => throw new InvalidOperationException($"Object store backend '{options.ObjectStore}' is not supported by this server.") - }; -} - static ICanonicalStore CreateCanonicalStore(CanonicalStoreOptions options) { return options.Provider switch @@ -2645,25 +2601,6 @@ static void ValidateCanonicalEvidenceBriefDocuments(CanonicalTransactionRequest } } -static FirestoreDb CreateFirestoreDb(ServerStorageOptions options) -{ - if (string.IsNullOrWhiteSpace(options.GoogleProjectId)) - { - throw new InvalidOperationException("Google Firestore store requires Google:ProjectId, Google:Firestore:ProjectId, GOOGLE_CLOUD_PROJECT, or VYRAL_GCP_PROJECT_ID."); - } - - var builder = new FirestoreDbBuilder - { - ProjectId = options.GoogleProjectId - }; - if (!string.IsNullOrWhiteSpace(options.GoogleFirestoreDatabaseId)) - { - builder.DatabaseId = options.GoogleFirestoreDatabaseId; - } - - return builder.Build(); -} - static ServerHealthStatus BuildServerHealth( IRecordCollectionStore recordStore, IObjectStore objectStore, @@ -3798,7 +3735,7 @@ static GoogleCloudExecutionRuntimeAdapter CreateGoogleExecutionRuntime( dispatchOptions.Validate(); var workerDispatchers = GetGoogleExecutionWorkerDispatchers(configuration, projectId, dispatchOptions); return new GoogleCloudExecutionRuntimeAdapter( - new FirestoreExecutionStateStore(CreateFirestoreDb(firestoreOptions), firestoreRoot!), + new FirestoreExecutionStateStore(ServerStorageFactory.CreateFirestoreDb(firestoreOptions), firestoreRoot!), new GoogleCloudExecutionDispatcher(new CloudTasksHttpJsonQueue(CloudTasksClient.Create()), dispatchOptions), new GoogleCloudExecutionRuntimeOptions { diff --git a/src/Vyral.Server/ServerStorageFactory.cs b/src/Vyral.Server/ServerStorageFactory.cs new file mode 100644 index 0000000..9c179f2 --- /dev/null +++ b/src/Vyral.Server/ServerStorageFactory.cs @@ -0,0 +1,90 @@ +using Google.Cloud.Firestore; +using Google.Cloud.Storage.V1; +using Vyral.Abstractions.Interfaces; +using Vyral.Cloudflare; +using Vyral.Google; +using Vyral.Local; + +namespace Vyral.Server; + +/// +/// Builds the storage adapters used by the server and by Vyral-owned hosted workers. +/// Hosts share portable storage behavior but receive separate deployment identities. +/// +public static class ServerStorageFactory +{ + public static async Task CreateRecordStoreAsync(ServerStorageOptions options) + { + switch (options.RecordStore) + { + case ServerStorageBackendIds.Sqlite: + { + var store = new SqliteRecordCollectionStore(options.DatabasePath); + await store.InitializeAsync(); + return store; + } + case ServerStorageBackendIds.GoogleFirestore: + return new FirestoreRecordCollectionStore(CreateFirestoreDb(options), options.GoogleFirestoreRootCollection); + case ServerStorageBackendIds.GoogleAlloyDb: + { + if (string.IsNullOrWhiteSpace(options.GoogleAlloyDbConnectionString)) + { + throw new InvalidOperationException("Google AlloyDB record store requires Google:AlloyDb:ConnectionString or VYRAL_ALLOYDB_CONNECTION_STRING."); + } + + var store = new AlloyDbRecordCollectionStore(options.GoogleAlloyDbConnectionString); + await store.InitializeAsync(); + return store; + } + default: + throw new InvalidOperationException($"Record store backend '{options.RecordStore}' is not supported by this host."); + } + } + + public static async Task CreateTraceStoreAsync(ServerStorageOptions options) + { + switch (options.TraceStore) + { + case ServerStorageBackendIds.Sqlite: + { + var store = new SqliteTraceStore(options.DatabasePath); + await store.InitializeAsync(); + return store; + } + case ServerStorageBackendIds.GoogleFirestore: + return new FirestoreTraceStore(CreateFirestoreDb(options), options.GoogleFirestoreRootCollection); + default: + throw new InvalidOperationException($"Trace store backend '{options.TraceStore}' is not supported by this host."); + } + } + + public static IObjectStore CreateObjectStore(ServerStorageOptions options) => options.ObjectStore switch + { + ServerStorageBackendIds.File => new FileObjectStore(options.ObjectsPath), + ServerStorageBackendIds.GoogleCloudStorage => new CloudStorageObjectStore(StorageClient.Create()), + ServerStorageBackendIds.CloudflareR2 => R2ObjectStore.Create(new CloudflareR2Options + { + AccountId = options.CloudflareAccountId, + AccessKeyId = options.CloudflareR2AccessKeyId, + SecretAccessKey = options.CloudflareR2SecretAccessKey, + ServiceUrl = options.CloudflareR2ServiceUrl + }), + _ => throw new InvalidOperationException($"Object store backend '{options.ObjectStore}' is not supported by this host.") + }; + + public static FirestoreDb CreateFirestoreDb(ServerStorageOptions options) + { + if (string.IsNullOrWhiteSpace(options.GoogleProjectId)) + { + throw new InvalidOperationException("Google Firestore store requires Google:ProjectId, Google:Firestore:ProjectId, GOOGLE_CLOUD_PROJECT, or VYRAL_GCP_PROJECT_ID."); + } + + var builder = new FirestoreDbBuilder { ProjectId = options.GoogleProjectId }; + if (!string.IsNullOrWhiteSpace(options.GoogleFirestoreDatabaseId)) + { + builder.DatabaseId = options.GoogleFirestoreDatabaseId; + } + + return builder.Build(); + } +} diff --git a/src/Vyral.Server/VyralExecutionAccess.cs b/src/Vyral.Server/VyralExecutionAccess.cs index 05ed091..98d3056 100644 --- a/src/Vyral.Server/VyralExecutionAccess.cs +++ b/src/Vyral.Server/VyralExecutionAccess.cs @@ -338,7 +338,17 @@ public sealed class GoogleExecutionTokenValidator : IGoogleExecutionTokenValidat public async Task ValidateAsync(string token, IReadOnlySet allowedAudiences, CancellationToken ct = default) { ct.ThrowIfCancellationRequested(); - var payload = await GoogleJsonWebSignature.ValidateAsync(token, new GoogleJsonWebSignature.ValidationSettings { Audience = allowedAudiences }); + GoogleJsonWebSignature.Payload payload; + try + { + payload = await GoogleJsonWebSignature.ValidateAsync( + token, + new GoogleJsonWebSignature.ValidationSettings { Audience = allowedAudiences }); + } + catch (InvalidJwtException) + { + throw new ExecutionAccessDeniedException("Google OIDC identity token is invalid."); + } if (!payload.EmailVerified || string.IsNullOrWhiteSpace(payload.Email)) throw new ExecutionAccessDeniedException("Google OIDC execution identity must contain a verified email."); return payload.Email.Trim(); } diff --git a/tests/Vyral.Tests.ExecutionWorkerClient/ExecutionWorkerClientTests.cs b/tests/Vyral.Tests.ExecutionWorkerClient/ExecutionWorkerClientTests.cs index 9666b82..a606da6 100644 --- a/tests/Vyral.Tests.ExecutionWorkerClient/ExecutionWorkerClientTests.cs +++ b/tests/Vyral.Tests.ExecutionWorkerClient/ExecutionWorkerClientTests.cs @@ -11,12 +11,16 @@ public sealed class ExecutionWorkerClientTests [Fact] public async Task Client_UsesEveryWorkerRouteWithoutLeakingLeaseOrBearerTokens() { - var requests = new List<(string Path, string Authorization, string Body)>(); + var requests = new List<(string Path, string Authorization, string ApiKey, string Body)>(); var telemetry = new List(); using var client = new HttpClient(new DelegateHandler(async request => { var body = request.Content is null ? string.Empty : await request.Content.ReadAsStringAsync(); - requests.Add((request.RequestUri!.AbsolutePath, request.Headers.Authorization?.ToString() ?? string.Empty, body)); + requests.Add(( + request.RequestUri!.AbsolutePath, + request.Headers.Authorization?.ToString() ?? string.Empty, + request.Headers.GetValues("X-Vyral-Api-Key").SingleOrDefault() ?? string.Empty, + body)); return request.RequestUri.AbsolutePath switch { "/execution/workers/leases" => Json(HttpStatusCode.OK, """{"leaseKey":"lease-a","leaseToken":"lease-secret","workerId":"worker-a","run":{"id":"run-a","handlerId":"handler-a","attempt":1,"status":"running"}}"""), @@ -37,6 +41,7 @@ public async Task Client_UsesEveryWorkerRouteWithoutLeakingLeaseOrBearerTokens() WorkerId = "worker-a", HandlerIds = ["handler-a"], TokenSource = new DelegateExecutionWorkerTokenSource(_ => Task.FromResult("identity-secret")), + ApiKey = "api-key-secret", Observe = telemetry.Add }); @@ -54,6 +59,7 @@ public async Task Client_UsesEveryWorkerRouteWithoutLeakingLeaseOrBearerTokens() Assert.Equal(9, requests.Count); Assert.All(requests, request => Assert.Equal("Bearer identity-secret", request.Authorization)); + Assert.All(requests, request => Assert.Equal("api-key-secret", request.ApiKey)); Assert.DoesNotContain("lease-secret", requests[0].Body, StringComparison.Ordinal); Assert.All(requests.Skip(1), request => Assert.Contains("lease-secret", request.Body, StringComparison.Ordinal)); Assert.All(telemetry, item => diff --git a/tests/Vyral.Tests.Google/GoogleCloudExecutionDispatchOptionsTests.cs b/tests/Vyral.Tests.Google/GoogleCloudExecutionDispatchOptionsTests.cs index 2194500..1b573ac 100644 --- a/tests/Vyral.Tests.Google/GoogleCloudExecutionDispatchOptionsTests.cs +++ b/tests/Vyral.Tests.Google/GoogleCloudExecutionDispatchOptionsTests.cs @@ -2,6 +2,22 @@ namespace Vyral.Tests.Google; public sealed class GoogleCloudExecutionDispatchOptionsTests { + [Fact] + public void DispatchMessage_UsesThePortableCamelCaseJsonContract() + { + const string json = """ + {"runId":"run-1","reason":"run_ready","scheduledAtUtc":null} + """; + + var message = System.Text.Json.JsonSerializer.Deserialize( + json, + Vyral.Execution.ExecutionJson.Options); + + Assert.NotNull(message); + Assert.Equal("run-1", message!.RunId); + Assert.Equal("run_ready", message.Reason); + } + [Fact] public void Validate_RequiresGoogleQueueAndCloudRunTarget() { diff --git a/tests/Vyral.Tests.Local/ExecutionRuntimeFactoryTests.cs b/tests/Vyral.Tests.Local/ExecutionRuntimeFactoryTests.cs index aa181a2..dc14bbe 100644 --- a/tests/Vyral.Tests.Local/ExecutionRuntimeFactoryTests.cs +++ b/tests/Vyral.Tests.Local/ExecutionRuntimeFactoryTests.cs @@ -13,6 +13,19 @@ namespace Vyral.Tests.Local; public sealed class ExecutionRuntimeFactoryTests { + [Fact] + public async Task GoogleExecutionTokenValidator_MapsMalformedTokensToAccessDenied() + { + var validator = new GoogleExecutionTokenValidator(); + + var error = await Assert.ThrowsAsync(() => + validator.ValidateAsync( + "not-a-google-identity-token", + new HashSet(StringComparer.Ordinal) { "https://vyral.example.test" })); + + Assert.Equal("Google OIDC identity token is invalid.", error.Message); + } + [Fact] public async Task Server_CanComposeConfiguredProviderFactoryWithoutChangingRuntimeSwitch() { diff --git a/tests/Vyral.Tests.Local/ExecutionRuntimeTests.cs b/tests/Vyral.Tests.Local/ExecutionRuntimeTests.cs index caff307..ff856be 100644 --- a/tests/Vyral.Tests.Local/ExecutionRuntimeTests.cs +++ b/tests/Vyral.Tests.Local/ExecutionRuntimeTests.cs @@ -5,6 +5,7 @@ using Vyral.Abstractions.Models; using Vyral.Execution; using Vyral.Execution.Local; +using Vyral.Execution.WorkerClient; using Vyral.Local; using Vyral.Primitives; using Vyral.Server; @@ -1260,6 +1261,77 @@ await runtime.GetHistoryAsync(accepted.Id), item => item.Type == ExecutionEventTypes.RetryScheduled); } + [Fact] + public async Task ArtifactRecordIngestion_HostedWorkerCompletesGenericExternalHandlerWithoutConsumerImplementation() + { + var runtime = CreateRuntime(); + var objects = new FileObjectStore(Path.Combine( + Path.GetTempPath(), + $"vyral-hosted-artifact-worker-{Guid.NewGuid():N}")); + var records = new SqliteRecordCollectionStore(Path.Combine( + Path.GetTempPath(), + $"vyral-hosted-artifact-records-{Guid.NewGuid():N}.sqlite")); + await records.InitializeAsync(); + await records.CreateCollectionAsync(new RecordCollectionPolicy { Name = "receipts" }); + var options = new ArtifactRecordIngestionOptions { StagingContainer = "vyral-system-objects" }; + var ingestion = new ArtifactRecordIngestionService(records, objects, options); + runtime.RegisterExternalHandler(ArtifactRecordIngestionHostedPlugin.CreateHandlerDescriptor()); + var admission = new ExecutionRuntimeArtifactRecordIngestionAdapter( + runtime, + objects, + ingestion, + options, + registerInProcessHandler: false); + var worker = new ExecutionPluginWorker( + new InProcessExecutionWorkerTransport( + runtime, + "vyral-hosted-worker", + [ArtifactRecordIngestionHostedPlugin.HandlerId]), + [new ArtifactRecordIngestionHostedPlugin(objects, ingestion)], + new ExecutionPluginWorkerOptions { HeartbeatInterval = Timeout.InfiniteTimeSpan }); + var manifest = new ArtifactRecordIngestManifest + { + Collection = "receipts", + Record = new VyralRecord + { + Id = "hosted-receipt-1", + PartitionKey = "tenant-a", + Type = "test.receipt" + }, + Artifact = new ArtifactRecordDescriptor + { + Container = "published", + Key = "receipts/hosted-receipt-1.json", + ContentType = "application/json" + } + }; + + await using var content = new MemoryStream("{\"hosted\":true}"u8.ToArray(), writable: false); + var accepted = await admission.StartAsync(manifest, content, "hosted-artifact-1"); + + Assert.Equal(ExecutionRunStatuses.Queued, accepted.Status); + Assert.False(accepted.Payload!.AsObject().ContainsKey("stagingEtag")); + var completed = await worker.RunOnceAsync(accepted.Id); + Assert.NotNull(completed); + Assert.Equal(ExecutionRunStatuses.Succeeded, completed!.Status); + Assert.NotNull(completed.Result); + Assert.True(completed.Result!["accepted"]!.GetValue()); + Assert.Null(await worker.RunOnceAsync(accepted.Id)); // Duplicate task delivery is harmless. + + var staged = await objects.GetObjectAsync(new ObjectReadRequest + { + Container = options.StagingContainer, + Key = "record-artifact/" + Convert.ToHexString(System.Security.Cryptography.SHA256.HashData("hosted-artifact-1"u8.ToArray())).ToLowerInvariant() + ".bin" + }); + Assert.Null(staged); + + await using var replayContent = new MemoryStream("{\"hosted\":true}"u8.ToArray(), writable: false); + var replayed = await admission.StartAsync(manifest, replayContent, "hosted-artifact-1"); + Assert.Equal(accepted.Id, replayed.Id); + Assert.True(replayed.AdmissionReplayed); + Assert.Equal(ExecutionRunStatuses.Succeeded, replayed.Status); + } + private static LocalExecutionRuntime CreateRuntime(ExecutionRuntimeLimits? limits = null) { var path = Path.Combine(Path.GetTempPath(), $"vyral-execution-{Guid.NewGuid():N}.sqlite"); diff --git a/tests/Vyral.Tests.Local/Vyral.Tests.Local.csproj b/tests/Vyral.Tests.Local/Vyral.Tests.Local.csproj index ae1505b..7d867ce 100644 --- a/tests/Vyral.Tests.Local/Vyral.Tests.Local.csproj +++ b/tests/Vyral.Tests.Local/Vyral.Tests.Local.csproj @@ -29,6 +29,7 @@ + From cd5e59ebf0859da8928e5c5d9d710bf3ad6ad333 Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Wed, 19 Aug 2026 00:28:15 -0400 Subject: [PATCH 2/3] Stabilize terminal trace conformance timing --- .../ExecutionRuntimeConformanceTests.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs b/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs index f640cb2..98b27c9 100644 --- a/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs +++ b/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs @@ -868,7 +868,19 @@ protected static async Task WaitForRunAsync(IExecutionRuntime runt run = await runtime.GetRunAsync(id); if (run?.Status == status) { - return run; + if (!ExecutionRunStatuses.IsTerminal(status)) + { + return run; + } + + var terminalEventType = status == ExecutionRunStatuses.Failed + ? ExecutionEventTypes.RunFailed + : ExecutionEventTypes.RunCompleted; + var observedHistory = await runtime.GetHistoryAsync(id); + if (observedHistory.Any(item => item.Type == terminalEventType)) + { + return run; + } } await Task.Delay(50); From 48e57161a77ca3827626e6dc328aead05fdd476c Mon Sep 17 00:00:00 2001 From: jeremydixon22 Date: Wed, 19 Aug 2026 00:36:44 -0400 Subject: [PATCH 3/3] Scope terminal trace waits to asserted evidence --- .../ExecutionRuntimeConformanceTests.cs | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs b/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs index 98b27c9..f8f2968 100644 --- a/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs +++ b/tests/Vyral.Tests.Conformance/ExecutionRuntimeConformanceTests.cs @@ -98,7 +98,7 @@ protected async Task RunExecutionRuntime_RunsPluginRecordsProgressArtifactsHisto Assert.Equal(1, completed.Progress); Assert.Equal(6, completed.Result!["total"]!.GetValue()); - var history = await runtime.GetHistoryAsync(first.Id); + var history = await WaitForHistoryEventAsync(runtime, first.Id, ExecutionEventTypes.RunCompleted); Assert.Contains(history, item => item.Type == ExecutionEventTypes.RunStarted); Assert.Contains(history, item => item.Type == ExecutionEventTypes.RunStatus); Assert.Contains(history, item => item.Type == ExecutionEventTypes.ArtifactWritten); @@ -269,7 +269,7 @@ protected async Task RunExecutionRuntime_ExposesConsumerErrorSemantics() var failed = await WaitForRunAsync(runtime, thrown.Id, ExecutionRunStatuses.Failed); Assert.Equal(ExecutionFailureClasses.Unknown, failed.FailureClass); Assert.Contains("consumer-visible failure", failed.Error); - var failedHistory = await runtime.GetHistoryAsync(failed.Id); + var failedHistory = await WaitForHistoryEventAsync(runtime, failed.Id, ExecutionEventTypes.RunFailed); Assert.Contains(failedHistory, item => item.Type == ExecutionEventTypes.RunFailed); runtime.RegisterHandler(new AlwaysFailHandler()); @@ -287,7 +287,7 @@ protected async Task RunExecutionRuntime_ExposesConsumerErrorSemantics() var exhausted = await WaitForRunAsync(runtime, retrying.Id, ExecutionRunStatuses.Failed); Assert.Equal(2, exhausted.Attempt); Assert.Equal(ExecutionFailureClasses.Transient, exhausted.FailureClass); - var exhaustedHistory = await runtime.GetHistoryAsync(exhausted.Id); + var exhaustedHistory = await WaitForHistoryEventAsync(runtime, exhausted.Id, ExecutionEventTypes.RunFailed); Assert.Single(exhaustedHistory, item => item.Type == ExecutionEventTypes.RetryScheduled); Assert.Contains(exhaustedHistory, item => item.Type == ExecutionEventTypes.RunFailed); @@ -406,7 +406,7 @@ protected async Task RunExecutionRuntime_StopsRetryingAfterMaxAttempts() Assert.Equal(2, handler.Attempts); Assert.Equal(ExecutionFailureClasses.Transient, failed.FailureClass); - var history = await runtime.GetHistoryAsync(failed.Id); + var history = await WaitForHistoryEventAsync(runtime, failed.Id, ExecutionEventTypes.RunFailed); Assert.Single(history, item => item.Type == ExecutionEventTypes.RetryScheduled); Assert.Contains(history, item => item.Type == ExecutionEventTypes.RunFailed); } @@ -868,19 +868,7 @@ protected static async Task WaitForRunAsync(IExecutionRuntime runt run = await runtime.GetRunAsync(id); if (run?.Status == status) { - if (!ExecutionRunStatuses.IsTerminal(status)) - { - return run; - } - - var terminalEventType = status == ExecutionRunStatuses.Failed - ? ExecutionEventTypes.RunFailed - : ExecutionEventTypes.RunCompleted; - var observedHistory = await runtime.GetHistoryAsync(id); - if (observedHistory.Any(item => item.Type == terminalEventType)) - { - return run; - } + return run; } await Task.Delay(50); @@ -897,6 +885,28 @@ protected static async Task WaitForRunAsync(IExecutionRuntime runt $"recent history: {recent}"); } + private static async Task> WaitForHistoryEventAsync( + IExecutionRuntime runtime, + string runId, + string eventType) + { + IReadOnlyList history = []; + for (var i = 0; i < 400; i++) + { + history = await runtime.GetHistoryAsync(runId); + if (history.Any(item => item.Type == eventType)) + { + return history; + } + + await Task.Delay(50); + } + + var observed = string.Join(", ", history.Select(item => item.Type).Distinct(StringComparer.Ordinal)); + throw new InvalidOperationException( + $"Run {runId} did not expose history event {eventType}. Observed: {observed}"); + } + private static void AssertRunAttempt(ExecutionRun run, int expected) { Assert.True(