Add hosted execution worker

Dockerfile

@@ -3,9 +3,15 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0@sha256:e1fc6e423f543119c406d24e2e687d67c5
 WORKDIR /src
 COPY . .
 RUN dotnet restore src/Vyral.Server/Vyral.Server.csproj --locked-mode --disable-parallel \
+    && dotnet restore src/Vyral.HostedWorker/Vyral.HostedWorker.csproj --locked-mode --disable-parallel \
     && dotnet publish src/Vyral.Server/Vyral.Server.csproj \
     -c Release \
-    -o /app/publish \
+    -o /app/publish/server \
+    --no-restore \
+    /p:UseAppHost=false \
+    && dotnet publish src/Vyral.HostedWorker/Vyral.HostedWorker.csproj \
+    -c Release \
+    -o /app/publish/worker \
     --no-restore \
     /p:UseAppHost=false \
     && mkdir -p /app/publish/.vyral
@@ -31,4 +37,6 @@ ENV ASPNETCORE_URLS=http://0.0.0.0:8080 \
 COPY --from=build --chown=1654:1654 /app/publish .
 USER 1654
 EXPOSE 8080
-ENTRYPOINT ["dotnet", "Vyral.Server.dll"]
+# The default is the public API server. Deploy the same pinned image as the
+# least-privilege generic worker with: dotnet worker/Vyral.HostedWorker.dll
+ENTRYPOINT ["dotnet", "server/Vyral.Server.dll"]

README.md

@@ -101,6 +101,11 @@ Vyral is most useful when an application needs to cross one or more provider
 seams without giving those providers ownership of its data and execution
 semantics.
 
+Vyral grows by portable capability, not by reproducing provider APIs. A shared
+contract earns its place when independent provider shapes—or an established
+application model—show a durable common core; provider-native behavior remains
+explicit at the edge.
+
 Vyral does not assume vector search is always the answer. For current,
 structured sources that an application can safely access, direct source search
 is often the simplest first path. Indexed lexical, vector, hybrid, and reranked
@@ -353,6 +358,11 @@ Vyral uses two canonical mutation shapes:
 - receipt-bound asynchronous operations durably admit work and return a run or
   job receipt whose status can be queried independently.
 
+Vyral-owned generic handlers can also run through a separately deployed,
+same-version hosted worker (preview), so consumers retain the public admission
+contract without reimplementing Vyral storage or lease behavior. The initial
+deployment shape is documented for [Google Cloud Run](deploy/README.md#vyral-hosted-generic-handlers).
+
 The local SQLite runtime is the reference implementation. Azure Durable,
 AWS, Google Cloud Tasks, and Temporal adapters implement different subsets and
 carry separate qualification evidence. External workers use leases,
@@ -392,7 +402,7 @@ Adapter authors should begin with the
 ## Repository map
 
 ```text
-src/           .NET contracts, runtimes, server, and provider adapters
+src/           .NET contracts, runtimes, API/hosted-worker services, and provider adapters
 clients/       Python and JavaScript HTTP SDKs; Go external-worker client
 runtimes/      Peer runtime implementations, currently Python
 contracts/     OpenAPI-derived public SDK catalog and JSON schemas

Vyral.sln

@@ -17,6 +17,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.Local", "src\Vyral.Lo
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.Server", "src\Vyral.Server\Vyral.Server.csproj", "{8EC7614A-6D41-4752-BC17-6858811C2B48}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.HostedWorker", "src\Vyral.HostedWorker\Vyral.HostedWorker.csproj", "{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{AAF8E491-F193-4C07-8669-9F3F823ED378}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Vyral.Tests.Azure", "tests\Vyral.Tests.Azure\Vyral.Tests.Azure.csproj", "{425659EC-721E-463D-85E4-764273695F21}"
@@ -183,6 +185,18 @@ Global
 		{8EC7614A-6D41-4752-BC17-6858811C2B48}.Release|x64.Build.0 = Release|Any CPU
 		{8EC7614A-6D41-4752-BC17-6858811C2B48}.Release|x86.ActiveCfg = Release|Any CPU
 		{8EC7614A-6D41-4752-BC17-6858811C2B48}.Release|x86.Build.0 = Release|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x64.Build.0 = Debug|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Debug|x86.Build.0 = Debug|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x64.ActiveCfg = Release|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x64.Build.0 = Release|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x86.ActiveCfg = Release|Any CPU
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490}.Release|x86.Build.0 = Release|Any CPU
 		{425659EC-721E-463D-85E4-764273695F21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{425659EC-721E-463D-85E4-764273695F21}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{425659EC-721E-463D-85E4-764273695F21}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -662,6 +676,7 @@ Global
 		{2D62965C-C36B-4EB8-BFE9-20EB6B23AEF5} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44}
 		{C9131552-2AC8-44F6-BE8D-17F223D0F5E1} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44}
 		{8EC7614A-6D41-4752-BC17-6858811C2B48} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44}
+		{A55D6B1E-6A81-4FCF-AEEA-7C319D6C0490} = {4BA473AE-AAF5-467B-9C6F-98151D2C2D44}
 		{425659EC-721E-463D-85E4-764273695F21} = {AAF8E491-F193-4C07-8669-9F3F823ED378}
 		{23B12913-7D75-46A6-A36B-F3755A1102BD} = {AAF8E491-F193-4C07-8669-9F3F823ED378}
 		{F248B3EE-1C18-48CF-89C4-72D3F93F2C57} = {AAF8E491-F193-4C07-8669-9F3F823ED378}

deploy/README.md

@@ -196,6 +196,43 @@ email. It rejects unscoped starts, worker-id impersonation, un-routed handlers,
 and maintenance calls without a dedicated policy. Keep test-only public worker
 fixtures out of the shared project.
 
+### Vyral-hosted generic handlers
+
+Consumer-neutral Vyral handlers may use the same external-worker protocol as a
+consumer worker. The initial handler, `vyral.artifacts.record-ingest`, preserves
+the public `POST /ingest/record-artifact` admission contract while a separately
+deployed Vyral worker performs the staged-object read, object publish, record
+upsert, and best-effort staging cleanup.
+
+Set `VYRAL_INGEST_STAGING_CONTAINER` identically on the API and worker. With
+GCS it is normally the Vyral artifact bucket, using a private
+`record-artifact/` object prefix; it is not a product-owned bucket. A manifest's
+published artifact container must likewise be one the Vyral worker is allowed to
+write.
+
+Deploy the same pinned Vyral image as a separate Cloud Run service with command
+`dotnet` and argument `worker/Vyral.HostedWorker.dll`. Configure the API route
+and external handler descriptor as shown in
+[`google-cloud-run.env.example`](google-cloud-run.env.example), and configure
+the worker from [`google-hosted-worker.env.example`](google-hosted-worker.env.example).
+The worker must have a distinct service account. The Cloud Tasks dispatch
+service account gets `run.invoker` only on the worker. The worker gets
+`run.invoker` only on the Vyral API, plus Firestore and object permissions for
+the Vyral storage plane; it does not need Cloud Tasks enqueue, execution-state,
+or consumer-deployment permissions. If the API retains API-key defense in
+depth, mount only that Vyral API-key secret into the worker. Each product policy
+that allows artifact admission must also allow `vyral.artifacts.record-ingest`
+and the hosted worker id. This keeps the generic worker from crossing a product
+scope merely because it can claim the handler.
+
+The queue message is only a run id and dispatch reason. The worker validates the
+Cloud Tasks OIDC callback, leases the run through Vyral, and returns success only
+after duplicate-safe lease completion. The original `202 Accepted` response and
+its `Location` execution-run receipt remain the consumer's status surface; no
+consumer implementation is required to complete the generic work. Do not route a
+generic Vyral handler to a consumer worker or grant consumers access to the
+Vyral admission-staging prefix.
+
 ### Execution deployment preflight
 
 Run the read-only preflight after deploying candidate Vyral and worker Cloud Run services, but
@@ -208,9 +245,9 @@ deletes a resource.
 ```bash
 VYRAL_EXECUTION_PROJECT_ID=your-gcp-project-id \
 VYRAL_EXECUTION_SERVER_SERVICE=vyral-server \
-VYRAL_EXECUTION_WORKER_SERVICE=product-example-worker \
-VYRAL_EXECUTION_WORKER_ID=product-example-worker \
-VYRAL_EXECUTION_HANDLER_IDS=product.example.job \
+VYRAL_EXECUTION_WORKER_SERVICE=vyral-hosted-worker \
+VYRAL_EXECUTION_WORKER_ID=vyral-hosted-artifact-worker \
+VYRAL_EXECUTION_HANDLER_IDS=vyral.artifacts.record-ingest \
 VYRAL_EXECUTION_CONFIG_FILE=deploy/google-cloud-run.env \
 deploy/preflight-google-execution.sh
 ```

deploy/google-cloud-run.env.example

@@ -19,6 +19,9 @@ VYRAL_TRACE_STORE=google-firestore
 VYRAL_OBJECT_STORE=google-cloud-storage
 VYRAL_GCS_BUCKET=your-gcp-project-id-vyral-artifacts
 VYRAL_OBJECT_PROBE_CONTAINER=your-gcp-project-id-vyral-artifacts
+# Both the API and the hosted worker use this private staging prefix in the shared Vyral bucket.
+# It is not a consumer artifact container or a second bucket to grant to product services.
+VYRAL_INGEST_STAGING_CONTAINER=your-gcp-project-id-vyral-artifacts
 VYRAL_FIRESTORE_ROOT_COLLECTION=vyral
 
 # Optional durable execution plane. Leave this unset to retain the default local SQLite
@@ -46,11 +49,23 @@ ExecutionRuntime__MaxRetainedTerminalRuns=500
 ExecutionRuntime__ExternalHandlers__0__HandlerId=product.example.job
 ExecutionRuntime__ExternalHandlers__0__PluginId=product.example
 ExecutionRuntime__ExternalHandlers__0__DisplayName=Product example job
+# Vyral-owned generic handlers are external only when an explicitly configured Vyral hosted
+# worker is deployed. Do not give consumers staging-object or generic storage access.
+ExecutionRuntime__Google__WorkerRoutes__1__HandlerId=vyral.artifacts.record-ingest
+ExecutionRuntime__Google__WorkerRoutes__1__WorkerUrl=https://replace-with-vyral-hosted-worker/tasks/execution
+ExecutionRuntime__Google__WorkerRoutes__1__OidcAudience=https://replace-with-vyral-hosted-worker
+ExecutionRuntime__ExternalHandlers__1__HandlerId=vyral.artifacts.record-ingest
+ExecutionRuntime__ExternalHandlers__1__PluginId=vyral.artifacts
+ExecutionRuntime__ExternalHandlers__1__DisplayName=Vyral artifact and record ingest
 # Product policy is the durable scope boundary that binds the verified worker identity below.
 ExecutionRuntime__ProductPolicies__0__ProductId=product-example
 ExecutionRuntime__ProductPolicies__0__AllowedTenantIds__0=tenant-example
 ExecutionRuntime__ProductPolicies__0__AllowedHandlerIds__0=product.example.job
 ExecutionRuntime__ProductPolicies__0__AllowedServiceIdentities__0=product-example-worker
+# A product that uses the public artifact admission route grants the generic handler and hosted
+# Vyral worker explicitly. Do not infer either permission from generic storage access.
+ExecutionRuntime__ProductPolicies__0__AllowedHandlerIds__1=vyral.artifacts.record-ingest
+ExecutionRuntime__ProductPolicies__0__AllowedServiceIdentities__1=vyral-hosted-artifact-worker
 
 # Shared execution requires a verified Cloud Run OIDC workload identity for every execution API
 # request. Bind each service account to exactly the product, tenants, handlers, and operations it
@@ -70,6 +85,14 @@ Server__ExecutionAccess__IdentityPolicies__1__WorkerId=product-example-worker
 Server__ExecutionAccess__IdentityPolicies__1__AllowedTenantIds__0=tenant-example
 Server__ExecutionAccess__IdentityPolicies__1__AllowedHandlerIds__0=product.example.job
 Server__ExecutionAccess__IdentityPolicies__1__AllowedOperations__0=worker
+# The generic Vyral worker is independently authenticated. Repeat this policy for every product
+# scope it may process, with that product's allowed tenants.
+Server__ExecutionAccess__IdentityPolicies__2__Principal=vyral-hosted-worker@your-gcp-project-id.iam.gserviceaccount.com
+Server__ExecutionAccess__IdentityPolicies__2__ProductId=product-example
+Server__ExecutionAccess__IdentityPolicies__2__WorkerId=vyral-hosted-artifact-worker
+Server__ExecutionAccess__IdentityPolicies__2__AllowedTenantIds__0=tenant-example
+Server__ExecutionAccess__IdentityPolicies__2__AllowedHandlerIds__0=vyral.artifacts.record-ingest
+Server__ExecutionAccess__IdentityPolicies__2__AllowedOperations__0=worker
 
 VYRAL_API_KEY_HEADER=X-Vyral-Api-Key
 

deploy/google-hosted-worker.env.example

@@ -0,0 +1,37 @@
+# Vyral hosted generic worker for Cloud Tasks -> Cloud Run. Run the same pinned Vyral image with
+# command `dotnet` and argument `worker/Vyral.HostedWorker.dll`.
+#
+# This worker owns only Vyral's generic staged artifact/record publish behavior. Consumer code,
+# collection policy, provenance, encryption, and domain retry choices remain outside this service.
+
+ASPNETCORE_ENVIRONMENT=Production
+GOOGLE_CLOUD_PROJECT=your-gcp-project-id
+VYRAL_GCP_PROJECT_ID=your-gcp-project-id
+
+# Match the Vyral API's generic storage plane. Grant this worker's service account only the
+# Firestore record and GCS object permissions required by this generic handler.
+VYRAL_RECORD_STORE=google-firestore
+VYRAL_OBJECT_STORE=google-cloud-storage
+VYRAL_GCS_BUCKET=your-gcp-project-id-vyral-artifacts
+VYRAL_FIRESTORE_ROOT_COLLECTION=vyral
+# Must exactly match the Vyral API's private generic staging container.
+VYRAL_INGEST_STAGING_CONTAINER=your-gcp-project-id-vyral-artifacts
+
+# The Vyral API worker protocol endpoint and this worker's stable identity. Both values must
+# match an API ExecutionAccess worker policy. Every product policy that allows artifact admission
+# must also allow this handler and worker id for its own tenant scope.
+HostedWorker__VyralUrl=https://replace-with-vyral-server
+HostedWorker__WorkerId=vyral-hosted-artifact-worker
+HostedWorker__HandlerIds__0=vyral.artifacts.record-ingest
+HostedWorker__LeaseTtlSeconds=60
+HostedWorker__HeartbeatSeconds=20
+# The worker always presents its Cloud Run identity to Vyral. If the Vyral API also requires an
+# API key, mount one Vyral API-key secret into this exact value; do not place its value in this file.
+# HostedWorker__ApiKey=mounted-secret-value
+# HostedWorker__ApiKeyHeader=X-Vyral-Api-Key
+
+# Cloud Tasks calls the worker with an OIDC token minted for the Cloud Run URL. Cloud Run IAM must
+# also grant that task service account run.invoker on this worker service.
+HostedWorker__TaskAuthentication__Mode=google-oidc
+HostedWorker__TaskAuthentication__AllowedAudiences__0=https://replace-with-vyral-hosted-worker
+HostedWorker__TaskAuthentication__AllowedPrincipals__0=vyral-cloud-tasks@your-gcp-project-id.iam.gserviceaccount.com

deploy/preflight-google-execution.sh

@@ -300,6 +300,7 @@ DEFAULT_OIDC_AUDIENCE="$(config_value "ExecutionRuntime__Google__OidcAudience")"
 ARTIFACT_OBJECT_CONTAINER="$(config_value "ExecutionRuntime__Google__ArtifactObjectContainer")"
 OBJECT_STORE="$(config_value "VYRAL_OBJECT_STORE")"
 GCS_BUCKET="$(config_value "VYRAL_GCS_BUCKET")"
+INGEST_STAGING_CONTAINER="$(config_value "VYRAL_INGEST_STAGING_CONTAINER")"
 AUTH_MODE="$(config_value "Server__ExecutionAccess__AuthenticationMode")"
 RECORD_ROOT="$(config_value "VYRAL_FIRESTORE_ROOT_COLLECTION")"
 RUNTIME_ADAPTER="$(config_value "ExecutionRuntime__Adapter")"
@@ -367,6 +368,15 @@ fi
 if has_run_invoker "$SERVER_SERVICE" "$WORKER_SERVICE_ACCOUNT"; then pass "worker service account can invoke Vyral server"; else fail "worker service account lacks roles/run.invoker on Vyral server"; fi
 if has_project_role "$SERVER_SERVICE_ACCOUNT" "roles/datastore.user"; then pass "Vyral runtime can use Firestore"; else fail "Vyral runtime lacks roles/datastore.user"; fi
 if has_project_role "$SERVER_SERVICE_ACCOUNT" "roles/cloudtasks.enqueuer"; then pass "Vyral runtime can enqueue Cloud Tasks"; else fail "Vyral runtime lacks roles/cloudtasks.enqueuer"; fi
+for handler_id in "${HANDLER_IDS[@]}"; do
+  if [[ "$handler_id" != "vyral.artifacts.record-ingest" ]]; then
+    continue
+  fi
+  config_value_is_real "VYRAL_INGEST_STAGING_CONTAINER" "$INGEST_STAGING_CONTAINER" || true
+  if [[ "$INGEST_STAGING_CONTAINER" == "$GCS_BUCKET" ]]; then pass "generic ingestion staging uses the Vyral object bucket"; else fail "VYRAL_INGEST_STAGING_CONTAINER must equal VYRAL_GCS_BUCKET for least-privilege hosted ingestion"; fi
+  if has_project_role "$WORKER_SERVICE_ACCOUNT" "roles/datastore.user"; then pass "Vyral hosted artifact worker can use Firestore records"; else fail "Vyral hosted artifact worker lacks roles/datastore.user"; fi
+  if [[ -n "$GCS_BUCKET" ]] && has_bucket_object_access "$GCS_BUCKET" "$WORKER_SERVICE_ACCOUNT"; then pass "Vyral hosted artifact worker can read and write generic objects"; else fail "Vyral hosted artifact worker lacks object access on $GCS_BUCKET"; fi
+done
 PROJECT_NUMBER="$(gcloud projects describe "$PROJECT_ID" --format='value(projectNumber)' 2>/dev/null || true)"
 TASKS_AGENT="service-${PROJECT_NUMBER}@gcp-sa-cloudtasks.iam.gserviceaccount.com"
 for tasks_service_account in $(printf '%s\n' "${ROUTE_TASKS_SERVICE_ACCOUNTS[@]}" | sort -u); do

docs/guides/consumer-handoff.md

@@ -87,9 +87,13 @@ Consumers decide whether and how to derive pseudonymous identifiers. Vyral
 stores only the metadata and content that their manifest supplies.
 
 For Cloud Run, keep specialized work request-bound and let Vyral own generic
-durable acceptance of the supplied artifact and record. A producer may select
-inline or queue-backed processing without changing the contract. Its integration
-owns the queue, callback identity, retry policy, and route.
+durable acceptance of the supplied artifact and record. The public operation
+returns a `202` admission receipt; a Vyral-hosted generic worker completes the
+configured external execution route. Consumer-owned workers remain for
+consumer-defined handlers, not Vyral's generic staging, storage publication, or
+lease-completion behavior. The Vyral deployment owns the generic queue, callback
+identity, and reconciliation. A producer owns a queue, callback identity, retry
+policy, and route only for its own consumer-defined handler.
 
 An upstream may attach a signed external context to its manifest. The host
 configures a public-key verifier and expected issuer, audience, and key id.