diff --git a/.gitignore b/.gitignore index 0413db346..01f2bcefc 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,7 @@ pr-stack-integrity-evidence.json /.r0-evidence*/ /keys/* + +# Umbrella subchart packaging is derived from the in-repo sources at render time. +apps/_infra/deploy-k8s/Chart.lock +apps/_infra/deploy-k8s/charts/ diff --git a/apps/_infra/deploy-k8s/Chart.lock b/apps/_infra/deploy-k8s/Chart.lock deleted file mode 100644 index ef78800e0..000000000 --- a/apps/_infra/deploy-k8s/Chart.lock +++ /dev/null @@ -1,48 +0,0 @@ -dependencies: -- name: k8s-platform - repository: file://platform - version: 0.9.2 -- name: opencrane-server - repository: file://../../opencrane/helm - version: 0.9.2 -- name: opencrane-ui - repository: file://../../opencrane-ui/helm - version: 0.9.2 -- name: opencrane-channel-proxy - repository: file://../../channel-proxy/helm - version: 0.9.2 -- name: opencrane-memory-gateway - repository: file://../../memory-gateway/helm - version: 0.9.2 -- name: opencrane-artifact-service - repository: file://../../artifact-service/helm - version: 0.8.0 -- name: opencrane-artifact-preprocessor - repository: file://../../artifact-preprocessor/helm - version: 0.9.2 -- name: opencrane-artifact-scanner - repository: file://../../artifact-scanner/helm - version: 0.9.2 -- name: opencrane-agent-controller - repository: file://../../agent-controller/helm - version: 0.9.2 -- name: opencrane-managed-agent-runtime - repository: file://../../managed-agent-runtime/helm - version: 0.8.0 -- name: opencrane-skill-authoring - repository: file://../../skill-authoring/helm - version: 0.8.0 -- name: opencrane-tool-runner - repository: file://../../tool-runner/helm - version: 0.8.0 -- name: opencrane-cognee - repository: file://../cognee/helm - version: 0.8.1 -- name: opencrane-litellm - repository: file://../litellm/helm - version: 0.8.0 -- name: opencrane-obot - repository: file://../obot/helm - version: 0.8.0 -digest: sha256:1670f21eb5487abeb34dcdf54087c3d14694d50bc432e6bee44d69a9cd46e951 -generated: "2026-08-19T15:41:04.77927+03:00" diff --git a/apps/_infra/deploy-k8s/Chart.yaml b/apps/_infra/deploy-k8s/Chart.yaml index 2c2c29794..f47cc2a08 100644 --- a/apps/_infra/deploy-k8s/Chart.yaml +++ b/apps/_infra/deploy-k8s/Chart.yaml @@ -17,53 +17,57 @@ keywords: maintainers: - name: OpenCrane dependencies: + # Every dependency is an in-repo file:// chart, so the checked-out commit is the only + # version authority. The open constraint always accepts the chart each app currently + # declares; helm dependency update packages the sources fresh at render time, and no + # lock file or vendored archive needs a matching bump. # Shared named-template library (labels, fullname, RBAC rule sets, env helpers). - name: k8s-platform - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://platform" # App-owned workload units export named templates; this umbrella only includes them # with the unchanged parent release context. - name: opencrane-server - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../opencrane/helm" - name: opencrane-ui - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../opencrane-ui/helm" - name: opencrane-channel-proxy - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../channel-proxy/helm" - name: opencrane-memory-gateway - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../memory-gateway/helm" - name: opencrane-artifact-service - version: 0.8.0 + version: ">=0.0.0-0" repository: "file://../../artifact-service/helm" - name: opencrane-artifact-preprocessor - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../artifact-preprocessor/helm" - name: opencrane-artifact-scanner - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../artifact-scanner/helm" - name: opencrane-agent-controller - version: 0.9.2 + version: ">=0.0.0-0" repository: "file://../../agent-controller/helm" - name: opencrane-managed-agent-runtime alias: managedAgentRuntimePlane - version: 0.8.0 + version: ">=0.0.0-0" repository: "file://../../managed-agent-runtime/helm" condition: agentController.enabled - name: opencrane-skill-authoring - version: 0.8.0 + version: ">=0.0.0-0" repository: "file://../../skill-authoring/helm" - name: opencrane-tool-runner - version: 0.8.0 + version: ">=0.0.0-0" repository: "file://../../tool-runner/helm" - name: opencrane-cognee - version: 0.8.1 + version: ">=0.0.0-0" repository: "file://../cognee/helm" - name: opencrane-litellm - version: 0.8.0 + version: ">=0.0.0-0" repository: "file://../litellm/helm" - name: opencrane-obot - version: 0.8.0 + version: ">=0.0.0-0" repository: "file://../obot/helm" diff --git a/apps/_infra/deploy-k8s/charts/k8s-platform-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/k8s-platform-0.9.2.tgz deleted file mode 100644 index 40fb5028b..000000000 Binary files a/apps/_infra/deploy-k8s/charts/k8s-platform-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz deleted file mode 100644 index be26a8acc..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz deleted file mode 100644 index b714996bb..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz deleted file mode 100644 index 8e7f3394c..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz deleted file mode 100644 index 786256d1b..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-channel-proxy-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-channel-proxy-0.9.2.tgz deleted file mode 100644 index 1034c4068..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-channel-proxy-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-cognee-0.8.1.tgz b/apps/_infra/deploy-k8s/charts/opencrane-cognee-0.8.1.tgz deleted file mode 100644 index daa69f90a..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-cognee-0.8.1.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-litellm-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-litellm-0.8.0.tgz deleted file mode 100644 index c918be5a2..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-litellm-0.8.0.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-managed-agent-runtime-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-managed-agent-runtime-0.8.0.tgz deleted file mode 100644 index 26dbbe0f1..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-managed-agent-runtime-0.8.0.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-memory-gateway-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-memory-gateway-0.9.2.tgz deleted file mode 100644 index a3b8f7ff1..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-memory-gateway-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-obot-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-obot-0.8.0.tgz deleted file mode 100644 index a90af8e46..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-obot-0.8.0.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-server-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-server-0.9.2.tgz deleted file mode 100644 index 0c41c5843..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-server-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-skill-authoring-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-skill-authoring-0.8.0.tgz deleted file mode 100644 index 07a8d5e12..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-skill-authoring-0.8.0.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-tool-runner-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-tool-runner-0.8.0.tgz deleted file mode 100644 index 4390d3476..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-tool-runner-0.8.0.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-ui-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-ui-0.9.2.tgz deleted file mode 100644 index e0fc803b5..000000000 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-ui-0.9.2.tgz and /dev/null differ diff --git a/apps/_infra/deploy-k8s/platform/current-chart-sources.sh b/apps/_infra/deploy-k8s/platform/current-chart-sources.sh index 738b09277..ae6fe79a4 100644 --- a/apps/_infra/deploy-k8s/platform/current-chart-sources.sh +++ b/apps/_infra/deploy-k8s/platform/current-chart-sources.sh @@ -13,6 +13,17 @@ current_chart_sources_root() cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd } +# Packages the umbrella's subcharts into the checkout itself, for the few contracts that render +# the repository chart directory rather than the disposable fixture. `charts/` and `Chart.lock` +# are derived artifacts (both gitignored), so writing them here is the intended model rather +# than churn. Repeat calls are cheap: Helm reuses archives whose version already matches. +ensure_umbrella_chart_dependencies() +{ + local umbrella + umbrella="$(current_chart_sources_root)/apps/_infra/deploy-k8s" + helm dependency update --skip-refresh "$umbrella" >/dev/null +} + prepare_current_chart_sources() { if [[ -n "$_CURRENT_CHART_SOURCES_FIXTURE" ]]; then @@ -44,7 +55,9 @@ prepare_current_chart_sources() # Keep the dependency layout identical to Chart.yaml while linking every # app-owned source directory. New local dependencies then need no fixture - # maintenance and Helm still verifies the committed Chart.lock digest. + # maintenance. Every dependency is an in-repo file:// chart, so the checked-out + # commit is the reproducibility authority and the update packages the current + # sources fresh — no lock file or vendored archive to keep in step. for app_dir in "$root/apps"/*; do [[ "$(basename "$app_dir")" == "_infra" ]] && continue ln -s "$app_dir" "$fixture_apps/$(basename "$app_dir")" @@ -54,7 +67,7 @@ prepare_current_chart_sources() ln -s "$infra_dir" "$fixture_apps/_infra/$(basename "$infra_dir")" done - helm dependency build --skip-refresh "$_CURRENT_CHART_SOURCES_DIR" >/dev/null + helm dependency update --skip-refresh "$_CURRENT_CHART_SOURCES_DIR" >/dev/null ); then cleanup_current_chart_sources return 1 diff --git a/apps/_infra/deploy-k8s/platform/k8s-deploy.sh b/apps/_infra/deploy-k8s/platform/k8s-deploy.sh index 5e4cd4c8b..5fedb9c66 100755 --- a/apps/_infra/deploy-k8s/platform/k8s-deploy.sh +++ b/apps/_infra/deploy-k8s/platform/k8s-deploy.sh @@ -135,6 +135,7 @@ fi POSTGRES_BASELINE_PUBLISHER="$SCRIPT_DIR/../../../postgres/scripts/publish-initdb-baseline-config-map.sh" POSTGRES_MIGRATION_PUBLISHER="$SCRIPT_DIR/../../../postgres/scripts/publish-database-migration-config-map.sh" DATABASE_TRANSITION_RESOLVER="$SCRIPT_DIR/../../../../scripts/release-versioning/database-transition.mjs" +DATABASE_SCHEMA_LINEAGE_RESOLVER="$SCRIPT_DIR/../../../../scripts/release-versioning/schema-lineage.mjs" POSTGRES_MIGRATION_BACKUP="$SCRIPT_DIR/../../../postgres/scripts/create-pre-migration-backup.sh" POSTGRES_BASELINE_FILE="$SCRIPT_DIR/../../../opencrane/prisma/bootstrap/target-baseline.sql" REPOSITORY_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" @@ -315,13 +316,13 @@ validate_unbacked_database_migration_override DATABASE_TARGET_SCHEMA_VERSION="$(jq -r '.targetSchemaVersion' <<<"$DATABASE_RELEASE_TRANSITION")" DATABASE_TARGET_BASELINE_SHA256="$(jq -r '.targetBaselineSha256' <<<"$DATABASE_RELEASE_TRANSITION")" DATABASE_CONVERGENCE_MIGRATION="$(jq '.migration' <<<"$DATABASE_RELEASE_TRANSITION")" -# A repair retry can name the current release while its database still needs the carried migration evidence. -if [[ "$DATABASE_CONVERGENCE_MIGRATION" == "null" ]]; then - previous_release_version="$(jq -r '.database.carriedForwardFromRepositoryVersion // .previousRepositoryVersion // empty' \ - "$REPOSITORY_ROOT/releases/$RELEASE_VERSION.json")" - if [[ -n "$previous_release_version" ]]; then - DATABASE_CONVERGENCE_MIGRATION="$(node "$DATABASE_TRANSITION_RESOLVER" "$REPOSITORY_ROOT" "$RELEASE_VERSION" "$previous_release_version" | jq '.migration')" - fi +# A release that changes no schema still meets databases that reached this schema through a real +# migration, and privilege reconciliation compares them against exactly that recorded transition. +# Walk the release chain for the migration that produced this schema. The previous form re-resolved +# the transition against this release's own previous version — another same-schema hop that always +# returned null again — so every same-schema patch failed the convergence gate on a migrated silo. +if [[ "$DATABASE_TRANSITION_KIND" == "current" && "$DATABASE_CONVERGENCE_MIGRATION" == "null" ]]; then + DATABASE_CONVERGENCE_MIGRATION="$(node "$DATABASE_SCHEMA_LINEAGE_RESOLVER" "$REPOSITORY_ROOT" "$RELEASE_VERSION")" || exit $? fi kubectl cluster-info >/dev/null 2>&1 || { err "kubectl can't reach a cluster. Point your context at the target cluster first."; exit 1; } KUBERNETES_CONTEXT="$(kubectl config current-context 2>/dev/null || true)" diff --git a/apps/_infra/deploy-k8s/platform/tests/current-chart-sources-contract.sh b/apps/_infra/deploy-k8s/platform/tests/current-chart-sources-contract.sh index be283241e..572b5ebad 100644 --- a/apps/_infra/deploy-k8s/platform/tests/current-chart-sources-contract.sh +++ b/apps/_infra/deploy-k8s/platform/tests/current-chart-sources-contract.sh @@ -35,7 +35,7 @@ helm() prepare_current_chart_sources prepared_chart="$(current_chart_sources_dir)" [[ "$prepared_chart" != "$AMBIENT_DIR" ]] -grep -Fq 'dependency build --skip-refresh' "$CALLS" +grep -Fq 'dependency update --skip-refresh' "$CALLS" [[ -L "$prepared_chart/../../opencrane" ]] cleanup_current_chart_sources [[ -f "$AMBIENT_FIXTURE/must-survive" ]] diff --git a/apps/_infra/deploy-k8s/platform/tests/post-deploy-health-contract.sh b/apps/_infra/deploy-k8s/platform/tests/post-deploy-health-contract.sh index aea63fa63..62063602f 100644 --- a/apps/_infra/deploy-k8s/platform/tests/post-deploy-health-contract.sh +++ b/apps/_infra/deploy-k8s/platform/tests/post-deploy-health-contract.sh @@ -4,20 +4,14 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" DEPLOY_SCRIPT="$ROOT_DIR/apps/_infra/deploy-k8s/platform/k8s-deploy.sh" VERIFY_SCRIPT="$ROOT_DIR/apps/_infra/deploy-k8s/platform/post-deploy-verify.sh" -CHART_DIR="$ROOT_DIR/apps/_infra/deploy-k8s" -CHART_FIXTURE="$(mktemp -d)" -trap 'rm -rf "$CHART_FIXTURE"' EXIT - grep -Fq 'source "$POST_DEPLOY_VERIFY"' "$DEPLOY_SCRIPT" source "$VERIFY_SCRIPT" -# Render against the current app-owned server chart, not the potentially stale committed archive. -cp -R "$CHART_DIR/." "$CHART_FIXTURE" -helm package "$ROOT_DIR/apps/opencrane/helm" --destination "$CHART_FIXTURE/charts" >/dev/null -rm -f "$CHART_FIXTURE/charts/opencrane-ui-"*.tgz -helm package "$ROOT_DIR/apps/opencrane-ui/helm" --destination "$CHART_FIXTURE/charts" >/dev/null -helm package "$ROOT_DIR/apps/channel-proxy/helm" --destination "$CHART_FIXTURE/charts" >/dev/null -helm package "$ROOT_DIR/apps/memory-gateway/helm" --destination "$CHART_FIXTURE/charts" >/dev/null +# The shared fixture packages every current app-owned chart source, the same way deploy.sh does. +source "$ROOT_DIR/apps/_infra/deploy-k8s/platform/current-chart-sources.sh" +prepare_current_chart_sources +trap 'cleanup_current_chart_sources' EXIT +CHART_FIXTURE="$(current_chart_sources_dir)" rendered_ingress="$(helm template opencrane-silo "$CHART_FIXTURE" \ --set-string 'memoryGateway.kubernetesApiServerCidrs[0]=10.43.0.1/32' \ --set-string 'memoryGateway.kubernetesApiServerEndpointCidrs[0]=172.18.0.2/32' \ diff --git a/apps/_infra/deploy-k8s/platform/tests/qualified-release-image-contract.sh b/apps/_infra/deploy-k8s/platform/tests/qualified-release-image-contract.sh index e2dcd90a7..d8e08d151 100755 --- a/apps/_infra/deploy-k8s/platform/tests/qualified-release-image-contract.sh +++ b/apps/_infra/deploy-k8s/platform/tests/qualified-release-image-contract.sh @@ -7,6 +7,8 @@ DEPLOY_CORE="$ROOT_DIR/apps/_infra/deploy-k8s/platform/k8s-deploy.sh" FINALIZATION="$ROOT_DIR/apps/_infra/deploy-k8s/platform/database-release-finalization.sh" source "$POLICY" +source "$ROOT_DIR/apps/_infra/deploy-k8s/platform/current-chart-sources.sh" +ensure_umbrella_chart_dependencies IMAGE_TAG="sha-f7d6771a4a5a075d424c7678d6165dd71c06b522" CP_TAG="sha-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" diff --git a/apps/_infra/deploy-k8s/platform/tests/silo-deploy-profile-contract.sh b/apps/_infra/deploy-k8s/platform/tests/silo-deploy-profile-contract.sh index 9fb329f8b..9469e707e 100644 --- a/apps/_infra/deploy-k8s/platform/tests/silo-deploy-profile-contract.sh +++ b/apps/_infra/deploy-k8s/platform/tests/silo-deploy-profile-contract.sh @@ -7,6 +7,9 @@ DEVELOP_SMOKE="$ROOT_DIR/apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh" MODEL_HELPER="$ROOT_DIR/apps/_infra/deploy-k8s/platform/initial-model-provider.sh" COGNEE_POLICY="$ROOT_DIR/apps/_infra/cognee/deploy/image-policy.sh" +source "$ROOT_DIR/apps/_infra/deploy-k8s/platform/current-chart-sources.sh" +ensure_umbrella_chart_dependencies + grep -Fq -- '--acme-email' "$DEPLOY_SCRIPT" grep -Fq -- '--first-user-email' "$DEPLOY_SCRIPT" grep -Fq -- '--initial-model-provider' "$DEPLOY_SCRIPT" diff --git a/apps/opencrane/prisma/migrations/tests/verify-postgres.sh b/apps/opencrane/prisma/migrations/tests/verify-postgres.sh index 3ea1565a3..0b0956d97 100755 --- a/apps/opencrane/prisma/migrations/tests/verify-postgres.sh +++ b/apps/opencrane/prisma/migrations/tests/verify-postgres.sh @@ -236,6 +236,40 @@ for database in migrated fresh; do done diff --unified "$WORK_DIR/fresh-schema.sql" "$WORK_DIR/migrated-schema.sql" +# A release that changes no schema still ships to silos in both shapes: bootstrapped fresh at the current +# baseline, and migrated into that schema by an earlier release. The deploy engine hands the classifier the +# owning release's evidence from the lineage resolver instead of a migration it must run, so every shape a +# silo can be in has to stay deployable rather than read as incompatible. +CURRENT_RELEASE_VERSION="$(jq -r '.version' "$ROOT/package.json")" +CURRENT_RELEASE_MANIFEST="$ROOT/releases/$CURRENT_RELEASE_VERSION.json" +SCHEMA_LINEAGE="$(node "$ROOT/scripts/release-versioning/schema-lineage.mjs" "$ROOT" "$CURRENT_RELEASE_VERSION")" +POSTGRES_BASELINE_SHA256="$(jq -r '.database.baselineSha256' "$CURRENT_RELEASE_MANIFEST")" +DATABASE_TARGET_BASELINE_SHA256="$POSTGRES_BASELINE_SHA256" +DATABASE_TARGET_SCHEMA_VERSION="$(jq -r '.database.schemaVersion' "$CURRENT_RELEASE_MANIFEST")" +DATABASE_PREVIOUS_MIGRATION_ID="$(jq -r '.id' <<<"$SCHEMA_LINEAGE")" +DATABASE_PREVIOUS_SCHEMA_VERSION="$(jq -r '.fromSchemaVersion' <<<"$SCHEMA_LINEAGE")" +DATABASE_PREVIOUS_TARGET_BASELINE_SHA256="$(jq -r '.sourceTargetBaselineSha256' <<<"$SCHEMA_LINEAGE")" +DATABASE_PREVIOUS_PROTECTED_BASELINE_SHA256S_JSON="$(jq -c '.sourceProtectedBaselineSha256s' <<<"$SCHEMA_LINEAGE")" +DATABASE_PREVIOUS_FRESH_PROTECTED_BASELINE_SHA256="$(jq -r '.freshSourceProtectedBaselineSha256' <<<"$SCHEMA_LINEAGE")" +DATABASE_SOURCE_HISTORY_LINEAGES_JSON="$(jq -c '.sourceHistoryLineages' <<<"$SCHEMA_LINEAGE")" +DATABASE_PREVIOUS_MIGRATION_SQL_SHA256="$(jq -r '.sqlSha256' <<<"$SCHEMA_LINEAGE")" +[[ "$DATABASE_TARGET_SCHEMA_VERSION" == "$(jq -r '.toSchemaVersion' <<<"$SCHEMA_LINEAGE")" ]] + +# A fresh bootstrap records where the database was born and writes no migration history. The postgres +# chart's bootstrap Job owns this row in a real silo. +psql_command fresh </dev/null +CREATE SCHEMA "opencrane_bootstrap"; +CREATE TABLE "opencrane_bootstrap"."target_baseline" ( + "singleton" BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK ("singleton"), + "baseline_sha256" TEXT NOT NULL CHECK ("baseline_sha256" ~ '^[0-9a-f]{64}$') +); +INSERT INTO "opencrane_bootstrap"."target_baseline" ("singleton", "baseline_sha256") +VALUES (TRUE, '$POSTGRES_BASELINE_SHA256'); +SQL +assert_classifier_state fresh "current|$POSTGRES_BASELINE_SHA256" +assert_classifier_state fresh_source "completed|$FRESH_PROTECTED_DIGEST" +assert_classifier_state migrated "completed|$PROTECTED_DIGEST" + psql_command migrated --tuples-only --no-align --command \ 'SELECT count(*) FROM "persona_questions" WHERE "question_set_id" = '\''personal-agent-onboarding'\'' AND "question_set_version" = 1;' \ | grep -qx '10' diff --git a/docs/agents/versioning.md b/docs/agents/versioning.md index 0041a2fb3..252557938 100644 --- a/docs/agents/versioning.md +++ b/docs/agents/versioning.md @@ -66,9 +66,12 @@ introduced chart has no predecessor and therefore no transition. A migration con shape once and emits only the current shape; do not retain compatibility aliases in templates or values. -The umbrella chart pins each local dependency to its owning chart version. Regenerate and review -`Chart.lock` and packaged dependencies after a chart stamp changes. PostgreSQL's chart version tracks -the OpenCrane wrapper; its `appVersion` remains the pinned PostgreSQL engine major. +The umbrella chart declares each local dependency with an open version constraint: every +dependency is an in-repo `file://` chart, so the checked-out commit is the version authority and +`helm dependency update` packages the sources fresh at render time. There is no `Chart.lock` or +vendored archive to keep in step — a chart stamp needs no umbrella edit at all. The release gate +still fails when a chart-bearing application is not declared in the umbrella. PostgreSQL's chart +version tracks the OpenCrane wrapper; its `appVersion` remains the pinned PostgreSQL engine major. ## Database migrations diff --git a/docs/ci-and-deploy.md b/docs/ci-and-deploy.md index c4acdaeb9..4b41d754f 100644 --- a/docs/ci-and-deploy.md +++ b/docs/ci-and-deploy.md @@ -112,7 +112,7 @@ apply` against a live cluster: ```mermaid flowchart TD A[apps/_infra/deploy-k8s/deploy.sh\nsilo profile: flags, presets] --> B[platform/k8s-deploy.sh\nthe install engine] - B --> C[current-chart-sources.sh\nhelm dependency build\nfrom Chart.lock] + B --> C[current-chart-sources.sh\npackages the in-repo\nsubchart sources] B --> D[database-migration-orchestrator.sh\nCNPG cluster, databases,\nmigration + privileges Jobs] B --> E[umbrella helm upgrade\nall app subcharts] E --> F[database-release-finalization.sh\ncredential-checksum roll,\nrollout waits, cert wait] @@ -140,10 +140,12 @@ flowchart TD - **CI green first.** Confirm the `docker.yml` run for the exact SHA is green before deploying; the deploy scripts pull published images and never build them. -- **`helm dependency build`, never `dependency update`, on the deploy path.** The engine resolves - subcharts from `Chart.lock` for reproducibility. `dependency update` re-resolves and can drift. - (Regenerating the lock/archives after a chart stamp is the one place `dependency update` is - correct — review the diff.) +- **Subchart packaging is derived, never committed.** Every umbrella dependency is an in-repo + `file://` chart, so the checked-out commit is the version authority: the deploy fixture runs + `helm dependency update --skip-refresh` and packages the current sources. There is no + `Chart.lock` or vendored archive to regenerate, and a chart version bump needs no umbrella + edit. (The bootstrap prerequisites are the opposite case: external charts stay pinned by + version and digest.) - **A green `helm template`/CI render does not prove a live `helm upgrade` works.** Stateful services need their PVC semantics, reconcile-retry, and Secret-change pod-roll trigger checked before deploying — see the live-upgrade checklist in the deploy ledger. @@ -177,8 +179,8 @@ summary. `appVersion` where a chart exists. Version-only mirror edits are "stamp-only" and do not count as changes themselves. - A **changed chart** bumps its chart version to the root version and adds exactly one - `helm/migrations/-to-.json` transition; the umbrella's `Chart.lock` and packaged - archives are then regenerated and reviewed. + `helm/migrations/-to-.json` transition. The umbrella needs no edit: it declares its + in-repo dependencies with open constraints and packages them fresh at render time. - A **database schema change** updates the clean target baseline and adds one adjacent, reviewed SQL transition under `apps/opencrane/prisma/migrations/-to-/`, bound by digest. - Adjacent minor trains (`0.8.x → 0.9.0`) are the only automatic transition. Patch, skipped-minor, diff --git a/scripts/__tests__/release-versioning-check.test.mjs b/scripts/__tests__/release-versioning-check.test.mjs index 5736b867b..bba99fe4b 100644 --- a/scripts/__tests__/release-versioning-check.test.mjs +++ b/scripts/__tests__/release-versioning-check.test.mjs @@ -9,7 +9,7 @@ import { __SelectDirectReleaseComparisonBase, validateWorkspace, } from "../release-versioning/core.mjs"; -import { resolveDatabaseTransition } from "../release-versioning/database-validation.mjs"; +import { resolveDatabaseTransition, resolveSchemaLineage } from "../release-versioning/database-validation.mjs"; import { isAdjacentMinor, isAdjacentPatch, parseSemver, sha256 } from "../release-versioning/version-utils.mjs"; function _WriteJson(path, value) @@ -365,7 +365,7 @@ test("rejects stale chart application metadata for a directly adapted app", asyn assert.ok(errors.some((error) => error.includes("directly adapted but chart appVersion"))); }); -test("rejects a checked-in dependency archive that differs from chart source", async () => +test("requires the umbrella to declare every chart-bearing application", async () => { const root = mkdtempSync(join(tmpdir(), "opencrane-release-packaging-")); mkdirSync(join(root, "apps/example/helm/templates"), { recursive: true }); @@ -380,7 +380,7 @@ test("rejects a checked-in dependency archive that differs from chart source", a writeFileSync(join(root, "apps/example/helm/Chart.yaml"), "apiVersion: v2\nname: example\ntype: application\nversion: 0.7.0\nappVersion: \"0.7.0\"\n"); writeFileSync(join(root, "apps/example/helm/values.yaml"), "replicas: 1\n"); writeFileSync(join(root, "apps/example/helm/templates/configmap.yaml"), "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: example\n"); - writeFileSync(join(root, "apps/_infra/deploy-k8s/Chart.yaml"), [ + const declaredUmbrella = [ "apiVersion: v2", "name: umbrella", "type: application", @@ -388,15 +388,11 @@ test("rejects a checked-in dependency archive that differs from chart source", a "appVersion: \"0.7.0\"", "dependencies:", " - name: example", - " version: 0.7.0", + " version: \">=0.0.0-0\"", " repository: file://../../example/helm", "", - ].join("\n")); - const dependencyResult = spawnSync("helm", ["dependency", "update", "apps/_infra/deploy-k8s"], { - cwd: root, - encoding: "utf8", - }); - assert.equal(dependencyResult.status, 0, dependencyResult.stderr); + ].join("\n"); + writeFileSync(join(root, "apps/_infra/deploy-k8s/Chart.yaml"), declaredUmbrella); const baselinePath = join(root, "apps/opencrane/prisma/bootstrap/target-baseline.sql"); writeFileSync(baselinePath, "SELECT 1;\n"); _WriteJson(join(root, "releases/0.7.0.json"), { @@ -413,15 +409,18 @@ test("rejects a checked-in dependency archive that differs from chart source", a example: { root: "apps/example", adaptedVersion: "0.7.0", chartVersion: "0.7.0" }, }, }); - writeFileSync(join(root, "apps/example/helm/values.yaml"), "replicas: 2\n"); const graph = { nodes: { "deploy-k8s": { data: { projectType: "application", root: "apps/_infra/deploy-k8s", metadata: { release: { adaptedVersion: "0.7.0" } } } }, example: { data: { projectType: "application", root: "apps/example", metadata: { release: { adaptedVersion: "0.7.0" } } } }, }, }; - const errors = await validateWorkspace(root, [], graph); - assert.ok(errors.some((error) => error.includes("differs from source"))); + // A declared dependency needs no lock file or vendored archive: the in-repo sources are the + // version authority and packaging is derived at render time. + assert.deepEqual(await validateWorkspace(root, [], graph), []); + writeFileSync(join(root, "apps/_infra/deploy-k8s/Chart.yaml"), declaredUmbrella.replace(/dependencies:[\s\S]*$/u, "")); + const missing = await validateWorkspace(root, [], graph); + assert.ok(missing.some((error) => error.includes("does not declare dependency example"))); }); test("rejects baseline changes inside an already adopted train", async () => @@ -497,6 +496,30 @@ test("carries one failed predecessor migration through its immediate repair patc assert.equal(resolveDatabaseTransition(fixture.root, "0.9.1", "0.9.1").kind, "current"); }); +test("recovers the migration that produced the schema for a same-schema release", () => +{ + const fixture = _CarryForwardFixture(); + // 0.9.1 changes no schema of its own, but a live database that reached 0.9.0 through the real + // 0.8.0-to-0.9.0 migration still records it, and privilege reconciliation compares against it. + assert.equal(resolveDatabaseTransition(fixture.root, "0.9.1", "0.9.0").migration, null); + const lineage = resolveSchemaLineage(fixture.root, "0.9.1"); + assert.equal(lineage.id, "0.8.0-to-0.9.0"); + assert.equal(lineage.fromSchemaVersion, "0.8.0"); + assert.equal(lineage.toSchemaVersion, "0.9.0"); + assert.equal(lineage.ownedByReleaseVersion, "0.9.0"); + // Reporting history must not authorise a carry-forward override. + assert.equal(lineage.carriedForwardThroughReleaseVersion, null); + assert.ok(lineage.sqlSha256); + assert.ok(Array.isArray(lineage.sourceProtectedBaselineSha256s)); + assert.ok(lineage.sourceProtectedBaselineSha256s.length > 0); +}); + +test("reports no lineage for a schema that was never migrated into", () => +{ + const fixture = _Fixture(); + assert.equal(resolveSchemaLineage(fixture.root, "0.7.0"), null); +}); + test("rejects undeclared and altered database carry-forward transitions", () => { const undeclared = _CarryForwardFixture(); diff --git a/scripts/release-versioning/chart-validation.mjs b/scripts/release-versioning/chart-validation.mjs index 33b6abbee..e0e7fcf1d 100644 --- a/scripts/release-versioning/chart-validation.mjs +++ b/scripts/release-versioning/chart-validation.mjs @@ -1,8 +1,5 @@ -import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { existsSync, readFileSync } from "node:fs"; import { join, relative } from "node:path"; -import { gunzipSync } from "node:zlib"; /** Read an optional app package version mirror. */ export function packageVersion(repositoryRoot, projectRoot) @@ -49,71 +46,6 @@ function _DependencyVersions(path) return versions; } -function _PackagedChartFiles(path) -{ - const archive = gunzipSync(readFileSync(path)); - const files = new Map(); - let offset = 0; - while (offset + 512 <= archive.length) - { - const header = archive.subarray(offset, offset + 512); - if (header.every((byte) => byte === 0)) break; - const name = header.subarray(0, 100).toString("utf8").replace(/\0.*$/u, ""); - const prefix = header.subarray(345, 500).toString("utf8").replace(/\0.*$/u, ""); - const fullName = prefix ? `${prefix}/${name}` : name; - const sizeText = header.subarray(124, 136).toString("utf8").replace(/\0.*$/u, "").trim(); - const size = Number.parseInt(sizeText || "0", 8); - if (!Number.isFinite(size)) throw new Error(`invalid tar entry size in '${path}'`); - const type = header.subarray(156, 157).toString("utf8"); - const relativeName = fullName.split("/").slice(1).join("/"); - if ((type === "" || type === "0") && relativeName) - files.set(relativeName, archive.subarray(offset + 512, offset + 512 + size)); - offset += 512 + Math.ceil(size / 512) * 512; - } - return files; -} - -function _ValidatePackagedChart(sourceRoot, packagePath, chartName, chartVersion, errors) -{ - if (!existsSync(packagePath)) return; - const temporaryRoot = mkdtempSync(join(tmpdir(), "opencrane-chart-package-")); - try - { - execFileSync("helm", ["package", sourceRoot, "--destination", temporaryRoot], { stdio: "ignore" }); - const generatedPath = join(temporaryRoot, `${chartName}-${chartVersion}.tgz`); - if (!existsSync(generatedPath)) - { - errors.push(`Helm packaged '${chartName}' under an unexpected name or version`); - return; - } - const expected = _PackagedChartFiles(generatedPath); - const actual = _PackagedChartFiles(packagePath); - const expectedNames = [...expected.keys()].sort(); - const actualNames = [...actual.keys()].sort(); - if (JSON.stringify(expectedNames) !== JSON.stringify(actualNames)) - { - errors.push(`packaged dependency ${chartName}-${chartVersion}.tgz file list differs from its chart source`); - return; - } - for (const name of expectedNames) - { - if (!expected.get(name).equals(actual.get(name))) - { - errors.push(`packaged dependency ${chartName}-${chartVersion}.tgz differs from source at '${name}'`); - return; - } - } - } - catch (error) - { - errors.push(`could not verify packaged dependency ${chartName}-${chartVersion}.tgz: ${error.message}`); - } - finally - { - rmSync(temporaryRoot, { recursive: true, force: true }); - } -} - /** Validate one declared chart transition. Only executable kinds may be admitted. */ export function validateHelmTransition(path, displayPath, fromVersion, toVersion, errors) { @@ -133,24 +65,31 @@ export function validateHelmTransition(path, displayPath, fromVersion, toVersion errors.push(`Helm transition '${displayPath}' must be a reviewed noop; executable value migrations require an implemented deploy consumer`); } -/** Verify umbrella pins, packages, and platform transition against app chart sources. */ +/** + * Verify the umbrella declares every chart-bearing app and the platform transition exists. + * + * Versions are deliberately not compared: every dependency is an in-repo file:// chart, so + * the checked-out commit already fixes the exact sources, the umbrella declares open + * constraints, and packaging is derived at render time. Only membership can drift — an app + * chart that falls out of the umbrella would silently stop deploying. + */ export function validateUmbrella(repositoryRoot, manifest, previousManifest, errors) { const umbrellaRoot = manifest.projects["deploy-k8s"]?.root; if (!umbrellaRoot) return; - const expected = new Map(); + const expected = new Set(); for (const project of Object.values(manifest.projects)) { if (!project.chartVersion || project.root === umbrellaRoot || project.root === "apps/postgres") continue; const chart = chartValues(repositoryRoot, project); - if (chart?.name) expected.set(chart.name, { root: join(repositoryRoot, project.root, "helm"), version: project.chartVersion }); + if (chart?.name) expected.add(chart.name); } const platformChart = chartValues(repositoryRoot, { root: `${umbrellaRoot}/platform` }); if (platformChart?.name) { if (platformChart.version !== manifest.projects["deploy-k8s"].chartVersion) errors.push(`k8s-platform chart version '${platformChart.version}' must track its deploy-k8s owner`); - expected.set(platformChart.name, { root: join(repositoryRoot, umbrellaRoot, "platform"), version: platformChart.version }); + expected.add(platformChart.name); const previousVersion = previousManifest?.projects?.["deploy-k8s"]?.chartVersion; if (previousVersion && previousVersion !== platformChart.version) { @@ -160,14 +99,8 @@ export function validateUmbrella(repositoryRoot, manifest, previousManifest, err } } const chartDependencies = _DependencyVersions(join(repositoryRoot, umbrellaRoot, "Chart.yaml")); - const lockDependencies = _DependencyVersions(join(repositoryRoot, umbrellaRoot, "Chart.lock")); - for (const [name, expectedChart] of expected) + for (const name of expected) { - const { root, version } = expectedChart; - if (chartDependencies.get(name) !== version) errors.push(`umbrella dependency ${name} does not pin chart version ${version}`); - if (lockDependencies.get(name) !== version) errors.push(`Chart.lock dependency ${name} does not pin chart version ${version}`); - const packagePath = join(repositoryRoot, umbrellaRoot, "charts", `${name}-${version}.tgz`); - if (!existsSync(packagePath)) errors.push(`packaged dependency ${name}-${version}.tgz is missing`); - else _ValidatePackagedChart(root, packagePath, name, version, errors); + if (!chartDependencies.has(name)) errors.push(`umbrella Chart.yaml does not declare dependency ${name}`); } } diff --git a/scripts/release-versioning/database-validation.mjs b/scripts/release-versioning/database-validation.mjs index 058cc1eed..9f960cec2 100644 --- a/scripts/release-versioning/database-validation.mjs +++ b/scripts/release-versioning/database-validation.mjs @@ -225,6 +225,61 @@ export function validateDatabase(repositoryRoot, manifest, previousManifest, cha * Called by: `database-transition.mjs`, which supplies this evidence to the deployment script. * @throws {Error} When either manifest or its database transition is invalid. */ +/** + * Finds the migration that last produced this release's database schema version. + * + * A same-schema patch train resolves to `current` with no migration of its own, but a live database + * that reached this schema through a real migration still records that transition, and privilege + * reconciliation compares the database against exactly that record. Walking the release chain + * recovers the evidence the database already carries; it never proposes a new migration, so the + * transition stays `current`. + * + * Called by: `scripts/release-versioning/schema-lineage.mjs`, which the deploy engine invokes when a + * transition carries no migration. + * @param repositoryRoot - Repository root holding `releases/` and the migration manifests. + * @param releaseVersion - The release being deployed. + * @returns The owning migration's evidence, or null when this schema was never migrated into. + * @see resolveDatabaseTransition + */ +export function resolveSchemaLineage(repositoryRoot, releaseVersion) +{ + const targetPath = join(repositoryRoot, "releases", `${releaseVersion}.json`); + if (!existsSync(targetPath)) throw new Error(`release manifest is missing: ${targetPath}`); + const target = readJson(targetPath); + const schemaVersion = target.database.schemaVersion; + let cursor = target; + while (cursor?.previousRepositoryVersion) + { + const predecessorPath = join(repositoryRoot, "releases", `${cursor.previousRepositoryVersion}.json`); + if (!existsSync(predecessorPath)) + throw new Error(`release manifest is missing: ${predecessorPath}`); + const predecessor = readJson(predecessorPath); + if (cursor.database.schemaVersion === schemaVersion && predecessor.database.schemaVersion !== schemaVersion) + { + const id = `${predecessor.database.schemaVersion}-to-${schemaVersion}`; + const migrationRoot = join(repositoryRoot, "apps/opencrane/prisma/migrations", id); + const migrationManifest = readJson(join(migrationRoot, "manifest.json")); + const sourceProtectedBaselineSha256s = _SourceProtectedBaselineDigests(migrationManifest); + return { + id, + fromSchemaVersion: predecessor.database.schemaVersion, + toSchemaVersion: schemaVersion, + sqlFile: join(migrationRoot, "migration.sql"), + sqlSha256: migrationManifest.sqlSha256, + sourceTargetBaselineSha256: migrationManifest.sourceTargetBaselineSha256, + sourceProtectedBaselineSha256s, + freshSourceProtectedBaselineSha256: _FreshSourceProtectedBaselineDigest(migrationManifest), + sourceHistoryLineages: _SourceHistoryLineages(repositoryRoot, predecessor, sourceProtectedBaselineSha256s), + // Reporting history, not authorizing a carry-forward override. + carriedForwardThroughReleaseVersion: null, + ownedByReleaseVersion: cursor.repositoryVersion, + }; + } + cursor = predecessor; + } + return null; +} + export function resolveDatabaseTransition(repositoryRoot, releaseVersion, fromReleaseVersion) { const rootVersion = readJson(join(repositoryRoot, "package.json")).version; diff --git a/scripts/release-versioning/schema-lineage.mjs b/scripts/release-versioning/schema-lineage.mjs new file mode 100755 index 000000000..65410f73e --- /dev/null +++ b/scripts/release-versioning/schema-lineage.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node +import { resolve } from "node:path"; +import { resolveSchemaLineage } from "./database-validation.mjs"; + +if (process.argv.length !== 4) +{ + console.error("usage: schema-lineage.mjs "); + process.exit(64); +} + +try +{ + const lineage = resolveSchemaLineage(resolve(process.argv[2]), process.argv[3]); + console.log(JSON.stringify(lineage)); +} +catch (error) +{ + console.error(`database schema lineage: ${error.message}`); + process.exit(1); +} diff --git a/website/contributing/deploying.md b/website/contributing/deploying.md index ab4d163e8..016a99e05 100644 --- a/website/contributing/deploying.md +++ b/website/contributing/deploying.md @@ -22,7 +22,7 @@ platform/k8s-deploy.sh the install engine │ ├──→ current-chart-sources.sh - │ helm dependency build from Chart.lock + │ packages the in-repo subchart sources │ ├──→ database-migration-orchestrator.sh │ CNPG cluster, databases, migration + privileges Jobs @@ -66,11 +66,11 @@ Confirm the `docker.yml` run for the exact SHA is green before deploying. The de published images and never build them — see [The CI pipeline](/contributing/ci-pipeline). ::: -::: warning `helm dependency build`, never `dependency update`, on the deploy path -The engine resolves subcharts from `Chart.lock` for reproducibility. `dependency update` -re-resolves and can drift. (Regenerating the lock and archives after a chart version stamp is the -one place `dependency update` is correct — review the diff. See -[Versions and migrations](/contributing/versions-and-migrations).) +::: tip Subchart packaging is derived, never committed +Every umbrella dependency is an in-repo `file://` chart, so the checked-out commit is the version +authority: the deploy fixture packages the current sources with `helm dependency update +--skip-refresh`. There is no `Chart.lock` or vendored archive to keep in step, and a chart +version bump needs no umbrella edit. External bootstrap charts stay pinned by version and digest. ::: ::: warning A green render does not prove a live upgrade works diff --git a/website/contributing/versions-and-migrations.md b/website/contributing/versions-and-migrations.md index 80e5af0cc..d71496a28 100644 --- a/website/contributing/versions-and-migrations.md +++ b/website/contributing/versions-and-migrations.md @@ -5,8 +5,8 @@ Helm charts and a shared database schema. This page covers how a change stamps i how chart and database migrations are recorded, and how CI enforces all of it. > See also: [The CI pipeline](/contributing/ci-pipeline) (where `check:release-versioning` runs), -> [Deploying](/contributing/deploying) (why `helm dependency build`, never `dependency update`, -> matters here), and the full policy in +> [Deploying](/contributing/deploying) (how the umbrella derives its subchart packaging), and the +> full policy in > [`docs/agents/versioning.md`](https://github.com/elewa-git/opencrane/blob/main/docs/agents/versioning.md). ## The repository train and the release manifest @@ -48,10 +48,11 @@ Version-only mirror edits are "stamp-only" and do not count as changes themselve ## Chart migrations A changed chart bumps its chart version to the root version and adds exactly one -`helm/migrations/-to-.json` transition. The umbrella's `Chart.lock` and packaged -archives are then regenerated and reviewed (with `helm dependency build`, never -`dependency update` — see [Deploying](/contributing/deploying)). A newly introduced chart has no -predecessor and therefore no transition. +`helm/migrations/-to-.json` transition. The umbrella needs no edit: it declares its +in-repo dependencies with open constraints and packages them fresh at render time (see +[Deploying](/contributing/deploying)). A newly introduced chart has no predecessor and therefore +no transition, but it must be declared as an umbrella dependency — the release gate fails when a +chart-bearing application is missing there. ## Database migrations