diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1b8094ffb..ed29cacac 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -45,11 +45,22 @@ permissions: env: REGISTRY: ghcr.io + # Image layer cache lives in registry repositories with one tag per deployable. The registry + # backend replaced the GitHub Actions cache backend because the 10GB Actions cache quota evicted + # the large layer blobs on almost every run, so every build started cold. + # + # Two repositories form a trust boundary: only integration-branch pushes write the trusted + # cache, and published image builds read only the trusted cache. Same-repository pull requests + # read both but write only the pull-request cache, so a layer produced by unreviewed code can + # never become part of a published image. + BUILD_CACHE_IMAGE: opencrane-buildcache + BUILD_CACHE_IMAGE_PR: opencrane-buildcache-pr jobs: prepare: name: Calculate affected work runs-on: ubuntu-latest + timeout-minutes: 15 outputs: nx_base: ${{ steps.affected.outputs.nx_base }} nx_head: ${{ steps.affected.outputs.nx_head }} @@ -126,6 +137,9 @@ jobs: test: name: Build, test, and lint affected projects runs-on: ubuntu-latest + # A hung step must fail fast: an untimed job once held a runner for hours on a stuck + # apt install and starved the whole queue. + timeout-minutes: 45 needs: prepare services: postgres: @@ -189,10 +203,21 @@ jobs: terraform init -backend=false -input=false -lockfile=readonly terraform validate -no-color + # The runner image already ships the PostgreSQL client. ripgrep comes as a pinned static + # binary because a full apt update/install spent ~30s per run and can hang on apt locks. - name: Install validation clients + env: + RIPGREP_VERSION: 14.1.1 + RIPGREP_SHA256: 4cf9f2741e6c465ffdb7c26f38056a59e2a2544b51f7cc128ef28337eeae4d8e run: | - sudo apt-get update - sudo apt-get install --yes postgresql-client ripgrep + if ! command -v psql >/dev/null; then + sudo apt-get update && sudo apt-get install --yes --no-install-recommends postgresql-client + fi + curl --location --fail --silent --show-error --output /tmp/ripgrep.tar.gz \ + "https://github.com/BurntSushi/ripgrep/releases/download/${RIPGREP_VERSION}/ripgrep-${RIPGREP_VERSION}-x86_64-unknown-linux-musl.tar.gz" + echo "${RIPGREP_SHA256} /tmp/ripgrep.tar.gz" | sha256sum --check + tar --extract --gzip --file /tmp/ripgrep.tar.gz --directory /tmp + sudo install --mode 0755 "/tmp/ripgrep-${RIPGREP_VERSION}-x86_64-unknown-linux-musl/rg" /usr/local/bin/rg - name: Enforce workload ownership and app composition run: npm run check:workload-ownership-app-composition @@ -229,12 +254,6 @@ jobs: - name: Prove the release-versioning classifier run: npm run test:release-versioning - - name: Enforce version-to-version database migration contracts - run: npm run check:database-migrations - - - name: Prove previous-to-current database convergence and rollback - run: npm run test:database-migrations - - name: Prove the PR-stack integrity classifier run: npm run test:pr-stack-integrity @@ -250,8 +269,78 @@ jobs: - name: Prove the Prisma-boundary classifier run: npm run test:prisma-boundaries - # The topology gate must inspect the checked-in app owners, before any - # generator can introduce transient files beneath an app root. + - name: Restore the Nx computation cache + uses: actions/cache@v4 + with: + path: .nx/cache + key: nx-${{ runner.os }}-${{ hashFiles('package-lock.json') }}-${{ github.sha }} + restore-keys: | + nx-${{ runner.os }}-${{ hashFiles('package-lock.json') }}- + + - name: Build, test, and lint affected projects + run: npx nx affected -t build test lint + + - name: Enforce monorepo dependency boundaries + run: npm run lint:boundaries + + # The PostgreSQL-bound proofs run beside the main test job instead of serially inside it: + # database generation, the target baseline, the SQL authority suites, and the migration + # convergence proofs together took minutes off the test job's critical path. + database: + name: Database authority and migration proofs + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: prepare + services: + postgres: + image: postgres:17 + env: + POSTGRES_DB: opencrane + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d opencrane" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + DATABASE_URL: postgresql://postgres:postgres@localhost:5432/opencrane + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Restore node_modules + id: node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-node24-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.node-modules.outputs.cache-hit != 'true' + run: npm ci + + # The runner image ships the PostgreSQL client; install it only if that ever changes. + - name: Install the PostgreSQL client + run: | + if ! command -v psql >/dev/null; then + sudo apt-get update && sudo apt-get install --yes --no-install-recommends postgresql-client + fi + + - name: Enforce version-to-version database migration contracts + run: npm run check:database-migrations + + - name: Prove previous-to-current database convergence and rollback + run: npm run test:database-migrations + - name: Generate the database client run: npm run db:generate -w @opencrane/server @@ -266,6 +355,33 @@ jobs: - name: Run every PostgreSQL authority suite run: npx nx run-many -t test:sql --parallel=1 + # The API contract check builds the full server, so when the contract changed it added + # minutes to the test job. It only needs the prepare outputs and runs beside it instead. + api_contract: + name: API reference and generated client + runs-on: ubuntu-latest + timeout-minutes: 20 + needs: prepare + if: needs.prepare.outputs.api_contract_changed == 'true' + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Restore node_modules + id: node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-node24-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.node-modules.outputs.cache-hit != 'true' + run: npm ci + - name: Restore the Nx computation cache uses: actions/cache@v4 with: @@ -274,8 +390,45 @@ jobs: restore-keys: | nx-${{ runner.os }}-${{ hashFiles('package-lock.json') }}- - - name: Build, test, and lint affected projects - run: npx nx affected -t build test lint + - name: Verify API reference and generated client + run: | + npx nx run opencrane:build + npm run sync-openapi -w @opencrane/website + git diff --exit-code -- website/public/openapi.json + npx nx run contracts:generate + git diff --exit-code -- libs/contracts/src/generated/api.ts + + # Storybook visual regressions run beside the main test job instead of serially after it: + # the Chromium install and the Storybook build/test chain used to extend the critical path + # by several minutes whenever a frontend project was affected. + storybook_visual: + name: Storybook component contracts + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: prepare + env: + NX_BASE: ${{ needs.prepare.outputs.nx_base }} + NX_HEAD: ${{ needs.prepare.outputs.nx_head }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Restore node_modules + id: node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-node24-${{ hashFiles('package-lock.json') }} + + - name: Install dependencies + if: steps.node-modules.outputs.cache-hit != 'true' + run: npm ci - name: Detect affected Storybook regression coverage id: storybook-regression @@ -288,8 +441,30 @@ jobs: echo "required=false" >> "$GITHUB_OUTPUT" fi - - name: Install Chromium for affected Storybook regressions + - name: Restore the Nx computation cache + if: steps.storybook-regression.outputs.required == 'true' + uses: actions/cache@v4 + with: + path: .nx/cache + key: nx-${{ runner.os }}-${{ hashFiles('package-lock.json') }}-${{ github.sha }} + restore-keys: | + nx-${{ runner.os }}-${{ hashFiles('package-lock.json') }}- + + # The browser binaries change only with the lockfile, so they restore from cache instead + # of downloading each run. On a cache hit the apt-driven install-deps is skipped entirely: + # the runner image already carries Chromium's system libraries, and the apt step has hung + # for tens of minutes on runner apt locks. + - name: Restore the Playwright browser cache if: steps.storybook-regression.outputs.required == 'true' + id: playwright-browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + + - name: Install Chromium for affected Storybook regressions + if: steps.storybook-regression.outputs.required == 'true' && steps.playwright-browsers.outputs.cache-hit != 'true' + timeout-minutes: 10 run: npx playwright install --with-deps chromium - name: Build and test affected Storybook component contracts @@ -306,18 +481,6 @@ jobs: if-no-files-found: ignore retention-days: 7 - - name: Enforce monorepo dependency boundaries - run: npm run lint:boundaries - - - name: Verify API reference and generated client when affected - if: needs.prepare.outputs.api_contract_changed == 'true' - run: | - npx nx run opencrane:build - npm run sync-openapi -w @opencrane/website - git diff --exit-code -- website/public/openapi.json - npx nx run contracts:generate - git diff --exit-code -- libs/contracts/src/generated/api.ts - develop_smoke: name: k3d current-silo smoke test runs-on: ubuntu-latest @@ -325,7 +488,10 @@ jobs: needs: prepare permissions: contents: read - packages: read + # Write lets the smoke publish its layer cache so the next run builds warm. GitHub + # downgrades the token to read-only for pull requests from forks, and the smoke then + # skips the cache export (see SMOKE_BUILD_CACHE_PUSH below). + packages: write # Every accepted develop commit proves the current deployment path. A pull request skips the # live cluster only when its exact base SHA already completed the same k3d job successfully. # Missing or malformed proof output deliberately compares unequal to true and runs k3d. @@ -384,20 +550,22 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - # Docker's GHA cache backend requires these runtime variables for an inline buildx command. - # Pin the helper by commit so the remote action cannot move underneath qualification. - - name: Expose the GitHub Actions cache runtime - uses: crazy-max/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487 - - name: Run the current-silo smoke run: ./apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh env: KEEP_CLUSTER: "0" - TIMEOUT_SECONDS: "300" + # 600 gives the loaded two-CPU runner headroom: the server pod's boot grew with the + # public health report and initial model seeding, and a healthy wait returns early. + TIMEOUT_SECONDS: "600" SMOKE_AFFECTED_PROJECTS: ${{ needs.prepare.outputs.develop_smoke_projects }} SMOKE_BASE_SHA: ${{ needs.prepare.outputs.nx_base }} SMOKE_REGISTRY: ${{ env.REGISTRY }}/${{ github.repository_owner }} SMOKE_STORAGE_MODE: ${{ needs.prepare.outputs.develop_smoke_storage_mode }} + SMOKE_BUILD_CACHE: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.BUILD_CACHE_IMAGE }} + SMOKE_BUILD_CACHE_UNTRUSTED: ${{ github.event_name == 'pull_request' && format('{0}/{1}/{2}', env.REGISTRY, github.repository_owner, env.BUILD_CACHE_IMAGE_PR) || '' }} + # Integration pushes refresh the trusted cache; same-repository pull requests export + # only the pull-request cache; fork pull requests (read-only token) export nothing. + SMOKE_BUILD_CACHE_EXPORT: ${{ github.event_name == 'push' && format('{0}/{1}/{2}', env.REGISTRY, github.repository_owner, env.BUILD_CACHE_IMAGE) || (github.event.pull_request.head.repo.full_name == github.repository && format('{0}/{1}/{2}', env.REGISTRY, github.repository_owner, env.BUILD_CACHE_IMAGE_PR) || '') }} image_smoke: name: Image smoke (${{ matrix.project }}) @@ -435,12 +603,16 @@ jobs: build-and-push: name: Build and publish affected images runs-on: ubuntu-latest - needs: [prepare, test, develop_smoke, image_smoke] + timeout-minutes: 30 + needs: [prepare, test, database, api_contract, storybook_visual, develop_smoke, image_smoke] if: >- ${{ always() && needs.prepare.outputs.has_deployables == 'true' && needs.test.result == 'success' && + needs.database.result == 'success' && + (needs.api_contract.result == 'success' || needs.api_contract.result == 'skipped') && + needs.storybook_visual.result == 'success' && (needs.develop_smoke.result == 'success' || needs.develop_smoke.result == 'skipped') && (needs.image_smoke.result == 'success' || needs.image_smoke.result == 'skipped') }} @@ -453,8 +625,9 @@ jobs: steps: - uses: actions/checkout@v6 + # Validation-only runs also log in: pulling the registry layer cache needs an + # authenticated token even when nothing is pushed. - name: Log in to GitHub Container Registry - if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish_deployables != 'none') uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} @@ -481,19 +654,28 @@ jobs: push: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish_deployables != 'none') }} tags: ${{ steps.metadata.outputs.tags }} labels: ${{ steps.metadata.outputs.labels }} - cache-from: type=gha,scope=${{ matrix.project }} - cache-to: type=gha,mode=max,scope=${{ matrix.project }} + # Publishable builds (push events) read only the trusted cache. Pull-request + # validation builds also read the pull-request cache and export only there; fork + # pull requests (read-only token) export nothing. + cache-from: | + type=registry,ref=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.BUILD_CACHE_IMAGE }}:${{ matrix.project }} + ${{ github.event_name == 'pull_request' && format('type=registry,ref={0}/{1}/{2}:{3}', env.REGISTRY, github.repository_owner, env.BUILD_CACHE_IMAGE_PR, matrix.project) || '' }} + cache-to: ${{ github.event_name == 'push' && format('type=registry,ref={0}/{1}/{2}:{3},mode=max', env.REGISTRY, github.repository_owner, env.BUILD_CACHE_IMAGE, matrix.project) || (github.event.pull_request.head.repo.full_name == github.repository && format('type=registry,ref={0}/{1}/{2}:{3},mode=max', env.REGISTRY, github.repository_owner, env.BUILD_CACHE_IMAGE_PR, matrix.project) || '') }} publish-develop-smoke-images: name: Publish develop smoke image (${{ matrix.project }}) runs-on: ubuntu-latest - needs: [prepare, test, develop_smoke, image_smoke, build-and-push] + timeout-minutes: 20 + needs: [prepare, test, database, api_contract, storybook_visual, develop_smoke, image_smoke, build-and-push] if: >- ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/develop' && needs.test.result == 'success' && + needs.database.result == 'success' && + (needs.api_contract.result == 'success' || needs.api_contract.result == 'skipped') && + needs.storybook_visual.result == 'success' && needs.develop_smoke.result == 'success' && (needs.image_smoke.result == 'success' || needs.image_smoke.result == 'skipped') && (needs['build-and-push'].result == 'success' || needs['build-and-push'].result == 'skipped') @@ -551,5 +733,5 @@ jobs: file: ${{ matrix.dockerfile }} push: true tags: ${{ steps.image-set.outputs.current_ref }} - cache-from: type=gha,scope=${{ matrix.project }} - cache-to: type=gha,mode=max,scope=${{ matrix.project }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.BUILD_CACHE_IMAGE }}:${{ matrix.project }} + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.BUILD_CACHE_IMAGE }}:${{ matrix.project }},mode=max diff --git a/AGENTS.md b/AGENTS.md index 17e6efaeb..7b2ba2fd1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ This file is the canonical agent instruction file for the repository. | **Build, Test & Infra** | [`docs/agents/infra.md`](docs/agents/infra.md) | building/testing, or editing Terraform/Helm/deploy under `platform/`. | | **Repository context** | [`docs/agents/repository-context.md`](docs/agents/repository-context.md) | starting or resuming a long-running task, tracing an unfamiliar capability, recording durable context, or deciding whether semantic context belongs in CI. | | **Workflow & Review Gate** | [`docs/agents/workflow.md`](docs/agents/workflow.md) | planning (`plan.md`/`CHANGELOG.md`), creating or updating a PR/stack, writing commit messages, or hitting the review gate. | -| **Release versions & migrations** | [`docs/agents/versioning.md`](docs/agents/versioning.md) | changing any application, shared dependency, Helm chart, database schema, release manifest, or version; every directly or dependency-adapted app and every upgrade path is recorded here. | +| **Release versions & migrations** | [`docs/agents/versioning.md`](docs/agents/versioning.md) | changing any application, shared dependency, Helm chart, database schema, release manifest, or version; every directly changed app and every upgrade path is recorded here. | | **Language-neutral maintainability** | [`docs/agents/maintainability.md`](docs/agents/maintainability.md) | adding substantial production code in any language, growing an already-large module, or reviewing cohesion and responsibility boundaries. | | **App-Specific** | [`docs/agents/app-specific.md`](docs/agents/app-specific.md) | working inside a specific `apps/*` or `libs/*` package; per-package map + API-first rule. | | **Package docs** | [`docs/agents/package-docs.md`](docs/agents/package-docs.md) | writing or editing any package `README.md`, or adding/moving/deleting a package — the README standard, the junior-dev voice, and the "update the README in the same change" rule. | diff --git a/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz index acc3826a0..be26a8acc 100644 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz and b/apps/_infra/deploy-k8s/charts/opencrane-agent-controller-0.9.2.tgz differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz index 0bf4be1e6..b714996bb 100644 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz and b/apps/_infra/deploy-k8s/charts/opencrane-artifact-preprocessor-0.9.2.tgz differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz b/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz index d357a5d1f..8e7f3394c 100644 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz and b/apps/_infra/deploy-k8s/charts/opencrane-artifact-scanner-0.9.2.tgz differ diff --git a/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz b/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz index 55103f9a1..786256d1b 100644 Binary files a/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz and b/apps/_infra/deploy-k8s/charts/opencrane-artifact-service-0.8.0.tgz differ diff --git a/apps/_infra/deploy-k8s/platform/database-release-finalization.sh b/apps/_infra/deploy-k8s/platform/database-release-finalization.sh index f4b465d7d..79d694bb8 100755 --- a/apps/_infra/deploy-k8s/platform/database-release-finalization.sh +++ b/apps/_infra/deploy-k8s/platform/database-release-finalization.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Owns application finalization after the database release transition. It restores the exact fenced -# Helm revision when un-fencing, Secret-triggered restarts, rollout waits, certificate readiness, or +# Helm revision when un-fencing, credential-checksum rolls, rollout waits, certificate readiness, or # verification fail. capture_fenced_main_release_revision() @@ -66,14 +66,33 @@ run_opencrane_finalization_stage() "$@" } -restart_database_consumers_for_finalization() +# Digests the published connection Secrets so the consumer roll below can tell whether the +# credentials the running pods loaded are still current. Credential bytes flow straight from +# kubectl into the digest through the pipe; the deploy shell never holds them in a variable +# or argument. +compute_database_connection_checksum() +{ + local namespace="$1" + shift + kubectl get secret "$@" -n "$namespace" \ + -o jsonpath='{range .items[*]}{.metadata.name}{":"}{.data}{"\n"}{end}' \ + | LC_ALL=C sort | sha256sum | cut -d' ' -f1 +} + +# Stamps the connection-Secret checksum onto each consumer Deployment's pod template. An +# unchanged checksum is a server-side no-op, so pods the preceding helm upgrade just started +# keep running; a changed checksum triggers exactly one rollout. The previous unconditional +# `rollout restart` here forced a second full startup of the heaviest workloads on every +# deploy, even when no credential changed. +roll_database_consumers_for_finalization() { local namespace="$1" local timeout="$2" + local checksum="$3" local command_status local deployment local deployment_resource - shift 2 + shift 3 for deployment in "$@"; do if deployment_resource="$(kubectl get "deployment/$deployment" -n "$namespace" --ignore-not-found -o name)"; then command_status=0 @@ -81,17 +100,18 @@ restart_database_consumers_for_finalization() command_status=$? fi if (( command_status != 0 )); then - err "Unable to inventory database consumer Deployment '$deployment' before restart." + err "Unable to inventory database consumer Deployment '$deployment' before the credential roll." return "$command_status" fi if [[ -n "$deployment_resource" ]]; then - if kubectl rollout restart "deployment/$deployment" -n "$namespace"; then + if kubectl patch "deployment/$deployment" -n "$namespace" --type merge \ + -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"opencrane.ai/database-connection-checksum\":\"$checksum\"}}}}}"; then command_status=0 else command_status=$? fi if (( command_status != 0 )); then - err "Unable to restart database consumer Deployment '$deployment'." + err "Unable to stamp the database connection checksum on Deployment '$deployment'." return "$command_status" fi fi @@ -103,7 +123,7 @@ restart_database_consumers_for_finalization() command_status=$? fi if (( command_status != 0 )); then - err "Unable to inventory database consumer Deployment '$deployment' after restart." + err "Unable to inventory database consumer Deployment '$deployment' after the credential roll." return "$command_status" fi if [[ -n "$deployment_resource" ]]; then @@ -113,7 +133,7 @@ restart_database_consumers_for_finalization() command_status=$? fi if (( command_status != 0 )); then - err "Database consumer Deployment '$deployment' did not complete its restart." + err "Database consumer Deployment '$deployment' did not complete its credential roll." return "$command_status" fi fi diff --git a/apps/_infra/deploy-k8s/platform/k8s-deploy.sh b/apps/_infra/deploy-k8s/platform/k8s-deploy.sh index 93828da79..5e4cd4c8b 100755 --- a/apps/_infra/deploy-k8s/platform/k8s-deploy.sh +++ b/apps/_infra/deploy-k8s/platform/k8s-deploy.sh @@ -978,11 +978,15 @@ helm_args+=( # --reuse-values → inherit last release verbatim (do NOT refresh chart defaults) # --reset-values → intentionally DROP prior overrides (start from chart defaults + this run) # A fresh install has nothing to reuse, so none of these apply. +RELEASE_PREEXISTED=0 +if helm status "$RELEASE" -n "$NAMESPACE" >/dev/null 2>&1; then + RELEASE_PREEXISTED=1 +fi if [[ -n "$REUSE_VALUES" ]]; then helm_args+=(--reuse-values) elif [[ -n "$RESET_VALUES" ]]; then helm_args+=(--reset-values) -elif helm status "$RELEASE" -n "$NAMESPACE" >/dev/null 2>&1; then +elif [[ "$RELEASE_PREEXISTED" == "1" ]]; then log "Existing release '$RELEASE' — using --reset-then-reuse-values so prior overrides are not silently dropped (pass --reset-values to start from chart defaults instead)." helm_args+=(--reset-then-reuse-values) fi @@ -1001,8 +1005,20 @@ fi append_authoritative_qualified_release_image_helm_args append_authoritative_cognee_image_helm_args run_opencrane_finalization_stage helm "${helm_args[@]}" || exit $? -run_opencrane_finalization_stage restart_database_consumers_for_finalization "$NAMESPACE" "$TIMEOUT" \ - "${RELEASE}-opencrane-server" "${RELEASE}-litellm" "${RELEASE}-mcp-gateway" || exit $? +# The database consumers load their connection Secrets at startup, and Helm does not roll pods +# when only a Secret published outside the chart changed. Stamping the Secret checksum onto the +# pod templates rolls the consumers exactly when the credentials changed, instead of restarting +# pods the upgrade above just started. A fresh install skips the roll entirely: its pods were +# born after this run published the Secrets, so there is nothing to propagate, and stamping +# mid-first-rollout forced a second boot of the heaviest workloads. +if [[ "$RELEASE_PREEXISTED" == "1" ]]; then + DATABASE_CONNECTION_CHECKSUM="$(compute_database_connection_checksum "$NAMESPACE" \ + "$POSTGRES_APP_SECRET" "$OBOT_POSTGRES_APP_SECRET" "$LITELLM_POSTGRES_APP_SECRET" \ + "$POSTGRES_ADMIN_APP_SECRET")" || exit $? + run_opencrane_finalization_stage roll_database_consumers_for_finalization "$NAMESPACE" "$TIMEOUT" \ + "$DATABASE_CONNECTION_CHECKSUM" \ + "${RELEASE}-opencrane-server" "${RELEASE}-litellm" "${RELEASE}-mcp-gateway" || exit $? +fi # 4. Wait for the core workloads. The database schema was created by CNPG initdb or converged by # the bounded deployment-owned migration Job; application startup never mutates it. diff --git a/apps/_infra/deploy-k8s/platform/tests/database-migration-deploy-contract.sh b/apps/_infra/deploy-k8s/platform/tests/database-migration-deploy-contract.sh index 1313aab28..e0ffd8876 100755 --- a/apps/_infra/deploy-k8s/platform/tests/database-migration-deploy-contract.sh +++ b/apps/_infra/deploy-k8s/platform/tests/database-migration-deploy-contract.sh @@ -319,9 +319,13 @@ export SUCCESS_CALLS deployment_name="${2#deployment/}" printf 'inventory %s\n' "$deployment_name" >>"$SUCCESS_CALLS" printf 'deployment.apps/%s\n' "$deployment_name" - elif [[ "$1 $2" == "rollout restart" ]]; then - deployment_name="${3#deployment/}" - printf 'restart %s\n' "$deployment_name" >>"$SUCCESS_CALLS" + elif [[ "$1 $2" == "patch deployment/"* ]]; then + deployment_name="${2#deployment/}" + if [[ "$*" != *'opencrane.ai/database-connection-checksum'* || "$*" != *'checksum-value'* ]]; then + printf 'unexpected finalization patch payload: %s\n' "$*" >&2 + return 1 + fi + printf 'patch %s\n' "$deployment_name" >>"$SUCCESS_CALLS" elif [[ "$1 $2" == "rollout status" ]]; then deployment_name="${3#deployment/}" printf 'rollout %s\n' "$deployment_name" >>"$SUCCESS_CALLS" @@ -342,8 +346,8 @@ export SUCCESS_CALLS BOUNDARY_PHASE=finalization run_opencrane_finalization_stage helm upgrade opencrane /chart \ --set clustertenantManager.replicas=2 --set migrationFence.active=false - run_opencrane_finalization_stage restart_database_consumers_for_finalization opencrane 37 \ - opencrane-opencrane-server opencrane-litellm + run_opencrane_finalization_stage roll_database_consumers_for_finalization opencrane 37 \ + checksum-value opencrane-opencrane-server opencrane-litellm run_opencrane_finalization_stage wait_for_final_deployment_if_present opencrane-clustertenant-manager run_opencrane_finalization_stage _wait_for_release_certificate run_opencrane_finalization_stage _post_deploy_verify @@ -362,9 +366,9 @@ printf '%s\n' \ helm-capture-fenced \ helm-unfence \ 'inventory opencrane-opencrane-server' \ - 'restart opencrane-opencrane-server' \ + 'patch opencrane-opencrane-server' \ 'inventory opencrane-litellm' \ - 'restart opencrane-litellm' \ + 'patch opencrane-litellm' \ 'inventory opencrane-opencrane-server' \ 'rollout opencrane-opencrane-server' \ 'inventory opencrane-litellm' \ @@ -844,7 +848,7 @@ set +e kubectl() { return 29; } helm() { printf 'helm %s\n' "$*" >>"$FINAL_CALLS"; } err() { :; } - run_opencrane_finalization_stage restart_database_consumers_for_finalization opencrane 37 opencrane-server + run_opencrane_finalization_stage roll_database_consumers_for_finalization opencrane 37 checksum-value opencrane-server ) final_inventory_status=$? set -e diff --git a/apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh b/apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh index b8661fead..70ab44893 100755 --- a/apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh +++ b/apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh @@ -153,11 +153,19 @@ _build_image() local image="$2" local dockerfile="$3" local cache_arguments=() - if [[ -n "${ACTIONS_RUNTIME_TOKEN:-}" \ - && ( -n "${ACTIONS_RESULTS_URL:-}" || -n "${ACTIONS_CACHE_URL:-}" ) ]]; then - cache_arguments=( - --cache-from "type=gha,scope=${project},timeout=2m" - ) + # CI shares registry layer caches per deployable with the publish jobs (see BUILD_CACHE_IMAGE + # in docker.yml). SMOKE_BUILD_CACHE is the trusted cache that integration pushes maintain; + # SMOKE_BUILD_CACHE_UNTRUSTED adds the pull-request cache as a second read source; and + # SMOKE_BUILD_CACHE_EXPORT names where this run may write its layers, so the next push builds + # warm. Local runs leave all three unset and build without a remote cache. + if [[ -n "${SMOKE_BUILD_CACHE:-}" ]]; then + cache_arguments+=(--cache-from "type=registry,ref=${SMOKE_BUILD_CACHE}:${project}") + fi + if [[ -n "${SMOKE_BUILD_CACHE_UNTRUSTED:-}" ]]; then + cache_arguments+=(--cache-from "type=registry,ref=${SMOKE_BUILD_CACHE_UNTRUSTED}:${project}") + fi + if [[ -n "${SMOKE_BUILD_CACHE_EXPORT:-}" ]]; then + cache_arguments+=(--cache-to "type=registry,ref=${SMOKE_BUILD_CACHE_EXPORT}:${project},mode=max") fi echo "[develop-smoke] Building $image" _retry 3 docker buildx build --load --file "$ROOT_DIR/$dockerfile" --tag "$image" \ @@ -377,6 +385,12 @@ EOF _wait_for_job "$job_name" } +# Proves the public health report is complete and every service the smoke can provision is +# healthy. Model routing is the one exception: CI holds no provider credentials, so LiteLLM +# serves an empty estate and the models probe reports unavailable. Seeding a placeholder key +# instead made the server fetch a BYOK Secret through the API server and exit fatally when that +# call failed, so the report is asserted as-is and models is allowed to be unavailable. Reporting +# an unconfigured estate as disabled rather than unavailable is tracked separately. _assert_ingress_health() { local health_url="https://${CONTROL_PLANE_HOST}:8443/healthz" @@ -385,10 +399,12 @@ _assert_ingress_health() until response="$(curl --connect-timeout 2 --max-time 5 --fail --silent --show-error --insecure \ --resolve "${CONTROL_PLANE_HOST}:8443:127.0.0.1" "$health_url" 2>/dev/null)" \ && jq -e ' - .status == "ok" - and .ready == true + .ready == true and (.services | keys == ["api", "channels", "database", "files", "integrations", "memory", "models"]) - and ([.services[]] | all(. == "available" or . == "disabled")) + and ([.services | to_entries[] | select(.key != "models") | .value] + | all(. == "available" or . == "disabled")) + and (.services.models == "available" or .services.models == "unavailable") + and (.status == "ok" or (.status == "degraded" and .services.models != "available")) ' >/dev/null <<<"$response"; do if [[ $(date +%s) -ge "$deadline" ]]; then echo "[develop-smoke] Timed out waiting for the complete public health report at $health_url; last response: $response" >&2 @@ -481,10 +497,6 @@ export OPENCRANE_OIDC_SESSION_SECRET="$(_random_secret)" # production deploy path still requires a UI digest; this explicit escape keeps the smoke honest. export OPENCRANE_ALLOW_TAG_FLOAT=1 export TIMEOUT_SECONDS -# The public health report only turns "models" available once LiteLLM lists a routable model. -# Seeding the initial provider with a placeholder key is safe here: registration writes the -# LiteLLM model row without calling the provider, and the health probe lists the estate. -export OPENCRANE_INITIAL_MODEL_API_KEY="sk-develop-smoke-placeholder" # Exercise the production wrapper's required contact and first-owner inputs. The disposable `.test` # host cannot complete public ACME, so the final --set flags deliberately restore its local issuer. "$ROOT_DIR/apps/_infra/deploy-k8s/deploy.sh" \ @@ -496,7 +508,6 @@ export OPENCRANE_INITIAL_MODEL_API_KEY="sk-develop-smoke-placeholder" --release "$RELEASE_NAME" \ --release-version "$(jq -r '.version' "$ROOT_DIR/package.json")" \ --from-release-version fresh \ - --initial-model-provider openai \ --image-tag develop-smoke \ --cognee-tag develop-smoke \ --storage-class "$SMOKE_STORAGE_CLASS" \ diff --git a/apps/agent-controller/helm/Chart.yaml b/apps/agent-controller/helm/Chart.yaml index a4185d84c..67addca00 100644 --- a/apps/agent-controller/helm/Chart.yaml +++ b/apps/agent-controller/helm/Chart.yaml @@ -3,4 +3,4 @@ name: opencrane-agent-controller description: App-owned named-template library for the personal-runtime workload controller. type: library version: 0.9.2 -appVersion: "0.9.2" +appVersion: "0.9.1" diff --git a/apps/agent-controller/package.json b/apps/agent-controller/package.json index 27f4e729b..f01ab5a04 100644 --- a/apps/agent-controller/package.json +++ b/apps/agent-controller/package.json @@ -1,6 +1,6 @@ { "name": "@opencrane/agent-controller", - "version": "0.9.2", + "version": "0.9.1", "private": true, "license": "AGPL-3.0-or-later", "type": "module", diff --git a/apps/agent-controller/project.json b/apps/agent-controller/project.json index c0972aa71..f4badb9bb 100644 --- a/apps/agent-controller/project.json +++ b/apps/agent-controller/project.json @@ -2,7 +2,7 @@ "name": "agent-controller", "$schema": "../../node_modules/nx/schemas/project-schema.json", "projectType": "application", - "metadata": { "release": { "adaptedVersion": "0.9.2" } }, + "metadata": { "release": { "adaptedVersion": "0.9.1" } }, "sourceRoot": "apps/agent-controller/src", "tags": ["type:app", "layer:entrypoint", "scope:agent-controller"], "targets": { diff --git a/apps/artifact-preprocessor/helm/Chart.yaml b/apps/artifact-preprocessor/helm/Chart.yaml index 8ead21564..35f2de7c0 100644 --- a/apps/artifact-preprocessor/helm/Chart.yaml +++ b/apps/artifact-preprocessor/helm/Chart.yaml @@ -3,4 +3,4 @@ name: opencrane-artifact-preprocessor description: App-owned named-template library for the isolated PDF preprocessing worker. type: library version: 0.9.2 -appVersion: "0.9.2" +appVersion: "0.9.1" diff --git a/apps/artifact-preprocessor/package.json b/apps/artifact-preprocessor/package.json index e5441e830..9b52dffa8 100644 --- a/apps/artifact-preprocessor/package.json +++ b/apps/artifact-preprocessor/package.json @@ -1,6 +1,6 @@ { "name": "@opencrane/artifact-preprocessor", - "version": "0.9.2", + "version": "0.9.1", "private": true, "license": "AGPL-3.0-or-later", "type": "module", diff --git a/apps/artifact-preprocessor/project.json b/apps/artifact-preprocessor/project.json index 53cc87653..2c43e76c3 100644 --- a/apps/artifact-preprocessor/project.json +++ b/apps/artifact-preprocessor/project.json @@ -2,7 +2,7 @@ "name": "artifact-preprocessor", "$schema": "../../node_modules/nx/schemas/project-schema.json", "projectType": "application", - "metadata": { "release": { "adaptedVersion": "0.9.2" } }, + "metadata": { "release": { "adaptedVersion": "0.9.1" } }, "sourceRoot": "apps/artifact-preprocessor/src", "tags": ["type:app", "layer:entrypoint", "scope:artifacts"], "targets": { diff --git a/apps/artifact-scanner/helm/Chart.yaml b/apps/artifact-scanner/helm/Chart.yaml index ed6602adc..10ed5e8c4 100644 --- a/apps/artifact-scanner/helm/Chart.yaml +++ b/apps/artifact-scanner/helm/Chart.yaml @@ -3,4 +3,4 @@ name: opencrane-artifact-scanner description: App-owned named-template library for the isolated malware-scanning worker. type: library version: 0.9.2 -appVersion: "0.9.2" +appVersion: "0.9.1" diff --git a/apps/artifact-scanner/package.json b/apps/artifact-scanner/package.json index e9dc81c95..a534e2e93 100644 --- a/apps/artifact-scanner/package.json +++ b/apps/artifact-scanner/package.json @@ -1,6 +1,6 @@ { "name": "@opencrane/artifact-scanner", - "version": "0.9.2", + "version": "0.9.1", "private": true, "license": "AGPL-3.0-or-later", "type": "module", diff --git a/apps/artifact-scanner/project.json b/apps/artifact-scanner/project.json index 9ef0a3590..889ced90a 100644 --- a/apps/artifact-scanner/project.json +++ b/apps/artifact-scanner/project.json @@ -2,7 +2,7 @@ "name": "artifact-scanner", "$schema": "../../node_modules/nx/schemas/project-schema.json", "projectType": "application", - "metadata": { "release": { "adaptedVersion": "0.9.2" } }, + "metadata": { "release": { "adaptedVersion": "0.9.1" } }, "sourceRoot": "apps/artifact-scanner/src", "tags": ["type:app", "layer:entrypoint", "scope:artifacts"], "targets": { diff --git a/apps/artifact-service/deploy/Dockerfile b/apps/artifact-service/deploy/Dockerfile index 3ee11e0a3..2db57730f 100644 --- a/apps/artifact-service/deploy/Dockerfile +++ b/apps/artifact-service/deploy/Dockerfile @@ -5,7 +5,9 @@ WORKDIR /app COPY package.json package-lock.json tsconfig.json nx.json ./ COPY libs libs COPY apps/artifact-service/package.json apps/artifact-service/project.json apps/artifact-service/tsconfig.json apps/artifact-service/ -RUN npm ci +# The cache mount keeps npm's download cache out of the image layers and shares it +# between stages, so the runtime-stage install reuses the packages this stage fetched. +RUN --mount=type=cache,target=/root/.npm npm ci COPY apps/artifact-service/src apps/artifact-service/src RUN npx nx run artifact-service:build @@ -16,7 +18,7 @@ WORKDIR /app ENV NODE_ENV=production COPY package.json package-lock.json ./ COPY apps/artifact-service/package.json apps/artifact-service/ -RUN npm ci --omit=dev --workspace=apps/artifact-service +RUN --mount=type=cache,target=/root/.npm npm ci --omit=dev --workspace=apps/artifact-service COPY --from=build /app/dist/apps/artifact-service dist/apps/artifact-service diff --git a/apps/artifact-service/helm/Chart.yaml b/apps/artifact-service/helm/Chart.yaml index 02be92ea8..4e58167e7 100644 --- a/apps/artifact-service/helm/Chart.yaml +++ b/apps/artifact-service/helm/Chart.yaml @@ -3,4 +3,4 @@ name: opencrane-artifact-service description: App-owned named-template library for canonical artifact bytes. type: library version: 0.8.0 -appVersion: "0.8.0" +appVersion: "0.9.2" diff --git a/apps/artifact-service/package.json b/apps/artifact-service/package.json index 31bf778bc..75835c186 100644 --- a/apps/artifact-service/package.json +++ b/apps/artifact-service/package.json @@ -1,6 +1,6 @@ { "name": "@opencrane/artifact-service", - "version": "0.8.0", + "version": "0.9.2", "private": true, "license": "AGPL-3.0-or-later", "type": "module", diff --git a/apps/artifact-service/project.json b/apps/artifact-service/project.json index cbd580ec8..86df09dac 100644 --- a/apps/artifact-service/project.json +++ b/apps/artifact-service/project.json @@ -2,7 +2,7 @@ "name": "artifact-service", "$schema": "../../node_modules/nx/schemas/project-schema.json", "projectType": "application", - "metadata": { "release": { "adaptedVersion": "0.8.0" } }, + "metadata": { "release": { "adaptedVersion": "0.9.2" } }, "sourceRoot": "apps/artifact-service/src", "tags": ["type:app", "layer:entrypoint", "scope:artifacts"], "targets": { diff --git a/apps/channel-proxy/deploy/Dockerfile b/apps/channel-proxy/deploy/Dockerfile index 9d3cdef70..acb50b84f 100644 --- a/apps/channel-proxy/deploy/Dockerfile +++ b/apps/channel-proxy/deploy/Dockerfile @@ -5,7 +5,9 @@ WORKDIR /app COPY package.json package-lock.json tsconfig.json nx.json ./ COPY libs libs COPY apps/channel-proxy/package.json apps/channel-proxy/project.json apps/channel-proxy/tsconfig.json apps/channel-proxy/ -RUN npm ci +# The cache mount keeps npm's download cache out of the image layers and shares it +# between stages, so the runtime-stage install reuses the packages this stage fetched. +RUN --mount=type=cache,target=/root/.npm npm ci COPY apps/channel-proxy/src apps/channel-proxy/src RUN npx nx run channel-proxy:build @@ -16,7 +18,7 @@ WORKDIR /app ENV NODE_ENV=production COPY package.json package-lock.json ./ COPY apps/channel-proxy/package.json apps/channel-proxy/ -RUN npm ci --omit=dev --workspace=apps/channel-proxy +RUN --mount=type=cache,target=/root/.npm npm ci --omit=dev --workspace=apps/channel-proxy COPY --from=build /app/dist/apps/channel-proxy dist/apps/channel-proxy diff --git a/apps/memory-gateway/deploy/Dockerfile b/apps/memory-gateway/deploy/Dockerfile index a5b47423c..4be4824d5 100644 --- a/apps/memory-gateway/deploy/Dockerfile +++ b/apps/memory-gateway/deploy/Dockerfile @@ -5,7 +5,9 @@ WORKDIR /app COPY package.json package-lock.json tsconfig.json nx.json ./ COPY libs libs COPY apps/memory-gateway/package.json apps/memory-gateway/project.json apps/memory-gateway/tsconfig.json apps/memory-gateway/ -RUN npm ci +# The cache mount keeps npm's download cache out of the image layers and shares it +# between stages, so the runtime-stage install reuses the packages this stage fetched. +RUN --mount=type=cache,target=/root/.npm npm ci COPY apps/memory-gateway/src apps/memory-gateway/src RUN npx nx run memory-gateway:build @@ -16,7 +18,7 @@ WORKDIR /app ENV NODE_ENV=production COPY package.json package-lock.json ./ COPY apps/memory-gateway/package.json apps/memory-gateway/ -RUN npm ci --omit=dev --workspace=apps/memory-gateway +RUN --mount=type=cache,target=/root/.npm npm ci --omit=dev --workspace=apps/memory-gateway # Shared conversation contracts keep these packages external in the compiled gateway bundle. # Resolve their public runtime entrypoints now so a production image cannot build without them. RUN node --input-type=module --eval \ diff --git a/apps/opencrane-ui/deploy/Dockerfile b/apps/opencrane-ui/deploy/Dockerfile index 7c327a4af..0446f1b9c 100644 --- a/apps/opencrane-ui/deploy/Dockerfile +++ b/apps/opencrane-ui/deploy/Dockerfile @@ -19,7 +19,9 @@ COPY package.json package-lock.json tsconfig.json tsconfig.frontend.json nx.json COPY libs libs COPY apps/opencrane-ui apps/opencrane-ui # npm workspaces install: root package.json's "workspaces" field includes apps/* and website -RUN npm ci +# The cache mount keeps npm's download cache out of the image layers and shares it +# between stages, so the runtime-stage install reuses the packages this stage fetched. +RUN --mount=type=cache,target=/root/.npm npm ci RUN npx nx build opencrane-ui diff --git a/apps/opencrane/deploy/Dockerfile b/apps/opencrane/deploy/Dockerfile index 4884bd25d..a3d650947 100644 --- a/apps/opencrane/deploy/Dockerfile +++ b/apps/opencrane/deploy/Dockerfile @@ -17,7 +17,9 @@ COPY apps/opencrane/scripts apps/opencrane/scripts COPY apps/opencrane/package.json apps/opencrane/tsconfig.json apps/opencrane/prisma.config.ts apps/opencrane/ COPY apps/opencrane/prisma apps/opencrane/prisma # npm workspaces install: root package.json's "workspaces" field includes apps/* and website -RUN npm ci +# The cache mount keeps npm's download cache out of the image layers and shares it +# between stages, so the runtime-stage install reuses the packages this stage fetched. +RUN --mount=type=cache,target=/root/.npm npm ci COPY apps/opencrane/src apps/opencrane/src # Generate Prisma client, build dependencies first, then the app @@ -39,7 +41,7 @@ COPY package.json package-lock.json ./ # database baseline are build/provisioning inputs; server startup never mutates schema. COPY apps/opencrane/package.json apps/opencrane/ # Install only the app workspace's runtime dependencies -RUN npm ci --omit=dev --workspace=apps/opencrane +RUN --mount=type=cache,target=/root/.npm npm ci --omit=dev --workspace=apps/opencrane # Shared conversation contracts keep these packages external in the compiled server bundle. # Resolve their public runtime entrypoints now so a production image cannot build without them. RUN node --input-type=module --eval \ diff --git a/docs/README.md b/docs/README.md index 0f40cfcff..8831baaf2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ belong in `CHANGELOG.md`, while only lasting design contracts remain under `docs | Folder | What it holds | |--------|---------------| +| [`ci-and-deploy.md`](ci-and-deploy.md) | The CI pipeline, its caching layers, the deploy engine, operational warnings, and the release-version and migration process. | | [`agents/`](agents/) | Canonical agent guidance, indexed by [`AGENTS.md`](../AGENTS.md). | | [`adr/`](adr/) | Accepted architecture decisions and the exact clauses they supersede. | | [`design/`](design/) | Durable product and platform contracts grounded in current implementations. | diff --git a/docs/agents/versioning.md b/docs/agents/versioning.md index 38bf2717e..0041a2fb3 100644 --- a/docs/agents/versioning.md +++ b/docs/agents/versioning.md @@ -5,11 +5,12 @@ ## Version authorities The root [`package.json`](../../package.json) version names the current repository train. Every Nx -application records `metadata.release.adaptedVersion`: the exact repository version in which that -application's production or deployment contract was last adapted, either directly or through a -changed project in its Nx dependency graph. Unchanged applications -retain their older value. Existing app `package.json` versions and app-owned Helm chart versions are -checked mirrors, never competing authorities. +application records `metadata.release.adaptedVersion`: the exact repository version in which files +under that application's own root last changed. An application whose own files did not change keeps +its latest released value, even when a shared library, the root dependency set, or the lockfile +moved underneath it — published images are pinned by commit SHA, so those shared changes reach the +application without a per-application stamp. Existing app `package.json` versions and app-owned +Helm chart versions are checked mirrors, never competing authorities. The initial `adoptionBaseline` is the explicit exception: it stamps the observed fleet composition at the train where this ledger was introduced, because older per-app adaptation history was not @@ -18,10 +19,10 @@ immutable. Once a repository-version tag exists, production or deployment compos change under that same version. [`releases/`](../../releases/README.md) is the immutable composition history. Each root version maps -the app, chart, and database schema versions known to work together. Use the Nx project graph to -start from directly changed project roots and stamp the applications that own those roots or depend -on them. Do not equate Nx's broader `affected` result with adaptation: named inputs such as shared -configuration can mark an app affected without changing its production or deployment contract. +the app, chart, and database schema versions known to work together. Stamp exactly the applications +that own a changed file under their project root. Do not equate Nx's `affected` result with +adaptation: a dependency-graph or shared-configuration change can mark an app affected without +requiring a new stamp. This follows Nx's independent-release principle—projects retain their own last meaningful version— without enabling `nx release` as a second version authority. Nx @@ -46,13 +47,13 @@ manifest contract; do not introduce parallel version files. predecessor database identity and may carry only that predecessor's exact source. This reuses the already reviewed SQL identity; it does not create a skipped-version migration or permit multi-hop carry-forward. -- A directly changed application, and an application depending on a changed project, is stamped to - the current full root version. This is a compatibility stamp, not a claim that every application - releases in lockstep. -- A semantic root dependency or lockfile change stamps every application because the shared runtime - boundary cannot be mapped safely to a smaller owner set. Root/version-only mirror edits do not. +- A directly changed application is stamped to the current full root version. This is a + compatibility stamp, not a claim that every application releases in lockstep. +- Shared library, root dependency, and lockfile changes stamp nothing on their own: the affected + applications keep their latest released version and receive the shared change through their + SHA-pinned images. - Never rewrite an older release manifest. Create the next manifest by carrying unchanged component - versions forward and updating only directly or dependency-adapted owners. + versions forward and updating only directly changed owners. ## Helm migrations diff --git a/docs/ci-and-deploy.md b/docs/ci-and-deploy.md new file mode 100644 index 000000000..c4acdaeb9 --- /dev/null +++ b/docs/ci-and-deploy.md @@ -0,0 +1,218 @@ +# CI and deployment + +This is the reference for how a change travels from a pull request to a running cluster: the CI +pipeline and its gates, the caching that keeps it fast, the deploy engine, the warnings that save +hours, and the release-version and migration process. The contributor-facing summary lives on the +website under Contributing; this file is the deeper repository-side reference. + +> See also: [`docs/agents/versioning.md`](agents/versioning.md) for the full release-version +> policy, [`docs/agents/infra.md`](agents/infra.md) for build/infra rules, and +> [`docs/agents/deploy-ledger.md`](agents/deploy-ledger.md) for the deploy fleet's cross-run notes. + +## The pipeline at a glance + +Three workflows run on every pull request: **Validate and publish affected deployables** +(`.github/workflows/docker.yml`, the main pipeline), **Enforce pull-request stack integrity** +(stacked-PR bookkeeping, under a minute), and **CodeQL** (static analysis, a few minutes). The +main pipeline is where the time goes: + +```mermaid +flowchart LR + P[prepare\ncalculate affected work\n~1-2 min] --> T[test\nbuild, test, lint,\npolicy guards\n~3-10 min] + P --> DB[database\nSQL authority suites,\nmigration proofs\n~2-4 min] + P --> A[api_contract\nOpenAPI + client sync\nonly when affected] + P --> S[storybook_visual\ncomponent contracts\nskips fast when\nnothing affected] + P --> K[develop_smoke\nk3d current-silo smoke\n~6-15 min] + P --> I[image_smoke\nper-image boot checks\n~1-2 min each] + T --> B[build-and-push\npublish affected images\npush events only] + DB --> B + A --> B + S --> B + K --> B + I --> B + B --> D[publish-develop-smoke-images\ncomplete the immutable\ndevelop image set] + K --> D +``` + +What each job owns: + +| Job | Purpose | Typical duration | +| --- | --- | --- | +| `prepare` | Computes the Nx affected graph, the deployable matrix, the guard comparison base, and whether the k3d smoke can be skipped. | 1–2 min | +| `test` | Builds, tests, and lints affected projects, and runs every policy guard: workload ownership, agent-domain boundary, mechanical style, module growth, release versioning, Prisma boundaries, config-docs coverage, dependency boundaries. | 3–10 min | +| `database` | Everything PostgreSQL-bound, beside `test` instead of inside it: the migration contracts and convergence proofs, the generated client, the target baseline, and the SQL authority suites. | 2–4 min | +| `api_contract` | Rebuilds the server and proves the OpenAPI reference and generated client are in sync. Runs only when the API contract changed. | skipped, or ~3–5 min | +| `storybook_visual` | Storybook build/behaviour/visual contracts for affected frontend projects, on cached Chromium. Runs beside `test`, not after it. | seconds when nothing affected; ~5 min otherwise | +| `develop_smoke` | Boots a disposable k3d cluster, deploys the full current silo through the real deploy scripts, and proves database isolation, TLS ingress, and workload health. The long pole of the pipeline. | 6–15 min | +| `image_smoke` | Boots individual images that declare an `image-smoke` target and checks they come up. | 1–2 min per image | +| `build-and-push` | Builds every affected deployable image and publishes `sha-` tags on push events. On pull requests it builds without pushing, as a proof. | 1–5 min warm | +| `publish-develop-smoke-images` | On develop pushes, completes the immutable image set for the commit: reuses the exact validated base image where nothing changed, copies tags forward, builds only what is missing. | seconds per image | + +A deploy to a live cluster then consumes the published `sha-` images — CI must be green +for the exact SHA before any deploy (the deploy scripts do not build images). + +## Caching layers + +Every job runs on a fresh runner, so anything not cached is paid on every run. The pipeline uses +these caches, from cheapest to most impactful: + +| Cache | Backend | Key | Used by | +| --- | --- | --- | --- | +| npm download cache | `actions/setup-node` | lockfile hash | all jobs | +| `node_modules` | `actions/cache` | lockfile hash | all jobs (skips `npm ci` entirely on a hit) | +| Nx computation cache | `actions/cache` (`.nx/cache`) | lockfile hash + commit, with prefix restore | `test`, `api_contract`, `storybook_visual` | +| Playwright Chromium | `actions/cache` (`~/.cache/ms-playwright`) | lockfile hash | `storybook_visual` | +| Docker image layers | **registry** (`ghcr.io//opencrane-buildcache:`) | buildx layer graph | `develop_smoke`, `build-and-push`, `publish-develop-smoke-images` | +| npm inside Dockerfiles | BuildKit cache mount (`/root/.npm`) | shared between build and runtime stages within one build | all Node images | + +Design decisions worth knowing: + +- **Image layers cache in the registry, not in the Actions cache.** The repository's 10GB Actions + cache quota was permanently over budget — single buildkit blobs reach ~700MB — so layer caches + were evicted almost immediately and every image built cold. Registry-backed caches + (`type=registry`) have no such quota, and moving them out also stops the node/Nx caches from + being evicted alongside. +- **The layer cache has a trust boundary.** Two cache repositories exist: + `opencrane-buildcache` (trusted) is written only by integration-branch pushes and is the only + cache a publishable build reads; `opencrane-buildcache-pr` is written and read by + same-repository pull-request validation builds. A layer produced by unreviewed code therefore + never becomes part of a published image. Fork pull requests read the trusted cache and write + nothing (their token is read-only). +- **Chromium installs from cache, and the apt step is capped.** The browser binaries restore from + the Actions cache; the apt-driven `--with-deps` install runs only on a cold cache and carries a + ten-minute step timeout, because a hung apt once held an untimed job (and its runner) for hours. +- **Every job has a `timeout-minutes`.** A hung job does not only lose its own time — it occupies + a concurrent-runner slot and starves every queued run behind it. + +## The k3d smoke and its skip proof + +The `develop_smoke` job is the pipeline's long pole and its most valuable gate: it exercises the +real deploy path end to end on every develop push. Two mechanisms keep its cost down: + +1. **Image reuse by digest.** Projects the affected graph did not select are pulled from the + validated base commit's published images instead of being rebuilt + (`develop-smoke.sh:_pull_baseline_image`). +2. **The skip proof.** A pull request skips the k3d smoke entirely when its exact base SHA + already completed the same k3d job successfully (`scripts/develop-smoke-baseline.mjs`). Any + ambiguity — API failure, missing proof, affected containers — deliberately runs the smoke. + +The skip proof is why **keeping develop green is a speed feature**: a red develop push means no +validated base exists, so every subsequent pull request pays the full smoke. A failure streak on +develop taxes every open PR in the repository. + +The smoke's storage tier also varies: touching `k8s-deploy.sh`, the smoke script, the workflow, +or `apps/postgres/` selects the `full` tier, which additionally exercises CSI volume expansion +(several extra minutes). Other changes run the `fast` tier. + +## Deploy infrastructure + +All cluster mutation goes through app-owned scripts — never bare `helm upgrade` or `kubectl +apply` against a live cluster: + +```mermaid +flowchart TD + A[apps/_infra/deploy-k8s/deploy.sh\nsilo profile: flags, presets] --> B[platform/k8s-deploy.sh\nthe install engine] + B --> C[current-chart-sources.sh\nhelm dependency build\nfrom Chart.lock] + B --> D[database-migration-orchestrator.sh\nCNPG cluster, databases,\nmigration + privileges Jobs] + B --> E[umbrella helm upgrade\nall app subcharts] + E --> F[database-release-finalization.sh\ncredential-checksum roll,\nrollout waits, cert wait] + F --> G[post-deploy-verify.sh\nlive health verification] +``` + +- `deploy.sh` installs one per-ClusterTenant silo: operator, channel proxy, LiteLLM, Cognee, + opencrane-ui, per-CT networking, and one app-owned PostgreSQL server with isolated logical + databases. Required flags: `--base-domain`, `--cluster-tenant`, `--acme-email`, + `--first-user-email`; fresh installs also need `--opencrane-ui-digest` and `--cognee-digest` + (immutable digests, never tags). +- Cluster-wide prerequisites (ingress-nginx, cert-manager, CloudNativePG) are installed once per + cluster by `bootstrap-prerequisites.sh` and are never part of a silo release. +- The PostgreSQL transition is resolved and schema-validated *before* the cluster is touched; + migration and privileges run as bounded Helm hook Jobs, with an automatic rollback path that + restores the exact fenced Helm revision on failure. +- After the umbrella upgrade, the engine stamps a checksum of the published database connection + Secrets onto the consumer Deployments (`opencrane-server`, `litellm`, `mcp-gateway`). An + unchanged checksum is a no-op; a changed one triggers exactly one rollout; and a fresh install + skips the roll entirely because its pods were born after the Secrets were published. (This + replaced an unconditional `rollout restart` that double-started the heaviest workloads on + every deploy.) + +## Warnings — read before deploying + +- **CI green first.** Confirm the `docker.yml` run for the exact SHA is green before deploying; + the deploy scripts pull published images and never build them. +- **`helm dependency build`, never `dependency update`, on the deploy path.** The engine resolves + subcharts from `Chart.lock` for reproducibility. `dependency update` re-resolves and can drift. + (Regenerating the lock/archives after a chart stamp is the one place `dependency update` is + correct — review the diff.) +- **A green `helm template`/CI render does not prove a live `helm upgrade` works.** Stateful + services need their PVC semantics, reconcile-retry, and Secret-change pod-roll trigger checked + before deploying — see the live-upgrade checklist in the deploy ledger. +- **Never deploy with floating tags.** Public releases require `sha-*` build tags or digests; the + qualified-release-image policy rejects `latest` and friends. Tag floating exists only for the + disposable local k3d smoke. +- **The tenant's openclaw version pin lives in `values.yaml`, not in code defaults.** +- **Watch the queue, not only the jobs.** The org has a fixed number of concurrent runners; a + workflow storm (or a hung job) can queue runs for 30+ minutes. If a run seems stuck before any + job started, it is queue starvation, not a slow job. +- **Develop red = every PR pays the k3d smoke.** See the skip proof above. Fixing develop first + is usually the fastest way to speed up everyone's PRs. +- **A cancelled develop push is normal.** The workflow's concurrency group replaces a queued push + run when a newer push arrives; only in-flight publishes are protected. + +## Release versions and migrations + +The full policy lives in [`docs/agents/versioning.md`](agents/versioning.md); this is the working +summary. + +- The root `package.json` version names the **repository train** (for example `0.9.2`). Each + train has an immutable manifest `releases/.json` recording every application's + `adaptedVersion`, chart version, and the database schema version that work together. +- **Only applications whose own files changed stamp to the root version.** An application that + changed only through a shared library, the root dependency set, or the lockfile keeps its + latest released version — published images are pinned by commit SHA, so the shared change + reaches it regardless. (Before this rule, every shared change failed CI until every manifest + entry was bumped by hand.) +- A directly changed application updates, together: its manifest entry, its `package.json` + version mirror, its `project.json` `metadata.release.adaptedVersion`, and its chart + `appVersion` where a chart exists. Version-only mirror edits are "stamp-only" and do not count + as changes themselves. +- A **changed chart** bumps its chart version to the root version and adds exactly one + `helm/migrations/-to-.json` transition; the umbrella's `Chart.lock` and packaged + archives are then regenerated and reviewed. +- A **database schema change** updates the clean target baseline and adds one adjacent, reviewed + SQL transition under `apps/opencrane/prisma/migrations/-to-/`, bound by digest. +- Adjacent minor trains (`0.8.x → 0.9.0`) are the only automatic transition. Patch, skipped-minor, + and major transitions require an approved `manualTransition` with a reason in the manifest. +- Once a version tag exists, that train's composition is frozen: any further change must advance + the train and create the next manifest. + +CI enforces all of this in the `test` job (`check:release-versioning`), diffed against the last +validated base, so a violation fails the PR — not the deploy. + +## Letting an AI agent manage your deployment + +The deploy path is scriptable end to end, which makes it a good fit for an agent-run loop (the +repository ships a `deploy` agent and a `/deploy-loop` skill that mutate clusters only through +the scripts above and triage every failure into a fix PR, an issue, or a design question). + +The one thing an agent must never see in plain text is credentials. The convention: + +- **`keys/` at the repository root is gitignored** (`/keys/*` in `.gitignore`). Put one secret + per file, named for what it is: + - `keys/initial-model-api-key` — the provider API key that seeds the first routable model. + The deploy reads it as an environment variable, never as a flag: + `OPENCRANE_INITIAL_MODEL_API_KEY="$(cat keys/initial-model-api-key)"` alongside + `--initial-model-provider `. + - `keys/zitadel-pat` — the Zitadel service-user PAT for organisation management once the + mode-scoped credential lands (tracked in the silo org-role issues); standalone silos get a + full-org credential, fleet-mode silos a claims-only one. +- The agent reads a key file straight into the environment of the one command that needs it and + never echoes it, logs it, or passes it as a command argument — the same custody rule the + scripts themselves follow (the API key is environment-only precisely to keep it out of command + history and Helm values). +- Everything else an agent needs is already non-secret: cluster context, base domain, tenant + name, image digests from the release manifest, and the deploy ledger for cross-run memory. + +With `keys/` populated, a fresh silo is one command the agent can compose, run, and verify — +and the post-deploy verification plus the run report tell it (and you) whether the cluster is +actually healthy. diff --git a/releases/0.9.2.json b/releases/0.9.2.json index ca2743238..d2d6b1224 100644 --- a/releases/0.9.2.json +++ b/releases/0.9.2.json @@ -15,15 +15,15 @@ "projects": { "@opencrane/website": { "root": "website", - "adaptedVersion": "0.9.1", - "packageVersion": "0.9.1" + "adaptedVersion": "0.9.2", + "packageVersion": "0.9.2" }, "agent-controller": { "root": "apps/agent-controller", - "adaptedVersion": "0.9.2", - "packageVersion": "0.9.2", + "adaptedVersion": "0.9.1", + "packageVersion": "0.9.1", "chartVersion": "0.9.2", - "chartAppVersion": "0.9.2" + "chartAppVersion": "0.9.1" }, "agent-runtime": { "root": "apps/agent-runtime", @@ -31,24 +31,24 @@ }, "artifact-preprocessor": { "root": "apps/artifact-preprocessor", - "adaptedVersion": "0.9.2", - "packageVersion": "0.9.2", + "adaptedVersion": "0.9.1", + "packageVersion": "0.9.1", "chartVersion": "0.9.2", - "chartAppVersion": "0.9.2" + "chartAppVersion": "0.9.1" }, "artifact-scanner": { "root": "apps/artifact-scanner", - "adaptedVersion": "0.9.2", - "packageVersion": "0.9.2", + "adaptedVersion": "0.9.1", + "packageVersion": "0.9.1", "chartVersion": "0.9.2", - "chartAppVersion": "0.9.2" + "chartAppVersion": "0.9.1" }, "artifact-service": { "root": "apps/artifact-service", - "adaptedVersion": "0.8.0", - "packageVersion": "0.8.0", + "adaptedVersion": "0.9.2", + "packageVersion": "0.9.2", "chartVersion": "0.8.0", - "chartAppVersion": "0.8.0" + "chartAppVersion": "0.9.2" }, "channel-proxy": { "root": "apps/channel-proxy", diff --git a/scripts/__tests__/affected-deployables.test.mjs b/scripts/__tests__/affected-deployables.test.mjs index 18a70d72e..82435db02 100644 --- a/scripts/__tests__/affected-deployables.test.mjs +++ b/scripts/__tests__/affected-deployables.test.mjs @@ -182,8 +182,9 @@ test("overlaps image preparation and imports one complete direct k3d batch", fun assert.match(smoke, /cert-manager jetstack\/cert-manager[\s\S]*?&[\s\S]*?CERT_MANAGER_INSTALL_PID=\$![\s\S]*?cnpg cnpg\/cloudnative-pg/u); assert.match(smoke, /if ! wait "\$CERT_MANAGER_INSTALL_PID"/u); assert.match(smoke, /docker buildx build --load/u); - assert.match(smoke, /--cache-from "type=gha,scope=\$\{project\},timeout=2m"/u); - assert.doesNotMatch(smoke, /--cache-to/u); + assert.match(smoke, /--cache-from "type=registry,ref=\$\{SMOKE_BUILD_CACHE\}:\$\{project\}"/u); + assert.match(smoke, /--cache-from "type=registry,ref=\$\{SMOKE_BUILD_CACHE_UNTRUSTED\}:\$\{project\}"/u); + assert.match(smoke, /--cache-to "type=registry,ref=\$\{SMOKE_BUILD_CACHE_EXPORT\}:\$\{project\},mode=max"/u); const imports = smoke.match(/k3d image import/g) ?? []; assert.equal(imports.length, 1); assert.match(smoke, /k3d image import "\$\{SMOKE_IMAGES\[@\]\}" --cluster "\$CLUSTER_NAME" --mode direct/u); @@ -212,7 +213,7 @@ test("keeps heavyweight remote qualification ahead of image publication", functi assert.match(workflow, /run: \.\/apps\/_infra\/deploy-k8s\/platform\/tests\/develop-smoke\.sh/u); assert.match(workflow, /inputs\.heavy_qualification == 'k3d'/u); assert.match(workflow, /inputs\.heavy_qualification == 'all'/u); - assert.match(workflow, /needs: \[prepare, test, develop_smoke, image_smoke\]/u); + assert.match(workflow, /needs: \[prepare, test, database, api_contract, storybook_visual, develop_smoke, image_smoke\]/u); assert.match(developSmokeJob[0], /needs: prepare/u); assert.match(developSmokeJob[0], /needs\.prepare\.outputs\.develop_smoke_can_skip != 'true'/u); assert.match(workflow, /continue-on-error: true[\s\S]*?run: node scripts\/develop-smoke-baseline\.mjs/u); @@ -230,13 +231,17 @@ test("keeps heavyweight remote qualification ahead of image publication", functi assert.match(developSmokeJob[0], /SMOKE_AFFECTED_PROJECTS: \$\{\{ needs\.prepare\.outputs\.develop_smoke_projects \}\}/u); assert.match(developSmokeJob[0], /SMOKE_BASE_SHA: \$\{\{ needs\.prepare\.outputs\.nx_base \}\}/u); assert.match(developSmokeJob[0], /SMOKE_STORAGE_MODE: \$\{\{ needs\.prepare\.outputs\.develop_smoke_storage_mode \}\}/u); - assert.match(developSmokeJob[0], /uses: crazy-max\/ghaction-github-runtime@04d248b84655b509d8c44dc1d6f990c879747487/u); + assert.match(developSmokeJob[0], /SMOKE_BUILD_CACHE: \$\{\{ env\.REGISTRY \}\}\/\$\{\{ github\.repository_owner \}\}\/\$\{\{ env\.BUILD_CACHE_IMAGE \}\}/u); + assert.match(developSmokeJob[0], /SMOKE_BUILD_CACHE_EXPORT: /u); assert.match(imageSmokeJob[0], /matrix: \$\{\{ fromJSON\(needs\.prepare\.outputs\.image_smokes\) \}\}/u); assert.match( imageSmokeJob[0], /IMAGE_SMOKE_PROJECT: \$\{\{ matrix\.project \}\}[\s\S]*?npx nx run "\$IMAGE_SMOKE_PROJECT:image-smoke"/u, ); - assert.match(workflow, /cache-from: type=gha,scope=\$\{\{ matrix\.project \}\}/u); + assert.match( + workflow, + /type=registry,ref=\$\{\{ env\.REGISTRY \}\}\/\$\{\{ github\.repository_owner \}\}\/\$\{\{ env\.BUILD_CACHE_IMAGE \}\}:\$\{\{ matrix\.project \}\}/u, + ); assert.match(workflow, /type=raw,value=sha-\$\{\{ github\.sha \}\}/u); assert.match(publishSmokeImagesJob[0], /github\.ref == 'refs\/heads\/develop'/u); assert.match(publishSmokeImagesJob[0], /matrix: \$\{\{ fromJSON\(needs\.prepare\.outputs\.develop_smoke_images\) \}\}/u); diff --git a/scripts/__tests__/release-versioning-check.test.mjs b/scripts/__tests__/release-versioning-check.test.mjs index 35568e66e..5736b867b 100644 --- a/scripts/__tests__/release-versioning-check.test.mjs +++ b/scripts/__tests__/release-versioning-check.test.mjs @@ -203,17 +203,17 @@ test("rejects a directly adapted project that retains an older stamp", async () { const fixture = _Fixture({ adaptedVersion: "0.6.2" }); const errors = await validateWorkspace(fixture.root, ["apps/example/src/index.ts"], fixture.graph); - assert.ok(errors.some((error) => error.includes("direct or dependency change"))); + assert.ok(errors.some((error) => error.includes("changed directly"))); }); test("counts test changes as direct application adaptations", async () => { const fixture = _Fixture({ adaptedVersion: "0.6.2" }); const errors = await validateWorkspace(fixture.root, ["apps/example/src/example.test.ts"], fixture.graph); - assert.ok(errors.some((error) => error.includes("direct or dependency change"))); + assert.ok(errors.some((error) => error.includes("changed directly"))); }); -test("uses the Nx dependency graph to stamp apps adapted through a changed library", async () => +test("keeps an application's stamp when only a depended-on library changed", async () => { const fixture = _Fixture({ adaptedVersion: "0.6.2" }); fixture.graph.nodes.contracts = { @@ -223,8 +223,7 @@ test("uses the Nx dependency graph to stamp apps adapted through a changed libra example: [{ source: "example", target: "contracts", type: "static" }], contracts: [], }; - const errors = await validateWorkspace(fixture.root, ["libs/contracts/src/index.ts"], fixture.graph); - assert.ok(errors.some((error) => error.includes("direct or dependency change"))); + assert.deepEqual(await validateWorkspace(fixture.root, ["libs/contracts/src/index.ts"], fixture.graph), []); }); test("rejects advancing an unaffected application's last-adapted version", async () => @@ -261,7 +260,7 @@ test("counts package and project configuration as direct adaptations", async () for (const file of ["apps/example/package.json", "apps/example/project.json"]) { const errors = await validateWorkspace(fixture.root, [file], fixture.graph); - assert.ok(errors.some((error) => error.includes("direct or dependency change"))); + assert.ok(errors.some((error) => error.includes("changed directly"))); } }); @@ -272,11 +271,10 @@ test("permits package and project stamp-only mirrors", async () => assert.deepEqual(await validateWorkspace(fixture.root, files, fixture.graph, files), []); }); -test("treats a semantic root dependency change as an adaptation of every application", async () => +test("keeps every application's stamp when only the root dependency set changed", async () => { const fixture = _Fixture({ adaptedVersion: "0.6.2" }); - const errors = await validateWorkspace(fixture.root, ["package.json"], fixture.graph); - assert.ok(errors.some((error) => error.includes("direct or dependency change"))); + assert.deepEqual(await validateWorkspace(fixture.root, ["package.json"], fixture.graph), []); }); test("permits root package and lockfile version-only mirrors", async () => diff --git a/scripts/release-versioning/core.mjs b/scripts/release-versioning/core.mjs index 6df13e942..236504799 100644 --- a/scripts/release-versioning/core.mjs +++ b/scripts/release-versioning/core.mjs @@ -94,47 +94,28 @@ function _ValidateTransition(manifest, errors) } } -function _AdaptedApplications(graph, changedFiles, stampOnlyFiles) +/** + * Selects the applications whose own files changed in this release. + * + * Only these applications must advance to the root version. An application that changed only + * through a shared library, the root dependency set, or the lockfile keeps its last released + * version: images are pinned by commit SHA at publication, so restating the root version on + * every untouched application recorded no extra information and failed CI on every shared + * change until each manifest entry was bumped by hand. + */ +function _DirectlyChangedApplications(graph, changedFiles, stampOnlyFiles) { const projectRoots = Object.entries(graph.nodes) .map(([name, node]) => [name, node.data.root]) .sort((left, right) => right[1].length - left[1].length); const touchedProjects = new Set(); - const rootRuntimeChanged = changedFiles.some((file) => - (file === "package.json" || file === "package-lock.json") && !stampOnlyFiles.has(file)); - if (rootRuntimeChanged) - { - for (const [name, node] of Object.entries(graph.nodes)) - if (node.data.projectType === "application") touchedProjects.add(name); - } for (const file of changedFiles) { if (stampOnlyFiles.has(file) || _IGNORED_TOUCH.some((pattern) => pattern.test(file))) continue; const owner = projectRoots.find(([, root]) => file === root || file.startsWith(`${root}/`)); if (owner) touchedProjects.add(owner[0]); } - const dependants = new Map(); - for (const [consumer, dependencies] of Object.entries(graph.dependencies ?? {})) - { - for (const dependency of dependencies) - { - const current = dependants.get(dependency.target) ?? []; - current.push(consumer); - dependants.set(dependency.target, current); - } - } - const affected = new Set(touchedProjects); - const queue = [...touchedProjects]; - while (queue.length > 0) - { - for (const dependant of dependants.get(queue.shift()) ?? []) - { - if (affected.has(dependant)) continue; - affected.add(dependant); - queue.push(dependant); - } - } - return new Set([...affected].filter((name) => graph.nodes[name]?.data.projectType === "application")); + return new Set([...touchedProjects].filter((name) => graph.nodes[name]?.data.projectType === "application")); } function _ValidateProject( @@ -175,7 +156,7 @@ function _ValidateProject( if (previousProject && compareSemver(project.adaptedVersion, previousProject.adaptedVersion) < 0) errors.push(`${name} adapted version regresses from '${previousProject.adaptedVersion}' to '${project.adaptedVersion}'`); if (directlyTouched && project.adaptedVersion !== rootVersion) - errors.push(`${name} is adapted by a direct or dependency change but remains at '${project.adaptedVersion}' instead of root '${rootVersion}'`); + errors.push(`${name} changed directly but remains at '${project.adaptedVersion}' instead of root '${rootVersion}'`); if (previousProject && !directlyTouched && project.adaptedVersion !== previousProject.adaptedVersion) errors.push(`${name} was not adapted but its version changed from '${previousProject.adaptedVersion}' to '${project.adaptedVersion}'`); const chartTouched = changedFiles.some((file) => file.startsWith(`${project.root}/helm/`) @@ -281,7 +262,7 @@ export async function validateWorkspace( if (previousManifest) errors.push(...validateReleaseManifest(previousManifest, "previous release manifest")); } const graph = suppliedGraph ?? await createProjectGraphAsync(); - const adaptedApplications = _AdaptedApplications(graph, changedFiles, stampOnlyFiles); + const adaptedApplications = _DirectlyChangedApplications(graph, changedFiles, stampOnlyFiles); const applicationNames = Object.entries(graph.nodes) .filter(([, node]) => node.data.projectType === "application") .map(([name]) => name) diff --git a/website/.vitepress/config.ts b/website/.vitepress/config.ts index 0e3a4d17c..325d90eff 100644 --- a/website/.vitepress/config.ts +++ b/website/.vitepress/config.ts @@ -51,6 +51,7 @@ export default defineConfig({ { text: 'API overview', link: '/reference/api-overview' }, ], }, + { text: 'Contributing', link: '/contributing/overview' }, { text: 'GitHub', link: REPO }, ], @@ -135,6 +136,17 @@ export default defineConfig({ { text: 'Running multiple instances', link: '/advanced/multi-instance' }, ], }, + { + text: 'Contributing', + collapsed: true, + items: [ + { text: 'Overview', link: '/contributing/overview' }, + { text: 'The CI pipeline', link: '/contributing/ci-pipeline' }, + { text: 'Deploying', link: '/contributing/deploying' }, + { text: 'Versions and migrations', link: '/contributing/versions-and-migrations' }, + { text: 'Letting an AI agent manage your deployment', link: '/contributing/ai-managed-deployment' }, + ], + }, ], socialLinks: [{ icon: 'github', link: REPO }], diff --git a/website/contributing/ai-managed-deployment.md b/website/contributing/ai-managed-deployment.md new file mode 100644 index 000000000..df6be6caa --- /dev/null +++ b/website/contributing/ai-managed-deployment.md @@ -0,0 +1,57 @@ +# Letting an AI agent manage your deployment + +The deploy path described in [Deploying](/contributing/deploying) is scriptable end to end, +which makes it a good fit for an agent-run loop. This page covers the credential convention that +keeps that safe, and the agent that uses it. + +> See also: [Deploying](/contributing/deploying) (the scripts the agent runs), [The CI pipeline]( +> /contributing/ci-pipeline) (what must be green before an agent deploys), and +> [Versions and migrations](/contributing/versions-and-migrations) (what the agent must resolve +> before mutating a cluster). + +The repository ships a `deploy` agent and a `/deploy-loop` skill that mutate clusters **only** +through the scripts covered in [Deploying](/contributing/deploying), and triage every failure +into a fix pull request, an issue, or a design question. The one thing an agent must never see +in plain text is a credential. + +## The `keys/` convention + +`keys/` at the repository root is gitignored (`/keys/*` in `.gitignore`). Put one secret per +file, named for what it is: + +| File | Contents | Consumed as | +| --- | --- | --- | +| `keys/initial-model-api-key` | The provider API key that seeds the first routable model | `OPENCRANE_INITIAL_MODEL_API_KEY="$(cat keys/initial-model-api-key)"`, alongside `--initial-model-provider ` | +| `keys/zitadel-pat` | The Zitadel service-user PAT for organisation management, once the mode-scoped credential lands 🔶 | Standalone silos get a full-org credential; fleet-mode silos get a claims-only one | + +::: warning +`keys/zitadel-pat` describes an upcoming credential shape, not something the deploy scripts +consume today. Treat it as an intended destination for the mode-scoped Zitadel credential, not a +currently wired flag. +::: + +## The custody rule + +The agent reads a key file straight into the environment of the one command that needs it, and +never echoes it, logs it, or passes it as a command argument — the same rule the deploy scripts +themselves follow. `OPENCRANE_INITIAL_MODEL_API_KEY` is environment-only precisely to keep the +key out of command history and Helm values. + +Everything else an agent needs is already non-secret: cluster context, base domain, tenant name, +image digests from the release manifest, and the deploy ledger for cross-run memory. + +## The deploy agent and `/deploy-loop` + +- The **`deploy` agent** mutates a cluster only through the deploy scripts, reads whatever it + needs for diagnosis (read-only `kubectl`, `helm status`, read-only SQL), and returns a + structured run report. +- The **`/deploy-loop` skill** orchestrates a full run: it resolves the exact release manifest + and database transition first, spawns one `deploy` agent, then triages every finding in the + report into a chart/script fix, a codebase issue, a data issue, or an infra/design question. + +With `keys/` populated, a fresh silo is one command an agent can compose, run and verify — and +the post-deploy verification plus the run report tell it, and you, whether the cluster is +actually healthy. + +Source: [`.claude/agents/deploy.md`](https://github.com/elewa-git/opencrane/blob/main/.claude/agents/deploy.md) +and [`.claude/commands/deploy-loop.md`](https://github.com/elewa-git/opencrane/blob/main/.claude/commands/deploy-loop.md). diff --git a/website/contributing/ci-pipeline.md b/website/contributing/ci-pipeline.md new file mode 100644 index 000000000..5cb1abdba --- /dev/null +++ b/website/contributing/ci-pipeline.md @@ -0,0 +1,142 @@ +# The CI pipeline + +Every pull request and every push to `develop` or `main` runs through **three GitHub Actions +workflows**. This page covers what each job gates and the caching that keeps the pipeline fast +enough to stay in the loop. + +> See also: [Contributing overview](/contributing/overview) (where this fits in the PR-to-cluster +> journey), [Deploying](/contributing/deploying) (what consumes the images this pipeline +> publishes), and [Versions and migrations](/contributing/versions-and-migrations) (the +> `check:release-versioning` gate enforced inside the `test` job). + +## The three workflows + +| Workflow | File | Purpose | Typical duration | +| --- | --- | --- | --- | +| Validate and publish affected deployables | [`docker.yml`](https://github.com/elewa-git/opencrane/blob/main/.github/workflows/docker.yml) | The main pipeline: build, test, smoke and publish | minutes (see below) | +| Enforce pull-request stack integrity | [`pr-stack-integrity.yml`](https://github.com/elewa-git/opencrane/blob/main/.github/workflows/pr-stack-integrity.yml) | Stacked-PR bookkeeping | under a minute | +| CodeQL | GitHub's default code-scanning setup | Static analysis | a few minutes | + +`docker.yml` is where the time goes, so the rest of this page is about its jobs. + +## The pipeline fan-out + +```text +pull request / push to develop, main + │ + ▼ + prepare + computes the affected graph, the deployable matrix, the guard + comparison base, and whether the k3d smoke can be skipped + │ + ├──→ test build, test, lint, every policy guard + ├──→ database SQL authority suites, migration proofs + ├──→ api_contract OpenAPI + generated client (when affected) + ├──→ storybook_visual component contracts, cached Chromium + ├──→ develop_smoke k3d silo smoke — the long pole + └──→ image_smoke per-image boot checks + │ + ▼ (all must pass) + build-and-push + publishes sha- images on push events; + on pull requests it builds without pushing, as a proof + │ + ▼ + publish-develop-smoke-images + develop pushes only: completes the immutable image + set for the commit, reusing untouched base images +``` + +## What each job gates + +| Job | Purpose | Typical duration | +| --- | --- | --- | +| `prepare` | Computes the Nx affected graph, the deployable matrix, the guard comparison base, and whether the k3d smoke can be skipped. | 1–2 min | +| `test` | Builds, tests and lints affected projects, and runs every policy guard: workload ownership, agent-domain boundary, mechanical style, module growth, release versioning, Prisma boundaries, config-docs coverage, dependency boundaries. | 3–10 min | +| `database` | Everything PostgreSQL-bound, beside `test` instead of inside it: the migration contracts and convergence proofs, the generated client, the target baseline, and the SQL authority suites. | 2–4 min | +| `api_contract` | Rebuilds the server and proves the OpenAPI reference and generated client are in sync. Runs only when the API contract changed. | skipped, or ~3–5 min | +| `storybook_visual` | Storybook build/behaviour/visual contracts for affected frontend projects, on cached Chromium. Runs beside `test`, not after it. | seconds when nothing affected; ~5 min otherwise | +| `develop_smoke` | Boots a disposable k3d cluster, deploys the full current silo through the real deploy scripts, and proves database isolation, TLS ingress and workload health. | 6–15 min | +| `image_smoke` | Boots individual images that declare an `image-smoke` target and checks they come up. | 1–2 min per image | +| `build-and-push` | Builds every affected deployable image and publishes `sha-` tags on push events. On pull requests it builds without pushing, as a proof. | 1–5 min warm | +| `publish-develop-smoke-images` | On develop pushes, completes the immutable image set for the commit: reuses the exact validated base image where nothing changed, copies tags forward, builds only what is missing. | seconds per image | + +A deploy to a live cluster consumes the published `sha-` images. CI must be green for the +exact SHA before any deploy — the deploy scripts pull images, never build them (see +[Deploying](/contributing/deploying)). + +## Caching layers + +Every job runs on a fresh runner, so anything not cached is paid on every run. + +| Cache | Backend | Key | Used by | +| --- | --- | --- | --- | +| npm download cache | `actions/setup-node` | lockfile hash | all jobs | +| `node_modules` | `actions/cache` | lockfile hash | all jobs (skips `npm ci` entirely on a hit) | +| Nx computation cache | `actions/cache` (`.nx/cache`) | lockfile hash + commit, with prefix restore | `test`, `api_contract`, `storybook_visual` | +| Playwright Chromium | `actions/cache` (`~/.cache/ms-playwright`) | lockfile hash | `storybook_visual` | +| Docker image layers | registry (`ghcr.io//opencrane-buildcache:`) | buildx layer graph | `develop_smoke`, `build-and-push`, `publish-develop-smoke-images` | +| npm inside Dockerfiles | BuildKit cache mount (`/root/.npm`) | shared between build and runtime stages within one build | all Node images | + +### Why image layers cache in the registry, not the Actions cache + +The repository's 10GB Actions cache quota was permanently over budget — single buildkit blobs +reach ~700MB — so layer caches were evicted almost immediately and every image built cold. +Registry-backed caches (`type=registry`) have no such quota, and moving them out of the Actions +cache also stops the node/Nx caches from being evicted alongside them. + +### The layer cache has a trust boundary + +Two cache repositories exist, and a layer produced by unreviewed code never becomes part of a +published image: + +```text +same-repository pull request build + reads ← opencrane-buildcache (trusted: written only by develop/main pushes) + writes → opencrane-buildcache-pr (pull-request cache, read back by later PR builds) + +fork pull request build + reads ← opencrane-buildcache (trusted; fork tokens are read-only) + writes → nothing + +develop / main push + reads ← opencrane-buildcache + writes → opencrane-buildcache (becomes the new trusted layer set) +``` + +`opencrane-buildcache` is the only cache a publishable build reads. Same-repository pull +requests write to the separate `opencrane-buildcache-pr` cache; fork pull requests read the +trusted cache and write nothing at all. + +::: info +Chromium binaries restore from the Actions cache; the apt-driven `--with-deps` install runs only +on a cold cache and carries a ten-minute step timeout, because a hung apt once held an untimed +job — and its runner — for hours. Every job in the pipeline has a `timeout-minutes`: a hung job +does not only lose its own time, it occupies a concurrent-runner slot and starves every queued +run behind it. +::: + +## The k3d smoke and its skip proof + +`develop_smoke` is the pipeline's long pole and its most valuable gate: it exercises the real +deploy path end to end on every develop push. Two mechanisms keep its cost down: + +1. **Image reuse by digest.** Projects the affected graph did not select are pulled from the + validated base commit's published images instead of being rebuilt + ([`develop-smoke.sh`](https://github.com/elewa-git/opencrane/blob/main/apps/_infra/deploy-k8s/platform/tests/develop-smoke.sh)`:_pull_baseline_image`). +2. **The skip proof.** A pull request skips the k3d smoke entirely when its exact base SHA + already completed the same k3d job successfully + ([`scripts/develop-smoke-baseline.mjs`](https://github.com/elewa-git/opencrane/blob/main/scripts/develop-smoke-baseline.mjs)). + Any ambiguity — API failure, missing proof, affected containers — deliberately runs the + smoke instead of skipping it. + +::: tip +The skip proof is why **keeping develop green is a speed feature**: a red develop push means no +validated base exists, so every subsequent pull request pays the full smoke. A failure streak on +develop taxes every open PR in the repository — fixing develop first is usually the fastest way +to speed up everyone else's work. +::: + +The smoke's storage tier also varies: touching `k8s-deploy.sh`, the smoke script, the workflow, +or `apps/postgres/` selects the `full` tier, which additionally exercises CSI volume expansion +(several extra minutes). Other changes run the `fast` tier. diff --git a/website/contributing/deploying.md b/website/contributing/deploying.md new file mode 100644 index 000000000..ab4d163e8 --- /dev/null +++ b/website/contributing/deploying.md @@ -0,0 +1,97 @@ +# Deploying + +All cluster mutation goes through **app-owned scripts** — never bare `helm upgrade` or `kubectl +apply` against a live cluster. This page covers the deploy chain, its prerequisites, and the +warnings that save hours of debugging. + +> See also: [The CI pipeline](/contributing/ci-pipeline) (the images this chain consumes), +> [Hosting and deployment](/operators/hosting) (the operator-facing install path), and +> [Runbook](/operators/runbook) (diagnosing a cluster once it is live). + +## The script-only rule + +Every deploy — from a contributor's k3d smoke to a production silo — runs through the same +entrypoints. There is no supported path that touches a live cluster by hand. + +```text +apps/_infra/deploy-k8s/deploy.sh + silo profile: flags, presets + │ + ▼ +platform/k8s-deploy.sh + the install engine + │ + ├──→ current-chart-sources.sh + │ helm dependency build from Chart.lock + │ + ├──→ database-migration-orchestrator.sh + │ CNPG cluster, databases, migration + privileges Jobs + │ + ├──→ umbrella helm upgrade + │ all app subcharts + │ │ + │ ▼ + │ database-release-finalization.sh + │ credential-checksum roll, rollout waits, cert wait + │ + └──→ post-deploy-verify.sh + live health verification +``` + +- `deploy.sh` installs one per-ClusterTenant silo: operator, channel proxy, LiteLLM, Cognee, + opencrane-ui, per-CT networking, and one app-owned PostgreSQL server with isolated logical + databases. Required flags: `--base-domain`, `--cluster-tenant`, `--acme-email`, + `--first-user-email`; fresh installs also need `--opencrane-ui-digest` and `--cognee-digest` + (immutable digests, never tags). +- Cluster-wide prerequisites (ingress-nginx, cert-manager, CloudNativePG) are installed once per + cluster by `bootstrap-prerequisites.sh` and are never part of a silo release. +- The PostgreSQL transition is resolved and schema-validated *before* the cluster is touched; + migration and privileges run as bounded Helm hook Jobs, with an automatic rollback path that + restores the exact fenced Helm revision on failure. +- After the umbrella upgrade, the engine stamps a checksum of the published database connection + Secrets onto the consumer Deployments (`opencrane-server`, `litellm`, `mcp-gateway`). An + unchanged checksum is a no-op; a changed one triggers exactly one rollout. This replaced an + unconditional `rollout restart` that double-started the heaviest workloads on every deploy. + +## Bootstrap prerequisites + +Before any silo installs, the cluster needs a default StorageClass, a `NetworkPolicy`-enforcing +CNI, and — via `bootstrap-prerequisites.sh` — ingress-nginx, cert-manager and CloudNativePG. These +are cluster-wide and installed once; the silo chart assumes them and never re-installs them. + +## Warnings — read before deploying + +::: warning CI green first +Confirm the `docker.yml` run for the exact SHA is green before deploying. The deploy scripts pull +published images and never build them — see [The CI pipeline](/contributing/ci-pipeline). +::: + +::: warning `helm dependency build`, never `dependency update`, on the deploy path +The engine resolves subcharts from `Chart.lock` for reproducibility. `dependency update` +re-resolves and can drift. (Regenerating the lock and archives after a chart version stamp is the +one place `dependency update` is correct — review the diff. See +[Versions and migrations](/contributing/versions-and-migrations).) +::: + +::: warning A green render does not prove a live upgrade works +A passing `helm template` or CI render does not prove a live `helm upgrade` works. Stateful +services need their PVC semantics, reconcile-retry, and Secret-change pod-roll trigger checked +before deploying — see the live-upgrade checklist referenced from the deploy ledger. +::: + +::: warning Never deploy with floating tags +Public releases require `sha-*` build tags or digests; the qualified-release-image policy rejects +`latest` and similar tags. Tag floating exists only for the disposable local k3d smoke. +::: + +- **The tenant's openclaw version pin lives in `values.yaml`, not in code defaults.** +- **Watch the queue, not only the jobs.** The organisation has a fixed number of concurrent + runners; a workflow storm (or a hung job) can queue runs for 30+ minutes. If a run seems stuck + before any job has started, that is queue starvation, not a slow job. +- **Develop red means every PR pays the k3d smoke.** See + [the skip proof](/contributing/ci-pipeline#the-k3d-smoke-and-its-skip-proof). Fixing develop + first is usually the fastest way to speed up everyone's pull requests. +- **A cancelled develop push is normal.** The workflow's concurrency group replaces a queued push + run when a newer push arrives; only in-flight publishes are protected. + +Source: [`apps/_infra/deploy-k8s`](https://github.com/elewa-git/opencrane/blob/main/apps/_infra/deploy-k8s/README.md). diff --git a/website/contributing/overview.md b/website/contributing/overview.md new file mode 100644 index 000000000..6bb20a997 --- /dev/null +++ b/website/contributing/overview.md @@ -0,0 +1,61 @@ +# Contributing to OpenCrane + +This section is for people changing the **OpenCrane codebase itself**: what happens between +opening a pull request and a change running on a live cluster, and how to work with that +pipeline instead of against it. + +> See also: [Cluster deployment](/guide/deploy-cluster) (the operator-facing install path this +> pipeline publishes images for) and [Hosting and deployment](/operators/hosting) (what a +> release owns once it is live). + +## The journey from pull request to running cluster + +A change passes through the same three stages whether it is a one-line fix or a new subsystem: + +```text +pull request opened + │ + ▼ + CI pipeline runs three workflows validate the change; + (see: The CI pipeline) the longest is the k3d silo smoke + │ + ▼ + merge to develop or main + │ + ▼ + images published sha- tags, immutable, never floating + │ + ▼ + deploy scripts run apps/_infra/deploy-k8s/deploy.sh → k8s-deploy.sh + (see: Deploying) pulls the exact published images, never builds them + │ + ▼ + live cluster, verified post-deploy-verify.sh checks real health +``` + +Two threads run underneath every step of that journey: + +- **Versioning.** Every directly changed application, chart and database schema stamps forward + together, and CI enforces the whole scheme before anything reaches the deploy path — see + [Versions and migrations](/contributing/versions-and-migrations). +- **Automation.** The same deploy scripts an operator would run by hand are scriptable enough + for an agent to run, verify and triage — see + [Letting an AI agent manage your deployment](/contributing/ai-managed-deployment). + +## In this section + +| Page | Covers | +| --- | --- | +| [The CI pipeline](/contributing/ci-pipeline) | The three workflows, what each `docker.yml` job gates, and the caching layers that keep it fast | +| [Deploying](/contributing/deploying) | The script-only rule, the deploy chain, bootstrap prerequisites, and the warnings that save hours | +| [Versions and migrations](/contributing/versions-and-migrations) | The repository train, the stamp rule, chart and database migrations, and how CI enforces them | +| [Letting an AI agent manage your deployment](/contributing/ai-managed-deployment) | The gitignored `keys/` convention, credential custody, and the deploy agent + `/deploy-loop` skill | + +::: tip +Keeping the `develop` branch green is the single highest-leverage thing a contributor can do: +a red `develop` means every open pull request pays the full k3d smoke instead of skipping it. +See [the skip proof](/contributing/ci-pipeline#the-k3d-smoke-and-its-skip-proof). +::: + +Source: [`docs/ci-and-deploy.md`](https://github.com/elewa-git/opencrane/blob/main/docs/ci-and-deploy.md) +is the deeper repository-side reference this section publishes from. diff --git a/website/contributing/versions-and-migrations.md b/website/contributing/versions-and-migrations.md new file mode 100644 index 000000000..80e5af0cc --- /dev/null +++ b/website/contributing/versions-and-migrations.md @@ -0,0 +1,75 @@ +# Versions and migrations + +OpenCrane ships one **repository train** made of many independently versioned applications, +Helm charts and a shared database schema. This page covers how a change stamps its version, +how chart and database migrations are recorded, and how CI enforces all of it. + +> See also: [The CI pipeline](/contributing/ci-pipeline) (where `check:release-versioning` runs), +> [Deploying](/contributing/deploying) (why `helm dependency build`, never `dependency update`, +> matters here), and the full policy in +> [`docs/agents/versioning.md`](https://github.com/elewa-git/opencrane/blob/main/docs/agents/versioning.md). + +## The repository train and the release manifest + +The root `package.json` version names the current train (for example `0.9.2`). Each train has an +immutable manifest, [`releases/.json`](https://github.com/elewa-git/opencrane/blob/main/releases), +recording every application's `adaptedVersion`, chart version and the database schema version +that work together. Once a version tag exists, that train's composition is frozen: any further +change must advance the train and create the next manifest. + +## The stamp rule + +Only applications whose own files changed stamp to the root version: + +```text +did this application's own project-root files change? + │ + yes ─┴─ no + │ │ + ▼ ▼ + stamp to keep its latest + the root released version + version │ + │ (a shared library, the root + ▼ dependency set, or the lockfile +update, together: moving underneath it does + · manifest entry not trigger a stamp — the + · package.json published image is pinned + version mirror by commit SHA, so the + · project.json shared change reaches the + adaptedVersion app regardless) + · chart appVersion + (if a chart exists) +``` + +Before this rule, every shared change failed CI until every manifest entry was bumped by hand. +Version-only mirror edits are "stamp-only" and do not count as changes themselves. + +## Chart migrations + +A changed chart bumps its chart version to the root version and adds exactly one +`helm/migrations/-to-.json` transition. The umbrella's `Chart.lock` and packaged +archives are then regenerated and reviewed (with `helm dependency build`, never +`dependency update` — see [Deploying](/contributing/deploying)). A newly introduced chart has no +predecessor and therefore no transition. + +## Database migrations + +A database schema change updates the clean target baseline and adds one adjacent, reviewed SQL +transition under +[`apps/opencrane/prisma/migrations/-to-/`](https://github.com/elewa-git/opencrane/blob/main/apps/opencrane/prisma/migrations), +bound by digest. + +- **Adjacent minor trains** (`0.8.x → 0.9.0`) are the only automatic transition. +- **Patch, skipped-minor, and major transitions** require an approved `manualTransition` with a + reason recorded in the manifest — tooling never guesses the upgrade path. + +## CI enforces the whole scheme + +::: tip +`check:release-versioning` runs inside the `test` job of [`docker.yml`](/contributing/ci-pipeline), +diffed against the last validated base. A violation fails the pull request — not the deploy. +::: + +Source: [`docs/agents/versioning.md`](https://github.com/elewa-git/opencrane/blob/main/docs/agents/versioning.md) +and [`scripts/release-versioning-check.mjs`](https://github.com/elewa-git/opencrane/blob/main/scripts/release-versioning-check.mjs). diff --git a/website/guide/deploy-cluster.md b/website/guide/deploy-cluster.md index 42876766f..42c6b18ee 100644 --- a/website/guide/deploy-cluster.md +++ b/website/guide/deploy-cluster.md @@ -55,3 +55,7 @@ Keep provider-specific identity and storage configuration outside the runtime au ## Next → [Set up your domain](/guide/dns) → [Set up your personal assistant](/guide/persona) + +Changing OpenCrane itself rather than installing it? See +[Contributing → Deploying](/contributing/deploying) for the CI-to-cluster pipeline this script +sits behind. diff --git a/website/operators/hosting.md b/website/operators/hosting.md index 7324b77db..3f052862c 100644 --- a/website/operators/hosting.md +++ b/website/operators/hosting.md @@ -5,8 +5,9 @@ umbrella chart composes the trusted services and separate, restricted Job namesp > See also: [Deployment configuration](/operators/deployment-configuration) (public Helm inputs), > [Organisation boundary](/operators/organisation-boundary) (what one silo serves), -> [Networking and isolation](/operators/networking) (allowed traffic), and -> [Runbook](/operators/runbook) (health and recovery). +> [Networking and isolation](/operators/networking) (allowed traffic), +> [Runbook](/operators/runbook) (health and recovery), and +> [Deploying](/contributing/deploying) (the script chain and CI gates behind a release). ## Deployment shape diff --git a/website/package.json b/website/package.json index 4d027aa2b..78c4bd674 100644 --- a/website/package.json +++ b/website/package.json @@ -1,6 +1,6 @@ { "name": "@opencrane/website", - "version": "0.9.1", + "version": "0.9.2", "private": true, "description": "OpenCrane documentation website (VitePress)", "license": "AGPL-3.0-or-later", diff --git a/website/project.json b/website/project.json index 482c88de0..763a9cfea 100644 --- a/website/project.json +++ b/website/project.json @@ -2,5 +2,5 @@ "name": "@opencrane/website", "$schema": "../node_modules/nx/schemas/project-schema.json", "projectType": "application", - "metadata": { "release": { "adaptedVersion": "0.9.1" } } + "metadata": { "release": { "adaptedVersion": "0.9.2" } } }