From a78f5c7a358e0ea8f34a7ee9a671a0cdbaa8aa6b Mon Sep 17 00:00:00 2001 From: levi <2421860260@qq.com> Date: Wed, 1 Jul 2026 18:03:51 +0800 Subject: [PATCH 1/2] [NA] [INFRA] chore: add Paperboy Cloud Build images --- .gcloudignore.paperboy-images | 30 ++++++ .github/workflows/paperboy-build-images.yml | 93 +++++++++++++++++ cloudbuild.paperboy-images.yaml | 107 ++++++++++++++++++++ 3 files changed, 230 insertions(+) create mode 100644 .gcloudignore.paperboy-images create mode 100644 .github/workflows/paperboy-build-images.yml create mode 100644 cloudbuild.paperboy-images.yaml diff --git a/.gcloudignore.paperboy-images b/.gcloudignore.paperboy-images new file mode 100644 index 00000000000..3883a273af1 --- /dev/null +++ b/.gcloudignore.paperboy-images @@ -0,0 +1,30 @@ +# Keep Cloud Build source uploads small and deterministic for Paperboy image builds. +* + +!.gcloudignore.paperboy-images +!cloudbuild.paperboy-images.yaml +!apps/ + +!apps/opik-backend/ +!apps/opik-backend/** + +!apps/opik-frontend/ +!apps/opik-frontend/** + +!apps/opik-python-backend/ +!apps/opik-python-backend/** + +!apps/opik-sandbox-executor-python/ +!apps/opik-sandbox-executor-python/** + +apps/opik-frontend/node_modules/ +apps/opik-frontend/dist/ +apps/opik-frontend/build/ +apps/opik-python-backend/opik-sandbox-executor-python.tar.gz + +**/.DS_Store +**/__pycache__/ +**/.mypy_cache/ +**/.pytest_cache/ +**/.ruff_cache/ +**/*.log diff --git a/.github/workflows/paperboy-build-images.yml b/.github/workflows/paperboy-build-images.yml new file mode 100644 index 00000000000..63a04947694 --- /dev/null +++ b/.github/workflows/paperboy-build-images.yml @@ -0,0 +1,93 @@ +name: Paperboy build Opik images + +on: + workflow_dispatch: + inputs: + ref: + description: "Source ref to build. Defaults to the dispatch ref." + required: false + type: string + tag: + description: "Image tag. Defaults to -paperboy-." + required: false + type: string + +concurrency: + group: paperboy-build-opik-images-${{ inputs.ref || github.ref_name }} + cancel-in-progress: false + +env: + GAR_LOCATION: us-central1 + GCP_PROJECT_ID: gen-lang-client-0436525842 + GAR_REPOSITORY: paperboy + +jobs: + build: + name: Submit Paperboy image build + runs-on: ubuntu-latest + environment: google-cloud-prod + permissions: + contents: read + outputs: + tag: ${{ steps.version.outputs.tag }} + registry: ${{ steps.version.outputs.registry }} + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v2 + + - name: Resolve image tag + id: version + env: + INPUT_TAG: ${{ inputs.tag }} + run: | + set -euo pipefail + + registry="${GAR_LOCATION}-docker.pkg.dev/${GCP_PROJECT_ID}/${GAR_REPOSITORY}" + if [[ -n "${INPUT_TAG}" ]]; then + tag="${INPUT_TAG}" + else + base_version="$(cat version.txt)" + short_sha="$(git rev-parse --short=12 HEAD)" + tag="${base_version}-paperboy-${short_sha}" + fi + + echo "registry=${registry}" >> "$GITHUB_OUTPUT" + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + echo "Submitting Cloud Build for ${registry}/:${tag}" + + - name: Submit Cloud Build + run: | + set -euo pipefail + + gcloud builds submit \ + --project="${GCP_PROJECT_ID}" \ + --config=cloudbuild.paperboy-images.yaml \ + --substitutions="_REGISTRY=${{ steps.version.outputs.registry }},_TAG=${{ steps.version.outputs.tag }}" \ + --ignore-file=.gcloudignore.paperboy-images \ + . + + - name: Output image refs + run: | + { + echo "### Paperboy Opik images" + echo + echo "**Tag:** \`${{ steps.version.outputs.tag }}\`" + echo + echo '```' + echo "${{ steps.version.outputs.registry }}/opik-backend:${{ steps.version.outputs.tag }}" + echo "${{ steps.version.outputs.registry }}/opik-python-backend:${{ steps.version.outputs.tag }}" + echo "${{ steps.version.outputs.registry }}/opik-sandbox-executor-python:${{ steps.version.outputs.tag }}" + echo "${{ steps.version.outputs.registry }}/opik-frontend:${{ steps.version.outputs.tag }}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/cloudbuild.paperboy-images.yaml b/cloudbuild.paperboy-images.yaml new file mode 100644 index 00000000000..9802fb831e8 --- /dev/null +++ b/cloudbuild.paperboy-images.yaml @@ -0,0 +1,107 @@ +timeout: 7200s + +options: + machineType: E2_HIGHCPU_32 + diskSizeGb: 200 + logging: CLOUD_LOGGING_ONLY + +substitutions: + _REGISTRY: us-central1-docker.pkg.dev/gen-lang-client-0436525842/paperboy + _TAG: local + +steps: + - id: build-sandbox-executor + name: gcr.io/cloud-builders/docker + dir: apps/opik-sandbox-executor-python + waitFor: ["-"] + env: + - DOCKER_BUILDKIT=1 + args: + - build + - --build-arg + - OPIK_VERSION=${_TAG} + - -t + - ${_REGISTRY}/opik-sandbox-executor-python:${_TAG} + - . + + - id: build-backend + name: gcr.io/cloud-builders/docker + dir: apps/opik-backend + waitFor: ["-"] + env: + - DOCKER_BUILDKIT=1 + args: + - build + - --build-arg + - OPIK_VERSION=${_TAG} + - -t + - ${_REGISTRY}/opik-backend:${_TAG} + - . + + - id: build-frontend + name: gcr.io/cloud-builders/docker + dir: apps/opik-frontend + waitFor: ["-"] + env: + - DOCKER_BUILDKIT=1 + args: + - build + - --build-arg + - OPIK_VERSION=${_TAG} + - --build-arg + - BUILD_MODE=production + - -t + - ${_REGISTRY}/opik-frontend:${_TAG} + - . + + - id: push-sandbox-executor + name: gcr.io/cloud-builders/docker + waitFor: ["build-sandbox-executor"] + args: + - push + - ${_REGISTRY}/opik-sandbox-executor-python:${_TAG} + + - id: package-sandbox-executor + name: gcr.io/cloud-builders/docker + waitFor: ["build-sandbox-executor"] + entrypoint: bash + args: + - -ceu + - | + docker save "${_REGISTRY}/opik-sandbox-executor-python:${_TAG}" \ + | gzip > apps/opik-python-backend/opik-sandbox-executor-python.tar.gz + + - id: build-python-backend + name: gcr.io/cloud-builders/docker + dir: apps/opik-python-backend + waitFor: ["package-sandbox-executor"] + env: + - DOCKER_BUILDKIT=1 + args: + - build + - --build-arg + - OPIK_VERSION=${_TAG} + - -t + - ${_REGISTRY}/opik-python-backend:${_TAG} + - . + + - id: push-backend + name: gcr.io/cloud-builders/docker + waitFor: ["build-backend"] + args: + - push + - ${_REGISTRY}/opik-backend:${_TAG} + + - id: push-frontend + name: gcr.io/cloud-builders/docker + waitFor: ["build-frontend"] + args: + - push + - ${_REGISTRY}/opik-frontend:${_TAG} + + - id: push-python-backend + name: gcr.io/cloud-builders/docker + waitFor: ["build-python-backend"] + args: + - push + - ${_REGISTRY}/opik-python-backend:${_TAG} From 7d0ea6124a66290af8a966b5fb12b49d89ae2373 Mon Sep 17 00:00:00 2001 From: levi <2421860260@qq.com> Date: Wed, 1 Jul 2026 18:07:12 +0800 Subject: [PATCH 2/2] docs(paperboy): document image build flow --- PAPERBOY_IMAGE_BUILD.md | 397 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 PAPERBOY_IMAGE_BUILD.md diff --git a/PAPERBOY_IMAGE_BUILD.md b/PAPERBOY_IMAGE_BUILD.md new file mode 100644 index 00000000000..3dbd439f071 --- /dev/null +++ b/PAPERBOY_IMAGE_BUILD.md @@ -0,0 +1,397 @@ +# Paperboy Opik Image Build Runbook + +This document is for the `paperboytm/opik` fork. Use it when Paperboy needs to +build and publish Opik application images from forked source code instead of +using upstream `ghcr.io/comet-ml/opik` images. + +The Paperboy deployment expects these images to stay on the same tag: + +```text +opik-backend +opik-python-backend +opik-frontend +opik-sandbox-executor-python +``` + +Registry: + +```text +us-central1-docker.pkg.dev/gen-lang-client-0436525842/paperboy +``` + +## Tag Format + +Use this format for production deployment tags: + +```text +-paperboy- +``` + +Example: + +```text +2.1.11-paperboy-d80c0bfb59b0 +``` + +The GitHub workflow uses this format automatically when the `tag` input is +empty: + +```bash +base_version="$(cat version.txt)" +short_sha="$(git rev-parse --short=12 HEAD)" +tag="${base_version}-paperboy-${short_sha}" +``` + +For production rollouts, pass the tag explicitly so Platform can pin the exact +same value in Helm values. + +## One-Time GCP Setup + +Cloud Build must be enabled in the Paperboy GCP project: + +```bash +gcloud services enable cloudbuild.googleapis.com \ + --project=gen-lang-client-0436525842 +``` + +The Cloud Build worker currently runs as: + +```text +634173089960-compute@developer.gserviceaccount.com +``` + +Grant it read access to the Cloud Build source bucket: + +```bash +gcloud storage buckets add-iam-policy-binding \ + gs://gen-lang-client-0436525842_cloudbuild \ + --member=serviceAccount:634173089960-compute@developer.gserviceaccount.com \ + --role=roles/storage.objectViewer +``` + +Grant it write access to the Paperboy Artifact Registry repository: + +```bash +gcloud artifacts repositories add-iam-policy-binding paperboy \ + --location=us-central1 \ + --project=gen-lang-client-0436525842 \ + --member=serviceAccount:634173089960-compute@developer.gserviceaccount.com \ + --role=roles/artifactregistry.writer +``` + +Grant log writing so Cloud Build step logs are visible: + +```bash +gcloud projects add-iam-policy-binding gen-lang-client-0436525842 \ + --member=serviceAccount:634173089960-compute@developer.gserviceaccount.com \ + --role=roles/logging.logWriter +``` + +## GitHub Actions Setup + +The workflow is: + +```text +.github/workflows/paperboy-build-images.yml +``` + +It does not build images on the GitHub runner. It only: + +1. checks out the requested ref; +2. authenticates to GCP; +3. resolves the Paperboy image tag; +4. submits `cloudbuild.paperboy-images.yaml` to Cloud Build. + +The workflow expects this secret: + +```text +GCP_SA_KEY +``` + +Configure it either as a repository secret or under the `google-cloud-prod` +GitHub environment. If this secret or environment is missing, manual `gcloud +builds submit` still works, but GitHub workflow dispatch will fail. + +## Manual Cloud Build + +Run commands from the repository root. + +Check the active GCP project and account: + +```bash +gcloud config get-value project +gcloud auth list --filter=status:ACTIVE --format='value(account)' +``` + +Set the tag and registry: + +```bash +TAG="2.1.11-paperboy-d80c0bfb59b0" +REGISTRY="us-central1-docker.pkg.dev/gen-lang-client-0436525842/paperboy" +``` + +Submit the build: + +```bash +gcloud builds submit \ + --project=gen-lang-client-0436525842 \ + --config=cloudbuild.paperboy-images.yaml \ + --substitutions="_REGISTRY=${REGISTRY},_TAG=${TAG}" \ + --ignore-file=.gcloudignore.paperboy-images \ + . +``` + +The build config uses: + +```yaml +options: + machineType: E2_HIGHCPU_32 + diskSizeGb: 200 + logging: CLOUD_LOGGING_ONLY +``` + +Every Docker build step sets: + +```text +DOCKER_BUILDKIT=1 +``` + +BuildKit is required because the frontend Dockerfile uses `COPY --chmod`. + +## What Cloud Build Does + +`cloudbuild.paperboy-images.yaml` builds and pushes the images in this order: + +1. Build `opik-sandbox-executor-python`. +2. Build `opik-backend`. +3. Build `opik-frontend`. +4. Push `opik-sandbox-executor-python`. +5. Save the sandbox executor image into + `apps/opik-python-backend/opik-sandbox-executor-python.tar.gz`. +6. Build `opik-python-backend` with that tarball in its Docker context. +7. Push `opik-backend`. +8. Push `opik-frontend`. +9. Push `opik-python-backend`. + +The sandbox executor tarball is not committed. It is generated inside the Cloud +Build worker for the python backend image build. + +## Source Upload Check + +The build uses a dedicated ignore file: + +```text +.gcloudignore.paperboy-images +``` + +It keeps the upload small by excluding local caches such as: + +```text +apps/opik-frontend/node_modules/ +apps/opik-frontend/dist/ +apps/opik-frontend/build/ +apps/opik-python-backend/opik-sandbox-executor-python.tar.gz +``` + +To inspect what Cloud Build would upload: + +```bash +if [[ -e .gcloudignore ]]; then + echo ".gcloudignore already exists; inspect it before overwriting" + exit 1 +fi + +cleanup() { rm -f .gcloudignore; } +trap cleanup EXIT + +cp .gcloudignore.paperboy-images .gcloudignore +gcloud meta list-files-for-upload . > /tmp/opik-paperboy-upload-files.txt +wc -l /tmp/opik-paperboy-upload-files.txt +rg -n 'node_modules|opik-sandbox-executor-python.tar.gz|cloudbuild|opik-backend/Dockerfile|opik-frontend/Dockerfile|opik-python-backend/Dockerfile|opik-sandbox-executor-python/Dockerfile' \ + /tmp/opik-paperboy-upload-files.txt +``` + +Expected result: + +- the four Dockerfiles and `cloudbuild.paperboy-images.yaml` are present; +- `node_modules` is absent; +- local sandbox executor tarballs are absent. + +## Verify Pushed Images + +After Cloud Build succeeds, verify every tag: + +```bash +TAG="2.1.11-paperboy-d80c0bfb59b0" + +for image in \ + opik-sandbox-executor-python \ + opik-backend \ + opik-python-backend \ + opik-frontend +do + echo "${image}" + gcloud artifacts docker tags list \ + "us-central1-docker.pkg.dev/gen-lang-client-0436525842/paperboy/${image}" \ + --project=gen-lang-client-0436525842 \ + --filter="tag:${TAG}" \ + --format='table(tag,version)' +done +``` + +For the verified `2.1.11-paperboy-d80c0bfb59b0` build, the digests were: + +```text +opik-sandbox-executor-python sha256:e0beed45de29f724fa67d12124538f5294f3a7b75fd98a726603b7bee667aa4a +opik-backend sha256:395cefa6e21b15efd470d9cb7809e121a9fedae8c647e0aac4631010f4f3a199 +opik-python-backend sha256:da561eeec69dcd2876e70d4dddd3191581389e2f1ad84be52fcd6e3c00c51725 +opik-frontend sha256:2155a9c952749390727fa3986c1fe4b554696c57b441b215d5cab3e1ee5d58cc +``` + +Successful Cloud Build example: + +```text +b2676833-d39f-467b-a712-d89d36f754de +duration: 3m58s +``` + +## Deploy Through Platform + +Platform pins the Paperboy images in: + +```text +gke/k8s/opik/values.yaml +``` + +The important fields are: + +```yaml +registry: us-central1-docker.pkg.dev/gen-lang-client-0436525842/paperboy + +component: + backend: + image: + repository: opik-backend + tag: + + python-backend: + image: + repository: opik-python-backend + tag: + env: + PYTHON_CODE_EXECUTOR_IMAGE_REGISTRY: us-central1-docker.pkg.dev/gen-lang-client-0436525842/paperboy + PYTHON_CODE_EXECUTOR_IMAGE_TAG: + + frontend: + image: + repository: opik-frontend + tag: +``` + +Before deploying, render the chart and confirm image refs: + +```bash +helm template opik opik/opik \ + --namespace opik \ + --version 2.1.11 \ + --values gke/k8s/opik/values.yaml \ + >/tmp/opik-paperboy-rendered.yaml + +rg -n 'image:|PYTHON_CODE_EXECUTOR_IMAGE_(REGISTRY|TAG)|opik-(backend|python-backend|frontend|sandbox)' \ + /tmp/opik-paperboy-rendered.yaml \ + -C 1 +``` + +## Troubleshooting + +### Cloud Build API disabled + +Error: + +```text +Cloud Build API has not been used in project ... before or it is disabled +``` + +Fix: + +```bash +gcloud services enable cloudbuild.googleapis.com \ + --project=gen-lang-client-0436525842 +``` + +### Source tarball permission denied + +Error: + +```text +storage.objects.get denied on ..._cloudbuild/source/... +``` + +Fix: + +```bash +gcloud storage buckets add-iam-policy-binding \ + gs://gen-lang-client-0436525842_cloudbuild \ + --member=serviceAccount:634173089960-compute@developer.gserviceaccount.com \ + --role=roles/storage.objectViewer +``` + +### No Cloud Build step logs + +Error: + +```text +does not have permission to write logs to Cloud Logging +``` + +Fix: + +```bash +gcloud projects add-iam-policy-binding gen-lang-client-0436525842 \ + --member=serviceAccount:634173089960-compute@developer.gserviceaccount.com \ + --role=roles/logging.logWriter +``` + +### Artifact Registry push denied + +Fix: + +```bash +gcloud artifacts repositories add-iam-policy-binding paperboy \ + --location=us-central1 \ + --project=gen-lang-client-0436525842 \ + --member=serviceAccount:634173089960-compute@developer.gserviceaccount.com \ + --role=roles/artifactregistry.writer +``` + +### Frontend fails on `COPY --chmod` + +Error: + +```text +the --chmod option requires BuildKit +``` + +Fix: + +Ensure every Cloud Build `docker build` step has: + +```yaml +env: + - DOCKER_BUILDKIT=1 +``` + +### Local Mac pulls the wrong architecture + +If you build `linux/amd64` images from macOS and then run `docker pull` without +a platform, Docker may try to pull `linux/arm64` and fail: + +```text +no matching manifest for linux/arm64/v8 +``` + +Prefer Cloud Build for production images. If debugging locally, use: + +```bash +docker pull --platform linux/amd64 "${REGISTRY}/opik-sandbox-executor-python:${TAG}" +```