diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml new file mode 100644 index 000000000..f013ab884 --- /dev/null +++ b/.github/workflows/build-and-deploy.yml @@ -0,0 +1,244 @@ +name: Build, push, and deploy images + +# Builds all 7 Rodan images, pushes them to the private GitHub Container Registry, and +# (on manual dispatch or version tags) deploys the app tier to the k3s cluster. +# Replaces the DockerHub autobuild hooks (hooks/build, hooks/push). +# +# Tags: +# git tag v* -> ghcr.io/ddmal/: (+ deploy) +# push to develop -> ghcr.io/ddmal/:nightly +# pull_request -> build only, no push (validates Dockerfiles) +# workflow_dispatch -> optional tag input (defaults to "nightly") (+ deploy) +# Every pushed build also gets an immutable ghcr.io/ddmal/:sha- tag, +# which the deploy job pins so the rollout is guaranteed to pull the new image. + +on: + push: + # k8s-migration is here temporarily to test image builds from this branch — remove before merge. + branches: [develop, k8s-migration] + tags: ['v*'] + workflow_dispatch: + inputs: + tag: + description: "Image tag to build/push (e.g. v3.4.0 or nightly)" + required: false + default: "nightly" + +permissions: + contents: read + packages: write + +concurrency: + group: build-and-deploy-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + OWNER: ddmal + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.meta.outputs.tag }} + push: ${{ steps.meta.outputs.push }} + short_sha: ${{ steps.meta.outputs.short_sha }} + steps: + - id: meta + env: + EVENT: ${{ github.event_name }} + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + PR_NUMBER: ${{ github.event.number }} + run: | + set -euo pipefail + if [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG"; PUSH=true + elif [ "$REF_TYPE" = "tag" ]; then + TAG="$REF_NAME"; PUSH=true # e.g. v3.4.0 + elif [ "$EVENT" = "pull_request" ]; then + TAG="pr-$PR_NUMBER"; PUSH=false # build only, never push + elif [ "$REF_NAME" = "develop" ]; then + TAG="nightly"; PUSH=true + else + TAG="$REF_NAME"; PUSH=true + fi + # Docker tags cannot contain "/" + TAG="$(printf '%s' "$TAG" | tr '/' '-')" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "push=$PUSH" >> "$GITHUB_OUTPUT" + echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + echo "Resolved tag=$TAG push=$PUSH sha=${GITHUB_SHA::7} (event=$EVENT ref_type=$REF_TYPE ref=$REF_NAME)" + + # Ordered dependency chain on ONE runner so the hardcoded + # `FROM ddmal/:${VERSION}` lines resolve to the locally-built images. + chain: + needs: setup + runs-on: ubuntu-latest + env: + TAG: ${{ needs.setup.outputs.tag }} + PUSH: ${{ needs.setup.outputs.push }} + SHA_TAG: sha-${{ needs.setup.outputs.short_sha }} + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + # Reclaims ~25-35 GB on ubuntu-latest. + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + large-packages: false + docker-images: false + + - name: Log in to GHCR + if: env.PUSH == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build rodan-python3-celery (base) + run: | + docker build \ + --build-arg BRANCHES=develop \ + --build-arg VERSION="$TAG" \ + -t "$REGISTRY/$OWNER/rodan-python3-celery:$TAG" \ + -t "$REGISTRY/$OWNER/rodan-python3-celery:$SHA_TAG" \ + -t "ddmal/rodan-python3-celery:$TAG" \ + -f python3-celery/Dockerfile \ + . + + - name: Build rodan-main (FROM rodan-python3-celery) + run: | + docker build \ + --build-arg BRANCHES=develop \ + --build-arg VERSION="$TAG" \ + --build-arg build_hash="${{ github.sha }}" \ + -t "$REGISTRY/$OWNER/rodan-main:$TAG" \ + -t "$REGISTRY/$OWNER/rodan-main:$SHA_TAG" \ + -t "ddmal/rodan-main:$TAG" \ + -f rodan-main/Dockerfile \ + . + + - name: Build nginx (FROM rodan-main, bakes static files) + run: | + docker build \ + --build-arg VERSION="$TAG" \ + -t "$REGISTRY/$OWNER/nginx:$TAG" \ + -t "$REGISTRY/$OWNER/nginx:$SHA_TAG" \ + ./nginx + + - name: Push chain images + if: env.PUSH == 'true' + run: | + docker push "$REGISTRY/$OWNER/rodan-python3-celery:$TAG" + docker push "$REGISTRY/$OWNER/rodan-python3-celery:$SHA_TAG" + docker push "$REGISTRY/$OWNER/rodan-main:$TAG" + docker push "$REGISTRY/$OWNER/rodan-main:$SHA_TAG" + docker push "$REGISTRY/$OWNER/nginx:$TAG" + docker push "$REGISTRY/$OWNER/nginx:$SHA_TAG" + + # Independent images — each on its own runner, in parallel. + independent: + needs: setup + runs-on: ubuntu-latest + env: + TAG: ${{ needs.setup.outputs.tag }} + PUSH: ${{ needs.setup.outputs.push }} + SHA_TAG: sha-${{ needs.setup.outputs.short_sha }} + strategy: + fail-fast: false + matrix: + include: + - name: rodan-client + dockerfile: rodan-client/Dockerfile + context: ./rodan-client + args: "--build-arg BRANCHES=develop" + - name: rodan-gpu-celery + dockerfile: gpu-celery/Dockerfile + context: . + args: "--build-arg BRANCHES=develop" + - name: postgres-plpython + dockerfile: postgres/Dockerfile + context: . + args: "" + - name: iipsrv + dockerfile: iipsrv/Dockerfile + context: ./iipsrv + args: "" + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + large-packages: false + docker-images: false + + - name: Log in to GHCR + if: env.PUSH == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build ${{ matrix.name }} + run: | + docker build ${{ matrix.args }} \ + --build-arg VERSION="$TAG" \ + -t "$REGISTRY/$OWNER/${{ matrix.name }}:$TAG" \ + -t "$REGISTRY/$OWNER/${{ matrix.name }}:$SHA_TAG" \ + -f "${{ matrix.dockerfile }}" \ + "${{ matrix.context }}" + + - name: Push ${{ matrix.name }} + if: env.PUSH == 'true' + run: | + docker push "$REGISTRY/$OWNER/${{ matrix.name }}:$TAG" + docker push "$REGISTRY/$OWNER/${{ matrix.name }}:$SHA_TAG" + + # Deploy the app tier by pinning the immutable sha- tag (guarantees a fresh pull + + # gives `kubectl rollout undo` rollback). Runs only on manual dispatch or version tags. + # postgres / redis / rabbitmq are intentionally left alone (don't bounce the DB on every deploy). + deploy: + needs: [setup, chain, independent] + if: needs.setup.outputs.push == 'true' && (github.event_name == 'workflow_dispatch' || github.ref_type == 'tag') + runs-on: ubuntu-latest + env: + NS: rodan + REG: ghcr.io/ddmal + SHA_TAG: sha-${{ needs.setup.outputs.short_sha }} + steps: + - name: Install kubectl + uses: azure/setup-kubectl@v4 + + - name: Configure kubeconfig from secret + env: + KUBECONFIG_DATA: ${{ secrets.KUBECONFIG }} + run: | + set -euo pipefail + printf '%s' "$KUBECONFIG_DATA" > "$RUNNER_TEMP/kubeconfig" + chmod 600 "$RUNNER_TEMP/kubeconfig" + echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV" + + - name: Roll app-tier images to the immutable SHA tag + run: | + set -euo pipefail + kubectl -n "$NS" set image deployment/rodan-main rodan-main="$REG/rodan-main:$SHA_TAG" + kubectl -n "$NS" set image deployment/celery celery="$REG/rodan-main:$SHA_TAG" + kubectl -n "$NS" set image deployment/py3-celery py3-celery="$REG/rodan-python3-celery:$SHA_TAG" + kubectl -n "$NS" set image deployment/gpu-celery gpu-celery="$REG/rodan-gpu-celery:$SHA_TAG" + kubectl -n "$NS" set image deployment/iipsrv iipsrv="$REG/iipsrv:$SHA_TAG" + kubectl -n "$NS" set image deployment/rodan-client rodan-client="$REG/rodan-client:$SHA_TAG" + kubectl -n "$NS" set image deployment/nginx nginx="$REG/nginx:$SHA_TAG" + + - name: Wait for rollouts + run: | + set -euo pipefail + for d in rodan-main celery py3-celery gpu-celery iipsrv rodan-client nginx; do + kubectl -n "$NS" rollout status deployment/"$d" --timeout=600s + done \ No newline at end of file diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml index ed4b3e1e8..ab0c33040 100644 --- a/.github/workflows/release-bot.yml +++ b/.github/workflows/release-bot.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Create output folder run: mkdir -p ${{ github.workspace }}/checklist_items - name: Write checklist items to files @@ -38,12 +40,18 @@ jobs: - name: Collect checklist items run: | - msg=$(awk 'FNR==1 && NR!=1 {print "---"}{print}' checklist_items/*.log) - echo "$msg" >> aggregated.log - sha="*SHA: ${{ github.event.pull_request.head.sha }}*" - sed -i "1i\\ - $sha - " aggregated.log + set -o pipefail + shopt -s nullglob + files=(checklist_items/*.log) + { + echo "*SHA: ${{ github.event.pull_request.head.sha }}*" + echo + if [ ${#files[@]} -eq 0 ]; then + echo "_No merge-commit checklist items found ahead of \`master\`._" + else + awk 'FNR==1 && NR!=1 {print "---"}{print}' "${files[@]}" + fi + } > aggregated.log - name: Update Pull Request uses: actions/github-script@v5 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 31abcc274..000000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM debian -# For more information about why this is here, check the build hooks. -RUN printenv \ No newline at end of file diff --git a/Makefile b/Makefile index 7f476cf85..125c1ac00 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -# TODO: Some of these should be converted into Ansible Playbooks when there's time. - -# Chose a makefile because its easier to read over a bunch of if statements inside a bash script. -# We are taking advantage of .PHONY that is available in makefiles to create this simple looking -# list of command shortcuts +# Local-development shortcuts (docker-compose). +# +# Image build/push and cluster deploy now live in .github/workflows/build-and-deploy.yml +# (builds to GHCR, deploys to the k3s `rodan` namespace). The old Docker Swarm / DockerHub +# targets were removed when Rodan migrated to k3s. # Portable replacement for `sed` or `gsed` # See https://unix.stackexchange.com/questions/92895/how-can-i-achieve-portability-with-sed-i-in-place-editing @@ -11,179 +11,27 @@ REPLACE := perl -i -pe RODAN_PATH := ./rodan-main/code/rodan JOBS_PATH := $(RODAN_PATH)/jobs -PROD_TAG := v3.3.1 - DOCKER_TAG := nightly -# Individual Commands - build: @echo "[-] Rebuilding Docker Images for Rodan..." - # Build py3-celery, because it's needed for Rodan and Celery images - # @docker-compose -f build.yml build --no-cache py3-celery # Sometimes it's better to use the - # no-cache option if something unexplicably broke with the py3-celery image (a cached build step perhaps) + # py3-celery first — rodan-main (the `rodan` service) and celery images are FROM it. @docker compose -f build.yml build --no-cache py3-celery - # Build rodan and rodan-client because they are needed for nginx + # rodan(-main) and rodan-client next — nginx is FROM rodan-main. @docker compose -f build.yml build --no-cache --parallel rodan rodan-client - # DockerHub is not intuitive. You won't be able to build from the source root folder in both build contextes. - # When you build locally, the COPY command is relative to the dockerfile. When you build on DockerHub, its relative to the source root. - # For this reason we replace the name to build locally because we build more often on DockerHub than on local. - @$(REPLACE) "s/COPY .\/postgres\/maintenance/COPY .\/maintenance/g" ./postgres/Dockerfile || $(REPLACE) "s/COPY .\/postgres\/maintenance/COPY .\/maintenance/g" ./postgres/Dockerfile - @docker compose -f build.yml build --no-cache --parallel nginx gpu-celery postgres hpc-rabbitmq - # Revert back the change to the COPY command so it will work on Docker Hub. - @$(REPLACE) "s/COPY .\/maintenance/COPY .\/postgres\/maintenance/g" ./postgres/Dockerfile || $(REPLACE) "s/COPY .\/maintenance/COPY .\/postgres\/maintenance/g" ./postgres/Dockerfile + @docker compose -f build.yml build --no-cache --parallel nginx gpu-celery postgres @echo "[+] Done." -backup_db: - @docker exec `docker ps -f name=rodan_postgres -q` backup - -restore_db: - @docker exec `docker ps -f name=rodan_postgres -q` restore - -# Keep in mind, you may need to deal with the postgres/maintenance/backup or backups files depending on setup - run: remote_jobs # Run local version for dev - # Hello, 2022 hires! @DOCKER_TAG=$(DOCKER_TAG) docker compose up -test_prod: pull_prod - # Test production Rodan images with specified tag - # May want to change test-prod-compose.yml if you want a - # different tag. - docker compose -f test-prod-compose.yml up - run_client: # Run Rodan-Client for dev (needs local dev up and running) @docker run -p 8080:9002 -v `pwd`/rodan-client/code:/code ddmal/rodan-client:nightly bash -deploy_staging: - # Can also be used to update a configuration (point to a different image.) - @echo "[-] Deploying Docker Swarm for: Rodan Staging" - @docker stack deploy --prune --with-registry-auth -c staging.yml rodan - @echo "[+] Done." - -deploy_production: - # Can also be used to update a configuration (point to a different image.) - @echo "[-] Deploying Docker Swarm for: Rodan Production" - @docker stack deploy --with-registry-auth -c production.yml rodan - @echo "[+] Done." - -copy_docker_tag: - # tag=v1.5.0rc0 make copy_docker_tag - @docker image tag $(docker images ddmal/rodan:nightly -q) ddmal/rodan:$(tag) - @docker image tag $(docker images ddmal/rodan-python3-celery:nightly -q) ddmal/rodan-python3-celery:$(tag) - @docker image tag $(docker images ddmal/rodan-gpu-celery:nightly -q) ddmal/rodan-gpu-celery:$(tag) - -pull_prod: - docker pull ddmal/iipsrv:nightly - docker pull ddmal/nginx:$(PROD_TAG) - docker pull ddmal/postgres-plpython:$(PROD_TAG) - docker pull ddmal/rodan-gpu-celery:$(PROD_TAG) - docker pull ddmal/rodan-main:$(PROD_TAG) - docker pull ddmal/rodan-python3-celery:$(PROD_TAG) - docker pull rabbitmq:alpine - docker pull redis:alpine - -pull_docker_tag: - # tag=v1.5.0rc0 make pull_docker_tag - @docker pull ddmal/rodan:$(tag) - @docker pull ddmal/rodan-python3-celery:$(tag) - @docker pull ddmal/rodan-gpu-celery:$(tag) - -push_docker_tag: - # tag=v1.5.0rc0 make push_docker_tag - @docker push ddmal/rodan:$(tag) - @docker push ddmal/rodan-python3-celery:$(tag) - @docker push ddmal/rodan-gpu-celery:$(tag) - -update: - # tag1=v1.5.0rc0 tag2=v1.3.1 make update - # This will update the nightly images forcefully - @echo "[-] Updating Docker Swarm images..." - # @docker-compose pull - - # DB First - @docker service update \ - --force \ - --update-order start-first \ - --update-delay 30s \ - --image ddmal/postgres-plpython:$(tag2) \ - rodan_postgres - - # You need to be logged in to docker for this one. - @docker service update \ - --force \ - --with-registry-auth \ - --update-order start-first \ - --update-delay 10m \ - --image ddmal/rodan:$(tag1) \ - rodan_rodan-main - - # These images might need time to update. - @docker service update \ - --force \ - --with-registry-auth \ - --update-order start-first \ - --stop-grace-period 9h \ - --update-delay 10m \ - --image ddmal/rodan:$(tag1) \ - rodan_celery - # These are public images - @docker service update \ - --force \ - --update-order start-first \ - --stop-grace-period 9h \ - --update-delay 30s \ - --image ddmal/rodan-python3-celery:$(tag1) \ - rodan_py3-celery - @docker service update \ - --force \ - --update-order start-first \ - --stop-grace-period 9h \ - --update-delay 30s \ - --image ddmal/rodan-gpu-celery:$(tag1) \ - rodan_gpu-celery - - # # TODO: Need to make rabbitmq durable and permanent - # # before we can make rolling updates for rabbitmq/hpc-rabbitmq. - # @docker service update \ - # --force \ - # --update-order start-first \ - # --update-delay 30s \ - # --image ddmal/hpc-rabbitmq:$(tag2) \ - # rodan_hpc-rabbitmq - @docker service update \ - --force \ - --update-order start-first \ - --update-delay 30s \ - --image ddmal/nginx:$(tag2) \ - rodan_nginx - - @echo "[+] Done." - -scale: - @docker service scale rodan_nginx=$(num) - @docker service scale rodan_rodan=$(num) - @docker service scale rodan_celery=$(num) - @docker service scale rodan_py3-celery=$(num) - # @docker service scale rodan_gpu-celery=$(num) - @docker service scale rodan_redis=$(num) - # @docker service scale rodan_postgres=$(num) - @docker service scale rodan_rabbitmq=$(num) - -health: - @docker inspect --format "{{json .State.Health }}" $(log) | jq - -renew_certbot: - @docker exec `docker ps -f name=rodan_nginx -q` certbot renew --no-random-sleep-on-renew - @docker exec `docker ps -f name=rodan_nginx -q` nginx -s reload - stop: - # This is the same command to stop docker swarm or docker compose - @echo "[-] Stopping all running docker containers and services..." - @docker service rm `docker service ls -q` >>/dev/null 2>&1 || echo "[+] No Services Running" - # @docker stop `docker ps -aq` >>/dev/null 2>&1 || echo "[+] No Containers Running" + @echo "[-] Stopping all running docker containers..." @docker stop `docker ps -aq | grep -v $$(docker ps -aq --filter "name=gpu_dont_kill_me")` >>/dev/null 2>&1 || echo "[+] No Containers Running" @echo "[+] Done." @@ -199,29 +47,8 @@ clean_git: @git pull @echo "[+] Done." - -clean_swarm: - # Not usually needed, but this will restart the swarm - @echo "[-] Exiting from Docker Swarm and recreating new Swarm Manager..." - @docker stack rm rodan || echo "[-] No stack to remove" - @docker swarm leave --force || echo "[-] Not a swarm manager" - @docker swarm init - @echo "[+] Done." - -debug_swarm: - @echo "[+] Creating a live service in the same network." - @docker service create --name statefulservice --network rodan_default --entrypoint="bash -c 'tail -f /dev/null'" --env-file=./scripts/staging.env ddmal/rodan:nightly bash - @docker exec -it `docker ps -f name=statefulservice -q` bash - -push: - @echo "[-] Pushing images to Docker Hub..." - @docker compose push - @echo "[+] Done." - -pull: - @echo "[-] Pulling docker images from Docker Hub..." - @DOCKER_TAG=$(DOCKER_TAG) docker compose pull - @echo "[+] Done." +health: + @docker inspect --format "{{json .State.Health }}" $(log) | jq $(JOBS_PATH)/neon_wrapper/Neon/package.json: @cd $(JOBS_PATH); \ @@ -235,32 +62,9 @@ $(JOBS_PATH)/neon_wrapper/static/editor.html: $(JOBS_PATH)/neon_wrapper/Neon/pac $(JOBS_PATH)/pixel_wrapper/package.json: @cd $(JOBS_PATH); git clone --recurse-submodules -b develop https://github.com/DDMAL/pixel_wrapper.git -remote_jobs: $(JOBS_PATH)/pixel_wrapper/package.json $(JOBS_PATH)/neon_wrapper/static/editor.html +remote_jobs: $(JOBS_PATH)/pixel_wrapper/package.json $(JOBS_PATH)/neon_wrapper/static/editor.html @cd $(RODAN_PATH); $(REPLACE) "s/#py3 //g" ./settings.py @cd $(RODAN_PATH); $(REPLACE) "s/#gpu //g" ./settings.py -gpu-celery_log: - @docker exec $$(docker ps -f name=rodan_gpu-celery --format "{{.ID}}") tail -f /code/Rodan/rodan-celery-GPU.log - -py3-celery_log: - @docker exec $$(docker ps -f name=rodan_py3-celery --format "{{.ID}}") tail -f /code/Rodan/rodan-celery-Python3.log - -celery_log: - @docker exec $$(docker ps -f name=rodan_celery --format "{{.ID}}") tail -f /code/Rodan/rodan-celery-celery.log - -rodan-main_log: - @docker exec $$(docker ps -f name=rodan_rodan-main --format "{{.ID}}") tail -f /code/Rodan/rodan.log - -update_prod_version_tag: - # old_tag=v2.x.x make update_prod_version_name - @$(REPLACE) "s/$(old_tag)/$(PROD_TAG)/g" ./production.yml - @$(REPLACE) "s/$(old_tag)/$(PROD_TAG)/g" ./rodan-main/code/rodan/__init__.py - # Command Groups -reset: stop clean pull run clean_reset: stop clean build run -upload: clean_reset push -deploy: clean_git pull run_swarm -reset_swarm: stop clean_git clean_swarm clean pull deploy_staging -update_swarm: clean_git update -staging: stop clean pull deploy_staging diff --git a/build.yml b/build.yml index b5672a8a1..8e8cdb6c0 100644 --- a/build.yml +++ b/build.yml @@ -5,6 +5,8 @@ services: build: context: ./nginx dockerfile: Dockerfile + args: + VERSION: nightly image: "ddmal/nginx:nightly" iipsrv: @@ -12,14 +14,15 @@ services: context: ./iipsrv dockerfile: Dockerfile image: "ddmal/iipsrv:nightly" - + rodan: build: context: . - dockerfile: Dockerfile + dockerfile: rodan-main/Dockerfile args: BRANCHES: develop - image: "ddmal/rodan:nightly" + VERSION: nightly + image: "ddmal/rodan-main:nightly" py3-celery: build: @@ -39,8 +42,8 @@ services: postgres: build: - context: ./postgres - dockerfile: Dockerfile + context: . + dockerfile: ./postgres/Dockerfile image: "ddmal/postgres-plpython:nightly" rodan-client: @@ -48,4 +51,3 @@ services: context: ./rodan-client dockerfile: Dockerfile image: "ddmal/rodan-client:nightly" - diff --git a/hooks/build b/hooks/build deleted file mode 100644 index 7a1193a9e..000000000 --- a/hooks/build +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# hooks/build -# https://docs.docker.com/docker-cloud/builds/advanced/ - -# RODAN_TAG=`cd rodan-main/code && git describe --tags --always` -# RODAN_CLIENT_TAG=`cd rodan-client/code && git describe --tags --always` -# RODAN_DOCKER_TAG=`git describe --tags --always` - -source ./hooks/helper.sh -DOCKER_TAG=$(GetDockerTag $DOCKER_TAG) || ret_code=$? -if [[ ret_code -eq 1 ]]; then - echo "[-] no branch detected, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -elif [[ ret_code -eq 2 ]]; then - echo "[-] no tag on release branch, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -fi - -echo "[+] Building images with tag: $DOCKER_TAG" - -set -o errexit - -# Fail building images -# Push to trigger webhook and send a message to slack -trap 'cleanup $?' EXIT -cleanup() { - if [ "$1" != "0" ]; then - echo "An error occurs when building images." - echo "Push to another repo to trigger webhook on failure with tag syntax: {repo name}-{build status}-{source branch}-{docker tag}" - - docker tag ddmal/docker-webhook:placeholder ddmal/docker-webhook:rodan-fail-${SOURCE_BRANCH}-${DOCKER_TAG} - docker push ddmal/docker-webhook:rodan-fail-${SOURCE_BRANCH}-${DOCKER_TAG} - fi -} - -# This "useless" image is build to prevent dockerhub from overwriting -# the nightly image with it's obligatory minimum of a build rule. If we want a build -# trigger with a HTTPS POST request, we must have at least 1 build rule. -docker build \ - --tag ddmal/docker-webhook:placeholder \ - . - -############################################################################### -# Stage 1 -# Build and push Python3-Celery image - -# Too many times did docker cache mess up a build. No more cache. -echo "[+] Building Python3-Celery" - -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/rodan-python3-celery:${DOCKER_TAG} \ - --file ./python3-celery/Dockerfile \ - . - -echo "[+] Pushing ddmal/rodan-python3-celery:${DOCKER_TAG}" -docker push ddmal/rodan-python3-celery:${DOCKER_TAG} - -############################################################################### -# Stage 2 -# Build and push Rodan and Rodan-Client - -echo "[+] Building Rodan & Celery Core" - -BUILD_HASH=`git rev-parse --verify HEAD` -# Don't remove --no-cache -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --build-arg build_hash=${BUILD_HASH} \ - --tag ddmal/rodan-main:${DOCKER_TAG} \ - --file ./rodan-main/Dockerfile \ - . - -echo "[+] Building Rodan-Client" - -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/rodan-client:${DOCKER_TAG} \ - ./rodan-client - -echo "[+] Pushing ddmal/rodan-main:${DOCKER_TAG}" -docker push ddmal/rodan-main:${DOCKER_TAG} - -echo "[+] Pushing ddmal/rodan-client:${DOCKER_TAG}" -docker push ddmal/rodan-client:${DOCKER_TAG} - -############################################################################## -# Stage 3 -# Build the rest - -echo "[+] Building GPU-Celery" - -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/rodan-gpu-celery:${DOCKER_TAG} \ - --file ./gpu-celery/Dockerfile \ - . - -echo "[+] Building Postgres" - -docker build \ - --no-cache \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/postgres-plpython:${DOCKER_TAG} \ - --file ./postgres/Dockerfile \ - . - -echo "[+] Building Nginx" - -docker build \ - --no-cache \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/nginx:${DOCKER_TAG} \ - ./nginx - -echo "[+] Building iipsrv" - -docker build \ - --no-cache \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/iipsrv:${DOCKER_TAG} \ - ./iipsrv - -echo "[+] Finished" diff --git a/hooks/helper.sh b/hooks/helper.sh deleted file mode 100644 index d0f8bc8cd..000000000 --- a/hooks/helper.sh +++ /dev/null @@ -1,27 +0,0 @@ -GetDockerTag() { - current_branch_full=$(git symbolic-ref HEAD) # ref/heads/ or pull/#PR-id/head - current_branch_short=$(git symbolic-ref --short HEAD) # or ? - _DOCKER_TAG=$1 - - # Release branch will always use git tag - if [[ $current_branch_short =~ ^release ]]; then - # Search for tag in release branch - tags=$(git tag --points-at HEAD) # git tags attached to theh HEAD - _DOCKER_TAG=$(echo $tags | sed -n '1p') - if [[ -z $_DOCKER_TAG ]]; then - # echo "no tag on release branch, fallback to use docker tag=release" - _DOCKER_TAG="release" - echo $_DOCKER_TAG - return 2 - fi - elif [[ $_DOCKER_TAG =~ ^placeholder$ ]]; then - # develop branch has a weird funny tag called placeholder, change it to nightly so commits and PRs will use the nightly tag - _DOCKER_TAG="nightly" - fi - # Finally, use custom tags send by docker hub. e.g.: a feature branch that has its build rule on docker hub - - # Replace invalid char / in docker tag with - - _DOCKER_TAG=$(echo $_DOCKER_TAG | sed 's/\//-/g') - echo $_DOCKER_TAG - return 0 -} \ No newline at end of file diff --git a/hooks/post_build b/hooks/post_build deleted file mode 100644 index 182d6d6fc..000000000 --- a/hooks/post_build +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# hooks/post_build - -source ./hooks/helper.sh -if [[ ret_code -eq 1 ]]; then - echo "[-] no branch detected, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -elif [[ ret_code -eq 2 ]]; then - echo "[-] no tag on release branch, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -fi - -# Pass testing a pull request -# Push to trigger webhook and send a message to slack -if [[ "$DOCKER_TAG" == "this" ]]; then - docker tag ddmal/docker-webhook:placeholder ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} - docker push ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} -fi diff --git a/hooks/post_checkout b/hooks/post_checkout deleted file mode 100644 index e83b801ec..000000000 --- a/hooks/post_checkout +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# hooks/post_checkout -# https://docs.docker.com/docker-cloud/builds/advanced/ - -# echo "[-] Changing branch" -# git checkout develop -# # git reset --hard HEAD -# # git clean -xffd -# # git pull origin develop - -# echo "[-] Fetching tags" -# # git fetch --tags --quiet origin \ No newline at end of file diff --git a/hooks/push b/hooks/push deleted file mode 100644 index f80eb8b34..000000000 --- a/hooks/push +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# hooks/post_push -# https://docs.docker.com/docker-cloud/builds/advanced/ - -source ./hooks/helper.sh -DOCKER_TAG=$(GetDockerTag $DOCKER_TAG) || ret_code=$? -if [[ ret_code -eq 1 ]]; then - echo "[-] no branch detected, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -elif [[ ret_code -eq 2 ]]; then - echo "[-] no tag on release branch, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -fi - -echo "[+] Building images with tag: $DOCKER_TAG" -echo "Docker tag: $DOCKER_TAG" - -# RODAN_TAG=`cd rodan-main/code && git describe --tags --always` -# RODAN_CLIENT_TAG=`cd rodan-client/code && git describe --tags --always` -# RODAN_DOCKER_TAG=`git describe --tags --always` - -echo "[+] Pushing ddmal/rodan-gpu-celery:${DOCKER_TAG}" -docker push ddmal/rodan-gpu-celery:${DOCKER_TAG} - -echo "[+] Pushing ddmal/postgres-plpython:${DOCKER_TAG}" -docker push ddmal/postgres-plpython:${DOCKER_TAG} - -echo "[+] Pushing ddmal/nginx:${DOCKER_TAG}" -docker push ddmal/nginx:${DOCKER_TAG} - -echo "[+] Pushing ddmal/iipsrv:${DOCKER_TAG}" -docker push ddmal/iipsrv:${DOCKER_TAG} - -# Pass building images -# Push to trigger webhook and send a message to slack -docker tag ddmal/docker-webhook:placeholder ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} -docker push ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} diff --git a/k8s/.gitignore b/k8s/.gitignore new file mode 100644 index 000000000..1f725e1bc --- /dev/null +++ b/k8s/.gitignore @@ -0,0 +1,2 @@ +# Real secret with filled-in values — never commit. Use 02-secret.template.yaml as the source. +02-secret.yaml \ No newline at end of file diff --git a/k8s/00-namespace.yaml b/k8s/00-namespace.yaml new file mode 100644 index 000000000..d866e4670 --- /dev/null +++ b/k8s/00-namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: rodan + labels: + app.kubernetes.io/part-of: rodan \ No newline at end of file diff --git a/k8s/01-configmap.yaml b/k8s/01-configmap.yaml new file mode 100644 index 000000000..b312684c4 --- /dev/null +++ b/k8s/01-configmap.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rodan-config + namespace: rodan +data: + # --- General --- + TZ: "America/Toronto" + SERVER_HOST: "rodan2.simssa.ca" + + # --- Django --- + # Matches the current production (DJANGO_ALLOWED_HOSTS=*). "*" also lets k8s httpGet probes + # (which send the pod IP as Host) pass. Tighten to a host list only if you also give the + # probes a matching Host header. + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_DEBUG_MODE: "False" + DJANGO_MEDIA_ROOT: "/rodan/data/" + # Obscured admin path (kept here as in the existing env files; move to Secret if you treat it as sensitive). + DJANGO_ADMIN_URL: "^api/random_secret_admin/" + DJANGO_ACCESS_LOG: "/code/Rodan/rodan.log" + DJANGO_DEBUG_LOG: "/code/Rodan/database.log" + + # --- IIP image server (public URL the client uses; TLS terminated at edge) --- + IIPSRV_URL: "https://rodan2.simssa.ca/fcgi-bin/iipsrv.fcgi/" + + # --- Postgres (non-secret connection coordinates; creds are in the Secret) --- + POSTGRES_HOST: "postgres" + POSTGRES_PORT: "5432" + POSTGRES_DB: "rodan" + POSTGRES_DATABASE_LOGFILE: "/code/Rodan/database.log" + + # --- Redis (cache + ws4redis websocket broker) --- + REDIS_HOST: "redis" + REDIS_PORT: "6379" + REDIS_DB: "0" + + # --- Celery --- + C_FORCE_ROOT: "true" \ No newline at end of file diff --git a/k8s/02-secret.template.yaml b/k8s/02-secret.template.yaml new file mode 100644 index 000000000..143ae816e --- /dev/null +++ b/k8s/02-secret.template.yaml @@ -0,0 +1,36 @@ +# TEMPLATE ONLY — do NOT commit real values. +# Copy to 02-secret.yaml, fill in real values, and apply that (02-secret.yaml is gitignored). +# stringData lets you paste plaintext; Kubernetes base64-encodes it on apply. +# +# NOTE: scripts/start reads ADMIN_PASS (the live production.env's ADMIN_PASSWORD is a bug — use ADMIN_PASS here). +# RABBITMQ_URL host/creds must match the rabbitmq Deployment's RABBITMQ_DEFAULT_USER/PASS: +# amqp://:@rabbitmq:5672// +apiVersion: v1 +kind: Secret +metadata: + name: rodan-secrets + namespace: rodan +type: Opaque +stringData: + # --- Django --- + DJANGO_SECRET_KEY: "CHANGE_ME_long_random_string" + + # --- Postgres credentials --- + POSTGRES_USER: "CHANGE_ME" + POSTGRES_PASSWORD: "CHANGE_ME" + + # --- RabbitMQ (broker URL + the credentials the broker is created with) --- + RABBITMQ_URL: "amqp://CHANGE_ME_user:CHANGE_ME_pass@rabbitmq:5672//" + RABBITMQ_DEFAULT_USER: "CHANGE_ME_user" + RABBITMQ_DEFAULT_PASS: "CHANGE_ME_pass" + + # --- Bootstrap superuser (created by scripts/start on first boot) --- + ADMIN_USER: "CHANGE_ME" + ADMIN_EMAIL: "admin@rodan2.simssa.ca" + ADMIN_PASS: "CHANGE_ME" + + # --- SMTP (optional; leave blank to fall back to console email backend) --- + EMAIL_HOST: "" + EMAIL_PORT: "" + EMAIL_HOST_USER: "" + EMAIL_HOST_PASSWORD: "" \ No newline at end of file diff --git a/k8s/10-pv-resources.yaml b/k8s/10-pv-resources.yaml new file mode 100644 index 000000000..1baf6c900 --- /dev/null +++ b/k8s/10-pv-resources.yaml @@ -0,0 +1,27 @@ +# Shared media volume (Django MEDIA_ROOT, /rodan/data, ~883 GB). RWX so all six +# consumers (nginx, rodan-main, iipsrv, celery, py3-celery, gpu-celery) mount it at once. +# Replace __ARBUTUS_NFS_IP__ and the export path with the real Arbutus NFS server. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: rodan-resources-pv + labels: + app.kubernetes.io/part-of: rodan +spec: + capacity: + storage: 2Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: "" # static binding; disables dynamic provisioning + mountOptions: + - nfsvers=4 + - soft # mirrors the current resources mount (soft,timeo=600,retrans=3) + - timeo=600 + - retrans=3 + nfs: + server: 192.168.236.124 + path: /srv/rodan-data/var/lib/docker/volumes/rodan_resources/_data + claimRef: + namespace: rodan + name: rodan-resources \ No newline at end of file diff --git a/k8s/11-pvc-resources.yaml b/k8s/11-pvc-resources.yaml new file mode 100644 index 000000000..f2e84bfad --- /dev/null +++ b/k8s/11-pvc-resources.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rodan-resources + namespace: rodan +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + resources: + requests: + storage: 2Ti + volumeName: rodan-resources-pv \ No newline at end of file diff --git a/k8s/12-pv-pgdata.yaml b/k8s/12-pv-pgdata.yaml new file mode 100644 index 000000000..567bd81c5 --- /dev/null +++ b/k8s/12-pv-pgdata.yaml @@ -0,0 +1,24 @@ +# Postgres PGDATA (~4.1 GB), PG 9.6, owned by uid 999. Mounted only by the postgres pod. +# 'hard' mount (DBs must not see truncated/soft-failed I/O). NFS export needs no_root_squash. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: rodan-pg-data-pv + labels: + app.kubernetes.io/part-of: rodan +spec: + capacity: + storage: 50Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + mountOptions: + - nfsvers=4 + - hard + nfs: + server: 192.168.236.124 + path: /srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data + claimRef: + namespace: rodan + name: rodan-pg-data \ No newline at end of file diff --git a/k8s/13-pvc-pgdata.yaml b/k8s/13-pvc-pgdata.yaml new file mode 100644 index 000000000..9a71d7be5 --- /dev/null +++ b/k8s/13-pvc-pgdata.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rodan-pg-data + namespace: rodan +spec: + accessModes: + - ReadWriteOnce + storageClassName: "" + resources: + requests: + storage: 50Gi + volumeName: rodan-pg-data-pv \ No newline at end of file diff --git a/k8s/14-pv-pgbackup.yaml b/k8s/14-pv-pgbackup.yaml new file mode 100644 index 000000000..159056d4c --- /dev/null +++ b/k8s/14-pv-pgbackup.yaml @@ -0,0 +1,23 @@ +# Postgres backups dir (/backups). Mounted by the postgres pod (and the backup CronJob if added). +apiVersion: v1 +kind: PersistentVolume +metadata: + name: rodan-pg-backup-pv + labels: + app.kubernetes.io/part-of: rodan +spec: + capacity: + storage: 500Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + mountOptions: + - nfsvers=4 + - hard + nfs: + server: 192.168.236.124 + path: /srv/rodan-data/var/lib/docker/volumes/rodan_pg_backup/_data + claimRef: + namespace: rodan + name: rodan-pg-backup \ No newline at end of file diff --git a/k8s/15-pvc-pgbackup.yaml b/k8s/15-pvc-pgbackup.yaml new file mode 100644 index 000000000..f711d02d9 --- /dev/null +++ b/k8s/15-pvc-pgbackup.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rodan-pg-backup + namespace: rodan +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + resources: + requests: + storage: 500Gi + volumeName: rodan-pg-backup-pv \ No newline at end of file diff --git a/k8s/20-postgres.yaml b/k8s/20-postgres.yaml new file mode 100644 index 000000000..5c9ce5344 --- /dev/null +++ b/k8s/20-postgres.yaml @@ -0,0 +1,81 @@ +# Postgres 9.6 + plpython3 (existing image). Single instance, PGDATA on the static NFS PVC. +# The existing PGDATA is already initialized, so the entrypoint skips initdb and the +# POSTGRES_USER/PASSWORD/DB env are effectively no-ops on an existing cluster (kept for first-init). +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: rodan +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: postgres + image: ghcr.io/ddmal/postgres-plpython:k8s-migration + ports: + - containerPort: 5432 + name: postgres + env: + - name: TZ + value: "America/Toronto" + - name: POSTGRES_DB + valueFrom: { configMapKeyRef: { name: rodan-config, key: POSTGRES_DB } } + - name: POSTGRES_USER + valueFrom: { secretKeyRef: { name: rodan-secrets, key: POSTGRES_USER } } + - name: POSTGRES_PASSWORD + valueFrom: { secretKeyRef: { name: rodan-secrets, key: POSTGRES_PASSWORD } } + volumeMounts: + - name: pg-data + mountPath: /var/lib/postgresql/data + - name: pg-backup + mountPath: /backups + readinessProbe: + exec: + command: ["sh", "-c", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""] + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-c", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""] + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "2Gi" + volumes: + - name: pg-data + persistentVolumeClaim: + claimName: rodan-pg-data + - name: pg-backup + persistentVolumeClaim: + claimName: rodan-pg-backup +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: rodan +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 + name: postgres \ No newline at end of file diff --git a/k8s/21-redis.yaml b/k8s/21-redis.yaml new file mode 100644 index 000000000..11915e5d2 --- /dev/null +++ b/k8s/21-redis.yaml @@ -0,0 +1,58 @@ +# Redis: Celery is NOT backed by redis here (broker+backend are RabbitMQ); redis is the +# ws4redis websocket broker + cache. Ephemeral (the current stack persists nothing for redis). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:alpine + ports: + - containerPort: 6379 + name: redis + env: + - name: TZ + value: "America/Toronto" + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "1Gi" +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: rodan +spec: + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 + name: redis \ No newline at end of file diff --git a/k8s/22-rabbitmq.yaml b/k8s/22-rabbitmq.yaml new file mode 100644 index 000000000..e10c38b7b --- /dev/null +++ b/k8s/22-rabbitmq.yaml @@ -0,0 +1,72 @@ +# RabbitMQ: Celery broker AND result backend (CELERY_RESULT_BACKEND="amqp" in settings.py). +# Ephemeral storage matches the current stack (rabbitmq durability is a known upstream TODO). +# DEFAULT_USER/PASS must match the credentials embedded in rodan-secrets/RABBITMQ_URL. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + # Pinned to 3.x: RabbitMQ 4.x removed `transient_nonexcl_queues`, which Celery 3.1.25 + # (Rodan's version) relies on. The floating `:alpine` tag now resolves to 4.x and breaks it. + image: rabbitmq:3.13-alpine + ports: + - containerPort: 5672 + name: amqp + env: + - name: TZ + value: "America/Toronto" + - name: RABBITMQ_DEFAULT_USER + valueFrom: { secretKeyRef: { name: rodan-secrets, key: RABBITMQ_DEFAULT_USER } } + - name: RABBITMQ_DEFAULT_PASS + valueFrom: { secretKeyRef: { name: rodan-secrets, key: RABBITMQ_DEFAULT_PASS } } + volumeMounts: + - name: data + mountPath: /var/lib/rabbitmq + readinessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 10 + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + resources: + requests: + cpu: "200m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" + volumes: + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + namespace: rodan +spec: + selector: + app: rabbitmq + ports: + - port: 5672 + targetPort: 5672 + name: amqp \ No newline at end of file diff --git a/k8s/30-rodan-main.yaml b/k8s/30-rodan-main.yaml new file mode 100644 index 000000000..d4ac6a457 --- /dev/null +++ b/k8s/30-rodan-main.yaml @@ -0,0 +1,80 @@ +# Django REST API (gunicorn :8000). Image ENTRYPOINT is /opt/entrypoint (waits for pg+redis), +# then runs /run/start which migrates, creates the superuser, collectstatic, then gunicorn. +# Single replica so migrations never run concurrently. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rodan-main + namespace: rodan +spec: + replicas: 1 + strategy: + type: Recreate # avoid two pods running migrations against the same DB at once + selector: + matchLabels: + app: rodan-main + template: + metadata: + labels: + app: rodan-main + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: rodan-main + image: ghcr.io/ddmal/rodan-main:k8s-migration + args: ["/run/start"] + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "None" + ports: + - containerPort: 8000 + name: http + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + httpGet: + path: /api/?format=json + port: 8000 + httpHeaders: + - name: User-Agent + value: k8s-healthcheck + periodSeconds: 10 + failureThreshold: 60 # up to ~10 min for image boot + migrate + collectstatic + readinessProbe: + httpGet: + path: /api/?format=json + port: 8000 + httpHeaders: + - name: User-Agent + value: k8s-healthcheck + periodSeconds: 15 + timeoutSeconds: 10 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "4Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: rodan-main + namespace: rodan +spec: + selector: + app: rodan-main + ports: + - port: 8000 + targetPort: 8000 + name: http diff --git a/k8s/31-celery.yaml b/k8s/31-celery.yaml new file mode 100644 index 000000000..4f497236d --- /dev/null +++ b/k8s/31-celery.yaml @@ -0,0 +1,56 @@ +# Default Celery worker (queue "celery"). Same image as the API; ENTRYPOINT /opt/entrypoint +# (waits for pg+redis), then /run/start-celery which also waits for rodan-main:8000 before +# launching the worker as node "celery@celery". +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: celery + template: + metadata: + labels: + app: celery + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: celery + image: ghcr.io/ddmal/rodan-main:k8s-migration + args: ["/run/start-celery"] + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "celery" + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] + periodSeconds: 45 + timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer + failureThreshold: 30 # tolerates the long wait-for rodan-main:8000 (up to 900s) + livenessProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] + periodSeconds: 60 + timeoutSeconds: 45 + failureThreshold: 3 + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "4" + memory: "12Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources diff --git a/k8s/32-py3-celery.yaml b/k8s/32-py3-celery.yaml new file mode 100644 index 000000000..f556d28c5 --- /dev/null +++ b/k8s/32-py3-celery.yaml @@ -0,0 +1,54 @@ +# Python3 Celery worker (queue "Python3"). This image's ENTRYPOINT is /run/start-celery +# directly (no /opt/entrypoint wrapper), so no args override is needed. Node "celery@Python3". +apiVersion: apps/v1 +kind: Deployment +metadata: + name: py3-celery + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: py3-celery + template: + metadata: + labels: + app: py3-celery + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: py3-celery + image: ghcr.io/ddmal/rodan-python3-celery:k8s-migration + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "Python3" + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] + periodSeconds: 45 + timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer + failureThreshold: 30 + livenessProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] + periodSeconds: 60 + timeoutSeconds: 45 + failureThreshold: 3 + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "4" + memory: "12Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources diff --git a/k8s/33-gpu-celery.yaml b/k8s/33-gpu-celery.yaml new file mode 100644 index 000000000..f1674f26c --- /dev/null +++ b/k8s/33-gpu-celery.yaml @@ -0,0 +1,66 @@ +# GPU Celery worker (queue "GPU") — the ONLY GPU-using workload. Pinned to the GPU node. +# ENTRYPOINT /opt/entrypoint (waits pg+redis) then /run/start-celery. Node "celery@GPU". +# Requires: GPU node labeled gpu=true and tainted nvidia.com/gpu=present:NoSchedule, +# plus the NVIDIA device plugin (60-nvidia-device-plugin.yaml) advertising nvidia.com/gpu. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-celery + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: gpu-celery + template: + metadata: + labels: + app: gpu-celery + spec: + imagePullSecrets: + - name: ghcr-pull-secret + nodeSelector: + gpu: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + # k3s exposes the NVIDIA runtime via this RuntimeClass (confirmed present in-cluster). + runtimeClassName: nvidia + containers: + - name: gpu-celery + image: ghcr.io/ddmal/rodan-gpu-celery:k8s-migration + args: ["/run/start-celery"] + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "GPU" + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] + periodSeconds: 45 + timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer + failureThreshold: 30 + livenessProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] + periodSeconds: 60 + timeoutSeconds: 45 + failureThreshold: 3 + resources: + requests: + cpu: "1" + memory: "4Gi" + limits: + cpu: "2" + memory: "16Gi" + nvidia.com/gpu: 1 + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources diff --git a/k8s/40-iipsrv.yaml b/k8s/40-iipsrv.yaml new file mode 100644 index 000000000..f78ecda9a --- /dev/null +++ b/k8s/40-iipsrv.yaml @@ -0,0 +1,62 @@ +# IIP image server (FastCGI :9003). Reads images from the shared resources volume +# (FILESYSTEM_PREFIX=/rodan/data/ is baked into the image). nginx fastcgi_pass's to it. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: iipsrv + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: iipsrv + template: + metadata: + labels: + app: iipsrv + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: iipsrv + image: ghcr.io/ddmal/iipsrv:k8s-migration + ports: + - containerPort: 9003 + name: fcgi + volumeMounts: + - name: resources + mountPath: /rodan/data + readinessProbe: + tcpSocket: + port: 9003 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 9003 + initialDelaySeconds: 15 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "2Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: iipsrv + namespace: rodan +spec: + selector: + app: iipsrv + ports: + - port: 9003 + targetPort: 9003 + name: fcgi diff --git a/k8s/41-rodan-client.yaml b/k8s/41-rodan-client.yaml new file mode 100644 index 000000000..100df8a32 --- /dev/null +++ b/k8s/41-rodan-client.yaml @@ -0,0 +1,108 @@ +# Backbone.js static client (nginx :80 inside the image). Its runtime configuration.json is +# injected via ConfigMap (the compose stack bind-mounts it at /client/configuration.json). +# Public access is https via the edge, so SERVER_HTTPS=true / port 443 / host rodan2.simssa.ca. +apiVersion: v1 +kind: ConfigMap +metadata: + name: rodan-client-config + namespace: rodan +data: + configuration.json: | + { + "SERVER_HOST": "rodan2.simssa.ca", + "SERVER_PORT": "443", + "SERVER_HTTPS": true, + "SERVER_SOCKET_AVAILABLE": false, + "SERVER_AUTHENTICATION_TYPE": "token", + "ADMIN_CLIENT": { + "NAME": "", + "EMAIL": "" + }, + "DEBUG": false, + "WORKFLOWBUILDERGUI": { + "USER_AGENT": "rodan-standard", + "GRID": { + "DIMENSION": 20, + "LINE_COLOR": "#606060", + "LINE_WIDTH": 0.5 + }, + "ZOOM_MAX": 3.0, + "ZOOM_MIN": 1.0, + "ZOOM_RATE": 0.05, + "ZOOM_INITIAL": 1.7, + "WORKFLOWJOB_WIDTH": 20, + "WORKFLOWJOB_HEIGHT": 22, + "PORT_WIDTH": 8, + "PORT_HEIGHT": 8, + "OUTPUTPORT_COLOR": "#00ff00", + "INPUTPORT_COLOR_SATISFIED": "#00ff00", + "INPUTPORT_COLOR_UNSATISFIED": "#ff0000", + "INPUTPORT_COLOR_CANDIDATE": "#00ff00", + "STROKE_COLOR": "#000000", + "FILL_COLOR": "#ccccff", + "WORKFLOWJOBGROUP_FILL_COLOR": "#8888ff", + "STROKE_WIDTH": 1, + "FONT_SIZE": 10, + "STROKE_COLOR_SELECTED": "#0000ff", + "STROKE_WIDTH_SELECTED": 2, + "CONNECTION_CIRCLE_RADIUS": 4, + "HOVER_TIME": 1000 + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rodan-client + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: rodan-client + template: + metadata: + labels: + app: rodan-client + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: rodan-client + image: ghcr.io/ddmal/rodan-client:k8s-migration + ports: + - containerPort: 80 + name: http + volumeMounts: + - name: client-config + mountPath: /client/configuration.json + subPath: configuration.json + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + volumes: + - name: client-config + configMap: + name: rodan-client-config +--- +apiVersion: v1 +kind: Service +metadata: + name: rodan-client + namespace: rodan +spec: + selector: + app: rodan-client + ports: + - port: 80 + targetPort: 80 + name: http diff --git a/k8s/50-nginx.yaml b/k8s/50-nginx.yaml new file mode 100644 index 000000000..a2f55b460 --- /dev/null +++ b/k8s/50-nginx.yaml @@ -0,0 +1,70 @@ +# Reverse proxy (kept as-is). Listens on :80 (plain HTTP — TLS terminated at the edge). +# Routes /api,/ht -> rodan-main:8000, / -> rodan-client, /fcgi-bin -> iipsrv:9003, +# /ws/ -> redis:6379, /uploads + /static -> filesystem. Serves baked static from /rodan/static. +# ENTRYPOINT is the base nginx /docker-entrypoint.sh; args run /run/start (waits for deps, then nginx). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: nginx + image: ghcr.io/ddmal/nginx:k8s-migration + args: ["/run/start"] + env: + - name: TZ + value: "America/Toronto" + - name: SERVER_HOST + valueFrom: { configMapKeyRef: { name: rodan-config, key: SERVER_HOST } } + ports: + - containerPort: 80 + name: http + volumeMounts: + - name: resources + mountPath: /rodan/data + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "1Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + namespace: rodan +spec: + selector: + app: nginx + ports: + - port: 80 + targetPort: 80 + name: http diff --git a/k8s/51-ingress.yaml b/k8s/51-ingress.yaml new file mode 100644 index 000000000..68ceb01f0 --- /dev/null +++ b/k8s/51-ingress.yaml @@ -0,0 +1,23 @@ +# Cluster entry point: k3s Traefik. HTTP only (web entrypoint) — TLS is terminated on the +# external edge server, which forwards plain HTTP with Host: rodan2.simssa.ca and +# X-Forwarded-Proto: https. All path routing stays inside the nginx container. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rodan + namespace: rodan + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - host: rodan2.simssa.ca + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: nginx + port: + number: 80 diff --git a/k8s/60-nvidia-device-plugin.yaml b/k8s/60-nvidia-device-plugin.yaml new file mode 100644 index 000000000..073a87fbb --- /dev/null +++ b/k8s/60-nvidia-device-plugin.yaml @@ -0,0 +1,49 @@ +# NVIDIA device plugin — advertises nvidia.com/gpu on the GPU node so gpu-celery can request it. +# Runs only on nodes labeled gpu=true and tolerates the GPU taint. +# Prereqs on the GPU node: NVIDIA driver (>= 460.x for the CUDA 11.2 image) + nvidia-container-toolkit, +# with the nvidia container runtime configured (k3s detects it and wires up containerd automatically). +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: nvidia-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + priorityClassName: system-node-critical + nodeSelector: + gpu: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + # k3s auto-creates the 'nvidia' RuntimeClass; the plugin must run under it so + # libnvidia-ml.so.1 (NVML) is injected, otherwise it detects 0 GPUs. + runtimeClassName: nvidia + containers: + - name: nvidia-device-plugin-ctr + image: nvcr.io/nvidia/k8s-device-plugin:v0.14.5 + env: + - name: FAIL_ON_INIT_ERROR + value: "false" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..1e06f050c --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,147 @@ +# Rodan on Kubernetes (k3s) + +Plain-YAML manifests to run the Rodan stack on a k3s cluster (target: Arbutus). This is a +**lift-and-shift** of the existing Docker Swarm stack — same published images (`v3.3.1`, plus +`nightly` for iipsrv/rodan-client), reorganized as k8s workloads. + +See the full design in `../.claude/plans/the-architecture-that-i-parallel-lampson.md`. + +## Topology + +- All services run on the k3s cluster. **Only `gpu-celery`** runs on a dedicated GPU node. +- Storage is **NFS on Arbutus**: `resources` (`/rodan/data`, ~883 GB, RWX), `pg_data` (PG 9.6 + PGDATA, RWO, `hard`), `pg_backup` (RWX). Static PV+PVC pairs, `Retain` reclaim. +- **Postgres runs in-cluster** (StatefulSet) on its NFS PGDATA. +- Entry point is **Traefik Ingress** for `rodan2.simssa.ca`, HTTP only — **TLS is terminated on + an external edge server** that forwards plain HTTP. The kept `nginx` container does all internal + routing. +- k8s Service names intentionally match the old Docker DNS names (`postgres`, `redis`, `rabbitmq`, + `rodan-main`, `rodan-client`, `iipsrv`) because they're hardcoded in `rodan.conf` / `wait-for-app`. + +## Files (apply in numeric order) + +| File | Purpose | +|---|---| +| `00-namespace.yaml` | namespace `rodan` | +| `01-configmap.yaml` | `rodan-config` (non-secret env) | +| `02-secret.template.yaml` | template → copy to `02-secret.yaml`, fill in, apply (gitignored) | +| `10..15-*` | NFS PVs + PVCs (resources, pg_data, pg_backup) | +| `20-postgres.yaml` | Postgres StatefulSet + Service | +| `21-redis.yaml` / `22-rabbitmq.yaml` | broker/cache Deployments + Services | +| `30-rodan-main.yaml` | Django API Deployment + Service | +| `31/32/33-*celery.yaml` | celery / py3-celery / gpu-celery workers | +| `40-iipsrv.yaml` / `41-rodan-client.yaml` | image server + static client | +| `50-nginx.yaml` | reverse proxy Deployment + ClusterIP Service | +| `51-ingress.yaml` | Traefik Ingress (`rodan2.simssa.ca`) | +| `60-nvidia-device-plugin.yaml` | advertises `nvidia.com/gpu` on the GPU node | + +## Before you apply — fill in placeholders + +1. **NFS server** — DONE. The three PV files (`10/12/14-pv-*.yaml`) already point at the Arbutus + data server **`192.168.236.124`** with the cloned paths + `/srv/rodan-data/var/lib/docker/volumes/rodan_{resources,pg_data,pg_backup}/_data`. + (That VM is a `dd` block-clone of the old data server's 2 TB disk, mounted at `/srv/rodan-data` + and NFS-exported to the `192.168.236.0/24` subnet.) +2. **Secret** — `cp 02-secret.template.yaml 02-secret.yaml`, set real values. Note: use + `ADMIN_PASS` (the live `production.env` calls it `ADMIN_PASSWORD`, which `scripts/start` does + **not** read). `RABBITMQ_URL` creds must match `RABBITMQ_DEFAULT_USER`/`PASS`. +3. **GPU node** — label + taint it (see below). + +## NFS server setup (Arbutus) — DONE, recorded here for reference + +The data server `192.168.236.124` (`/srv/rodan-data` = `dd` clone of the old 2 TB disk) exports the +three cloned dirs to the k3s subnet (`no_root_squash` is required for the postgres uid 999): +```sh +# /etc/exports on 192.168.236.124 +/srv/rodan-data/var/lib/docker/volumes/rodan_resources/_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +/srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +/srv/rodan-data/var/lib/docker/volumes/rodan_pg_backup/_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +# sudo exportfs -ra && sudo exportfs -v +``` + +**Every k3s node (incl. the GPU node) must have the NFS client installed**, or pods that mount these +PVCs get stuck in `ContainerCreating`: +```sh +sudo apt update && sudo apt install -y nfs-common +``` +Per-node check (resources export verified working from k3s-node-4): +```sh +sudo mount -t nfs4 192.168.236.124:/srv/rodan-data/var/lib/docker/volumes/rodan_resources/_data /mnt && ls /mnt && sudo umount /mnt +``` +Also open **TCP 2049** from the k3s nodes in the Arbutus security group. + +## GPU node setup + +1. Install the NVIDIA driver (**≥ 460.x**, required by the CUDA 11.2 `gpu-celery` image) and the + nvidia-container-toolkit; join the node to k3s. k3s auto-detects the nvidia runtime in containerd. +2. Label and taint: + ```sh + kubectl label node gpu=true + kubectl taint node nvidia.com/gpu=present:NoSchedule + ``` +3. `60-nvidia-device-plugin.yaml` then makes the node advertise `nvidia.com/gpu`. If k3s exposes + the runtime only as a RuntimeClass (not the node default), uncomment `runtimeClassName: nvidia` + in `33-gpu-celery.yaml` and the device plugin. + +## Data migration + +Run with the old stack still serving, then a final pass during a short maintenance window. + +```sh +# resources (~883 GB) — old NFS export -> new Arbutus export +rsync -aHAX --info=progress2 \ + root@192.168.17.244:/var/lib/docker/volumes/rodan_resources/_data/ \ + /export/resources/ +# repeat for a quick incremental pass right before cutover + +# pg_data (~4.1 GB, PG 9.6 preserved) — STOP the old postgres first, then raw-copy: +rsync -aHAX root@192.168.17.244:/var/lib/docker/volumes/rodan_pg_data/_data/ /export/pg_data/ +# Fallback / verification: pg_dump on the old DB and restore into the new postgres pod instead. +``` +After copying, sanity-check: `ls /export/resources/projects | wc -l` should be ~309. + +## Apply order + +```sh +kubectl apply -f 00-namespace.yaml +kubectl apply -f 01-configmap.yaml -f 02-secret.yaml +kubectl apply -f 10-pv-resources.yaml -f 11-pvc-resources.yaml \ + -f 12-pv-pgdata.yaml -f 13-pvc-pgdata.yaml \ + -f 14-pv-pgbackup.yaml -f 15-pvc-pgbackup.yaml +kubectl -n rodan get pvc # all Bound before continuing + +kubectl apply -f 20-postgres.yaml -f 21-redis.yaml -f 22-rabbitmq.yaml +kubectl apply -f 60-nvidia-device-plugin.yaml +kubectl apply -f 30-rodan-main.yaml # watch logs: migrate -> collectstatic -> gunicorn +kubectl apply -f 31-celery.yaml -f 32-py3-celery.yaml -f 33-gpu-celery.yaml +kubectl apply -f 40-iipsrv.yaml -f 41-rodan-client.yaml -f 50-nginx.yaml -f 51-ingress.yaml +``` +(Or just `kubectl apply -f .` once `02-secret.yaml` and placeholders are filled — ordering is +self-healing via the `wait-for-app` logic baked into the images, but applying backends first is cleaner.) + +## Verify + +```sh +kubectl -n rodan get pods,svc,pvc,ingress +kubectl -n rodan exec deploy/rodan-main -- curl -s -o /dev/null -w '%{http_code}\n' \ + -H 'User-Agent: k8s' localhost:8000/api/?format=json # 200 +kubectl -n rodan exec deploy/celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@celery +kubectl -n rodan exec deploy/py3-celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@Python3 +kubectl -n rodan exec deploy/gpu-celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@GPU +kubectl -n rodan exec deploy/gpu-celery -- nvidia-smi +kubectl describe node | grep nvidia.com/gpu # allocatable: 1 +# external (through the edge): https://rodan2.simssa.ca — log in, open a project/image, run a workflow +``` + +## Notes / gotchas + +- **Postgres on NFS** is preserved per decision — keep replicas=1 and the `hard` mount. Don't scale it. +- **Celery node names** are fixed (`-n "${CELERY_JOB_QUEUE}"`); fine at 1 replica each. To scale a + worker, change the image's `start-celery` to use a per-pod name (`%h`) first. +- **Large uploads** (nginx allows 700m): if uploads fail at the edge/Traefik, add a Traefik + middleware raising `maxRequestBodyBytes` (Traefik streams by default, so usually fine). +- **WebSockets** (`/ws/`): if long-lived sockets drop, raise Traefik's responding read timeout. +- **TLS**: none inside the cluster — the edge must send `X-Forwarded-Proto: https` and the correct + `Host` so Django builds https URLs (nginx already sets `X-Scheme: https` on `/api`). +- **Backups**: the old monthly pg-backup cron appears stopped (~Sep 2024). Consider a k8s CronJob + invoking the image's `backup` script against `/backups`. diff --git a/production.sample b/production.sample deleted file mode 100644 index f23ed9dc6..000000000 --- a/production.sample +++ /dev/null @@ -1,294 +0,0 @@ -version: "3.4" - -services: - - nginx: - image: "ddmal/nginx:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "0.25" - memory: 0.5G - limits: - cpus: "0.25" - memory: 0.5G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD", "/usr/sbin/service", "nginx", "status"] - interval: "30s" - timeout: "10s" - retries: 10 - start_period: "5m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - TLS: 1 - ports: - - "80:80" - - "443:443" - - "5671:5671" - - "9002:9002" - volumes: - - "resources_nfs:/rodan/data" - - rodan-main: - image: "ddmal/rodan-main:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1.5" - memory: 6G - limits: - cpus: "1.5" - memory: 6G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD-SHELL", "/usr/bin/curl -H 'User-Agent: docker-healthcheck' http://localhost:8000/api/?format=json || exit 1"] - interval: "30s" - timeout: "30s" - retries: 5 - start_period: "15m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: None - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - rodan-client: - image: "ddmal/rodan-client:nightly" - deploy: - placement: - constraints: - - node.role == worker - volumes: - - "./rodan-client/config/configuration.json:/client/configuration.json" - - iipsrv: - image: "ddmal/iipsrv:nightly" - volumes: - - "resources_nfs:/rodan/data" - - celery: - image: "ddmal/rodan-main:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 6G - limits: - cpus: "2" - memory: 6G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] - interval: "30s" - timeout: "30s" - start_period: "10m" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: celery - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - py3-celery: - image: "ddmal/rodan-python3-celery:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 5G - limits: - cpus: "2" - memory: 5G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: Python3 - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - gpu-celery: - image: "ddmal/rodan-gpu-celery:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 18G - limits: - cpus: "1" - memory: 18G - placement: - constraints: - - node.role == manager - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: GPU - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - redis: - image: "redis:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == worker - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - - postgres: - image: "ddmal/postgres-plpython:v3.2.0" - deploy: - replicas: 1 - endpoint_mode: dnsrr - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD-SHELL", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - volumes: - - "pg_data_nfs:/var/lib/postgresql/data" - - "pg_backup_nfs:/backups" - env_file: - - ./scripts/production.env - - rabbitmq: - image: "rabbitmq:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 4G - limits: - cpus: "2" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == worker - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] - interval: "30s" - timeout: "3s" - retries: 3 - environment: - TZ: America/Toronto - env_file: - - ./scripts/production.env - -volumes: - resources_nfs: - driver: local - driver_opts: - type: nfs - o: addr=[ip address],rw,nfsvers=4,soft,timeo=600,retrans=3 - device: "[ip address]:/var/lib/docker/volumes/rodan_resources/_data" - pg_backup_nfs: - driver: local - driver_opts: - type: nfs - o: addr=[ip address],rw,nfsvers=4 - device: "[ip address]:/var/lib/docker/volumes/rodan_pg_backup/_data" - pg_data_nfs: - driver: local - driver_opts: - type: nfs - o: addr=[ip address],rw,nfsvers=4 - device: "[ip address]:/var/lib/docker/volumes/rodan_pg_data/_data" \ No newline at end of file diff --git a/production.yml b/production.yml deleted file mode 100644 index 6c1b04150..000000000 --- a/production.yml +++ /dev/null @@ -1,254 +0,0 @@ -version: "3.4" - -services: - - nginx: - image: "ddmal/nginx:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "0.5" - memory: 1G - limits: - cpus: "0.5" - memory: 1G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "/usr/sbin/service", "nginx", "status"] - interval: "30s" - timeout: "10s" - retries: 10 - start_period: "5m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - TLS: 1 - ports: - - "80:80" - - "443:443" - - "5671:5671" - - "9002:9002" - volumes: - - "resources:/rodan/data" - - rodan-main: - image: "ddmal/rodan-main:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 4G - limits: - cpus: "1" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD-SHELL", "/usr/bin/curl -H 'User-Agent: docker-healthcheck' http://localhost:8000/api/?format=json || exit 1"] - interval: "30s" - timeout: "30s" - retries: 5 - start_period: "2m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: None - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - rodan-client: - image: "ddmal/rodan-client:nightly" - volumes: - - "./rodan-client/config/configuration.json:/client/configuration.json" - - iipsrv: - image: "ddmal/iipsrv:nightly" - volumes: - - "resources:/rodan/data" - - celery: - image: "ddmal/rodan-main:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "6" - memory: 12G - limits: - cpus: "6" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] - interval: "30s" - timeout: "30s" - start_period: "1m" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: celery - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - py3-celery: - image: "ddmal/rodan-python3-celery:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "6" - memory: 12G - limits: - cpus: "6" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: Python3 - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - gpu-celery: - image: "ddmal/rodan-gpu-celery:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 45G - limits: - cpus: "2" - memory: 45G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.labels.queue == GPU - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: GPU - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - redis: - image: "redis:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - - postgres: - image: "ddmal/postgres-plpython:v3.3.1" - deploy: - replicas: 1 - endpoint_mode: dnsrr - resources: - reservations: - cpus: "4" - memory: 10G - limits: - cpus: "4" - memory: 10G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD-SHELL", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - volumes: - - "pg_data:/var/lib/postgresql/data" - - "pg_backup:/backups" - env_file: - - ./scripts/production.env - - rabbitmq: - image: "rabbitmq:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] - interval: "30s" - timeout: "3s" - retries: 3 - environment: - TZ: America/Toronto - env_file: - - ./scripts/production.env - -volumes: - resources: - pg_backup: - pg_data: diff --git a/runLog.txt b/runLog.txt deleted file mode 100644 index 0f5889d7b..000000000 --- a/runLog.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Run build_arm first if you don't have the NGINX container. -# Launch ARM instance -nightly: Pulling from ddmal/rodan-python3-celery diff --git a/run_containers b/run_containers deleted file mode 100644 index f46879527..000000000 --- a/run_containers +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# parallel silence if you want -# parallel --citation - -parallel --jobs 4 < starter.txt >> runLog.txt diff --git a/scripts/production.sample b/scripts/production.sample deleted file mode 100644 index 624e0c798..000000000 --- a/scripts/production.sample +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Django Configuration -############################################################################### - -ADMIN_USER=someadmin -ADMIN_EMAIL=admin@rodan2.simssa.ca -ADMIN_PASS=123456Seven -DJANGO_DEBUG_MODE=False -DJANGO_SECRET_KEY=AVeryLongSecretKeyThatWasGeneratedByOurSpecialScript_DoNotForgetToChangeThis -DJANGO_MEDIA_ROOT=/rodan/data/ -# You can specify multiple hosts with a comma -DJANGO_ALLOWED_HOSTS=* -DJANGO_ADMIN_URL=^api/random_secret_admin/ -IIPSRV_URL=https://rodan2.simssa.ca/fcgi-bin/iipsrv.fcgi/ -DJANGO_ACCESS_LOG=/code/Rodan/rodan.log -DJANGO_DEBUG_LOG=/code/Rodan/database.log -SERVER_HOST=rodan2.simssa.ca - -############################################################################### -# SMTP Configuration -############################################################################### - -EMAIL_HOST= -EMAIL_PORT= -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= - -############################################################################### -# Celery Configuration -############################################################################### - -C_FORCE_ROOT=true - -############################################################################### -# Database Configuration (Postgres) -############################################################################### - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=rodan -POSTGRES_USER=someadmin -POSTGRES_PASSWORD=123456Seven -POSTGRES_DATABASE_LOGFILE=/code/Rodan/database.log - -############################################################################### -# Messenger Configuration (RabbitMQ) -############################################################################### - -# The format is -> $Protocol :// $User : $Password @ $Host : $Port / $Vhost -RABBITMQ_URL=amqp://someadmin:123456Seven@rabbitmq:5672// -RABBITMQ_DEFAULT_USER=someadmin -RABBITMQ_DEFAULT_PASS=123456Seven - -############################################################################### -# Websocket Configuration (Redis) -############################################################################### - -REDIS_HOST=redis -REDIS_PORT=6379 -REDIS_DB=0 diff --git a/scripts/staging.env b/scripts/staging.env deleted file mode 100644 index fec111a1c..000000000 --- a/scripts/staging.env +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################### -# Django Configuration -############################################################################### - -ADMIN_USER=someadmin -ADMIN_EMAIL=someadmin@rodan2.simssa.ca -ADMIN_PASS=123456Seven -DJANGO_DEBUG_MODE=False -DJANGO_SECRET_KEY=AVeryLongSecretKeyThatWasGeneratedByOurSpecialScript_DoNotForgetToChangeThis -DJANGO_MEDIA_ROOT=/rodan/data/ -# You can specify multiple hosts with a comma -DJANGO_ALLOWED_HOSTS=* -DJANGO_ADMIN_URL=^api/random_secret_admin/ -IIPSRV_URL=http://rodan.staging.simssa.ca/fcgi-bin/iipsrv.fcgi/ -DJANGO_ACCESS_LOG=/code/Rodan/rodan.log -DJANGO_DEBUG_LOG=/code/Rodan/database.log -SERVER_HOST=rodan.staging.simssa.ca -############################################################################### -# Celery Configuration -############################################################################### - -C_FORCE_ROOT=true - -############################################################################### -# Database Configuration (Postgres) -############################################################################### - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=rodan -POSTGRES_USER=someadmin -POSTGRES_PASSWORD=123456Seven -POSTGRES_DATABASE_LOGFILE=/code/Rodan/database.log - -############################################################################### -# Messenger Configuration (RabbitMQ) -############################################################################### - -# The format is -> $Protocol :// $User : $Password @ $Host : $Port / $Vhost -RABBITMQ_URL=amqp://someadmin:123456Seven@rabbitmq:5672// -RABBITMQ_DEFAULT_USER=someadmin -RABBITMQ_DEFAULT_PASS=123456Seven - -############################################################################### -# Websocket Configuration (Redis) -############################################################################### - -REDIS_HOST=redis -REDIS_PORT=6379 -REDIS_DB=0 diff --git a/scripts/staging.sample b/scripts/staging.sample deleted file mode 100644 index 0d5e184ab..000000000 --- a/scripts/staging.sample +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Django Configuration -############################################################################### - -ADMIN_USER=someadmin -ADMIN_EMAIL=admin@rodan2.simssa.ca -ADMIN_PASS=123456Seven -DJANGO_DEBUG_MODE=False -DJANGO_SECRET_KEY=AVeryLongSecretKeyThatWasGeneratedByOurSpecialScript_DoNotForgetToChangeThis -DJANGO_MEDIA_ROOT=/rodan/data/ -# You can specify multiple hosts with a comma -DJANGO_ALLOWED_HOSTS=* -DJANGO_ADMIN_URL=^api/random_secret_admin/ -IIPSRV_URL=http://rodan-staging.simssa.ca/fcgi-bin/iipsrv.fcgi/ -DJANGO_ACCESS_LOG=/code/Rodan/rodan.log -DJANGO_DEBUG_LOG=/code/Rodan/database.log -SERVER_HOST=rodan-staging.simssa.ca - -############################################################################### -# SMTP Configuration -############################################################################### - -EMAIL_HOST= -EMAIL_PORT= -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= - -############################################################################### -# Celery Configuration -############################################################################### - -C_FORCE_ROOT=true - -############################################################################### -# Database Configuration (Postgres) -############################################################################### - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=rodan -POSTGRES_USER=someadmin -POSTGRES_PASSWORD=123456Seven -POSTGRES_DATABASE_LOGFILE=/code/Rodan/database.log - -############################################################################### -# Messenger Configuration (RabbitMQ) -############################################################################### - -# The format is -> $Protocol :// $User : $Password @ $Host : $Port / $Vhost -RABBITMQ_URL=amqp://someadmin:123456Seven@rabbitmq:5672// -RABBITMQ_DEFAULT_USER=someadmin -RABBITMQ_DEFAULT_PASS=123456Seven - -############################################################################### -# Websocket Configuration (Redis) -############################################################################### - -REDIS_HOST=redis -REDIS_PORT=6379 -REDIS_DB=0 diff --git a/staging.yml b/staging.yml deleted file mode 100644 index f15eaf8ee..000000000 --- a/staging.yml +++ /dev/null @@ -1,263 +0,0 @@ -version: "3.4" - -services: - - nginx: - image: "ddmal/nginx:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "0.5" - memory: 1G - limits: - cpus: "0.5" - memory: 1G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "/usr/sbin/service", "nginx", "status"] - interval: "30s" - timeout: "10s" - retries: 10 - start_period: "5m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - env_file: - - ./scripts/staging.env - ports: - - "80:80" - - "5671:5671" - - "9002:9002" - volumes: - - "resources:/rodan/data" - - rodan-main: - image: "ddmal/rodan-main:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 4G - limits: - cpus: "1" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - # TODO: Make the healthcheck a service status call instead of adding to the logfile. - # https://github.com/DDMAL/rodan-docker/issues/61 - healthcheck: - test: ["CMD-SHELL", "/usr/bin/curl -H 'User-Agent: docker-healthcheck' http://localhost:8000/api/?format=json || exit 1"] - interval: "30s" - timeout: "30s" - retries: 5 - start_period: "2m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: None - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - rodan-client: - image: "ddmal/rodan-client:nightly" - volumes: - - "./rodan-client/config/configuration.json:/client/configuration.json" - - iipsrv: - image: "ddmal/iipsrv:nightly" - volumes: - - "resources:/rodan/data" - - celery: - image: "ddmal/rodan-main:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 12G - limits: - cpus: "2" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] - # first run interval seconds after the container is started, and then again interval seconds after each previous - interval: "30s" - # How long to wait for the healthcheck to succeed - timeout: "30s" - # Ignore failures during the start_period - start_period: "1m" - # accept 3 consecutive failures before - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: celery - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - py3-celery: - image: "ddmal/rodan-python3-celery:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 12G - limits: - cpus: "2" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: Python3 - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - gpu-celery: - image: "ddmal/rodan-gpu-celery:nightly" - deploy: - replicas: 1 - resources: - limits: - cpus: '1' - memory: 8G - reservations: - cpus: '1' - memory: 8G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: GPU - depends_on: - - postgres - - rodan - - rabbitmq - - redis - - celery - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - redis: - image: "redis:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - - postgres: - image: "ddmal/postgres-plpython:nightly" - deploy: - replicas: 1 - endpoint_mode: dnsrr - resources: - reservations: - cpus: "2" - memory: 4G - limits: - cpus: "2" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD-SHELL", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - volumes: - - "pg_data:/var/lib/postgresql/data" - - "pg_backup:/backups" - env_file: - - ./scripts/staging.env - - rabbitmq: - image: "rabbitmq:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] - interval: "30s" - timeout: "3s" - retries: 3 - environment: - TZ: America/Toronto - env_file: - - ./scripts/staging.env - -volumes: - resources: - pg_backup: - pg_data: diff --git a/starter.sh b/starter.sh deleted file mode 100755 index 47c7b657e..000000000 --- a/starter.sh +++ /dev/null @@ -1,32 +0,0 @@ -# runs the commands in new terminal tabs you have to open a new separate window of iterm to start with -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "t" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && make run_arm -"' -sleep 30 -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "d" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && docker compose exec rodan-main /run/start -"' -sleep 30 -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "d" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && docker compose exec celery /run/start-celery -"' -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "d" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && docker compose exec py3-celery /run/start-celery -"' -echo "######### DEPLOYED #########" -sleep 20 -osascript -e 'tell application "Google Chrome" to open location "http://localhost"' - -# wait for the key q and terminate and close all the tabs if q is entered -echo "enter q to terminate the process" -read key -while [[ $key != "q" ]] -do - read key -done -if [[ $key = q ]] -then - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "2" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - printf "\nQuitting from the program\n" -fi -# for the other features regarding the demands of your own environment you can modify the code above -# Author: Shahrad Mohammadzadeh