diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml new file mode 100644 index 000000000..81acf2424 --- /dev/null +++ b/.github/workflows/build-and-deploy.yml @@ -0,0 +1,243 @@ +name: Build, push, and deploy images + +# Builds all 7 Rodan images, pushes them to the private GitHub Container Registry, and +# (on manual dispatch or version tags) deploys the app tier to the k3s cluster. +# Replaces the DockerHub autobuild hooks (hooks/build, hooks/push). +# +# Tags: +# git tag v* -> ghcr.io/ddmal/: (+ deploy) +# push to develop -> ghcr.io/ddmal/:nightly +# pull_request -> build only, no push (validates Dockerfiles) +# workflow_dispatch -> optional tag input (defaults to "nightly") (+ deploy) +# Every pushed build also gets an immutable ghcr.io/ddmal/:sha- tag, +# which the deploy job pins so the rollout is guaranteed to pull the new image. + +on: + push: + branches: [develop] + tags: ['v*'] + workflow_dispatch: + inputs: + tag: + description: "Image tag to build/push (e.g. v3.4.0 or nightly)" + required: false + default: "nightly" + +permissions: + contents: read + packages: write + +concurrency: + group: build-and-deploy-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + OWNER: ddmal + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.meta.outputs.tag }} + push: ${{ steps.meta.outputs.push }} + short_sha: ${{ steps.meta.outputs.short_sha }} + steps: + - id: meta + env: + EVENT: ${{ github.event_name }} + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + PR_NUMBER: ${{ github.event.number }} + run: | + set -euo pipefail + if [ "$EVENT" = "workflow_dispatch" ] && [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG"; PUSH=true + elif [ "$REF_TYPE" = "tag" ]; then + TAG="$REF_NAME"; PUSH=true # e.g. v3.4.0 + elif [ "$EVENT" = "pull_request" ]; then + TAG="pr-$PR_NUMBER"; PUSH=false # build only, never push + elif [ "$REF_NAME" = "develop" ]; then + TAG="nightly"; PUSH=true + else + TAG="$REF_NAME"; PUSH=true + fi + # Docker tags cannot contain "/" + TAG="$(printf '%s' "$TAG" | tr '/' '-')" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "push=$PUSH" >> "$GITHUB_OUTPUT" + echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + echo "Resolved tag=$TAG push=$PUSH sha=${GITHUB_SHA::7} (event=$EVENT ref_type=$REF_TYPE ref=$REF_NAME)" + + # Ordered dependency chain on ONE runner so the hardcoded + # `FROM ddmal/:${VERSION}` lines resolve to the locally-built images. + chain: + needs: setup + runs-on: ubuntu-latest + env: + TAG: ${{ needs.setup.outputs.tag }} + PUSH: ${{ needs.setup.outputs.push }} + SHA_TAG: sha-${{ needs.setup.outputs.short_sha }} + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + # Reclaims ~25-35 GB on ubuntu-latest. + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + large-packages: false + docker-images: false + + - name: Log in to GHCR + if: env.PUSH == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build rodan-python3-celery (base) + run: | + docker build \ + --build-arg BRANCHES=develop \ + --build-arg VERSION="$TAG" \ + -t "$REGISTRY/$OWNER/rodan-python3-celery:$TAG" \ + -t "$REGISTRY/$OWNER/rodan-python3-celery:$SHA_TAG" \ + -t "ddmal/rodan-python3-celery:$TAG" \ + -f python3-celery/Dockerfile \ + . + + - name: Build rodan-main (FROM rodan-python3-celery) + run: | + docker build \ + --build-arg BRANCHES=develop \ + --build-arg VERSION="$TAG" \ + --build-arg build_hash="${{ github.sha }}" \ + -t "$REGISTRY/$OWNER/rodan-main:$TAG" \ + -t "$REGISTRY/$OWNER/rodan-main:$SHA_TAG" \ + -t "ddmal/rodan-main:$TAG" \ + -f rodan-main/Dockerfile \ + . + + - name: Build nginx (FROM rodan-main, bakes static files) + run: | + docker build \ + --build-arg VERSION="$TAG" \ + -t "$REGISTRY/$OWNER/nginx:$TAG" \ + -t "$REGISTRY/$OWNER/nginx:$SHA_TAG" \ + ./nginx + + - name: Push chain images + if: env.PUSH == 'true' + run: | + docker push "$REGISTRY/$OWNER/rodan-python3-celery:$TAG" + docker push "$REGISTRY/$OWNER/rodan-python3-celery:$SHA_TAG" + docker push "$REGISTRY/$OWNER/rodan-main:$TAG" + docker push "$REGISTRY/$OWNER/rodan-main:$SHA_TAG" + docker push "$REGISTRY/$OWNER/nginx:$TAG" + docker push "$REGISTRY/$OWNER/nginx:$SHA_TAG" + + # Independent images — each on its own runner, in parallel. + independent: + needs: setup + runs-on: ubuntu-latest + env: + TAG: ${{ needs.setup.outputs.tag }} + PUSH: ${{ needs.setup.outputs.push }} + SHA_TAG: sha-${{ needs.setup.outputs.short_sha }} + strategy: + fail-fast: false + matrix: + include: + - name: rodan-client + dockerfile: rodan-client/Dockerfile + context: ./rodan-client + args: "--build-arg BRANCHES=develop" + - name: rodan-gpu-celery + dockerfile: gpu-celery/Dockerfile + context: . + args: "--build-arg BRANCHES=develop" + - name: postgres-plpython + dockerfile: postgres/Dockerfile + context: . + args: "" + - name: iipsrv + dockerfile: iipsrv/Dockerfile + context: ./iipsrv + args: "" + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + large-packages: false + docker-images: false + + - name: Log in to GHCR + if: env.PUSH == 'true' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build ${{ matrix.name }} + run: | + docker build ${{ matrix.args }} \ + --build-arg VERSION="$TAG" \ + -t "$REGISTRY/$OWNER/${{ matrix.name }}:$TAG" \ + -t "$REGISTRY/$OWNER/${{ matrix.name }}:$SHA_TAG" \ + -f "${{ matrix.dockerfile }}" \ + "${{ matrix.context }}" + + - name: Push ${{ matrix.name }} + if: env.PUSH == 'true' + run: | + docker push "$REGISTRY/$OWNER/${{ matrix.name }}:$TAG" + docker push "$REGISTRY/$OWNER/${{ matrix.name }}:$SHA_TAG" + + # Deploy the app tier by pinning the immutable sha- tag (guarantees a fresh pull + + # gives `kubectl rollout undo` rollback). Runs only on manual dispatch or version tags. + # postgres / redis / rabbitmq are intentionally left alone (don't bounce the DB on every deploy). + deploy: + needs: [setup, chain, independent] + if: needs.setup.outputs.push == 'true' && (github.event_name == 'workflow_dispatch' || github.ref_type == 'tag') + runs-on: ubuntu-latest + env: + NS: rodan + REG: ghcr.io/ddmal + SHA_TAG: sha-${{ needs.setup.outputs.short_sha }} + steps: + - name: Install kubectl + uses: azure/setup-kubectl@v4 + + - name: Configure kubeconfig from secret + env: + KUBECONFIG_DATA: ${{ secrets.KUBECONFIG }} + run: | + set -euo pipefail + printf '%s' "$KUBECONFIG_DATA" > "$RUNNER_TEMP/kubeconfig" + chmod 600 "$RUNNER_TEMP/kubeconfig" + echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV" + + - name: Roll app-tier images to the immutable SHA tag + run: | + set -euo pipefail + kubectl -n "$NS" set image deployment/rodan-main rodan-main="$REG/rodan-main:$SHA_TAG" + kubectl -n "$NS" set image deployment/celery celery="$REG/rodan-main:$SHA_TAG" + kubectl -n "$NS" set image deployment/py3-celery py3-celery="$REG/rodan-python3-celery:$SHA_TAG" + kubectl -n "$NS" set image deployment/gpu-celery gpu-celery="$REG/rodan-gpu-celery:$SHA_TAG" + kubectl -n "$NS" set image deployment/iipsrv iipsrv="$REG/iipsrv:$SHA_TAG" + kubectl -n "$NS" set image deployment/rodan-client rodan-client="$REG/rodan-client:$SHA_TAG" + kubectl -n "$NS" set image deployment/nginx nginx="$REG/nginx:$SHA_TAG" + + - name: Wait for rollouts + run: | + set -euo pipefail + for d in rodan-main celery py3-celery gpu-celery iipsrv rodan-client nginx; do + kubectl -n "$NS" rollout status deployment/"$d" --timeout=600s + done \ No newline at end of file diff --git a/.github/workflows/release-bot.yml b/.github/workflows/release-bot.yml index ed4b3e1e8..ab0c33040 100644 --- a/.github/workflows/release-bot.yml +++ b/.github/workflows/release-bot.yml @@ -10,6 +10,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Create output folder run: mkdir -p ${{ github.workspace }}/checklist_items - name: Write checklist items to files @@ -38,12 +40,18 @@ jobs: - name: Collect checklist items run: | - msg=$(awk 'FNR==1 && NR!=1 {print "---"}{print}' checklist_items/*.log) - echo "$msg" >> aggregated.log - sha="*SHA: ${{ github.event.pull_request.head.sha }}*" - sed -i "1i\\ - $sha - " aggregated.log + set -o pipefail + shopt -s nullglob + files=(checklist_items/*.log) + { + echo "*SHA: ${{ github.event.pull_request.head.sha }}*" + echo + if [ ${#files[@]} -eq 0 ]; then + echo "_No merge-commit checklist items found ahead of \`master\`._" + else + awk 'FNR==1 && NR!=1 {print "---"}{print}' "${files[@]}" + fi + } > aggregated.log - name: Update Pull Request uses: actions/github-script@v5 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 31abcc274..000000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM debian -# For more information about why this is here, check the build hooks. -RUN printenv \ No newline at end of file diff --git a/Makefile b/Makefile index 7f476cf85..125c1ac00 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -# TODO: Some of these should be converted into Ansible Playbooks when there's time. - -# Chose a makefile because its easier to read over a bunch of if statements inside a bash script. -# We are taking advantage of .PHONY that is available in makefiles to create this simple looking -# list of command shortcuts +# Local-development shortcuts (docker-compose). +# +# Image build/push and cluster deploy now live in .github/workflows/build-and-deploy.yml +# (builds to GHCR, deploys to the k3s `rodan` namespace). The old Docker Swarm / DockerHub +# targets were removed when Rodan migrated to k3s. # Portable replacement for `sed` or `gsed` # See https://unix.stackexchange.com/questions/92895/how-can-i-achieve-portability-with-sed-i-in-place-editing @@ -11,179 +11,27 @@ REPLACE := perl -i -pe RODAN_PATH := ./rodan-main/code/rodan JOBS_PATH := $(RODAN_PATH)/jobs -PROD_TAG := v3.3.1 - DOCKER_TAG := nightly -# Individual Commands - build: @echo "[-] Rebuilding Docker Images for Rodan..." - # Build py3-celery, because it's needed for Rodan and Celery images - # @docker-compose -f build.yml build --no-cache py3-celery # Sometimes it's better to use the - # no-cache option if something unexplicably broke with the py3-celery image (a cached build step perhaps) + # py3-celery first — rodan-main (the `rodan` service) and celery images are FROM it. @docker compose -f build.yml build --no-cache py3-celery - # Build rodan and rodan-client because they are needed for nginx + # rodan(-main) and rodan-client next — nginx is FROM rodan-main. @docker compose -f build.yml build --no-cache --parallel rodan rodan-client - # DockerHub is not intuitive. You won't be able to build from the source root folder in both build contextes. - # When you build locally, the COPY command is relative to the dockerfile. When you build on DockerHub, its relative to the source root. - # For this reason we replace the name to build locally because we build more often on DockerHub than on local. - @$(REPLACE) "s/COPY .\/postgres\/maintenance/COPY .\/maintenance/g" ./postgres/Dockerfile || $(REPLACE) "s/COPY .\/postgres\/maintenance/COPY .\/maintenance/g" ./postgres/Dockerfile - @docker compose -f build.yml build --no-cache --parallel nginx gpu-celery postgres hpc-rabbitmq - # Revert back the change to the COPY command so it will work on Docker Hub. - @$(REPLACE) "s/COPY .\/maintenance/COPY .\/postgres\/maintenance/g" ./postgres/Dockerfile || $(REPLACE) "s/COPY .\/maintenance/COPY .\/postgres\/maintenance/g" ./postgres/Dockerfile + @docker compose -f build.yml build --no-cache --parallel nginx gpu-celery postgres @echo "[+] Done." -backup_db: - @docker exec `docker ps -f name=rodan_postgres -q` backup - -restore_db: - @docker exec `docker ps -f name=rodan_postgres -q` restore - -# Keep in mind, you may need to deal with the postgres/maintenance/backup or backups files depending on setup - run: remote_jobs # Run local version for dev - # Hello, 2022 hires! @DOCKER_TAG=$(DOCKER_TAG) docker compose up -test_prod: pull_prod - # Test production Rodan images with specified tag - # May want to change test-prod-compose.yml if you want a - # different tag. - docker compose -f test-prod-compose.yml up - run_client: # Run Rodan-Client for dev (needs local dev up and running) @docker run -p 8080:9002 -v `pwd`/rodan-client/code:/code ddmal/rodan-client:nightly bash -deploy_staging: - # Can also be used to update a configuration (point to a different image.) - @echo "[-] Deploying Docker Swarm for: Rodan Staging" - @docker stack deploy --prune --with-registry-auth -c staging.yml rodan - @echo "[+] Done." - -deploy_production: - # Can also be used to update a configuration (point to a different image.) - @echo "[-] Deploying Docker Swarm for: Rodan Production" - @docker stack deploy --with-registry-auth -c production.yml rodan - @echo "[+] Done." - -copy_docker_tag: - # tag=v1.5.0rc0 make copy_docker_tag - @docker image tag $(docker images ddmal/rodan:nightly -q) ddmal/rodan:$(tag) - @docker image tag $(docker images ddmal/rodan-python3-celery:nightly -q) ddmal/rodan-python3-celery:$(tag) - @docker image tag $(docker images ddmal/rodan-gpu-celery:nightly -q) ddmal/rodan-gpu-celery:$(tag) - -pull_prod: - docker pull ddmal/iipsrv:nightly - docker pull ddmal/nginx:$(PROD_TAG) - docker pull ddmal/postgres-plpython:$(PROD_TAG) - docker pull ddmal/rodan-gpu-celery:$(PROD_TAG) - docker pull ddmal/rodan-main:$(PROD_TAG) - docker pull ddmal/rodan-python3-celery:$(PROD_TAG) - docker pull rabbitmq:alpine - docker pull redis:alpine - -pull_docker_tag: - # tag=v1.5.0rc0 make pull_docker_tag - @docker pull ddmal/rodan:$(tag) - @docker pull ddmal/rodan-python3-celery:$(tag) - @docker pull ddmal/rodan-gpu-celery:$(tag) - -push_docker_tag: - # tag=v1.5.0rc0 make push_docker_tag - @docker push ddmal/rodan:$(tag) - @docker push ddmal/rodan-python3-celery:$(tag) - @docker push ddmal/rodan-gpu-celery:$(tag) - -update: - # tag1=v1.5.0rc0 tag2=v1.3.1 make update - # This will update the nightly images forcefully - @echo "[-] Updating Docker Swarm images..." - # @docker-compose pull - - # DB First - @docker service update \ - --force \ - --update-order start-first \ - --update-delay 30s \ - --image ddmal/postgres-plpython:$(tag2) \ - rodan_postgres - - # You need to be logged in to docker for this one. - @docker service update \ - --force \ - --with-registry-auth \ - --update-order start-first \ - --update-delay 10m \ - --image ddmal/rodan:$(tag1) \ - rodan_rodan-main - - # These images might need time to update. - @docker service update \ - --force \ - --with-registry-auth \ - --update-order start-first \ - --stop-grace-period 9h \ - --update-delay 10m \ - --image ddmal/rodan:$(tag1) \ - rodan_celery - # These are public images - @docker service update \ - --force \ - --update-order start-first \ - --stop-grace-period 9h \ - --update-delay 30s \ - --image ddmal/rodan-python3-celery:$(tag1) \ - rodan_py3-celery - @docker service update \ - --force \ - --update-order start-first \ - --stop-grace-period 9h \ - --update-delay 30s \ - --image ddmal/rodan-gpu-celery:$(tag1) \ - rodan_gpu-celery - - # # TODO: Need to make rabbitmq durable and permanent - # # before we can make rolling updates for rabbitmq/hpc-rabbitmq. - # @docker service update \ - # --force \ - # --update-order start-first \ - # --update-delay 30s \ - # --image ddmal/hpc-rabbitmq:$(tag2) \ - # rodan_hpc-rabbitmq - @docker service update \ - --force \ - --update-order start-first \ - --update-delay 30s \ - --image ddmal/nginx:$(tag2) \ - rodan_nginx - - @echo "[+] Done." - -scale: - @docker service scale rodan_nginx=$(num) - @docker service scale rodan_rodan=$(num) - @docker service scale rodan_celery=$(num) - @docker service scale rodan_py3-celery=$(num) - # @docker service scale rodan_gpu-celery=$(num) - @docker service scale rodan_redis=$(num) - # @docker service scale rodan_postgres=$(num) - @docker service scale rodan_rabbitmq=$(num) - -health: - @docker inspect --format "{{json .State.Health }}" $(log) | jq - -renew_certbot: - @docker exec `docker ps -f name=rodan_nginx -q` certbot renew --no-random-sleep-on-renew - @docker exec `docker ps -f name=rodan_nginx -q` nginx -s reload - stop: - # This is the same command to stop docker swarm or docker compose - @echo "[-] Stopping all running docker containers and services..." - @docker service rm `docker service ls -q` >>/dev/null 2>&1 || echo "[+] No Services Running" - # @docker stop `docker ps -aq` >>/dev/null 2>&1 || echo "[+] No Containers Running" + @echo "[-] Stopping all running docker containers..." @docker stop `docker ps -aq | grep -v $$(docker ps -aq --filter "name=gpu_dont_kill_me")` >>/dev/null 2>&1 || echo "[+] No Containers Running" @echo "[+] Done." @@ -199,29 +47,8 @@ clean_git: @git pull @echo "[+] Done." - -clean_swarm: - # Not usually needed, but this will restart the swarm - @echo "[-] Exiting from Docker Swarm and recreating new Swarm Manager..." - @docker stack rm rodan || echo "[-] No stack to remove" - @docker swarm leave --force || echo "[-] Not a swarm manager" - @docker swarm init - @echo "[+] Done." - -debug_swarm: - @echo "[+] Creating a live service in the same network." - @docker service create --name statefulservice --network rodan_default --entrypoint="bash -c 'tail -f /dev/null'" --env-file=./scripts/staging.env ddmal/rodan:nightly bash - @docker exec -it `docker ps -f name=statefulservice -q` bash - -push: - @echo "[-] Pushing images to Docker Hub..." - @docker compose push - @echo "[+] Done." - -pull: - @echo "[-] Pulling docker images from Docker Hub..." - @DOCKER_TAG=$(DOCKER_TAG) docker compose pull - @echo "[+] Done." +health: + @docker inspect --format "{{json .State.Health }}" $(log) | jq $(JOBS_PATH)/neon_wrapper/Neon/package.json: @cd $(JOBS_PATH); \ @@ -235,32 +62,9 @@ $(JOBS_PATH)/neon_wrapper/static/editor.html: $(JOBS_PATH)/neon_wrapper/Neon/pac $(JOBS_PATH)/pixel_wrapper/package.json: @cd $(JOBS_PATH); git clone --recurse-submodules -b develop https://github.com/DDMAL/pixel_wrapper.git -remote_jobs: $(JOBS_PATH)/pixel_wrapper/package.json $(JOBS_PATH)/neon_wrapper/static/editor.html +remote_jobs: $(JOBS_PATH)/pixel_wrapper/package.json $(JOBS_PATH)/neon_wrapper/static/editor.html @cd $(RODAN_PATH); $(REPLACE) "s/#py3 //g" ./settings.py @cd $(RODAN_PATH); $(REPLACE) "s/#gpu //g" ./settings.py -gpu-celery_log: - @docker exec $$(docker ps -f name=rodan_gpu-celery --format "{{.ID}}") tail -f /code/Rodan/rodan-celery-GPU.log - -py3-celery_log: - @docker exec $$(docker ps -f name=rodan_py3-celery --format "{{.ID}}") tail -f /code/Rodan/rodan-celery-Python3.log - -celery_log: - @docker exec $$(docker ps -f name=rodan_celery --format "{{.ID}}") tail -f /code/Rodan/rodan-celery-celery.log - -rodan-main_log: - @docker exec $$(docker ps -f name=rodan_rodan-main --format "{{.ID}}") tail -f /code/Rodan/rodan.log - -update_prod_version_tag: - # old_tag=v2.x.x make update_prod_version_name - @$(REPLACE) "s/$(old_tag)/$(PROD_TAG)/g" ./production.yml - @$(REPLACE) "s/$(old_tag)/$(PROD_TAG)/g" ./rodan-main/code/rodan/__init__.py - # Command Groups -reset: stop clean pull run clean_reset: stop clean build run -upload: clean_reset push -deploy: clean_git pull run_swarm -reset_swarm: stop clean_git clean_swarm clean pull deploy_staging -update_swarm: clean_git update -staging: stop clean pull deploy_staging diff --git a/build.yml b/build.yml index b5672a8a1..8e8cdb6c0 100644 --- a/build.yml +++ b/build.yml @@ -5,6 +5,8 @@ services: build: context: ./nginx dockerfile: Dockerfile + args: + VERSION: nightly image: "ddmal/nginx:nightly" iipsrv: @@ -12,14 +14,15 @@ services: context: ./iipsrv dockerfile: Dockerfile image: "ddmal/iipsrv:nightly" - + rodan: build: context: . - dockerfile: Dockerfile + dockerfile: rodan-main/Dockerfile args: BRANCHES: develop - image: "ddmal/rodan:nightly" + VERSION: nightly + image: "ddmal/rodan-main:nightly" py3-celery: build: @@ -39,8 +42,8 @@ services: postgres: build: - context: ./postgres - dockerfile: Dockerfile + context: . + dockerfile: ./postgres/Dockerfile image: "ddmal/postgres-plpython:nightly" rodan-client: @@ -48,4 +51,3 @@ services: context: ./rodan-client dockerfile: Dockerfile image: "ddmal/rodan-client:nightly" - diff --git a/hooks/build b/hooks/build deleted file mode 100644 index 7a1193a9e..000000000 --- a/hooks/build +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# hooks/build -# https://docs.docker.com/docker-cloud/builds/advanced/ - -# RODAN_TAG=`cd rodan-main/code && git describe --tags --always` -# RODAN_CLIENT_TAG=`cd rodan-client/code && git describe --tags --always` -# RODAN_DOCKER_TAG=`git describe --tags --always` - -source ./hooks/helper.sh -DOCKER_TAG=$(GetDockerTag $DOCKER_TAG) || ret_code=$? -if [[ ret_code -eq 1 ]]; then - echo "[-] no branch detected, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -elif [[ ret_code -eq 2 ]]; then - echo "[-] no tag on release branch, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -fi - -echo "[+] Building images with tag: $DOCKER_TAG" - -set -o errexit - -# Fail building images -# Push to trigger webhook and send a message to slack -trap 'cleanup $?' EXIT -cleanup() { - if [ "$1" != "0" ]; then - echo "An error occurs when building images." - echo "Push to another repo to trigger webhook on failure with tag syntax: {repo name}-{build status}-{source branch}-{docker tag}" - - docker tag ddmal/docker-webhook:placeholder ddmal/docker-webhook:rodan-fail-${SOURCE_BRANCH}-${DOCKER_TAG} - docker push ddmal/docker-webhook:rodan-fail-${SOURCE_BRANCH}-${DOCKER_TAG} - fi -} - -# This "useless" image is build to prevent dockerhub from overwriting -# the nightly image with it's obligatory minimum of a build rule. If we want a build -# trigger with a HTTPS POST request, we must have at least 1 build rule. -docker build \ - --tag ddmal/docker-webhook:placeholder \ - . - -############################################################################### -# Stage 1 -# Build and push Python3-Celery image - -# Too many times did docker cache mess up a build. No more cache. -echo "[+] Building Python3-Celery" - -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/rodan-python3-celery:${DOCKER_TAG} \ - --file ./python3-celery/Dockerfile \ - . - -echo "[+] Pushing ddmal/rodan-python3-celery:${DOCKER_TAG}" -docker push ddmal/rodan-python3-celery:${DOCKER_TAG} - -############################################################################### -# Stage 2 -# Build and push Rodan and Rodan-Client - -echo "[+] Building Rodan & Celery Core" - -BUILD_HASH=`git rev-parse --verify HEAD` -# Don't remove --no-cache -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --build-arg build_hash=${BUILD_HASH} \ - --tag ddmal/rodan-main:${DOCKER_TAG} \ - --file ./rodan-main/Dockerfile \ - . - -echo "[+] Building Rodan-Client" - -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/rodan-client:${DOCKER_TAG} \ - ./rodan-client - -echo "[+] Pushing ddmal/rodan-main:${DOCKER_TAG}" -docker push ddmal/rodan-main:${DOCKER_TAG} - -echo "[+] Pushing ddmal/rodan-client:${DOCKER_TAG}" -docker push ddmal/rodan-client:${DOCKER_TAG} - -############################################################################## -# Stage 3 -# Build the rest - -echo "[+] Building GPU-Celery" - -docker build \ - --no-cache \ - --build-arg BRANCHES="develop" \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/rodan-gpu-celery:${DOCKER_TAG} \ - --file ./gpu-celery/Dockerfile \ - . - -echo "[+] Building Postgres" - -docker build \ - --no-cache \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/postgres-plpython:${DOCKER_TAG} \ - --file ./postgres/Dockerfile \ - . - -echo "[+] Building Nginx" - -docker build \ - --no-cache \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/nginx:${DOCKER_TAG} \ - ./nginx - -echo "[+] Building iipsrv" - -docker build \ - --no-cache \ - --build-arg VERSION=${DOCKER_TAG} \ - --tag ddmal/iipsrv:${DOCKER_TAG} \ - ./iipsrv - -echo "[+] Finished" diff --git a/hooks/helper.sh b/hooks/helper.sh deleted file mode 100644 index d0f8bc8cd..000000000 --- a/hooks/helper.sh +++ /dev/null @@ -1,27 +0,0 @@ -GetDockerTag() { - current_branch_full=$(git symbolic-ref HEAD) # ref/heads/ or pull/#PR-id/head - current_branch_short=$(git symbolic-ref --short HEAD) # or ? - _DOCKER_TAG=$1 - - # Release branch will always use git tag - if [[ $current_branch_short =~ ^release ]]; then - # Search for tag in release branch - tags=$(git tag --points-at HEAD) # git tags attached to theh HEAD - _DOCKER_TAG=$(echo $tags | sed -n '1p') - if [[ -z $_DOCKER_TAG ]]; then - # echo "no tag on release branch, fallback to use docker tag=release" - _DOCKER_TAG="release" - echo $_DOCKER_TAG - return 2 - fi - elif [[ $_DOCKER_TAG =~ ^placeholder$ ]]; then - # develop branch has a weird funny tag called placeholder, change it to nightly so commits and PRs will use the nightly tag - _DOCKER_TAG="nightly" - fi - # Finally, use custom tags send by docker hub. e.g.: a feature branch that has its build rule on docker hub - - # Replace invalid char / in docker tag with - - _DOCKER_TAG=$(echo $_DOCKER_TAG | sed 's/\//-/g') - echo $_DOCKER_TAG - return 0 -} \ No newline at end of file diff --git a/hooks/post_build b/hooks/post_build deleted file mode 100644 index 182d6d6fc..000000000 --- a/hooks/post_build +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# hooks/post_build - -source ./hooks/helper.sh -if [[ ret_code -eq 1 ]]; then - echo "[-] no branch detected, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -elif [[ ret_code -eq 2 ]]; then - echo "[-] no tag on release branch, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -fi - -# Pass testing a pull request -# Push to trigger webhook and send a message to slack -if [[ "$DOCKER_TAG" == "this" ]]; then - docker tag ddmal/docker-webhook:placeholder ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} - docker push ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} -fi diff --git a/hooks/post_checkout b/hooks/post_checkout deleted file mode 100644 index e83b801ec..000000000 --- a/hooks/post_checkout +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# hooks/post_checkout -# https://docs.docker.com/docker-cloud/builds/advanced/ - -# echo "[-] Changing branch" -# git checkout develop -# # git reset --hard HEAD -# # git clean -xffd -# # git pull origin develop - -# echo "[-] Fetching tags" -# # git fetch --tags --quiet origin \ No newline at end of file diff --git a/hooks/push b/hooks/push deleted file mode 100644 index f80eb8b34..000000000 --- a/hooks/push +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# hooks/post_push -# https://docs.docker.com/docker-cloud/builds/advanced/ - -source ./hooks/helper.sh -DOCKER_TAG=$(GetDockerTag $DOCKER_TAG) || ret_code=$? -if [[ ret_code -eq 1 ]]; then - echo "[-] no branch detected, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -elif [[ ret_code -eq 2 ]]; then - echo "[-] no tag on release branch, fallback to use docker tag=$DOCKER_TAG" - echo "[-] Stop building due to wrong tag: $DOCKER_TAG" - exit 1 -fi - -echo "[+] Building images with tag: $DOCKER_TAG" -echo "Docker tag: $DOCKER_TAG" - -# RODAN_TAG=`cd rodan-main/code && git describe --tags --always` -# RODAN_CLIENT_TAG=`cd rodan-client/code && git describe --tags --always` -# RODAN_DOCKER_TAG=`git describe --tags --always` - -echo "[+] Pushing ddmal/rodan-gpu-celery:${DOCKER_TAG}" -docker push ddmal/rodan-gpu-celery:${DOCKER_TAG} - -echo "[+] Pushing ddmal/postgres-plpython:${DOCKER_TAG}" -docker push ddmal/postgres-plpython:${DOCKER_TAG} - -echo "[+] Pushing ddmal/nginx:${DOCKER_TAG}" -docker push ddmal/nginx:${DOCKER_TAG} - -echo "[+] Pushing ddmal/iipsrv:${DOCKER_TAG}" -docker push ddmal/iipsrv:${DOCKER_TAG} - -# Pass building images -# Push to trigger webhook and send a message to slack -docker tag ddmal/docker-webhook:placeholder ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} -docker push ddmal/docker-webhook:rodan-pass-${SOURCE_BRANCH}-${DOCKER_TAG} diff --git a/k8s/.gitignore b/k8s/.gitignore new file mode 100644 index 000000000..1f725e1bc --- /dev/null +++ b/k8s/.gitignore @@ -0,0 +1,2 @@ +# Real secret with filled-in values — never commit. Use 02-secret.template.yaml as the source. +02-secret.yaml \ No newline at end of file diff --git a/k8s/00-namespace.yaml b/k8s/00-namespace.yaml new file mode 100644 index 000000000..d866e4670 --- /dev/null +++ b/k8s/00-namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: rodan + labels: + app.kubernetes.io/part-of: rodan \ No newline at end of file diff --git a/k8s/01-configmap.yaml b/k8s/01-configmap.yaml new file mode 100644 index 000000000..b312684c4 --- /dev/null +++ b/k8s/01-configmap.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rodan-config + namespace: rodan +data: + # --- General --- + TZ: "America/Toronto" + SERVER_HOST: "rodan2.simssa.ca" + + # --- Django --- + # Matches the current production (DJANGO_ALLOWED_HOSTS=*). "*" also lets k8s httpGet probes + # (which send the pod IP as Host) pass. Tighten to a host list only if you also give the + # probes a matching Host header. + DJANGO_ALLOWED_HOSTS: "*" + DJANGO_DEBUG_MODE: "False" + DJANGO_MEDIA_ROOT: "/rodan/data/" + # Obscured admin path (kept here as in the existing env files; move to Secret if you treat it as sensitive). + DJANGO_ADMIN_URL: "^api/random_secret_admin/" + DJANGO_ACCESS_LOG: "/code/Rodan/rodan.log" + DJANGO_DEBUG_LOG: "/code/Rodan/database.log" + + # --- IIP image server (public URL the client uses; TLS terminated at edge) --- + IIPSRV_URL: "https://rodan2.simssa.ca/fcgi-bin/iipsrv.fcgi/" + + # --- Postgres (non-secret connection coordinates; creds are in the Secret) --- + POSTGRES_HOST: "postgres" + POSTGRES_PORT: "5432" + POSTGRES_DB: "rodan" + POSTGRES_DATABASE_LOGFILE: "/code/Rodan/database.log" + + # --- Redis (cache + ws4redis websocket broker) --- + REDIS_HOST: "redis" + REDIS_PORT: "6379" + REDIS_DB: "0" + + # --- Celery --- + C_FORCE_ROOT: "true" \ No newline at end of file diff --git a/k8s/02-secret.template.yaml b/k8s/02-secret.template.yaml new file mode 100644 index 000000000..143ae816e --- /dev/null +++ b/k8s/02-secret.template.yaml @@ -0,0 +1,36 @@ +# TEMPLATE ONLY — do NOT commit real values. +# Copy to 02-secret.yaml, fill in real values, and apply that (02-secret.yaml is gitignored). +# stringData lets you paste plaintext; Kubernetes base64-encodes it on apply. +# +# NOTE: scripts/start reads ADMIN_PASS (the live production.env's ADMIN_PASSWORD is a bug — use ADMIN_PASS here). +# RABBITMQ_URL host/creds must match the rabbitmq Deployment's RABBITMQ_DEFAULT_USER/PASS: +# amqp://:@rabbitmq:5672// +apiVersion: v1 +kind: Secret +metadata: + name: rodan-secrets + namespace: rodan +type: Opaque +stringData: + # --- Django --- + DJANGO_SECRET_KEY: "CHANGE_ME_long_random_string" + + # --- Postgres credentials --- + POSTGRES_USER: "CHANGE_ME" + POSTGRES_PASSWORD: "CHANGE_ME" + + # --- RabbitMQ (broker URL + the credentials the broker is created with) --- + RABBITMQ_URL: "amqp://CHANGE_ME_user:CHANGE_ME_pass@rabbitmq:5672//" + RABBITMQ_DEFAULT_USER: "CHANGE_ME_user" + RABBITMQ_DEFAULT_PASS: "CHANGE_ME_pass" + + # --- Bootstrap superuser (created by scripts/start on first boot) --- + ADMIN_USER: "CHANGE_ME" + ADMIN_EMAIL: "admin@rodan2.simssa.ca" + ADMIN_PASS: "CHANGE_ME" + + # --- SMTP (optional; leave blank to fall back to console email backend) --- + EMAIL_HOST: "" + EMAIL_PORT: "" + EMAIL_HOST_USER: "" + EMAIL_HOST_PASSWORD: "" \ No newline at end of file diff --git a/k8s/10-pv-resources.yaml b/k8s/10-pv-resources.yaml new file mode 100644 index 000000000..1baf6c900 --- /dev/null +++ b/k8s/10-pv-resources.yaml @@ -0,0 +1,27 @@ +# Shared media volume (Django MEDIA_ROOT, /rodan/data, ~883 GB). RWX so all six +# consumers (nginx, rodan-main, iipsrv, celery, py3-celery, gpu-celery) mount it at once. +# Replace __ARBUTUS_NFS_IP__ and the export path with the real Arbutus NFS server. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: rodan-resources-pv + labels: + app.kubernetes.io/part-of: rodan +spec: + capacity: + storage: 2Ti + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: "" # static binding; disables dynamic provisioning + mountOptions: + - nfsvers=4 + - soft # mirrors the current resources mount (soft,timeo=600,retrans=3) + - timeo=600 + - retrans=3 + nfs: + server: 192.168.236.124 + path: /srv/rodan-data/var/lib/docker/volumes/rodan_resources/_data + claimRef: + namespace: rodan + name: rodan-resources \ No newline at end of file diff --git a/k8s/11-pvc-resources.yaml b/k8s/11-pvc-resources.yaml new file mode 100644 index 000000000..f2e84bfad --- /dev/null +++ b/k8s/11-pvc-resources.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rodan-resources + namespace: rodan +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + resources: + requests: + storage: 2Ti + volumeName: rodan-resources-pv \ No newline at end of file diff --git a/k8s/12-pv-pgdata.yaml b/k8s/12-pv-pgdata.yaml new file mode 100644 index 000000000..567bd81c5 --- /dev/null +++ b/k8s/12-pv-pgdata.yaml @@ -0,0 +1,24 @@ +# Postgres PGDATA (~4.1 GB), PG 9.6, owned by uid 999. Mounted only by the postgres pod. +# 'hard' mount (DBs must not see truncated/soft-failed I/O). NFS export needs no_root_squash. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: rodan-pg-data-pv + labels: + app.kubernetes.io/part-of: rodan +spec: + capacity: + storage: 50Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + mountOptions: + - nfsvers=4 + - hard + nfs: + server: 192.168.236.124 + path: /srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data + claimRef: + namespace: rodan + name: rodan-pg-data \ No newline at end of file diff --git a/k8s/13-pvc-pgdata.yaml b/k8s/13-pvc-pgdata.yaml new file mode 100644 index 000000000..9a71d7be5 --- /dev/null +++ b/k8s/13-pvc-pgdata.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rodan-pg-data + namespace: rodan +spec: + accessModes: + - ReadWriteOnce + storageClassName: "" + resources: + requests: + storage: 50Gi + volumeName: rodan-pg-data-pv \ No newline at end of file diff --git a/k8s/14-pv-pgbackup.yaml b/k8s/14-pv-pgbackup.yaml new file mode 100644 index 000000000..159056d4c --- /dev/null +++ b/k8s/14-pv-pgbackup.yaml @@ -0,0 +1,23 @@ +# Postgres backups dir (/backups). Mounted by the postgres pod (and the backup CronJob if added). +apiVersion: v1 +kind: PersistentVolume +metadata: + name: rodan-pg-backup-pv + labels: + app.kubernetes.io/part-of: rodan +spec: + capacity: + storage: 500Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: "" + mountOptions: + - nfsvers=4 + - hard + nfs: + server: 192.168.236.124 + path: /srv/rodan-data/var/lib/docker/volumes/rodan_pg_backup/_data + claimRef: + namespace: rodan + name: rodan-pg-backup \ No newline at end of file diff --git a/k8s/15-pvc-pgbackup.yaml b/k8s/15-pvc-pgbackup.yaml new file mode 100644 index 000000000..f711d02d9 --- /dev/null +++ b/k8s/15-pvc-pgbackup.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rodan-pg-backup + namespace: rodan +spec: + accessModes: + - ReadWriteMany + storageClassName: "" + resources: + requests: + storage: 500Gi + volumeName: rodan-pg-backup-pv \ No newline at end of file diff --git a/k8s/20-postgres.yaml b/k8s/20-postgres.yaml new file mode 100644 index 000000000..5c9ce5344 --- /dev/null +++ b/k8s/20-postgres.yaml @@ -0,0 +1,81 @@ +# Postgres 9.6 + plpython3 (existing image). Single instance, PGDATA on the static NFS PVC. +# The existing PGDATA is already initialized, so the entrypoint skips initdb and the +# POSTGRES_USER/PASSWORD/DB env are effectively no-ops on an existing cluster (kept for first-init). +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: rodan +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: postgres + image: ghcr.io/ddmal/postgres-plpython:k8s-migration + ports: + - containerPort: 5432 + name: postgres + env: + - name: TZ + value: "America/Toronto" + - name: POSTGRES_DB + valueFrom: { configMapKeyRef: { name: rodan-config, key: POSTGRES_DB } } + - name: POSTGRES_USER + valueFrom: { secretKeyRef: { name: rodan-secrets, key: POSTGRES_USER } } + - name: POSTGRES_PASSWORD + valueFrom: { secretKeyRef: { name: rodan-secrets, key: POSTGRES_PASSWORD } } + volumeMounts: + - name: pg-data + mountPath: /var/lib/postgresql/data + - name: pg-backup + mountPath: /backups + readinessProbe: + exec: + command: ["sh", "-c", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""] + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-c", "pg_isready -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\""] + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "2Gi" + volumes: + - name: pg-data + persistentVolumeClaim: + claimName: rodan-pg-data + - name: pg-backup + persistentVolumeClaim: + claimName: rodan-pg-backup +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: rodan +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 + name: postgres \ No newline at end of file diff --git a/k8s/21-redis.yaml b/k8s/21-redis.yaml new file mode 100644 index 000000000..11915e5d2 --- /dev/null +++ b/k8s/21-redis.yaml @@ -0,0 +1,58 @@ +# Redis: Celery is NOT backed by redis here (broker+backend are RabbitMQ); redis is the +# ws4redis websocket broker + cache. Ephemeral (the current stack persists nothing for redis). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:alpine + ports: + - containerPort: 6379 + name: redis + env: + - name: TZ + value: "America/Toronto" + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "1Gi" +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: rodan +spec: + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 + name: redis \ No newline at end of file diff --git a/k8s/22-rabbitmq.yaml b/k8s/22-rabbitmq.yaml new file mode 100644 index 000000000..e10c38b7b --- /dev/null +++ b/k8s/22-rabbitmq.yaml @@ -0,0 +1,72 @@ +# RabbitMQ: Celery broker AND result backend (CELERY_RESULT_BACKEND="amqp" in settings.py). +# Ephemeral storage matches the current stack (rabbitmq durability is a known upstream TODO). +# DEFAULT_USER/PASS must match the credentials embedded in rodan-secrets/RABBITMQ_URL. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + # Pinned to 3.x: RabbitMQ 4.x removed `transient_nonexcl_queues`, which Celery 3.1.25 + # (Rodan's version) relies on. The floating `:alpine` tag now resolves to 4.x and breaks it. + image: rabbitmq:3.13-alpine + ports: + - containerPort: 5672 + name: amqp + env: + - name: TZ + value: "America/Toronto" + - name: RABBITMQ_DEFAULT_USER + valueFrom: { secretKeyRef: { name: rodan-secrets, key: RABBITMQ_DEFAULT_USER } } + - name: RABBITMQ_DEFAULT_PASS + valueFrom: { secretKeyRef: { name: rodan-secrets, key: RABBITMQ_DEFAULT_PASS } } + volumeMounts: + - name: data + mountPath: /var/lib/rabbitmq + readinessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 10 + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 3 + resources: + requests: + cpu: "200m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" + volumes: + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + namespace: rodan +spec: + selector: + app: rabbitmq + ports: + - port: 5672 + targetPort: 5672 + name: amqp \ No newline at end of file diff --git a/k8s/30-rodan-main.yaml b/k8s/30-rodan-main.yaml new file mode 100644 index 000000000..d4ac6a457 --- /dev/null +++ b/k8s/30-rodan-main.yaml @@ -0,0 +1,80 @@ +# Django REST API (gunicorn :8000). Image ENTRYPOINT is /opt/entrypoint (waits for pg+redis), +# then runs /run/start which migrates, creates the superuser, collectstatic, then gunicorn. +# Single replica so migrations never run concurrently. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rodan-main + namespace: rodan +spec: + replicas: 1 + strategy: + type: Recreate # avoid two pods running migrations against the same DB at once + selector: + matchLabels: + app: rodan-main + template: + metadata: + labels: + app: rodan-main + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: rodan-main + image: ghcr.io/ddmal/rodan-main:k8s-migration + args: ["/run/start"] + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "None" + ports: + - containerPort: 8000 + name: http + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + httpGet: + path: /api/?format=json + port: 8000 + httpHeaders: + - name: User-Agent + value: k8s-healthcheck + periodSeconds: 10 + failureThreshold: 60 # up to ~10 min for image boot + migrate + collectstatic + readinessProbe: + httpGet: + path: /api/?format=json + port: 8000 + httpHeaders: + - name: User-Agent + value: k8s-healthcheck + periodSeconds: 15 + timeoutSeconds: 10 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "2" + memory: "4Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: rodan-main + namespace: rodan +spec: + selector: + app: rodan-main + ports: + - port: 8000 + targetPort: 8000 + name: http diff --git a/k8s/31-celery.yaml b/k8s/31-celery.yaml new file mode 100644 index 000000000..4f497236d --- /dev/null +++ b/k8s/31-celery.yaml @@ -0,0 +1,56 @@ +# Default Celery worker (queue "celery"). Same image as the API; ENTRYPOINT /opt/entrypoint +# (waits for pg+redis), then /run/start-celery which also waits for rodan-main:8000 before +# launching the worker as node "celery@celery". +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: celery + template: + metadata: + labels: + app: celery + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: celery + image: ghcr.io/ddmal/rodan-main:k8s-migration + args: ["/run/start-celery"] + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "celery" + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] + periodSeconds: 45 + timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer + failureThreshold: 30 # tolerates the long wait-for rodan-main:8000 (up to 900s) + livenessProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] + periodSeconds: 60 + timeoutSeconds: 45 + failureThreshold: 3 + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "4" + memory: "12Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources diff --git a/k8s/32-py3-celery.yaml b/k8s/32-py3-celery.yaml new file mode 100644 index 000000000..f556d28c5 --- /dev/null +++ b/k8s/32-py3-celery.yaml @@ -0,0 +1,54 @@ +# Python3 Celery worker (queue "Python3"). This image's ENTRYPOINT is /run/start-celery +# directly (no /opt/entrypoint wrapper), so no args override is needed. Node "celery@Python3". +apiVersion: apps/v1 +kind: Deployment +metadata: + name: py3-celery + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: py3-celery + template: + metadata: + labels: + app: py3-celery + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: py3-celery + image: ghcr.io/ddmal/rodan-python3-celery:k8s-migration + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "Python3" + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] + periodSeconds: 45 + timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer + failureThreshold: 30 + livenessProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] + periodSeconds: 60 + timeoutSeconds: 45 + failureThreshold: 3 + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + cpu: "4" + memory: "12Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources diff --git a/k8s/33-gpu-celery.yaml b/k8s/33-gpu-celery.yaml new file mode 100644 index 000000000..f1674f26c --- /dev/null +++ b/k8s/33-gpu-celery.yaml @@ -0,0 +1,66 @@ +# GPU Celery worker (queue "GPU") — the ONLY GPU-using workload. Pinned to the GPU node. +# ENTRYPOINT /opt/entrypoint (waits pg+redis) then /run/start-celery. Node "celery@GPU". +# Requires: GPU node labeled gpu=true and tainted nvidia.com/gpu=present:NoSchedule, +# plus the NVIDIA device plugin (60-nvidia-device-plugin.yaml) advertising nvidia.com/gpu. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-celery + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: gpu-celery + template: + metadata: + labels: + app: gpu-celery + spec: + imagePullSecrets: + - name: ghcr-pull-secret + nodeSelector: + gpu: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + # k3s exposes the NVIDIA runtime via this RuntimeClass (confirmed present in-cluster). + runtimeClassName: nvidia + containers: + - name: gpu-celery + image: ghcr.io/ddmal/rodan-gpu-celery:k8s-migration + args: ["/run/start-celery"] + envFrom: + - configMapRef: { name: rodan-config } + - secretRef: { name: rodan-secrets } + env: + - name: CELERY_JOB_QUEUE + value: "GPU" + volumeMounts: + - name: resources + mountPath: /rodan/data + startupProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] + periodSeconds: 45 + timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer + failureThreshold: 30 + livenessProbe: + exec: + command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] + periodSeconds: 60 + timeoutSeconds: 45 + failureThreshold: 3 + resources: + requests: + cpu: "1" + memory: "4Gi" + limits: + cpu: "2" + memory: "16Gi" + nvidia.com/gpu: 1 + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources diff --git a/k8s/40-iipsrv.yaml b/k8s/40-iipsrv.yaml new file mode 100644 index 000000000..f78ecda9a --- /dev/null +++ b/k8s/40-iipsrv.yaml @@ -0,0 +1,62 @@ +# IIP image server (FastCGI :9003). Reads images from the shared resources volume +# (FILESYSTEM_PREFIX=/rodan/data/ is baked into the image). nginx fastcgi_pass's to it. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: iipsrv + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: iipsrv + template: + metadata: + labels: + app: iipsrv + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: iipsrv + image: ghcr.io/ddmal/iipsrv:k8s-migration + ports: + - containerPort: 9003 + name: fcgi + volumeMounts: + - name: resources + mountPath: /rodan/data + readinessProbe: + tcpSocket: + port: 9003 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 9003 + initialDelaySeconds: 15 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1" + memory: "2Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: iipsrv + namespace: rodan +spec: + selector: + app: iipsrv + ports: + - port: 9003 + targetPort: 9003 + name: fcgi diff --git a/k8s/41-rodan-client.yaml b/k8s/41-rodan-client.yaml new file mode 100644 index 000000000..100df8a32 --- /dev/null +++ b/k8s/41-rodan-client.yaml @@ -0,0 +1,108 @@ +# Backbone.js static client (nginx :80 inside the image). Its runtime configuration.json is +# injected via ConfigMap (the compose stack bind-mounts it at /client/configuration.json). +# Public access is https via the edge, so SERVER_HTTPS=true / port 443 / host rodan2.simssa.ca. +apiVersion: v1 +kind: ConfigMap +metadata: + name: rodan-client-config + namespace: rodan +data: + configuration.json: | + { + "SERVER_HOST": "rodan2.simssa.ca", + "SERVER_PORT": "443", + "SERVER_HTTPS": true, + "SERVER_SOCKET_AVAILABLE": false, + "SERVER_AUTHENTICATION_TYPE": "token", + "ADMIN_CLIENT": { + "NAME": "", + "EMAIL": "" + }, + "DEBUG": false, + "WORKFLOWBUILDERGUI": { + "USER_AGENT": "rodan-standard", + "GRID": { + "DIMENSION": 20, + "LINE_COLOR": "#606060", + "LINE_WIDTH": 0.5 + }, + "ZOOM_MAX": 3.0, + "ZOOM_MIN": 1.0, + "ZOOM_RATE": 0.05, + "ZOOM_INITIAL": 1.7, + "WORKFLOWJOB_WIDTH": 20, + "WORKFLOWJOB_HEIGHT": 22, + "PORT_WIDTH": 8, + "PORT_HEIGHT": 8, + "OUTPUTPORT_COLOR": "#00ff00", + "INPUTPORT_COLOR_SATISFIED": "#00ff00", + "INPUTPORT_COLOR_UNSATISFIED": "#ff0000", + "INPUTPORT_COLOR_CANDIDATE": "#00ff00", + "STROKE_COLOR": "#000000", + "FILL_COLOR": "#ccccff", + "WORKFLOWJOBGROUP_FILL_COLOR": "#8888ff", + "STROKE_WIDTH": 1, + "FONT_SIZE": 10, + "STROKE_COLOR_SELECTED": "#0000ff", + "STROKE_WIDTH_SELECTED": 2, + "CONNECTION_CIRCLE_RADIUS": 4, + "HOVER_TIME": 1000 + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rodan-client + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: rodan-client + template: + metadata: + labels: + app: rodan-client + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: rodan-client + image: ghcr.io/ddmal/rodan-client:k8s-migration + ports: + - containerPort: 80 + name: http + volumeMounts: + - name: client-config + mountPath: /client/configuration.json + subPath: configuration.json + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + volumes: + - name: client-config + configMap: + name: rodan-client-config +--- +apiVersion: v1 +kind: Service +metadata: + name: rodan-client + namespace: rodan +spec: + selector: + app: rodan-client + ports: + - port: 80 + targetPort: 80 + name: http diff --git a/k8s/50-nginx.yaml b/k8s/50-nginx.yaml new file mode 100644 index 000000000..a2f55b460 --- /dev/null +++ b/k8s/50-nginx.yaml @@ -0,0 +1,70 @@ +# Reverse proxy (kept as-is). Listens on :80 (plain HTTP — TLS terminated at the edge). +# Routes /api,/ht -> rodan-main:8000, / -> rodan-client, /fcgi-bin -> iipsrv:9003, +# /ws/ -> redis:6379, /uploads + /static -> filesystem. Serves baked static from /rodan/static. +# ENTRYPOINT is the base nginx /docker-entrypoint.sh; args run /run/start (waits for deps, then nginx). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + namespace: rodan +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + imagePullSecrets: + - name: ghcr-pull-secret + containers: + - name: nginx + image: ghcr.io/ddmal/nginx:k8s-migration + args: ["/run/start"] + env: + - name: TZ + value: "America/Toronto" + - name: SERVER_HOST + valueFrom: { configMapKeyRef: { name: rodan-config, key: SERVER_HOST } } + ports: + - containerPort: 80 + name: http + volumeMounts: + - name: resources + mountPath: /rodan/data + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "1Gi" + volumes: + - name: resources + persistentVolumeClaim: + claimName: rodan-resources +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + namespace: rodan +spec: + selector: + app: nginx + ports: + - port: 80 + targetPort: 80 + name: http diff --git a/k8s/51-ingress.yaml b/k8s/51-ingress.yaml new file mode 100644 index 000000000..68ceb01f0 --- /dev/null +++ b/k8s/51-ingress.yaml @@ -0,0 +1,23 @@ +# Cluster entry point: k3s Traefik. HTTP only (web entrypoint) — TLS is terminated on the +# external edge server, which forwards plain HTTP with Host: rodan2.simssa.ca and +# X-Forwarded-Proto: https. All path routing stays inside the nginx container. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: rodan + namespace: rodan + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web +spec: + ingressClassName: traefik + rules: + - host: rodan2.simssa.ca + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: nginx + port: + number: 80 diff --git a/k8s/60-nvidia-device-plugin.yaml b/k8s/60-nvidia-device-plugin.yaml new file mode 100644 index 000000000..073a87fbb --- /dev/null +++ b/k8s/60-nvidia-device-plugin.yaml @@ -0,0 +1,49 @@ +# NVIDIA device plugin — advertises nvidia.com/gpu on the GPU node so gpu-celery can request it. +# Runs only on nodes labeled gpu=true and tolerates the GPU taint. +# Prereqs on the GPU node: NVIDIA driver (>= 460.x for the CUDA 11.2 image) + nvidia-container-toolkit, +# with the nvidia container runtime configured (k3s detects it and wires up containerd automatically). +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nvidia-device-plugin-daemonset + namespace: kube-system +spec: + selector: + matchLabels: + name: nvidia-device-plugin-ds + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + name: nvidia-device-plugin-ds + spec: + priorityClassName: system-node-critical + nodeSelector: + gpu: "true" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + # k3s auto-creates the 'nvidia' RuntimeClass; the plugin must run under it so + # libnvidia-ml.so.1 (NVML) is injected, otherwise it detects 0 GPUs. + runtimeClassName: nvidia + containers: + - name: nvidia-device-plugin-ctr + image: nvcr.io/nvidia/k8s-device-plugin:v0.14.5 + env: + - name: FAIL_ON_INIT_ERROR + value: "false" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: device-plugin + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: device-plugin + hostPath: + path: /var/lib/kubelet/device-plugins diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..1e06f050c --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,147 @@ +# Rodan on Kubernetes (k3s) + +Plain-YAML manifests to run the Rodan stack on a k3s cluster (target: Arbutus). This is a +**lift-and-shift** of the existing Docker Swarm stack — same published images (`v3.3.1`, plus +`nightly` for iipsrv/rodan-client), reorganized as k8s workloads. + +See the full design in `../.claude/plans/the-architecture-that-i-parallel-lampson.md`. + +## Topology + +- All services run on the k3s cluster. **Only `gpu-celery`** runs on a dedicated GPU node. +- Storage is **NFS on Arbutus**: `resources` (`/rodan/data`, ~883 GB, RWX), `pg_data` (PG 9.6 + PGDATA, RWO, `hard`), `pg_backup` (RWX). Static PV+PVC pairs, `Retain` reclaim. +- **Postgres runs in-cluster** (StatefulSet) on its NFS PGDATA. +- Entry point is **Traefik Ingress** for `rodan2.simssa.ca`, HTTP only — **TLS is terminated on + an external edge server** that forwards plain HTTP. The kept `nginx` container does all internal + routing. +- k8s Service names intentionally match the old Docker DNS names (`postgres`, `redis`, `rabbitmq`, + `rodan-main`, `rodan-client`, `iipsrv`) because they're hardcoded in `rodan.conf` / `wait-for-app`. + +## Files (apply in numeric order) + +| File | Purpose | +|---|---| +| `00-namespace.yaml` | namespace `rodan` | +| `01-configmap.yaml` | `rodan-config` (non-secret env) | +| `02-secret.template.yaml` | template → copy to `02-secret.yaml`, fill in, apply (gitignored) | +| `10..15-*` | NFS PVs + PVCs (resources, pg_data, pg_backup) | +| `20-postgres.yaml` | Postgres StatefulSet + Service | +| `21-redis.yaml` / `22-rabbitmq.yaml` | broker/cache Deployments + Services | +| `30-rodan-main.yaml` | Django API Deployment + Service | +| `31/32/33-*celery.yaml` | celery / py3-celery / gpu-celery workers | +| `40-iipsrv.yaml` / `41-rodan-client.yaml` | image server + static client | +| `50-nginx.yaml` | reverse proxy Deployment + ClusterIP Service | +| `51-ingress.yaml` | Traefik Ingress (`rodan2.simssa.ca`) | +| `60-nvidia-device-plugin.yaml` | advertises `nvidia.com/gpu` on the GPU node | + +## Before you apply — fill in placeholders + +1. **NFS server** — DONE. The three PV files (`10/12/14-pv-*.yaml`) already point at the Arbutus + data server **`192.168.236.124`** with the cloned paths + `/srv/rodan-data/var/lib/docker/volumes/rodan_{resources,pg_data,pg_backup}/_data`. + (That VM is a `dd` block-clone of the old data server's 2 TB disk, mounted at `/srv/rodan-data` + and NFS-exported to the `192.168.236.0/24` subnet.) +2. **Secret** — `cp 02-secret.template.yaml 02-secret.yaml`, set real values. Note: use + `ADMIN_PASS` (the live `production.env` calls it `ADMIN_PASSWORD`, which `scripts/start` does + **not** read). `RABBITMQ_URL` creds must match `RABBITMQ_DEFAULT_USER`/`PASS`. +3. **GPU node** — label + taint it (see below). + +## NFS server setup (Arbutus) — DONE, recorded here for reference + +The data server `192.168.236.124` (`/srv/rodan-data` = `dd` clone of the old 2 TB disk) exports the +three cloned dirs to the k3s subnet (`no_root_squash` is required for the postgres uid 999): +```sh +# /etc/exports on 192.168.236.124 +/srv/rodan-data/var/lib/docker/volumes/rodan_resources/_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +/srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +/srv/rodan-data/var/lib/docker/volumes/rodan_pg_backup/_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +# sudo exportfs -ra && sudo exportfs -v +``` + +**Every k3s node (incl. the GPU node) must have the NFS client installed**, or pods that mount these +PVCs get stuck in `ContainerCreating`: +```sh +sudo apt update && sudo apt install -y nfs-common +``` +Per-node check (resources export verified working from k3s-node-4): +```sh +sudo mount -t nfs4 192.168.236.124:/srv/rodan-data/var/lib/docker/volumes/rodan_resources/_data /mnt && ls /mnt && sudo umount /mnt +``` +Also open **TCP 2049** from the k3s nodes in the Arbutus security group. + +## GPU node setup + +1. Install the NVIDIA driver (**≥ 460.x**, required by the CUDA 11.2 `gpu-celery` image) and the + nvidia-container-toolkit; join the node to k3s. k3s auto-detects the nvidia runtime in containerd. +2. Label and taint: + ```sh + kubectl label node gpu=true + kubectl taint node nvidia.com/gpu=present:NoSchedule + ``` +3. `60-nvidia-device-plugin.yaml` then makes the node advertise `nvidia.com/gpu`. If k3s exposes + the runtime only as a RuntimeClass (not the node default), uncomment `runtimeClassName: nvidia` + in `33-gpu-celery.yaml` and the device plugin. + +## Data migration + +Run with the old stack still serving, then a final pass during a short maintenance window. + +```sh +# resources (~883 GB) — old NFS export -> new Arbutus export +rsync -aHAX --info=progress2 \ + root@192.168.17.244:/var/lib/docker/volumes/rodan_resources/_data/ \ + /export/resources/ +# repeat for a quick incremental pass right before cutover + +# pg_data (~4.1 GB, PG 9.6 preserved) — STOP the old postgres first, then raw-copy: +rsync -aHAX root@192.168.17.244:/var/lib/docker/volumes/rodan_pg_data/_data/ /export/pg_data/ +# Fallback / verification: pg_dump on the old DB and restore into the new postgres pod instead. +``` +After copying, sanity-check: `ls /export/resources/projects | wc -l` should be ~309. + +## Apply order + +```sh +kubectl apply -f 00-namespace.yaml +kubectl apply -f 01-configmap.yaml -f 02-secret.yaml +kubectl apply -f 10-pv-resources.yaml -f 11-pvc-resources.yaml \ + -f 12-pv-pgdata.yaml -f 13-pvc-pgdata.yaml \ + -f 14-pv-pgbackup.yaml -f 15-pvc-pgbackup.yaml +kubectl -n rodan get pvc # all Bound before continuing + +kubectl apply -f 20-postgres.yaml -f 21-redis.yaml -f 22-rabbitmq.yaml +kubectl apply -f 60-nvidia-device-plugin.yaml +kubectl apply -f 30-rodan-main.yaml # watch logs: migrate -> collectstatic -> gunicorn +kubectl apply -f 31-celery.yaml -f 32-py3-celery.yaml -f 33-gpu-celery.yaml +kubectl apply -f 40-iipsrv.yaml -f 41-rodan-client.yaml -f 50-nginx.yaml -f 51-ingress.yaml +``` +(Or just `kubectl apply -f .` once `02-secret.yaml` and placeholders are filled — ordering is +self-healing via the `wait-for-app` logic baked into the images, but applying backends first is cleaner.) + +## Verify + +```sh +kubectl -n rodan get pods,svc,pvc,ingress +kubectl -n rodan exec deploy/rodan-main -- curl -s -o /dev/null -w '%{http_code}\n' \ + -H 'User-Agent: k8s' localhost:8000/api/?format=json # 200 +kubectl -n rodan exec deploy/celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@celery +kubectl -n rodan exec deploy/py3-celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@Python3 +kubectl -n rodan exec deploy/gpu-celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@GPU +kubectl -n rodan exec deploy/gpu-celery -- nvidia-smi +kubectl describe node | grep nvidia.com/gpu # allocatable: 1 +# external (through the edge): https://rodan2.simssa.ca — log in, open a project/image, run a workflow +``` + +## Notes / gotchas + +- **Postgres on NFS** is preserved per decision — keep replicas=1 and the `hard` mount. Don't scale it. +- **Celery node names** are fixed (`-n "${CELERY_JOB_QUEUE}"`); fine at 1 replica each. To scale a + worker, change the image's `start-celery` to use a per-pod name (`%h`) first. +- **Large uploads** (nginx allows 700m): if uploads fail at the edge/Traefik, add a Traefik + middleware raising `maxRequestBodyBytes` (Traefik streams by default, so usually fine). +- **WebSockets** (`/ws/`): if long-lived sockets drop, raise Traefik's responding read timeout. +- **TLS**: none inside the cluster — the edge must send `X-Forwarded-Proto: https` and the correct + `Host` so Django builds https URLs (nginx already sets `X-Scheme: https` on `/api`). +- **Backups**: the old monthly pg-backup cron appears stopped (~Sep 2024). Consider a k8s CronJob + invoking the image's `backup` script against `/backups`. diff --git a/production.sample b/production.sample deleted file mode 100644 index f23ed9dc6..000000000 --- a/production.sample +++ /dev/null @@ -1,294 +0,0 @@ -version: "3.4" - -services: - - nginx: - image: "ddmal/nginx:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "0.25" - memory: 0.5G - limits: - cpus: "0.25" - memory: 0.5G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD", "/usr/sbin/service", "nginx", "status"] - interval: "30s" - timeout: "10s" - retries: 10 - start_period: "5m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - TLS: 1 - ports: - - "80:80" - - "443:443" - - "5671:5671" - - "9002:9002" - volumes: - - "resources_nfs:/rodan/data" - - rodan-main: - image: "ddmal/rodan-main:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1.5" - memory: 6G - limits: - cpus: "1.5" - memory: 6G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD-SHELL", "/usr/bin/curl -H 'User-Agent: docker-healthcheck' http://localhost:8000/api/?format=json || exit 1"] - interval: "30s" - timeout: "30s" - retries: 5 - start_period: "15m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: None - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - rodan-client: - image: "ddmal/rodan-client:nightly" - deploy: - placement: - constraints: - - node.role == worker - volumes: - - "./rodan-client/config/configuration.json:/client/configuration.json" - - iipsrv: - image: "ddmal/iipsrv:nightly" - volumes: - - "resources_nfs:/rodan/data" - - celery: - image: "ddmal/rodan-main:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 6G - limits: - cpus: "2" - memory: 6G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] - interval: "30s" - timeout: "30s" - start_period: "10m" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: celery - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - py3-celery: - image: "ddmal/rodan-python3-celery:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 5G - limits: - cpus: "2" - memory: 5G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: Python3 - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - gpu-celery: - image: "ddmal/rodan-gpu-celery:v3.2.0" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 18G - limits: - cpus: "1" - memory: 18G - placement: - constraints: - - node.role == manager - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: GPU - env_file: - - ./scripts/production.env - volumes: - - "resources_nfs:/rodan/data" - - redis: - image: "redis:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == worker - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - - postgres: - image: "ddmal/postgres-plpython:v3.2.0" - deploy: - replicas: 1 - endpoint_mode: dnsrr - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == manager - healthcheck: - test: ["CMD-SHELL", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - volumes: - - "pg_data_nfs:/var/lib/postgresql/data" - - "pg_backup_nfs:/backups" - env_file: - - ./scripts/production.env - - rabbitmq: - image: "rabbitmq:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 4G - limits: - cpus: "2" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.role == worker - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] - interval: "30s" - timeout: "3s" - retries: 3 - environment: - TZ: America/Toronto - env_file: - - ./scripts/production.env - -volumes: - resources_nfs: - driver: local - driver_opts: - type: nfs - o: addr=[ip address],rw,nfsvers=4,soft,timeo=600,retrans=3 - device: "[ip address]:/var/lib/docker/volumes/rodan_resources/_data" - pg_backup_nfs: - driver: local - driver_opts: - type: nfs - o: addr=[ip address],rw,nfsvers=4 - device: "[ip address]:/var/lib/docker/volumes/rodan_pg_backup/_data" - pg_data_nfs: - driver: local - driver_opts: - type: nfs - o: addr=[ip address],rw,nfsvers=4 - device: "[ip address]:/var/lib/docker/volumes/rodan_pg_data/_data" \ No newline at end of file diff --git a/production.yml b/production.yml deleted file mode 100644 index 6c1b04150..000000000 --- a/production.yml +++ /dev/null @@ -1,254 +0,0 @@ -version: "3.4" - -services: - - nginx: - image: "ddmal/nginx:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "0.5" - memory: 1G - limits: - cpus: "0.5" - memory: 1G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "/usr/sbin/service", "nginx", "status"] - interval: "30s" - timeout: "10s" - retries: 10 - start_period: "5m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - TLS: 1 - ports: - - "80:80" - - "443:443" - - "5671:5671" - - "9002:9002" - volumes: - - "resources:/rodan/data" - - rodan-main: - image: "ddmal/rodan-main:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 4G - limits: - cpus: "1" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD-SHELL", "/usr/bin/curl -H 'User-Agent: docker-healthcheck' http://localhost:8000/api/?format=json || exit 1"] - interval: "30s" - timeout: "30s" - retries: 5 - start_period: "2m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: None - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - rodan-client: - image: "ddmal/rodan-client:nightly" - volumes: - - "./rodan-client/config/configuration.json:/client/configuration.json" - - iipsrv: - image: "ddmal/iipsrv:nightly" - volumes: - - "resources:/rodan/data" - - celery: - image: "ddmal/rodan-main:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "6" - memory: 12G - limits: - cpus: "6" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] - interval: "30s" - timeout: "30s" - start_period: "1m" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: celery - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - py3-celery: - image: "ddmal/rodan-python3-celery:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "6" - memory: 12G - limits: - cpus: "6" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: Python3 - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - gpu-celery: - image: "ddmal/rodan-gpu-celery:v3.3.1" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 45G - limits: - cpus: "2" - memory: 45G - restart_policy: - condition: any - delay: 5s - window: 30s - placement: - constraints: - - node.labels.queue == GPU - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan2.simssa.ca - CELERY_JOB_QUEUE: GPU - env_file: - - ./scripts/production.env - volumes: - - "resources:/rodan/data" - - redis: - image: "redis:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - - postgres: - image: "ddmal/postgres-plpython:v3.3.1" - deploy: - replicas: 1 - endpoint_mode: dnsrr - resources: - reservations: - cpus: "4" - memory: 10G - limits: - cpus: "4" - memory: 10G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD-SHELL", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - volumes: - - "pg_data:/var/lib/postgresql/data" - - "pg_backup:/backups" - env_file: - - ./scripts/production.env - - rabbitmq: - image: "rabbitmq:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] - interval: "30s" - timeout: "3s" - retries: 3 - environment: - TZ: America/Toronto - env_file: - - ./scripts/production.env - -volumes: - resources: - pg_backup: - pg_data: diff --git a/readme.md b/readme.md index ea2e27b88..46f121ea3 100644 --- a/readme.md +++ b/readme.md @@ -4,7 +4,7 @@ - Master Branch ![GitHub last commit (branch)](https://img.shields.io/github/last-commit/ddmal/rodan/master) - Develop Branch ![GitHub last commit (branch)](https://img.shields.io/github/last-commit/ddmal/rodan/develop) -This repository contains Docker images that can be used to set up [Rodan](https://github.com/DDMAL/rodan) locally for development. These images can also be used in the future with slight modifications for deployment to a swarm production environment. Please see the wiki for more information about deploying Rodan. [Rodan Wiki](https://github.com/DDMAL/Rodan/wiki) +This repository contains Docker images that can be used to set up [Rodan](https://github.com/DDMAL/rodan) locally for development. The same images are deployed to **Kubernetes (k3s)** in production — see [`k8s/`](./k8s) (and [`k8s/README.md`](./k8s/README.md)) for the manifests and deployment runbook. For more general information, see the [Rodan Wiki](https://github.com/DDMAL/Rodan/wiki). #### Objectives @@ -40,15 +40,14 @@ A similar concept to using `exec` is using SSH to connect to another computer. W Consult the documentation of the [Docker command line](https://docs.docker.com/engine/reference/commandline/cli/) for additional information. -## Automated Build +## CI/CD -The images are rebuilt and pushed automatically on a nightly basis at 2am. This accomplished with a cron job. You must point the cron job to the nightly script on one of the staging virtual machines. Any account will do and no authentication required, add this line to the crontab. Docker hub will send a Slack notification if the image has built. We should expect 5 new images daily, or more if there was a new tagged release of any of them. +Image builds, pushes, and deploys are handled by GitHub Actions in [`.github/workflows/build-and-deploy.yml`](./.github/workflows/build-and-deploy.yml): -```shell -0 2 * * 1-5 /srv/webapps/rodan-docker/scripts/nightly -``` +- **Build & push** — all seven images are built and pushed to the private GitHub Container Registry at `ghcr.io/ddmal/`. Triggers: push to `develop` → `:nightly`; a `v*` git tag → `:`; pull requests build only (no push). Every pushed build also gets an immutable `:sha-` tag. +- **Deploy** — on a `v*` tag or manual `workflow_dispatch`, the app-tier Deployments in the k3s `rodan` namespace are rolled to the new `:sha-` images (via `kubectl set image`). `postgres`/`redis`/`rabbitmq` are left untouched. -You may also force Docker Cloud to rebuild new images when new commits are pushed to a Git repository. Unfortunately, we had problems connecting the `rodan-docker` GitHub repository to Docker Cloud due to authentication issues, so we set up a private repository on Bitbucket instead. +See [`k8s/README.md`](./k8s/README.md) for the full deployment/runbook details. ## Additional Information diff --git a/runLog.txt b/runLog.txt deleted file mode 100644 index 0f5889d7b..000000000 --- a/runLog.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Run build_arm first if you don't have the NGINX container. -# Launch ARM instance -nightly: Pulling from ddmal/rodan-python3-celery diff --git a/run_containers b/run_containers deleted file mode 100644 index f46879527..000000000 --- a/run_containers +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# parallel silence if you want -# parallel --citation - -parallel --jobs 4 < starter.txt >> runLog.txt diff --git a/scripts/healthCheckFromUrl.sh b/scripts/healthCheckFromUrl.sh deleted file mode 100755 index a99961e34..000000000 --- a/scripts/healthCheckFromUrl.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Usage: bash healthCheckFromUrl.sh -# Use curl to access a URL. If it fails, send a notification email to addresses in MAIL_LIST. -# This script is located in /root and scheduled in /etc/crontab -# -# Steps to use this script (only do this in production): -# 1. Add your email address to MAIL_LIST with format ;;:... -# 2. chmod 744 healthCheckFromUrl.sh -# 3. chown root accessFromUrl.sh -# 4. chgrp root accessFromUrl.sh -# 5. move this script to /root -# 6. Add this to /etc/crontab: 0 0,6,12,18 * * * root /root/healthCheckFromUrl.sh https://rodan-staging.simssa.ca/ -# 7. Add this to /etc/crontab: 0 0,6,12,18 * * * root /root/healthCheckFromUrl.sh https://rodan2.simssa.ca/ - -URL=$1 -MAIL_LIST="wan.y.lin@mail.mcgill.ca" - -if curl -sSf $URL > /dev/null 2>&1 --connect-timeout 10; then - logger -t HealthCheck "$URL is up!" -else - IFS=';' read -ra ARRAY <<< "$MAIL_LIST" - for MAIL in "${ARRAY[@]}"; do - DATE=$(date) - mail -s "Cannot access $URL" -r rodan@production-rodan2-gpu $MAIL <<< "Timestamp: $DATE" - done -fi diff --git a/scripts/nightly b/scripts/nightly deleted file mode 100644 index 66c239225..000000000 --- a/scripts/nightly +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# This script is a nightly cronjob to tell dockerhub to build all of Rodan's many containers - -set -o errexit # Exit immediately if a command exits with a non-zero status. -set -o xtrace # Print commands and their arguments as they are executed. -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -echo `date` >> /var/log/docker/rodan-build.log - -# jq nicely indents the json response from dockerhub. -jq || apt install jq -yq -http || apt install httpie -yq - -http POST https://hub.docker.com/api/build/v1/source/5b0f11c1-d438-44a4-aee3-de94c9628755/trigger/7776337e-a10b-46df-b33c-6b36208888a8/call/ | jq '.' >> /var/log/docker/rodan-build.log diff --git a/scripts/production.sample b/scripts/production.sample deleted file mode 100644 index 624e0c798..000000000 --- a/scripts/production.sample +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Django Configuration -############################################################################### - -ADMIN_USER=someadmin -ADMIN_EMAIL=admin@rodan2.simssa.ca -ADMIN_PASS=123456Seven -DJANGO_DEBUG_MODE=False -DJANGO_SECRET_KEY=AVeryLongSecretKeyThatWasGeneratedByOurSpecialScript_DoNotForgetToChangeThis -DJANGO_MEDIA_ROOT=/rodan/data/ -# You can specify multiple hosts with a comma -DJANGO_ALLOWED_HOSTS=* -DJANGO_ADMIN_URL=^api/random_secret_admin/ -IIPSRV_URL=https://rodan2.simssa.ca/fcgi-bin/iipsrv.fcgi/ -DJANGO_ACCESS_LOG=/code/Rodan/rodan.log -DJANGO_DEBUG_LOG=/code/Rodan/database.log -SERVER_HOST=rodan2.simssa.ca - -############################################################################### -# SMTP Configuration -############################################################################### - -EMAIL_HOST= -EMAIL_PORT= -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= - -############################################################################### -# Celery Configuration -############################################################################### - -C_FORCE_ROOT=true - -############################################################################### -# Database Configuration (Postgres) -############################################################################### - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=rodan -POSTGRES_USER=someadmin -POSTGRES_PASSWORD=123456Seven -POSTGRES_DATABASE_LOGFILE=/code/Rodan/database.log - -############################################################################### -# Messenger Configuration (RabbitMQ) -############################################################################### - -# The format is -> $Protocol :// $User : $Password @ $Host : $Port / $Vhost -RABBITMQ_URL=amqp://someadmin:123456Seven@rabbitmq:5672// -RABBITMQ_DEFAULT_USER=someadmin -RABBITMQ_DEFAULT_PASS=123456Seven - -############################################################################### -# Websocket Configuration (Redis) -############################################################################### - -REDIS_HOST=redis -REDIS_PORT=6379 -REDIS_DB=0 diff --git a/scripts/staging.env b/scripts/staging.env deleted file mode 100644 index fec111a1c..000000000 --- a/scripts/staging.env +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################### -# Django Configuration -############################################################################### - -ADMIN_USER=someadmin -ADMIN_EMAIL=someadmin@rodan2.simssa.ca -ADMIN_PASS=123456Seven -DJANGO_DEBUG_MODE=False -DJANGO_SECRET_KEY=AVeryLongSecretKeyThatWasGeneratedByOurSpecialScript_DoNotForgetToChangeThis -DJANGO_MEDIA_ROOT=/rodan/data/ -# You can specify multiple hosts with a comma -DJANGO_ALLOWED_HOSTS=* -DJANGO_ADMIN_URL=^api/random_secret_admin/ -IIPSRV_URL=http://rodan.staging.simssa.ca/fcgi-bin/iipsrv.fcgi/ -DJANGO_ACCESS_LOG=/code/Rodan/rodan.log -DJANGO_DEBUG_LOG=/code/Rodan/database.log -SERVER_HOST=rodan.staging.simssa.ca -############################################################################### -# Celery Configuration -############################################################################### - -C_FORCE_ROOT=true - -############################################################################### -# Database Configuration (Postgres) -############################################################################### - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=rodan -POSTGRES_USER=someadmin -POSTGRES_PASSWORD=123456Seven -POSTGRES_DATABASE_LOGFILE=/code/Rodan/database.log - -############################################################################### -# Messenger Configuration (RabbitMQ) -############################################################################### - -# The format is -> $Protocol :// $User : $Password @ $Host : $Port / $Vhost -RABBITMQ_URL=amqp://someadmin:123456Seven@rabbitmq:5672// -RABBITMQ_DEFAULT_USER=someadmin -RABBITMQ_DEFAULT_PASS=123456Seven - -############################################################################### -# Websocket Configuration (Redis) -############################################################################### - -REDIS_HOST=redis -REDIS_PORT=6379 -REDIS_DB=0 diff --git a/scripts/staging.sample b/scripts/staging.sample deleted file mode 100644 index 0d5e184ab..000000000 --- a/scripts/staging.sample +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################### -# Django Configuration -############################################################################### - -ADMIN_USER=someadmin -ADMIN_EMAIL=admin@rodan2.simssa.ca -ADMIN_PASS=123456Seven -DJANGO_DEBUG_MODE=False -DJANGO_SECRET_KEY=AVeryLongSecretKeyThatWasGeneratedByOurSpecialScript_DoNotForgetToChangeThis -DJANGO_MEDIA_ROOT=/rodan/data/ -# You can specify multiple hosts with a comma -DJANGO_ALLOWED_HOSTS=* -DJANGO_ADMIN_URL=^api/random_secret_admin/ -IIPSRV_URL=http://rodan-staging.simssa.ca/fcgi-bin/iipsrv.fcgi/ -DJANGO_ACCESS_LOG=/code/Rodan/rodan.log -DJANGO_DEBUG_LOG=/code/Rodan/database.log -SERVER_HOST=rodan-staging.simssa.ca - -############################################################################### -# SMTP Configuration -############################################################################### - -EMAIL_HOST= -EMAIL_PORT= -EMAIL_HOST_USER= -EMAIL_HOST_PASSWORD= - -############################################################################### -# Celery Configuration -############################################################################### - -C_FORCE_ROOT=true - -############################################################################### -# Database Configuration (Postgres) -############################################################################### - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_DB=rodan -POSTGRES_USER=someadmin -POSTGRES_PASSWORD=123456Seven -POSTGRES_DATABASE_LOGFILE=/code/Rodan/database.log - -############################################################################### -# Messenger Configuration (RabbitMQ) -############################################################################### - -# The format is -> $Protocol :// $User : $Password @ $Host : $Port / $Vhost -RABBITMQ_URL=amqp://someadmin:123456Seven@rabbitmq:5672// -RABBITMQ_DEFAULT_USER=someadmin -RABBITMQ_DEFAULT_PASS=123456Seven - -############################################################################### -# Websocket Configuration (Redis) -############################################################################### - -REDIS_HOST=redis -REDIS_PORT=6379 -REDIS_DB=0 diff --git a/scripts/update b/scripts/update deleted file mode 100644 index a12c61f16..000000000 --- a/scripts/update +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# This script is a nightly cronjob to update the docker swarm deployment. - -set -o errexit # Exit immediately if a command exits with a non-zero status. -set -o xtrace # Print commands and their arguments as they are executed. - -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -cd /srv/webapps/Rodan -echo `date` >> /var/log/docker/rodan-update.log -make update tag1=nightly tag2=nightly >> /var/log/docker/rodan-update.log diff --git a/staging.yml b/staging.yml deleted file mode 100644 index f15eaf8ee..000000000 --- a/staging.yml +++ /dev/null @@ -1,263 +0,0 @@ -version: "3.4" - -services: - - nginx: - image: "ddmal/nginx:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "0.5" - memory: 1G - limits: - cpus: "0.5" - memory: 1G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "/usr/sbin/service", "nginx", "status"] - interval: "30s" - timeout: "10s" - retries: 10 - start_period: "5m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - env_file: - - ./scripts/staging.env - ports: - - "80:80" - - "5671:5671" - - "9002:9002" - volumes: - - "resources:/rodan/data" - - rodan-main: - image: "ddmal/rodan-main:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 4G - limits: - cpus: "1" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - # TODO: Make the healthcheck a service status call instead of adding to the logfile. - # https://github.com/DDMAL/rodan-docker/issues/61 - healthcheck: - test: ["CMD-SHELL", "/usr/bin/curl -H 'User-Agent: docker-healthcheck' http://localhost:8000/api/?format=json || exit 1"] - interval: "30s" - timeout: "30s" - retries: 5 - start_period: "2m" - command: /run/start - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: None - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - rodan-client: - image: "ddmal/rodan-client:nightly" - volumes: - - "./rodan-client/config/configuration.json:/client/configuration.json" - - iipsrv: - image: "ddmal/iipsrv:nightly" - volumes: - - "resources:/rodan/data" - - celery: - image: "ddmal/rodan-main:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 12G - limits: - cpus: "2" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] - # first run interval seconds after the container is started, and then again interval seconds after each previous - interval: "30s" - # How long to wait for the healthcheck to succeed - timeout: "30s" - # Ignore failures during the start_period - start_period: "1m" - # accept 3 consecutive failures before - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: celery - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - py3-celery: - image: "ddmal/rodan-python3-celery:nightly" - deploy: - replicas: 1 - resources: - reservations: - cpus: "2" - memory: 12G - limits: - cpus: "2" - memory: 12G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: Python3 - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - gpu-celery: - image: "ddmal/rodan-gpu-celery:nightly" - deploy: - replicas: 1 - resources: - limits: - cpus: '1' - memory: 8G - reservations: - cpus: '1' - memory: 8G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] - interval: "30s" - timeout: "30s" - retries: 5 - command: /run/start-celery - environment: - TZ: America/Toronto - SERVER_HOST: rodan.staging.simssa.ca - CELERY_JOB_QUEUE: GPU - depends_on: - - postgres - - rodan - - rabbitmq - - redis - - celery - env_file: - - ./scripts/staging.env - volumes: - - "resources:/rodan/data" - - redis: - image: "redis:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - - postgres: - image: "ddmal/postgres-plpython:nightly" - deploy: - replicas: 1 - endpoint_mode: dnsrr - resources: - reservations: - cpus: "2" - memory: 4G - limits: - cpus: "2" - memory: 4G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD-SHELL", "pg_isready", "-U", "postgres"] - interval: 10s - timeout: 5s - retries: 5 - environment: - TZ: America/Toronto - volumes: - - "pg_data:/var/lib/postgresql/data" - - "pg_backup:/backups" - env_file: - - ./scripts/staging.env - - rabbitmq: - image: "rabbitmq:alpine" - deploy: - replicas: 1 - resources: - reservations: - cpus: "1" - memory: 2G - limits: - cpus: "1" - memory: 2G - restart_policy: - condition: any - delay: 5s - window: 30s - healthcheck: - test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] - interval: "30s" - timeout: "3s" - retries: 3 - environment: - TZ: America/Toronto - env_file: - - ./scripts/staging.env - -volumes: - resources: - pg_backup: - pg_data: diff --git a/starter.sh b/starter.sh deleted file mode 100755 index 47c7b657e..000000000 --- a/starter.sh +++ /dev/null @@ -1,32 +0,0 @@ -# runs the commands in new terminal tabs you have to open a new separate window of iterm to start with -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "t" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && make run_arm -"' -sleep 30 -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "d" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && docker compose exec rodan-main /run/start -"' -sleep 30 -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "d" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && docker compose exec celery /run/start-celery -"' -osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "d" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "cd ~/Desktop/Rodan && docker compose exec py3-celery /run/start-celery -"' -echo "######### DEPLOYED #########" -sleep 20 -osascript -e 'tell application "Google Chrome" to open location "http://localhost"' - -# wait for the key q and terminate and close all the tabs if q is entered -echo "enter q to terminate the process" -read key -while [[ $key != "q" ]] -do - read key -done -if [[ $key = q ]] -then - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "2" using command down' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - osascript -e 'tell application "iTerm" to activate' -e 'tell application "System Events" to tell process "iTerm" to keystroke "w" using command down' - printf "\nQuitting from the program\n" -fi -# for the other features regarding the demands of your own environment you can modify the code above -# Author: Shahrad Mohammadzadeh