diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 42c678092..000000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,5 +0,0 @@ -Resolves: (#ID-of-the-issue) -- [ ] I passed the docker hub test, and images can be built successfully. -- [ ] I passed the GitHub CI test, so rodan functionalities and jobs work. - -(Describe the changes you've made and the purpose of this PR) \ No newline at end of file diff --git a/.github/workflows/build-ci-jobs.yml b/.github/workflows/build-ci-jobs.yml new file mode 100644 index 000000000..a3e1dc685 --- /dev/null +++ b/.github/workflows/build-ci-jobs.yml @@ -0,0 +1,44 @@ +name: Build & push ci-jobs +on: + workflow_dispatch: + inputs: + tag: + description: "Docker Hub tag to push (ddmal/ci-jobs:)" + required: true + default: "django-v4.2.16" + branch: + description: "BRANCH build-arg — ref used to clone neon_wrapper/pixel_wrapper" + required: true + default: "develop" + +jobs: + build-ci-jobs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Free disk space + # ci-jobs is a heavy image (Grok compile + gamera/musicstaves + postgres + deps). + uses: jlumbroso/free-disk-space@v1.3.1 + with: + tool-cache: true + large-packages: false + docker-images: false + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build ci-jobs + run: | + docker build \ + --build-arg BRANCH="${{ github.event.inputs.branch }}" \ + -t "ddmal/ci-jobs:${{ github.event.inputs.tag }}" \ + -f ci-jobs/Dockerfile \ + . + + - name: Push ci-jobs + run: docker push "ddmal/ci-jobs:${{ github.event.inputs.tag }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0415fae96..f3f9eda9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: rodan-CI-test: runs-on: ubuntu-22.04 container: - image: ddmal/ci-jobs:django-v2.0.13 + image: ddmal/ci-jobs:django-v4.2.16 env: CELERY_JOB_QUEUE: None ADMIN_USER: rodan diff --git a/ci-jobs/Dockerfile b/ci-jobs/Dockerfile index a06f96ba1..96df71554 100644 --- a/ci-jobs/Dockerfile +++ b/ci-jobs/Dockerfile @@ -29,10 +29,11 @@ RUN yarn install RUN yarn build # Make Gamera files accessible to the main container. -FROM ddmal/gamera4:fix_IC AS gamera +FROM ddmal/gamera4:py311 AS gamera -# This release is based on Debian 11 "Bullseye" -FROM python:3.7-slim +# Debian 12 "Bookworm" with Python 3.11 (matches python3-celery; Django 4.2 needs py>=3.8 +# and the py3.11-ported Gamera source uses the 3.9+ C-API). +FROM python:3.11-slim # Copy various scripts and make them executable. # COPY ./scripts/start-celery /run/ @@ -65,8 +66,9 @@ RUN apt-get install -y \ wget \ vim -# https://stackoverflow.com/questions/69100275/error-while-downloading-the-requirements-using-pip-install-setup-command-use-2 -RUN pip3 install setuptools==58 +# setuptools <80 still supports the legacy `setup.py install` that Gamera/Musicstaves use; +# py3.11 keeps stdlib distutils, which Gamera's setup.py imports directly. +RUN pip3 install "setuptools<80" wheel RUN pip3 install -r /code/Rodan/requirements.txt # Install Gamera and Musicstaves. @@ -74,9 +76,12 @@ WORKDIR /gamera4-rodan/gamera-4 RUN python3 setup.py --nowx install WORKDIR /gamera4-rodan/musicstaves RUN python3 setup.py install - -# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. -RUN pip install opencv-python==4.6.0.66 +# Musicstaves installs as an isolated egg whose gamera/toolkits/ does not merge with +# gamera's (gamera's toolkits/__init__.py is empty, no extend_path). Move its package +# into the gamera egg so `import gamera.toolkits.musicstaves` resolves. Self-checks. +RUN cp -r /usr/local/lib/python3.11/site-packages/musicstaves-*.egg/gamera/toolkits/musicstaves \ + /usr/local/lib/python3.11/site-packages/gamera-*.egg/gamera/toolkits/ \ + && python3 -W ignore -c "import gamera.toolkits.musicstaves.stafffinder_miyao; from gamera.toolkits import musicstaves" # ============================ # from ./rodan-main/Dockerfile @@ -108,13 +113,9 @@ RUN apt-get install -y \ wget \ vim -# https://stackoverflow.com/questions/69100275/error-while-downloading-the-requirements-using-pip-install-setup-command-use-2 -RUN pip3 install setuptools==58 +RUN pip3 install "setuptools<80" wheel RUN pip3 install -r /code/Rodan/requirements.txt -# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. -RUN pip install opencv-python==4.6.0.66 - # From rodan-main RUN set -x \ && mkdir /vendor || echo "Already exists" \ @@ -137,7 +138,10 @@ RUN cd /vendor \ && git checkout 4932d86b19d9008a28ae898035cfa43340e636e6 \ && mkdir build \ && cd build \ - && /vendor/cmake-3.17.0-Linux-x86_64/bin/cmake -DBUILD_THIRDPARTY=ON .. \ + # GCC 12+ (Debian bookworm, via the py3.11 base) no longer transitively includes + # , so Grok's pinned source fails on undeclared uint8_t. Force-include it + # for every C++ TU instead of patching the third-party headers. + && /vendor/cmake-3.17.0-Linux-x86_64/bin/cmake -DBUILD_THIRDPARTY=ON -DCMAKE_CXX_FLAGS="-include cstdint" .. \ && make -j3 # ============================ diff --git a/gpu-celery/Dockerfile b/gpu-celery/Dockerfile index fd44c14f2..a36d16f22 100644 --- a/gpu-celery/Dockerfile +++ b/gpu-celery/Dockerfile @@ -1,189 +1,74 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# GPU celery worker — TensorFlow 2.15 on CUDA 12 for Hopper (sm_90). # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# The CUDA 12 runtime libraries are pulled as pip wheels via `tensorflow[and-cuda]`, so the +# image bakes in NO CUDA toolkit and only needs the host NVIDIA driver (supplied by the k8s +# `nvidia` RuntimeClass — see k8s/33-gpu-celery.yaml). This replaces the old hand-assembled +# nvidia/cuda:11.2 + TF 2.5.1 + py3.7 base, which cannot run on Hopper. # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -# -# THIS IS A GENERATED DOCKERFILE. -# -# This file was assembled from multiple pieces, whose use is documented -# throughout. Please refer to the TensorFlow dockerfiles documentation -# for more information. - -ARG UBUNTU_VERSION=18.04 -ARG CUDA=11.2 -FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.2-base-ubuntu${UBUNTU_VERSION} as base -# ARCH and CUDA are specified again because the FROM directive resets ARGs -# (but their default value is retained if set previously) -ARG ARCH -ARG CUDA -ARG CUDNN=8.1.0.77-1 -ARG CUDNN_MAJOR_VERSION=8 -ARG LIB_DIR_PREFIX=x86_64 -ARG LIBNVINFER=7.2.2-1 -ARG LIBNVINFER_MAJOR_VERSION=7 +# This worker serves only the BASE + GPU job queues (CELERY_JOB_QUEUE=GPU), so it needs no +# Gamera. GPU jobs: Paco_classifier, background_removal, SAE_binarization (TF 2.15), +# staff_distance (skimage/cv2), and text_alignment (Calamari-OCR / ocrd-fork-tfaip 1.2.7). +FROM python:3.11-slim -# The following two arguments are rodan-specific ARG BRANCHES ARG VERSION -# Needed for string substitution -SHELL ["/bin/bash", "-c"] -# Pick up some TF dependencies -#RUN apt-get update - -#RUN rm -rf /etc/apt/sources.list.d/cuda.list - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-${CUDA/./-} \ - libcublas-${CUDA/./-} \ - cuda-nvrtc-${CUDA/./-} \ - libcufft-${CUDA/./-} \ - libcurand-${CUDA/./-} \ - libcusolver-${CUDA/./-} \ - libcusparse-${CUDA/./-} \ - curl \ - libcudnn8=${CUDNN}+cuda${CUDA} \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - pkg-config \ - software-properties-common \ - unzip \ - wget -# added wget - -# Install TensorRT if not building for PowerPC -# NOTE: libnvinfer uses cuda11.1 versions -# RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \ -# apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ -# libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/*; } - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda-11.1/lib64:$LD_LIBRARY_PATH - -# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure -# dynamic linker run-time bindings -RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \ - && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \ - && ldconfig - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# This section differs from the default tensorflow2.5.1 Dockerfile, because we specifically add python 3.7; -ARG PYTHON=python3.7 -ARG TF_PACKAGE=tensorflow -ARG TF_PACKAGE_VERSION=2.5.1 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - python3.7 \ - python3-pip \ - && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* \ - && wget https://bootstrap.pypa.io/pip/3.7/get-pip.py \ - && ${PYTHON} get-pip.py \ - && ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 \ - && ln -sf /usr/local/bin/pip /usr/local/bin/pip3 \ - && pip3 --no-cache-dir install --upgrade pip setuptools==57.0.0 \ - # Some TF tools expect a "python" binary - && ln -s $(which python3) /usr/local/bin/python \ - && python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}} - -# RUN ln -s $(which python3) /usr/local/bin/python - -# Options: -# tensorflow -# tensorflow-gpu -# tf-nightly -# tf-nightly-gpu -# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version. -# Installs the latest version by default. - -# COPY bashrc /etc/bash.bashrc -# RUN chmod a+rwx /etc/bash.bashrc - -# This ends the material obtained from TensorFlow's dockerfile. the remainder is rodan-docker-specific setup. - -# FROM base -RUN set -e \ - && apt-get update \ - && DEBIAN_FRONTEND="noninteractive" apt-get install -yqq \ - git \ - # Python lxml dependencies - python3.7-dev \ - python3-opencv \ - libxml2-dev \ - libxslt1-dev \ - zlib1g-dev \ - lib32ncurses5-dev \ - # Psycopg2 dependencies - libpq-dev \ - # For resource identification - libmagic-dev \ - unzip \ - # Remove when done - vim - -RUN rm -rf /var/lib/apt/lists/* - -# Bandaid fix for the cannot import name '_registerMatType' from 'cv2.cv2' issue -#RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30 - -# Install GPU Rodan Jobs +# Scripts (entrypoint waits for postgres/redis then execs the arg = /run/start-celery). COPY ./scripts/install_gpu_rodan_jobs /opt/ - -# Install Rodan -# Runs on both Rodan service, and Rodan-Celery COPY ./scripts/entrypoint /opt/ COPY ./scripts/start-celery /run/ COPY ./scripts/wait-for-app /run/ -# Copying rodan core from build context into container -# Rodan folder MUST be uppercase, otherwise many unittests fail. +# Rodan core. The folder MUST be uppercase or many unittests fail. COPY ./rodan-main/code /code/Rodan -# necessary for scikit-image > 0.17, or else it will try to make a cache directory -# in a place where the www-data user does not have permissions to do so -ENV SKIMAGE_DATADIR "/tmp/.skimage_cache" +# scikit-image otherwise caches under $HOME, which www-data cannot create. +ENV SKIMAGE_DATADIR="/tmp/.skimage_cache" + +# The old nvidia/cuda base set this; python:3.11-slim does not. The NVIDIA container +# runtime reads it to decide which driver features to inject — "compute" is required for +# CUDA (without it you get nvidia-smi but cuInit -> CUDA_ERROR_NO_DEVICE). NVIDIA_VISIBLE_DEVICES +# is intentionally left to the k8s device plugin. (Note: the GPU also needs a valid vGPU +# license on the host, or CUDA is blocked regardless.) +ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility + +# System deps: git (clones the GPU job repos), build-essential (native pip builds), +# libmagic1 (python-magic), libpq-dev (psycopg2), and the runtime libs opencv-python needs. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + curl \ + git \ + libmagic1 \ + libpq-dev \ + libgl1 \ + libglib2.0-0 \ + libsm6 \ + libxext6 \ + libxrender1 \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# setuptools<80 keeps the legacy `setup.py install` used by the GPU job packages working. +RUN pip3 install --no-cache-dir "setuptools<80" wheel + +# TensorFlow 2.15 with the bundled CUDA 12 wheels (Hopper sm_90 support). +RUN pip3 install --no-cache-dir "tensorflow[and-cuda]==2.15.1" +# Rodan Python deps. +RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt -RUN set -x \ - # Create Folders +# GPU job packages (py311 branches + text_alignment's calamari). Their `tensorflow==2.15.1` +# pin is already satisfied by tensorflow[and-cuda] above, so pip does not reinstall TF. +RUN set -e \ && mkdir -p /code/jobs \ - # Install GPU Jobs && chmod +x /opt/install_gpu_rodan_jobs \ - && /opt/install_gpu_rodan_jobs \ - # Install Rodan - && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \ - # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \ - # Add Entrypoints - && sed -i 's/\r//' /opt/entrypoint \ - && chmod +x /opt/entrypoint \ - # Add Celery script - && chmod +x /run/start-celery \ - # Change the concurency for gpu jobs because Calvo is very expensive - && sed -i "s/=10/=1/g" /run/start-celery \ - # Script to wait for postgres and redis to be running before attempting to connect to them. - && chmod +x /run/wait-for-app + && /opt/install_gpu_rodan_jobs -# Install Rodan. -RUN pip3 install -r /code/Rodan/requirements.txt - -RUN pip3 uninstall -y opencv-python opencv-python-headless -RUN pip3 install opencv-python-headless==4.1.2.30 +# GPU jobs are expensive — one task per worker at a time. +RUN sed -i "s/--concurrency=10/--concurrency=1/g" /run/start-celery \ + && sed -i 's/\r//' /opt/entrypoint \ + && chmod +x /opt/entrypoint /run/start-celery /run/wait-for-app ENTRYPOINT ["/opt/entrypoint"] diff --git a/k8s/12-pv-pgdata.yaml b/k8s/12-pv-pgdata.yaml index 567bd81c5..baccabd07 100644 --- a/k8s/12-pv-pgdata.yaml +++ b/k8s/12-pv-pgdata.yaml @@ -1,5 +1,6 @@ -# Postgres PGDATA (~4.1 GB), PG 9.6, owned by uid 999. Mounted only by the postgres pod. -# 'hard' mount (DBs must not see truncated/soft-failed I/O). NFS export needs no_root_squash. +# Postgres 16 PGDATA (dump/restored from the old 9.6 clone — Django 4.2 needs PG>=12), owned by +# uid 999. Mounted only by the postgres pod. 'hard' mount (DBs must not see truncated/soft-failed +# I/O). NFS export needs no_root_squash. See the PG9.6->16 runbook before first apply. apiVersion: v1 kind: PersistentVolume metadata: @@ -18,7 +19,7 @@ spec: - hard nfs: server: 192.168.236.124 - path: /srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data + path: /srv/rodan-data/pg16_data claimRef: namespace: rodan name: rodan-pg-data \ No newline at end of file diff --git a/k8s/20-postgres.yaml b/k8s/20-postgres.yaml index 5c9ce5344..3d2bfc231 100644 --- a/k8s/20-postgres.yaml +++ b/k8s/20-postgres.yaml @@ -1,6 +1,6 @@ -# Postgres 9.6 + plpython3 (existing image). Single instance, PGDATA on the static NFS PVC. -# The existing PGDATA is already initialized, so the entrypoint skips initdb and the -# POSTGRES_USER/PASSWORD/DB env are effectively no-ops on an existing cluster (kept for first-init). +# Postgres 16 + plpython3. Single instance, PGDATA on the static NFS PVC (pg16_data, restored +# from the old 9.6 dump). The PGDATA is already initialized, so the entrypoint skips initdb and +# the POSTGRES_USER/PASSWORD/DB env are effectively no-ops (kept for first-init). apiVersion: apps/v1 kind: StatefulSet metadata: @@ -21,7 +21,7 @@ spec: - name: ghcr-pull-secret containers: - name: postgres - image: ghcr.io/ddmal/postgres-plpython:k8s-migration + image: ghcr.io/ddmal/postgres-plpython:nightly ports: - containerPort: 5432 name: postgres diff --git a/k8s/30-rodan-main.yaml b/k8s/30-rodan-main.yaml index d4ac6a457..27c4a381e 100644 --- a/k8s/30-rodan-main.yaml +++ b/k8s/30-rodan-main.yaml @@ -22,7 +22,7 @@ spec: - name: ghcr-pull-secret containers: - name: rodan-main - image: ghcr.io/ddmal/rodan-main:k8s-migration + image: ghcr.io/ddmal/rodan-main:nightly args: ["/run/start"] envFrom: - configMapRef: { name: rodan-config } diff --git a/k8s/31-celery.yaml b/k8s/31-celery.yaml index 4f497236d..190981a45 100644 --- a/k8s/31-celery.yaml +++ b/k8s/31-celery.yaml @@ -20,7 +20,7 @@ spec: - name: ghcr-pull-secret containers: - name: celery - image: ghcr.io/ddmal/rodan-main:k8s-migration + image: ghcr.io/ddmal/rodan-main:nightly args: ["/run/start-celery"] envFrom: - configMapRef: { name: rodan-config } @@ -33,13 +33,13 @@ spec: mountPath: /rodan/data startupProbe: exec: - command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] + command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@celery", "-t", "30"] periodSeconds: 45 timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer failureThreshold: 30 # tolerates the long wait-for rodan-main:8000 (up to 900s) livenessProbe: exec: - command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@celery", "-t", "30"] + command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@celery", "-t", "30"] periodSeconds: 60 timeoutSeconds: 45 failureThreshold: 3 diff --git a/k8s/32-py3-celery.yaml b/k8s/32-py3-celery.yaml index f556d28c5..8a861edc8 100644 --- a/k8s/32-py3-celery.yaml +++ b/k8s/32-py3-celery.yaml @@ -19,7 +19,7 @@ spec: - name: ghcr-pull-secret containers: - name: py3-celery - image: ghcr.io/ddmal/rodan-python3-celery:k8s-migration + image: ghcr.io/ddmal/rodan-python3-celery:nightly envFrom: - configMapRef: { name: rodan-config } - secretRef: { name: rodan-secrets } @@ -31,13 +31,13 @@ spec: mountPath: /rodan/data startupProbe: exec: - command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] + command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@Python3", "-t", "30"] periodSeconds: 45 timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer failureThreshold: 30 livenessProbe: exec: - command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@Python3", "-t", "30"] + command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@Python3", "-t", "30"] periodSeconds: 60 timeoutSeconds: 45 failureThreshold: 3 diff --git a/k8s/33-gpu-celery.yaml b/k8s/33-gpu-celery.yaml index f1674f26c..1d9131280 100644 --- a/k8s/33-gpu-celery.yaml +++ b/k8s/33-gpu-celery.yaml @@ -19,8 +19,17 @@ spec: spec: imagePullSecrets: - name: ghcr-pull-secret + # MIG-backed vGPU workaround: the device plugin (migStrategy=none, the only mode that + # works in a vGPU guest) hands the container the PARENT GPU, but CUDA needs the MIG + # compute instance — whose access node /dev/nvidia-caps/nvidia-cap4 the plugin doesn't + # inject, so cuInit -> CUDA_ERROR_NO_DEVICE. Instead we pin to a specific node and let + # the NVIDIA runtime inject the MIG instance directly via NVIDIA_VISIBLE_DEVICES (which + # DOES inject cap4). This means NOT requesting nvidia.com/gpu (that would make the plugin + # overwrite NVIDIA_VISIBLE_DEVICES with the parent). The MIG UUID is per-node, hence the + # hostname pin. Long-term fix: a non-MIG (time-sliced) vGPU profile removes all of this. nodeSelector: gpu: "true" + kubernetes.io/hostname: k3s-gpu-node-2 tolerations: - key: nvidia.com/gpu operator: Exists @@ -29,7 +38,7 @@ spec: runtimeClassName: nvidia containers: - name: gpu-celery - image: ghcr.io/ddmal/rodan-gpu-celery:k8s-migration + image: ghcr.io/ddmal/rodan-gpu-celery:nightly args: ["/run/start-celery"] envFrom: - configMapRef: { name: rodan-config } @@ -37,21 +46,40 @@ spec: env: - name: CELERY_JOB_QUEUE value: "GPU" + # MIG compute-instance UUID on k3s-gpu-node-2 (nvidia-smi -L). The runtime + # injects this exact MIG device (incl. /dev/nvidia-caps/nvidia-cap4) so CUDA can + # open it. This is node-specific — it must match the node pinned above. + - name: NVIDIA_VISIBLE_DEVICES + value: "MIG-a724b14d-5057-529c-b9ef-87ebd0d2d92d" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "compute,utility" + # TF 2.15.1 has no sm_90 (Hopper) cubins, so kernels JIT-compile from PTX on first use + # (gpu_device.cc warns "could take 30 minutes"). Persist that JIT cache on the resources + # PVC (instead of the ephemeral /root/.nv) and raise its cap to the 4 GiB max so large + # fused kernels don't evict — the compile then happens once, not on every pod restart. + - name: CUDA_CACHE_PATH + value: "/rodan/data/.cuda-cache" + - name: CUDA_CACHE_MAXSIZE + value: "4294967296" # 4 GiB (CUDA hard max); default is only ~1 GiB volumeMounts: - name: resources mountPath: /rodan/data startupProbe: exec: - command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] + command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@GPU", "-t", "30"] periodSeconds: 45 timeoutSeconds: 40 # default probe timeout is 1s; celery inspect ping needs far longer failureThreshold: 30 livenessProbe: exec: - command: ["celery", "inspect", "ping", "-A", "rodan", "--workdir", "/code/Rodan", "-d", "celery@GPU", "-t", "30"] + command: ["celery", "-A", "rodan", "--workdir", "/code/Rodan", "inspect", "ping", "-d", "celery@GPU", "-t", "30"] periodSeconds: 60 timeoutSeconds: 45 failureThreshold: 3 + # NOTE: no `nvidia.com/gpu` request here on purpose — the GPU is selected via + # NVIDIA_VISIBLE_DEVICES above. Requesting it would let the device plugin overwrite + # that env with the parent GPU UUID and re-break CUDA. The hostname pin + taint + # toleration keep this the only GPU workload on the node. resources: requests: cpu: "1" @@ -59,7 +87,6 @@ spec: limits: cpu: "2" memory: "16Gi" - nvidia.com/gpu: 1 volumes: - name: resources persistentVolumeClaim: diff --git a/k8s/40-iipsrv.yaml b/k8s/40-iipsrv.yaml index f78ecda9a..bdbad0217 100644 --- a/k8s/40-iipsrv.yaml +++ b/k8s/40-iipsrv.yaml @@ -19,7 +19,7 @@ spec: - name: ghcr-pull-secret containers: - name: iipsrv - image: ghcr.io/ddmal/iipsrv:k8s-migration + image: ghcr.io/ddmal/iipsrv:nightly ports: - containerPort: 9003 name: fcgi diff --git a/k8s/41-rodan-client.yaml b/k8s/41-rodan-client.yaml index 100df8a32..65c9f8270 100644 --- a/k8s/41-rodan-client.yaml +++ b/k8s/41-rodan-client.yaml @@ -69,7 +69,7 @@ spec: - name: ghcr-pull-secret containers: - name: rodan-client - image: ghcr.io/ddmal/rodan-client:k8s-migration + image: ghcr.io/ddmal/rodan-client:nightly ports: - containerPort: 80 name: http diff --git a/k8s/50-nginx.yaml b/k8s/50-nginx.yaml index a2f55b460..bfccc6c35 100644 --- a/k8s/50-nginx.yaml +++ b/k8s/50-nginx.yaml @@ -21,7 +21,7 @@ spec: - name: ghcr-pull-secret containers: - name: nginx - image: ghcr.io/ddmal/nginx:k8s-migration + image: ghcr.io/ddmal/nginx:nightly args: ["/run/start"] env: - name: TZ diff --git a/k8s/60-nvidia-device-plugin.yaml b/k8s/60-nvidia-device-plugin.yaml index 073a87fbb..58d9f28b4 100644 --- a/k8s/60-nvidia-device-plugin.yaml +++ b/k8s/60-nvidia-device-plugin.yaml @@ -1,7 +1,9 @@ # NVIDIA device plugin — advertises nvidia.com/gpu on the GPU node so gpu-celery can request it. # Runs only on nodes labeled gpu=true and tolerates the GPU taint. -# Prereqs on the GPU node: NVIDIA driver (>= 460.x for the CUDA 11.2 image) + nvidia-container-toolkit, -# with the nvidia container runtime configured (k3s detects it and wires up containerd automatically). +# Prereqs on the GPU node: NVIDIA driver (>= 525 for the CUDA 12.x / TF 2.15 image; nodes run +# 580.x) + nvidia-container-toolkit, with the nvidia container runtime configured (k3s detects +# it and wires up containerd automatically). The H100 vGPUs are MIG-backed, so migStrategy=single +# is required below (see the container args) for CUDA to see the MIG instance. apiVersion: apps/v1 kind: DaemonSet metadata: @@ -33,6 +35,11 @@ spec: containers: - name: nvidia-device-plugin-ctr image: nvcr.io/nvidia/k8s-device-plugin:v0.14.5 + # NOTE: migStrategy stays "none" (the default). These are MIG-backed *vGPUs*, and + # the plugin's MIG path fails in a vGPU guest ("error getting parent memory info: + # Insufficient Permissions") because the guest can't introspect the parent GPU. + # So the plugin exposes the vGPU/parent UUID; CUDA is pointed at the MIG instance + # via CUDA_VISIBLE_DEVICES in the gpu-celery pod instead (see k8s/33-gpu-celery.yaml). env: - name: FAIL_ON_INIT_ERROR value: "false" diff --git a/k8s/POSTGRES_UPGRADE.md b/k8s/POSTGRES_UPGRADE.md new file mode 100644 index 000000000..eed6bdd37 --- /dev/null +++ b/k8s/POSTGRES_UPGRADE.md @@ -0,0 +1,80 @@ +# PostgreSQL 9.6 → 16 upgrade runbook + +Django 4.2 dropped support for PostgreSQL < 12, so the modernized stack requires Postgres 16. +The image is already on 16 (`postgres/Dockerfile` → `postgres:16-bookworm` + `postgresql-plpython3-16` ++ `python3-redis`). Because 9.6 and 16 on-disk formats are incompatible, PG16 can't start on the +9.6 data files — the data must be **logically dumped from 9.6 and restored into 16**. + +This is a **one-time cutover**, run on the NFS/data server **192.168.236.124** (has Docker; the +`dd`-cloned 9.6 cluster lives there). It produces a new `pg16_data` dir alongside the 9.6 clone, +which is kept untouched as rollback. DB `rodan`, role `someadmin`, ~4.1 GB. + +> The k8s manifests already target the new datadir: `12-pv-pgdata.yaml` `nfs.path` = `/srv/rodan-data/pg16_data`. +> Do **not** `kubectl apply` the postgres PV/StatefulSet until steps 1–4 below are done. + +## 1. Quiesce (never run two servers on one datadir) +```sh +kubectl -n rodan scale statefulset/postgres --replicas=0 +kubectl -n rodan scale deploy/rodan-main deploy/celery deploy/py3-celery deploy/gpu-celery --replicas=0 +kubectl -n rodan get pods # confirm no postgres pod +``` + +## 2. Dump the 9.6 DB (from a copy, keeping the clone pristine) +```sh +cp -a /srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data /srv/rodan-data/pg96_src +docker run -d --name pg96 -v /srv/rodan-data/pg96_src:/var/lib/postgresql/data ddmal/postgres-plpython:v3.3.1 +until docker exec pg96 pg_isready -U someadmin -q; do sleep 1; done +docker exec pg96 pg_dump -U someadmin -d rodan --no-owner --no-privileges -Fc -f /tmp/rodan.dump +docker cp pg96:/tmp/rodan.dump /srv/rodan-data/rodan.dump +docker rm -f pg96 +``` + +## 3. Init PG16 on a new datadir and restore +```sh +mkdir -p /srv/rodan-data/pg16_data +docker run -d --name pg16 \ + -e POSTGRES_USER=someadmin -e POSTGRES_PASSWORD=123456Seven -e POSTGRES_DB=rodan \ + -v /srv/rodan-data/pg16_data:/var/lib/postgresql/data ddmal/postgres-plpython:py311test +until docker exec pg16 pg_isready -U someadmin -q; do sleep 1; done +docker cp /srv/rodan-data/rodan.dump pg16:/tmp/rodan.dump +docker exec pg16 pg_restore -U someadmin -d rodan --no-owner --no-privileges /tmp/rodan.dump +docker exec pg16 psql -U someadmin -d rodan -c "SELECT count(*) FROM auth_user;" # expect 129 +docker rm -f pg16 +``` +- The dump's `publish_message` (plpython3u) + `object_notify` triggers restore fine on PG16 (it has + `plpython3-16` + `python3-redis`). If any plpython object errors, ignore it — `rodan/models/__init__.py` + re-creates the function/triggers at app startup. + +## 4. Export the new datadir + recreate the k8s PV/PVC +On 192.168.236.124 `/etc/exports` (then `sudo exportfs -ra`): +``` +/srv/rodan-data/pg16_data 192.168.236.0/24(rw,sync,no_subtree_check,no_root_squash) +``` +The NFS source on a PV is immutable, so recreate (Retain policy leaves both data dirs intact): +```sh +kubectl -n rodan delete pvc rodan-pg-data ; kubectl delete pv rodan-pg-data-pv +kubectl apply -f k8s/12-pv-pgdata.yaml -f k8s/13-pvc-pgdata.yaml +``` + +## 5. Deploy PG16 image + bring the stack back +Ensure `20-postgres.yaml`'s image is the CI-published PG16 tag (`ghcr.io/ddmal/postgres-plpython:`). +```sh +kubectl apply -f k8s/20-postgres.yaml +kubectl -n rodan scale statefulset/postgres --replicas=1 +kubectl -n rodan rollout status statefulset/postgres +kubectl -n rodan scale deploy/rodan-main deploy/celery deploy/py3-celery --replicas=1 +``` +`scripts/start` runs `manage.py migrate` → applies **0006** (widens `User.first_name`; the `view_` rows +already exist so `create_permissions` is a no-op) and re-establishes the plpython trigger. + +## Verify +```sh +kubectl -n rodan exec statefulset/postgres -- psql -U someadmin -d rodan -c "SHOW server_version;" # 16.x +kubectl -n rodan exec statefulset/postgres -- psql -U someadmin -d rodan -c "SELECT count(*) FROM auth_user;" # 129 +kubectl -n rodan exec deploy/rodan-main -- curl -s -o /dev/null -w '%{http_code}\n' -H 'User-Agent: k8s' localhost:8000/api/?format=json # 200 +``` + +## Rollback +The original 9.6 datadir (`.../rodan_pg_data/_data`) is never modified (dump ran from `pg96_src`; the PV +is `Retain`). Reverting means re-pointing the PV to it **and** reverting the `modernize-backend-py311` +branch (the Django-4.2 app cannot run on 9.6). diff --git a/k8s/README.md b/k8s/README.md index 1e06f050c..fc320db04 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -125,9 +125,9 @@ self-healing via the `wait-for-app` logic baked into the images, but applying ba kubectl -n rodan get pods,svc,pvc,ingress kubectl -n rodan exec deploy/rodan-main -- curl -s -o /dev/null -w '%{http_code}\n' \ -H 'User-Agent: k8s' localhost:8000/api/?format=json # 200 -kubectl -n rodan exec deploy/celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@celery -kubectl -n rodan exec deploy/py3-celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@Python3 -kubectl -n rodan exec deploy/gpu-celery -- celery inspect ping -A rodan --workdir /code/Rodan -d celery@GPU +kubectl -n rodan exec deploy/celery -- celery -A rodan --workdir /code/Rodan inspect ping -d celery@celery +kubectl -n rodan exec deploy/py3-celery -- celery -A rodan --workdir /code/Rodan inspect ping -d celery@Python3 +kubectl -n rodan exec deploy/gpu-celery -- celery -A rodan --workdir /code/Rodan inspect ping -d celery@GPU kubectl -n rodan exec deploy/gpu-celery -- nvidia-smi kubectl describe node | grep nvidia.com/gpu # allocatable: 1 # external (through the edge): https://rodan2.simssa.ca — log in, open a project/image, run a workflow diff --git a/nginx/config/rodan.conf b/nginx/config/rodan.conf index 5e48eef6f..4b8096d15 100644 --- a/nginx/config/rodan.conf +++ b/nginx/config/rodan.conf @@ -34,13 +34,18 @@ server { # Rodan API alias /rodan/data; } - location /ws/ { # Redis Reload Caching - proxy_http_version 1.1; + location /ws/ { # WebSocket broadcast (Django Channels ASGI on rodan-main; replaces ws4redis) + proxy_pass http://rodan-main:8000; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; proxy_connect_timeout 10; - proxy_read_timeout 60; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - return 101 http://redis:6379; + proxy_read_timeout 3600; # long-lived broadcast socket + proxy_send_timeout 3600; } location /ht { # Django Health Check diff --git a/postgres/Dockerfile b/postgres/Dockerfile index 5a1040261..9457ffe62 100644 --- a/postgres/Dockerfile +++ b/postgres/Dockerfile @@ -1,10 +1,14 @@ -# FROM postgres:latest -FROM postgres:9.6-bullseye +# PostgreSQL 16 — Django 4.2 requires PostgreSQL >= 12 (9.6 is no longer supported). +# The 9.6 -> 16 upgrade means the existing PGDATA must be dump/restored (pg_dump | pg_restore), +# not started in place. +FROM postgres:16-bookworm EXPOSE 5432 +# postgresql-plpython3-16 provides the plpython3u language used by the publish_message trigger; +# python3-redis gives that trigger's `import redis` (bookworm is PEP-668 externally-managed, so +# use the Debian package rather than pip). RUN apt-get update \ - # && apt-get install -y postgresql-plpython-${PG_MAJOR} python-pip \ - && apt-get install -y postgresql-plpython3-9.6 python3-pip \ + && apt-get install -y postgresql-plpython3-16 python3-redis \ && rm -rf /var/lib/apt/lists/* # Installing the postgresql-plpython overrides the postgresql.conf.sample file generated by the postgres Dockerfile, @@ -14,8 +18,6 @@ RUN apt-get update \ RUN rm /usr/share/postgresql/${PG_MAJOR}/postgresql.conf.sample \ && ln -sv /usr/share/postgresql/postgresql.conf.sample /usr/share/postgresql/$PG_MAJOR/ -RUN pip3 install redis - COPY ./postgres/maintenance /usr/local/bin/maintenance RUN chmod +x /usr/local/bin/maintenance/* RUN mv /usr/local/bin/maintenance/* /usr/local/bin \ diff --git a/python3-celery/Dockerfile b/python3-celery/Dockerfile index 083ef3f69..44d8828fd 100644 --- a/python3-celery/Dockerfile +++ b/python3-celery/Dockerfile @@ -26,10 +26,10 @@ RUN yarn install RUN yarn build # Make Gamera files accessible to the main container. -FROM ddmal/gamera4:2.0.0 AS gamera +FROM ddmal/gamera4:py311 AS gamera -# This release is based on Debian 11 "Bullseye" -FROM python:3.7-slim +# Debian 12 "Bookworm" with Python 3.11. +FROM python:3.11-slim # Copy various scripts and make them executable. COPY ./scripts/start-celery /run/ @@ -62,8 +62,9 @@ RUN apt-get install -y \ wget \ vim -# https://stackoverflow.com/questions/69100275/error-while-downloading-the-requirements-using-pip-install-setup-command-use-2 -RUN pip3 install setuptools==58 +# setuptools <80 still supports the legacy `setup.py install` that Gamera/Musicstaves use; +# py3.11 keeps stdlib distutils, which Gamera's setup.py imports directly. +RUN pip3 install "setuptools<80" wheel RUN pip3 install -r /code/Rodan/requirements.txt # Install Gamera and Musicstaves. @@ -71,9 +72,12 @@ WORKDIR /gamera4-rodan/gamera-4 RUN python3 setup.py --nowx install WORKDIR /gamera4-rodan/musicstaves RUN python3 setup.py install - -# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved. -RUN pip install opencv-python==4.6.0.66 +# Musicstaves installs as an isolated egg whose gamera/toolkits/ does not merge with +# gamera's (gamera's toolkits/__init__.py is empty, no extend_path). Move its package +# into the gamera egg so `import gamera.toolkits.musicstaves` resolves. Self-checks. +RUN cp -r /usr/local/lib/python3.11/site-packages/musicstaves-*.egg/gamera/toolkits/musicstaves \ + /usr/local/lib/python3.11/site-packages/gamera-*.egg/gamera/toolkits/ \ + && python3 -W ignore -c "import gamera.toolkits.musicstaves.stafffinder_miyao; from gamera.toolkits import musicstaves" # Change the concurency for python3 jobs RUN sed -i "s/concurrency=10/concurrency=3/g" /run/start-celery diff --git a/rodan-main/Dockerfile b/rodan-main/Dockerfile index 8201bdd75..57d830124 100644 --- a/rodan-main/Dockerfile +++ b/rodan-main/Dockerfile @@ -29,7 +29,10 @@ RUN \ && git checkout 4932d86b19d9008a28ae898035cfa43340e636e6 \ && mkdir build \ && cd build \ - && /vendor/cmake-3.17.0-Linux-x86_64/bin/cmake -DBUILD_THIRDPARTY=ON .. \ + # GCC 12+ (Debian bookworm, via the py3.11 base) no longer transitively includes + # , so Grok's pinned source fails on undeclared uint8_t. Force-include it + # for every C++ TU instead of patching the third-party headers. + && /vendor/cmake-3.17.0-Linux-x86_64/bin/cmake -DBUILD_THIRDPARTY=ON -DCMAKE_CXX_FLAGS="-include cstdint" .. \ && make -j3 # This script gets the gpu jobs into celery diff --git a/rodan-main/code/requirements.txt b/rodan-main/code/requirements.txt index baeac571c..0e154a63c 100755 --- a/rodan-main/code/requirements.txt +++ b/rodan-main/code/requirements.txt @@ -1,48 +1,67 @@ -backports.ssl-match-hostname==3.4.0.2 -celery==4.4.7 -certifi==14.05.14 -django-cors-headers==3.4.0 #up to django 3.0 -django-decorator-include==3.0 #only supports up to 3.0 django (bottleneck?) -django-environ==0.5.0 #added support for django 2.1 and 2.2 -django-extensions==1.9.9 #updated -django-filter==2.1.0 #supports up to django 2.2 -django-guardian==1.5.1 #updated -django-health-check==2.2.0 #supports up to django 2.2 -django-sortedm2m==2.0 #django 1.11 to 2.2 -django-websocket-redis==0.5.2 -Django==2.0.13 -djoser==2.0.5 -djangorestframework==3.11 -flower==0.9.3 -gevent==1.3.6 -gunicorn==19.10.0 -jsonfield==2.1.0 -jsonschema==2.5.1 -lxml==4.6.3 -Markdown==3.0.1 -mimeparse==0.1.3 -numpy==1.17.3 -model-mommy==1.6.0 -pathlib2==2.3.6 -pika==1.1.0 -pillow==8.2.0 -protobuf==3.17.3 -psycopg2==2.8.4 --e git+https://github.com/deepio/pybagit.git@4eea813148bc590896000f57cde35fb86443d2f4#egg=pybagit ; python_version >= '3.0' -opencv-python==4.6.0.66 -pyparsing==2.0.3 -pypng==0.0.18 -python-dateutil==2.5 -python-magic==0.4.22 -PyYAML==5.4 -redis==2.10.3 -uWSGI==2.0.18 -Werkzeug==1.0.1 -importlib-metadata==4.12.0 - -# text_alignment -#numpy==1.17.3; python_version > "3.4" -#tensorflow==2.3.2; python_version > "3.4" -#scikit-image==0.15.0; python_version > "3.4" -#calamari-ocr==1.0.5; python_version > "3.4" -#Unidecode==1.0.22; python_version > "3.4" +# Modernized for Python 3.11 / Django 4.2 LTS / Celery 5.3 / TF-2.15-compatible libs. +# TARGET pins — must be confirmed with an iterative build (`pip install` + `manage.py check` +# + ci-jobs suite). Lines marked "VERIFY" are the aging Django plugins most likely to need +# a newer release, a fork, or replacement on Django 4.2. + +# --- Django + REST --- +Django==4.2.16 +djangorestframework==3.15.2 +djoser==2.2.3 +django-cors-headers==4.4.0 +django-filter==24.3 +django-guardian==2.4.0 # VERIFY on Django 4.2 (last release is 2.4.0) +django-health-check==3.18.3 +django-extensions==3.2.3 +django-environ==0.11.2 +django-sortedm2m==3.1.1 +django-decorator-include==3.0 # VERIFY on Django 4.2 (no newer release) +jsonfield==3.1.0 + +# --- Celery / broker --- +celery==5.3.6 +flower==2.0.1 +pika==1.3.2 + +# --- WebSockets (replaces dead django-websocket-redis; wired up in Phase 1b) --- +channels==4.1.0 +channels-redis==4.2.0 +daphne==4.1.2 + +# --- Servers --- +gunicorn==22.0.0 +uvicorn[standard]==0.30.5 +gevent==24.2.1 + +# --- Core libraries --- +numpy==1.26.4 # TF 2.15 compatible; cp311 wheels +protobuf==4.25.4 # TF 2.15 needs >=3.20.3,<5 +Pillow==10.4.0 +psycopg2==2.9.9 +redis==5.0.8 +lxml==5.2.2 +PyYAML==6.0.2 +python-dateutil==2.9.0.post0 +python-magic==0.4.27 +opencv-python==4.10.0.84 +Markdown==3.6 +jsonschema==4.23.0 +pyparsing==3.1.2 +pypng==0.20220715.0 +python-mimeparse==1.6.0 +Werkzeug==3.0.3 +certifi==2024.7.4 +model-bakery==1.17.0 # was model-mommy; test imports change model_mommy -> model_bakery + +# pybagit — small helper; VERIFY it imports on py3.11 (may have py2-isms) +-e git+https://github.com/deepio/pybagit.git@4eea813148bc590896000f57cde35fb86443d2f4#egg=pybagit + +# Dropped (py2 backports / superseded): backports.ssl-match-hostname, pathlib2, uWSGI, +# importlib-metadata (stdlib on 3.11), model-mommy. + +# scikit-image is used directly by the staff_distance GPU job (skimage.segmentation / +# filters / morphology / transform); 0.22 supports py3.11 + numpy 1.26. +scikit-image==0.22.0 + +# text_alignment's OCR engine (calamari-ocr==2.3.1, py3.11 + TF2.15 via ocrd-fork-tfaip +# 1.2.7) is installed on the GPU worker by scripts/install_gpu_rodan_jobs, not from here — +# the job registers on non-GPU images via its guarded import. diff --git a/rodan-main/code/rodan/asgi.py b/rodan-main/code/rodan/asgi.py new file mode 100644 index 000000000..b09426483 --- /dev/null +++ b/rodan-main/code/rodan/asgi.py @@ -0,0 +1,41 @@ +""" +ASGI config for Rodan. + +Serves HTTP (Django) and WebSocket (Channels) from one application. The websocket +side replaces the retired django-websocket-redis (ws4redis): the BroadcastConsumer +relays the Redis pub/sub channel that the Postgres plpython3 trigger publishes to. +""" +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rodan.settings") + +from django.core.asgi import get_asgi_application # noqa: E402 + +# Initialise Django (populates apps) before importing anything that touches models/settings. +django_asgi_app = get_asgi_application() + +# Force the root URLconf to import now, at startup, in this synchronous context. Django +# otherwise resolves the URLconf lazily on the first request; under ASGI that runs in the +# event loop, and rodan/urls.py imports rodan.jobs.load, which executes a synchronous ORM +# query (ResourceType.objects.all()) at import time -> SynchronousOnlyOperation. Doing it +# here runs that one-time job/resource-type registration once per worker at boot instead. +from django.urls import get_resolver # noqa: E402 + +get_resolver().url_patterns + +from channels.routing import ProtocolTypeRouter, URLRouter # noqa: E402 +from django.urls import re_path # noqa: E402 + +from rodan.consumers import BroadcastConsumer # noqa: E402 + +application = ProtocolTypeRouter( + { + "http": django_asgi_app, + # Matches the client's connection to /ws/rodan?subscribe-broadcast&... (SocketUpdater.js). + "websocket": URLRouter( + [ + re_path(r"^ws/", BroadcastConsumer.as_asgi()), + ] + ), + } +) diff --git a/rodan-main/code/rodan/celery.py b/rodan-main/code/rodan/celery.py index 60bde4eb4..05afff1fe 100644 --- a/rodan-main/code/rodan/celery.py +++ b/rodan-main/code/rodan/celery.py @@ -9,7 +9,7 @@ django.setup() app = Celery("rodan") -app.config_from_object("django.conf:settings") +app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) @@ -25,14 +25,15 @@ ) from rodan.jobs.master_task import master_task # noqa -# Core Rodan Tasks -app.tasks.register(create_resource()) -app.tasks.register(create_workflowrun()) +# Core Rodan Tasks. +# create_resource / create_workflowrun are Task *subclasses* (not auto-registered), so +# register an instance of each explicitly. +app.register_task(create_resource()) +app.register_task(create_workflowrun()) -app.tasks.register(cancel_workflowrun) -app.tasks.register(create_diva) -app.tasks.register(redo_runjob_tree) -app.tasks.register(retry_workflowrun) -app.tasks.register(send_email) -app.tasks.register(send_templated_email) -app.tasks.register(master_task) +# cancel_workflowrun, create_diva, redo_runjob_tree, retry_workflowrun, send_email, +# send_templated_email and master_task are @shared_task functions. Importing them (above) +# already auto-registers them with this app. They must NOT be manually registered: a +# @shared_task is a promise proxy, and registering the proxy stores it under its own name +# in the registry, so a later lookup resolves the proxy to itself — an infinite loop +# (RecursionError in celery.local). Celery 5 binds shared tasks to the app on finalize. diff --git a/rodan-main/code/rodan/consumers.py b/rodan-main/code/rodan/consumers.py new file mode 100644 index 000000000..195ca395d --- /dev/null +++ b/rodan-main/code/rodan/consumers.py @@ -0,0 +1,64 @@ +""" +WebSocket consumer that replaces django-websocket-redis (ws4redis). + +Rodan's live status updates are produced by a Postgres ``plpython3u`` trigger +(see ``rodan/models/__init__.py``) that ``redis.publish``es row-change events to the +``WEBSOCKET_BROADCAST_CHANNEL`` Redis channel. ws4redis used to relay that channel to +browser websockets; this consumer does the same with Django Channels: on connect it +subscribes to the Redis channel and forwards every published message to the client. + +Server -> client only (status broadcasts); inbound frames are ignored, which preserves +the behaviour the Backbone client relies on (it only subscribes to broadcasts). +""" +import asyncio + +import redis.asyncio as aioredis +from channels.generic.websocket import AsyncWebsocketConsumer +from django.conf import settings + + +class BroadcastConsumer(AsyncWebsocketConsumer): + async def connect(self): + await self.accept() + conn = settings.WS4REDIS_CONNECTION + self._redis = aioredis.Redis( + host=conn["host"], + port=int(conn["port"]), + db=int(conn["db"]), + ) + self._pubsub = self._redis.pubsub() + channel = getattr(settings, "WEBSOCKET_BROADCAST_CHANNEL", "rodan:broadcast:rodan") + await self._pubsub.subscribe(channel) + self._reader_task = asyncio.create_task(self._relay()) + + async def _relay(self): + try: + async for message in self._pubsub.listen(): + if message and message.get("type") == "message": + data = message["data"] + if isinstance(data, (bytes, bytearray)): + data = data.decode("utf-8", "replace") + await self.send(text_data=data) + except asyncio.CancelledError: + pass + + async def receive(self, text_data=None, bytes_data=None): + # Status stream is server -> client only; ignore anything the client sends. + return + + async def disconnect(self, code): + task = getattr(self, "_reader_task", None) + if task is not None: + task.cancel() + pubsub = getattr(self, "_pubsub", None) + if pubsub is not None: + try: + await pubsub.aclose() + except Exception: + pass + client = getattr(self, "_redis", None) + if client is not None: + try: + await client.aclose() + except Exception: + pass diff --git a/rodan-main/code/rodan/jobs/MEI_encoding/MEI_encoding.py b/rodan-main/code/rodan/jobs/MEI_encoding/MEI_encoding.py index 340cff171..f8538d01a 100644 --- a/rodan-main/code/rodan/jobs/MEI_encoding/MEI_encoding.py +++ b/rodan-main/code/rodan/jobs/MEI_encoding/MEI_encoding.py @@ -106,6 +106,7 @@ def run_my_task(self, inputs, settings, outputs): def test_my_task(self, testcase): import re + import xml.etree.ElementTree as ET input_jsomr = "/code/Rodan/rodan/test/files/mei-encoding-test-hpf.json" input_text = "/code/Rodan/rodan/test/files/mei-encoding-test-ta.json" input_col_data = "/code/Rodan/rodan/test/files/mei-encoding-test-csd.json" @@ -129,21 +130,18 @@ def test_my_task(self, testcase): # Read the gt and predicted result with open(output_path, "r") as fp: - predicted = [l.strip() for l in fp.readlines()] + predicted = fp.read() with open(gt_output_path, "r") as fp: - gt = [l.strip() for l in fp.readlines()] + gt = fp.read() - # The number lines should be identical - testcase.assertEqual(len(gt), len(predicted)) - - # also each line should be identical to its counterpart - # Since mei encoding creates unique ids, we use regex to replace each id with an underscore + # MEI encoding creates unique ids, so replace each id with an underscore. pattern = re.compile(r"m-\w{8}-\w{4}-\w{4}-\w{4}-\w{12}") - for i, (gt_line, pred_line) in enumerate(zip(gt, predicted)): - # Replace ids - gt_line = pattern.sub("_", gt_line) - pred_line = pattern.sub("_", pred_line) - # and compare if two meis are identical to each other - testcase.assertEqual(gt_line, pred_line, "Line {}".format(i)) + predicted = pattern.sub("_", predicted) + gt = pattern.sub("_", gt) + + # Compare canonicalized XML so attribute ordering does not matter. Python < 3.8 + # serialized attributes alphabetically while >= 3.8 preserves insertion order; the + # stored fixture predates 3.8. C14N normalizes ordering on both sides. + testcase.assertEqual(ET.canonicalize(gt), ET.canonicalize(predicted)) del predicted, gt \ No newline at end of file diff --git a/rodan-main/code/rodan/jobs/SAE_binarization/SAE_binarization.py b/rodan-main/code/rodan/jobs/SAE_binarization/SAE_binarization.py index 9c6212652..a6f48dcb6 100644 --- a/rodan-main/code/rodan/jobs/SAE_binarization/SAE_binarization.py +++ b/rodan-main/code/rodan/jobs/SAE_binarization/SAE_binarization.py @@ -39,5 +39,24 @@ def run_my_task(self, inputs, settings, outputs): os.rename(save_image_path,outputs['RGB PNG image'][0]['resource_path']) return True + def test_my_task(self, testcase): + # The SAE engine (TensorFlow + model weights) is only installed in the GPU + # worker image; skip gracefully elsewhere (e.g. the CPU-only CI image) so + # test_all_jobs stays green. Real coverage runs on the GPU worker. + try: + from SAE_binarization.binarize.binarize import run_binarize # noqa: F401 + except ImportError: + return + import cv2 + input_image = "/code/Rodan/rodan/test/files/lenna_convert2Rgb-png_greyscale-png_output.png" + output_path = testcase.new_available_path() + inputs = {'Image': [{'resource_type': 'image/rgb+png', 'resource_path': input_image}]} + outputs = {'RGB PNG image': [{'resource_type': 'image/rgba+png', 'resource_path': output_path}]} + self.run_my_task(inputs, {}, outputs) + testcase.assertTrue(os.path.isfile(output_path)) + result = cv2.imread(output_path, cv2.IMREAD_UNCHANGED) + testcase.assertIsNotNone(result) + testcase.assertGreater(result.size, 0) + def my_error_information(self, exc, traceback): return diff --git a/rodan-main/code/rodan/jobs/base.py b/rodan-main/code/rodan/jobs/base.py index 6ec4e1030..4da84da5e 100644 --- a/rodan-main/code/rodan/jobs/base.py +++ b/rodan-main/code/rodan/jobs/base.py @@ -13,7 +13,7 @@ import time import uuid -from celery import Task, registry +from celery import Task, current_app as registry from celery.app.task import TaskType from django.conf import settings as rodan_settings from django.core.files import File @@ -86,16 +86,16 @@ def __init__(cls, clsname, bases, attrs): # check the number of arguments of implemented function if "run_my_task" in attrs: - argspec = inspect.getargspec(attrs["run_my_task"]) + argspec = inspect.getfullargspec(attrs["run_my_task"]) assert len(argspec.args) == 4, "run_my_task" if "get_my_interface" in attrs: - argspec = inspect.getargspec(attrs["get_my_interface"]) + argspec = inspect.getfullargspec(attrs["get_my_interface"]) assert len(argspec.args) == 3, "get_my_interface" if "validate_my_user_input" in attrs: - argspec = inspect.getargspec(attrs["validate_my_user_input"]) + argspec = inspect.getfullargspec(attrs["validate_my_user_input"]) assert len(argspec.args) == 4, "validate_my_user_input" if "test_my_task" in attrs: - argspec = inspect.getargspec(attrs["test_my_task"]) + argspec = inspect.getfullargspec(attrs["test_my_task"]) assert len(argspec.args) == 2, "test_my_task" # not the abstract class @@ -520,11 +520,22 @@ def check_port_types(which): check_port_types("in") check_port_types("out") - # Process done - from rodan.jobs.load import job_list - - if attrs["name"] in job_list: - job_list.remove(attrs["name"]) + # Process done. + # + # `job_list` is load.py's running tally of catalogue jobs still awaiting a code + # module; each job crosses itself off here so load.py can flag the leftovers as + # orphaned. Only do this when load.py is the active orchestrator (already in + # sys.modules). Importing it from a standalone job-module import (e.g. a bare + # `import rodan.jobs..`) would run load.py's module-level registration + # while THIS class is still mid-definition; that registration then getattrs the + # not-yet-bound class off its half-imported module and raises a spurious + # AttributeError (swallowed by register_all_jobs.py). Skipping the tally when + # load.py isn't orchestrating is harmless — there is no orphan sweep to feed. + if "rodan.jobs.load" in sys.modules: + from rodan.jobs.load import job_list + + if attrs["name"] in job_list: + job_list.remove(attrs["name"]) @staticmethod def _resolve_resource_types(value): @@ -1117,7 +1128,12 @@ def get_interface(self, runjob_id): else: with open(template_file, "r") as f: t = Template(f.read()) - _django_template_cache = t + # NB: index into the cache dict — assigning `_django_template_cache = t` + # would replace the dict with a Template, so the NEXT interactive + # `get_interface` in the same worker process hits `template_file in