Develop

.github/workflows/build-ci-jobs.yml

@@ -0,0 +1,44 @@
+name: Build & push ci-jobs
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker Hub tag to push (ddmal/ci-jobs:<tag>)"
+        required: true
+        default: "django-v4.2.16"
+      branch:
+        description: "BRANCH build-arg — ref used to clone neon_wrapper/pixel_wrapper"
+        required: true
+        default: "develop"
+
+jobs:
+  build-ci-jobs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Free disk space
+        # ci-jobs is a heavy image (Grok compile + gamera/musicstaves + postgres + deps).
+        uses: jlumbroso/free-disk-space@v1.3.1
+        with:
+          tool-cache: true
+          large-packages: false
+          docker-images: false
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          registry: docker.io
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build ci-jobs
+        run: |
+          docker build \
+            --build-arg BRANCH="${{ github.event.inputs.branch }}" \
+            -t "ddmal/ci-jobs:${{ github.event.inputs.tag }}" \
+            -f ci-jobs/Dockerfile \
+            .
+
+      - name: Push ci-jobs
+        run: docker push "ddmal/ci-jobs:${{ github.event.inputs.tag }}"

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
   rodan-CI-test:
     runs-on: ubuntu-22.04
     container:
-        image: ddmal/ci-jobs:django-v2.0.13
+        image: ddmal/ci-jobs:django-v4.2.16
         env: 
           CELERY_JOB_QUEUE: None
           ADMIN_USER: rodan

ci-jobs/Dockerfile

@@ -29,10 +29,11 @@ RUN yarn install
 RUN yarn build
 
 # Make Gamera files accessible to the main container.
-FROM ddmal/gamera4:fix_IC AS gamera
+FROM ddmal/gamera4:py311 AS gamera
 
-# This release is based on Debian 11 "Bullseye"
-FROM python:3.7-slim
+# Debian 12 "Bookworm" with Python 3.11 (matches python3-celery; Django 4.2 needs py>=3.8
+# and the py3.11-ported Gamera source uses the 3.9+ C-API).
+FROM python:3.11-slim
 
 # Copy various scripts and make them executable.
 # COPY ./scripts/start-celery /run/
@@ -65,18 +66,22 @@ RUN apt-get install -y \
   wget \
   vim
 
-# https://stackoverflow.com/questions/69100275/error-while-downloading-the-requirements-using-pip-install-setup-command-use-2
-RUN pip3 install setuptools==58
+# setuptools <80 still supports the legacy `setup.py install` that Gamera/Musicstaves use;
+# py3.11 keeps stdlib distutils, which Gamera's setup.py imports directly.
+RUN pip3 install "setuptools<80" wheel
 RUN pip3 install -r /code/Rodan/requirements.txt
 
 # Install Gamera and Musicstaves.
 WORKDIR /gamera4-rodan/gamera-4
 RUN python3 setup.py --nowx install
 WORKDIR /gamera4-rodan/musicstaves
 RUN python3 setup.py install
-
-# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved.
-RUN pip install opencv-python==4.6.0.66
+# Musicstaves installs as an isolated egg whose gamera/toolkits/ does not merge with
+# gamera's (gamera's toolkits/__init__.py is empty, no extend_path). Move its package
+# into the gamera egg so `import gamera.toolkits.musicstaves` resolves. Self-checks.
+RUN cp -r /usr/local/lib/python3.11/site-packages/musicstaves-*.egg/gamera/toolkits/musicstaves \
+          /usr/local/lib/python3.11/site-packages/gamera-*.egg/gamera/toolkits/ \
+ && python3 -W ignore -c "import gamera.toolkits.musicstaves.stafffinder_miyao; from gamera.toolkits import musicstaves"
 
 # ============================
 # from ./rodan-main/Dockerfile
@@ -108,13 +113,9 @@ RUN apt-get install -y \
   wget \
   vim
 
-# https://stackoverflow.com/questions/69100275/error-while-downloading-the-requirements-using-pip-install-setup-command-use-2
-RUN pip3 install setuptools==58
+RUN pip3 install "setuptools<80" wheel
 RUN pip3 install -r /code/Rodan/requirements.txt
 
-# Temporary fix for OpenCV until https://github.com/DDMAL/Rodan/issues/639 is resolved.
-RUN pip install opencv-python==4.6.0.66
-
 # From rodan-main
 RUN set -x \
     && mkdir /vendor || echo "Already exists" \
@@ -137,7 +138,10 @@ RUN cd /vendor \
   && git checkout 4932d86b19d9008a28ae898035cfa43340e636e6 \
   && mkdir build \
   && cd build \
-  && /vendor/cmake-3.17.0-Linux-x86_64/bin/cmake -DBUILD_THIRDPARTY=ON ..  \
+  # GCC 12+ (Debian bookworm, via the py3.11 base) no longer transitively includes
+  # <cstdint>, so Grok's pinned source fails on undeclared uint8_t. Force-include it
+  # for every C++ TU instead of patching the third-party headers.
+  && /vendor/cmake-3.17.0-Linux-x86_64/bin/cmake -DBUILD_THIRDPARTY=ON -DCMAKE_CXX_FLAGS="-include cstdint" ..  \
   && make -j3
 
 # ============================

gpu-celery/Dockerfile

@@ -1,189 +1,74 @@
-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# GPU celery worker — TensorFlow 2.15 on CUDA 12 for Hopper (sm_90).
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# The CUDA 12 runtime libraries are pulled as pip wheels via `tensorflow[and-cuda]`, so the
+# image bakes in NO CUDA toolkit and only needs the host NVIDIA driver (supplied by the k8s
+# `nvidia` RuntimeClass — see k8s/33-gpu-celery.yaml). This replaces the old hand-assembled
+# nvidia/cuda:11.2 + TF 2.5.1 + py3.7 base, which cannot run on Hopper.
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-#
-# THIS IS A GENERATED DOCKERFILE.
-#
-# This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the TensorFlow dockerfiles documentation
-# for more information.
-
-ARG UBUNTU_VERSION=18.04
-ARG CUDA=11.2
-FROM nvidia/cuda${ARCH:+-$ARCH}:${CUDA}.2-base-ubuntu${UBUNTU_VERSION} as base
-# ARCH and CUDA are specified again because the FROM directive resets ARGs
-# (but their default value is retained if set previously)
-ARG ARCH
-ARG CUDA
-ARG CUDNN=8.1.0.77-1
-ARG CUDNN_MAJOR_VERSION=8
-ARG LIB_DIR_PREFIX=x86_64
-ARG LIBNVINFER=7.2.2-1
-ARG LIBNVINFER_MAJOR_VERSION=7
+# This worker serves only the BASE + GPU job queues (CELERY_JOB_QUEUE=GPU), so it needs no
+# Gamera. GPU jobs: Paco_classifier, background_removal, SAE_binarization (TF 2.15),
+# staff_distance (skimage/cv2), and text_alignment (Calamari-OCR / ocrd-fork-tfaip 1.2.7).
+FROM python:3.11-slim
 
-# The following two arguments are rodan-specific
 ARG BRANCHES
 ARG VERSION
 
-# Needed for string substitution
-SHELL ["/bin/bash", "-c"]
-# Pick up some TF dependencies
-#RUN apt-get update 
-
-#RUN rm  -rf /etc/apt/sources.list.d/cuda.list
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        build-essential \
-        cuda-command-line-tools-${CUDA/./-} \
-        libcublas-${CUDA/./-} \
-        cuda-nvrtc-${CUDA/./-} \
-        libcufft-${CUDA/./-} \
-        libcurand-${CUDA/./-} \
-        libcusolver-${CUDA/./-} \
-        libcusparse-${CUDA/./-} \
-        curl \
-        libcudnn8=${CUDNN}+cuda${CUDA} \
-        libfreetype6-dev \
-        libhdf5-serial-dev \
-        libzmq3-dev \
-        pkg-config \
-        software-properties-common \
-        unzip \
-		wget
-# added wget		
-
-# Install TensorRT if not building for PowerPC
-# NOTE: libnvinfer uses cuda11.1 versions
-# RUN [[ "${ARCH}" = "ppc64le" ]] || { apt-get update && \
-#         apt-get install -y --no-install-recommends libnvinfer${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \
-#         libnvinfer-plugin${LIBNVINFER_MAJOR_VERSION}=${LIBNVINFER}+cuda11.1 \
-#         && apt-get clean \
-#         && rm -rf /var/lib/apt/lists/*; }
-
-# For CUDA profiling, TensorFlow requires CUPTI.
-ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/cuda-11.1/lib64:$LD_LIBRARY_PATH
-
-# Link the libcuda stub to the location where tensorflow is searching for it and reconfigure
-# dynamic linker run-time bindings
-RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 \
-    && echo "/usr/local/cuda/lib64/stubs" > /etc/ld.so.conf.d/z-cuda-stubs.conf \
-    && ldconfig
-
-# See http://bugs.python.org/issue19846
-ENV LANG C.UTF-8
-
-# This section differs from the default tensorflow2.5.1 Dockerfile, because we specifically add python 3.7;
-ARG PYTHON=python3.7
-ARG TF_PACKAGE=tensorflow
-ARG TF_PACKAGE_VERSION=2.5.1
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    python3.7 \
-    python3-pip \
-    && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* \
-  && wget https://bootstrap.pypa.io/pip/3.7/get-pip.py \
-  && ${PYTHON} get-pip.py \
-  && ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 \
-  && ln -sf /usr/local/bin/pip /usr/local/bin/pip3 \
-  && pip3 --no-cache-dir install --upgrade pip setuptools==57.0.0 \
-  # Some TF tools expect a "python" binary
-  && ln -s $(which python3) /usr/local/bin/python \
-  && python3 -m pip install --no-cache-dir ${TF_PACKAGE}${TF_PACKAGE_VERSION:+==${TF_PACKAGE_VERSION}}
-
-# RUN ln -s $(which python3) /usr/local/bin/python
-
-# Options:
-#   tensorflow
-#   tensorflow-gpu
-#   tf-nightly
-#   tf-nightly-gpu
-# Set --build-arg TF_PACKAGE_VERSION=1.11.0rc0 to install a specific version.
-# Installs the latest version by default.
-
-# COPY bashrc /etc/bash.bashrc
-# RUN chmod a+rwx /etc/bash.bashrc
-
-# This ends the material obtained from TensorFlow's dockerfile. the remainder is rodan-docker-specific setup.
-
-# FROM base
-RUN set -e \
-  && apt-get update \
-  && DEBIAN_FRONTEND="noninteractive" apt-get install -yqq \
-    git \
-    # Python lxml dependencies
-    python3.7-dev \
-    python3-opencv \
-    libxml2-dev \
-    libxslt1-dev \
-    zlib1g-dev \
-    lib32ncurses5-dev \
-    # Psycopg2 dependencies
-    libpq-dev \
-    # For resource identification
-    libmagic-dev \
-    unzip \
-    # Remove when done
-    vim
-
-RUN rm -rf /var/lib/apt/lists/*
-
-# Bandaid fix for the cannot import name '_registerMatType' from 'cv2.cv2' issue
-#RUN pip uninstall opencv-python-headless && pip install opencv-python-headless==4.1.2.30
-
-# Install GPU Rodan Jobs
+# Scripts (entrypoint waits for postgres/redis then execs the arg = /run/start-celery).
 COPY ./scripts/install_gpu_rodan_jobs /opt/
-
-# Install Rodan
-# Runs on both Rodan service, and Rodan-Celery
 COPY ./scripts/entrypoint /opt/
 COPY ./scripts/start-celery /run/
 COPY ./scripts/wait-for-app /run/
 
-# Copying rodan core from build context into container
-# Rodan folder MUST be uppercase, otherwise many unittests fail.
+# Rodan core. The folder MUST be uppercase or many unittests fail.
 COPY ./rodan-main/code /code/Rodan
 
-# necessary for scikit-image > 0.17, or else it will try to make a cache directory
-# in a place where the www-data user does not have permissions to do so
-ENV SKIMAGE_DATADIR "/tmp/.skimage_cache"
+# scikit-image otherwise caches under $HOME, which www-data cannot create.
+ENV SKIMAGE_DATADIR="/tmp/.skimage_cache"
+
+# The old nvidia/cuda base set this; python:3.11-slim does not. The NVIDIA container
+# runtime reads it to decide which driver features to inject — "compute" is required for
+# CUDA (without it you get nvidia-smi but cuInit -> CUDA_ERROR_NO_DEVICE). NVIDIA_VISIBLE_DEVICES
+# is intentionally left to the k8s device plugin. (Note: the GPU also needs a valid vGPU
+# license on the host, or CUDA is blocked regardless.)
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+# System deps: git (clones the GPU job repos), build-essential (native pip builds),
+# libmagic1 (python-magic), libpq-dev (psycopg2), and the runtime libs opencv-python needs.
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    bash \
+    build-essential \
+    curl \
+    git \
+    libmagic1 \
+    libpq-dev \
+    libgl1 \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender1 \
+    wget \
+  && rm -rf /var/lib/apt/lists/*
+
+# setuptools<80 keeps the legacy `setup.py install` used by the GPU job packages working.
+RUN pip3 install --no-cache-dir "setuptools<80" wheel
+
+# TensorFlow 2.15 with the bundled CUDA 12 wheels (Hopper sm_90 support).
+RUN pip3 install --no-cache-dir "tensorflow[and-cuda]==2.15.1"
 
+# Rodan Python deps.
+RUN pip3 install --no-cache-dir -r /code/Rodan/requirements.txt
 
-RUN set -x \
-  # Create Folders
+# GPU job packages (py311 branches + text_alignment's calamari). Their `tensorflow==2.15.1`
+# pin is already satisfied by tensorflow[and-cuda] above, so pip does not reinstall TF.
+RUN set -e \
   && mkdir -p /code/jobs \
-  # Install GPU Jobs
   && chmod +x /opt/install_gpu_rodan_jobs \
-  && /opt/install_gpu_rodan_jobs \
-  # Install Rodan
-  && sed -i "s/lxml/#lxml/g" /code/Rodan/requirements.txt \
-  # && sed -i "s/pybagit==1.5.0/-e git:\/\/github.com\/deepio\/pybagit.git@a27c9e0fc3bdf99dab8bd327f3ce9ea884abd6b4#egg=pybagit/g" /code/Rodan/requirements.txt \
-  # Add Entrypoints
-  && sed -i 's/\r//' /opt/entrypoint \
-  && chmod +x /opt/entrypoint \
-  # Add Celery script
-  && chmod +x /run/start-celery \
-  # Change the concurency for gpu jobs because Calvo is very expensive
-  && sed -i "s/=10/=1/g" /run/start-celery \
-  # Script to wait for postgres and redis to be running before attempting to connect to them.
-  && chmod +x /run/wait-for-app
+  && /opt/install_gpu_rodan_jobs
 
-# Install Rodan.
-RUN pip3 install -r /code/Rodan/requirements.txt
-
-RUN pip3 uninstall -y opencv-python opencv-python-headless
-RUN pip3 install opencv-python-headless==4.1.2.30
+# GPU jobs are expensive — one task per worker at a time.
+RUN sed -i "s/--concurrency=10/--concurrency=1/g" /run/start-celery \
+  && sed -i 's/\r//' /opt/entrypoint \
+  && chmod +x /opt/entrypoint /run/start-celery /run/wait-for-app
 
 ENTRYPOINT ["/opt/entrypoint"]

k8s/12-pv-pgdata.yaml

@@ -1,5 +1,6 @@
-# Postgres PGDATA (~4.1 GB), PG 9.6, owned by uid 999. Mounted only by the postgres pod.
-# 'hard' mount (DBs must not see truncated/soft-failed I/O). NFS export needs no_root_squash.
+# Postgres 16 PGDATA (dump/restored from the old 9.6 clone — Django 4.2 needs PG>=12), owned by
+# uid 999. Mounted only by the postgres pod. 'hard' mount (DBs must not see truncated/soft-failed
+# I/O). NFS export needs no_root_squash. See the PG9.6->16 runbook before first apply.
 apiVersion: v1
 kind: PersistentVolume
 metadata:
@@ -18,7 +19,7 @@ spec:
     - hard
   nfs:
     server: 192.168.236.124
-    path: /srv/rodan-data/var/lib/docker/volumes/rodan_pg_data/_data
+    path: /srv/rodan-data/pg16_data
   claimRef:
     namespace: rodan
     name: rodan-pg-data