Django 5 upgrade

.github/workflows/test.yml

@@ -1,15 +1,13 @@
 name: CI
-
 on:
   workflow_dispatch:
   pull_request:
   push:
     branches:
       - master
       - staging
-
 env:
-  PYENV_VERSION: '3.10'
+  PYENV_VERSION: '3.12'
   DJANGO_SECRET_KEY: ${{secrets.SECRET_KEY}}
   DJANGO_SETTINGS_MODULE: config.settings.test
   DATABASE_URL: postgres://${PGUSER}:${PGPASSWORD}@127.0.0.1:5432/test
@@ -38,15 +36,10 @@ env:
   DOCUMENT_BUCKET: ${{secrets.DOCUMENT_BUCKET}}
   REDIS_PROCESSING_PASSWORD: ""
   SAM_CLI_TELEMETRY: 0
-
-
-
 
-
 jobs:
   check:
     runs-on: ubuntu-latest
-
     services:
       redis:
         image: redis
@@ -63,30 +56,19 @@ jobs:
             submodules: recursive
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           cache: "pip" # caching pip dependencies
-
-      - name: Install pip
-        run: |
-          wget https://bootstrap.pypa.io/pip/3.6/get-pip.py
-          python3 get-pip.py
-
       - name: Install
         run: |
           pip install -r requirements/local.txt
-
       - name: Lint
         run: pylint documentcloud
-
       - name: Isort
         run: isort --check-only -rc documentcloud
-
       - name: Formatting
         run: black --check documentcloud --exclude migrations
-
   test:
     runs-on: ubuntu-latest
-
     services:
       redis:
         image: redis
@@ -97,7 +79,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-
       postgres:
         image: postgres
         env:
@@ -113,33 +94,21 @@ jobs:
         ports:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
-
     steps:
-
       - name: Checkout
         uses: actions/checkout@v4
-
       - name: Install Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.12"
           cache: "pip" # caching pip dependencies
-
-      - name: Install pip
-        run: |
-          wget https://bootstrap.pypa.io/pip/3.6/get-pip.py
-          python3 get-pip.py
-
       - name: Install requirements
         run: |
           pip install -r requirements/local.txt
-
-
       - name: Test
         run:  pytest documentcloud -m "not solr"
         env:
           # use the credentials for the service container
           PG_USER: test
           PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
-          DATABASE_URL: postgres://test:postgres@127.0.0.1:5432/test
-
+          DATABASE_URL: postgres://test:postgres@127.0.0.1:5432/test

.gitignore

@@ -304,5 +304,5 @@ scratch/
 
 CLAUDE.md
 .claude
-
 rootCA.pem
+tests.txt

.pylintrc

@@ -1,7 +1,6 @@
 [MASTER]
-# https://stackoverflow.com/a/39207275/10952222
-init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.join(os.path.dirname(find_pylintrc()), 'documentcloud'))"
-load-plugins=pylint_django, pylint_celery
+init-hook="import sys, os; sys.path.insert(0, '/app'); os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local')"
+load-plugins=pylint_django
 ignore=migrations
 
 [FORMAT]
@@ -11,7 +10,7 @@ max-args=6
 
 [MESSAGES CONTROL]
 enable=useless-suppression
-disable=missing-docstring,too-many-ancestors,too-few-public-methods,no-else-return,no-member,attribute-defined-outside-init,similarities,import-outside-toplevel,cyclic-import,raise-missing-from,django-not-configured
+disable=missing-docstring,too-many-ancestors,too-few-public-methods,no-else-return,no-member,attribute-defined-outside-init,similarities,import-outside-toplevel,cyclic-import,raise-missing-from,django-not-configured,wrong-import-order,wrong-import-position
 
 [TYPECHECK]
 generated-members=REQUEST,acl_users,aq_parent,"[a-zA-Z]+_set{1,2}",save,delete

compose/local/django/Dockerfile

@@ -1,33 +1,27 @@
-# FROM python:3.6-alpine
-FROM matthewfeickert/docker-python3-ubuntu:3.10.5
-
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
-
 USER root
 
 RUN apt-get -qq -y update && \
   apt-get -qq -y install \
-  # Pip dependencies
-  python3-pip \
+  # Build dependencies
+  build-essential \
   # Postgres dependencies
   libpq-dev postgresql-client python3-psycopg2 \
   # Tesseract dependencies
-  libjpeg-turbo8 libtiff5 \
+  libjpeg62-turbo libtiff6 \
   # LibreOffice dependencies
-  libnss3-dev libcurl4-nss-dev libxslt1-dev libpixman-1-0 libxcb-render0-dev && \
-  # Symlink bash and python
-  ln -sf bash /bin/sh && rm -f /usr/bin/python && \
-  ln -s /usr/bin/python3 /usr/bin/python && \
-  curl https://bootstrap.pypa.io/pip/3.6/get-pip.py -o get-pip.py && \
-  python3 get-pip.py --force-reinstall
+  libnss3-dev libcurl4-openssl-dev libxslt1-dev libpixman-1-0 libxcb-render0-dev \
+  # Utilities
+  curl git
 
 # set up makecert root CA
 RUN curl http://localhost/rootCA.pem > /usr/local/share/ca-certificates/rootCA.crt && update-ca-certificates
 
 # Requirements are installed here to ensure they will be cached.
 COPY ./requirements /requirements
-# RUN pip install --upgrade pip && pip install -r /requirements/local.txt
-RUN pip install -r /requirements/local.txt
+
+RUN pip install --upgrade "pip>=26,<27" && pip install -r /requirements/local.txt
 
 COPY ./compose/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r//' /entrypoint && chmod +x /entrypoint
@@ -48,11 +42,8 @@ RUN sed -i 's/\r//' /start-flower && chmod +x /start-flower
 # Entry point
 # =-=-=-=-=-=
 
-# Temporary measure to get pip-compile to work
-# RUN pip install 'pip<19.2'
-
 WORKDIR /app
 
 ENV LD_LIBRARY_PATH /app/documentcloud/documents/processing/ocr/tesseract
 
-ENTRYPOINT ["/entrypoint"]
+ENTRYPOINT ["/entrypoint"]

compose/production/django/Dockerfile

@@ -1,23 +1,20 @@
-
-FROM python:3.6-alpine
-
+FROM python:3.12-slim
 ENV PYTHONUNBUFFERED 1
 
-RUN apk update \
+RUN apt-get -qq -y update && apt-get -qq -y install \
   # psycopg2 dependencies
-  && apk add --virtual build-deps gcc python3-dev musl-dev \
-  && apk add postgresql-dev \
+  gcc python3-dev libpq-dev \
   # Pillow dependencies
-  && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \
+  libjpeg62-turbo zlib1g-dev libfreetype6-dev liblcms2-dev libopenjp2-7-dev libtiff6 tk-dev tcl-dev \
   # CFFI dependencies
-  && apk add libffi-dev py-cffi
+  libffi-dev python3-cffi
 
-RUN addgroup -S django \
-    && adduser -S -G django django
+RUN groupadd -r django \
+    && useradd -r -g django django
 
-# Requirements are installed here to ensure they will be cached.
+# Requirements are installed here to ensure they will be cached. 
 COPY ./requirements /requirements
-RUN pip install --no-cache-dir -r /requirements/production.txt \
+RUN pip install --upgrade "pip>=26,<27" && pip install --no-cache-dir -r /requirements/production.txt \
     && rm -rf /requirements
 
 COPY ./compose/production/django/entrypoint /entrypoint

config/settings/base.py

@@ -93,7 +93,6 @@
     "documentcloud.addons.apps.AddOnsConfig",
     "documentcloud.projects.apps.ProjectsConfig",
     "documentcloud.statistics.apps.StatisticsConfig",
-    "documentcloud.sidekick.apps.SidekickConfig",
     "documentcloud.users.apps.UsersConfig",
     "documentcloud.entities.apps.EntitiesConfig",
 ]
@@ -497,7 +496,7 @@
 PROGRESS_URL = env("PROGRESS_URL", default="")
 IMPORT_URL = env("IMPORT_URL", default="")
 PROGRESS_TIMEOUT = env.int("PROGRESS_TIMEOUT", default=1)
-SIDEKICK_PROCESSING_URL = env("SIDEKICK_PROCESSING_URL", default="")
+# SIDEKICK_PROCESSING_URL = env("SIDEKICK_PROCESSING_URL", default="")
 
 # Auth
 LOGIN_URL = "/accounts/login/squarelet"

config/settings/local.py

@@ -53,7 +53,7 @@
         "debug_toolbar.panels.profiling.ProfilingPanel",
     ],
     "SHOW_TEMPLATE_CONTEXT": True,
-    "SHOW_TOOLBAR_CALLBACK": lambda _: True,
+    "SHOW_TOOLBAR_CALLBACK": lambda _: False,
 }
 
 # Celery

config/urls.py

@@ -49,8 +49,6 @@
     ProjectMembershipViewSet,
     ProjectViewSet,
 )
-from documentcloud.sidekick.routers import SidekickRouter
-from documentcloud.sidekick.views import SidekickViewSet
 from documentcloud.statistics.views import StatisticsViewSet
 from documentcloud.users.views import MessageView, UserViewSet
 
@@ -76,8 +74,12 @@ class BulkNestedDefaultRouter(BulkRouterMixin, NestedDefaultRouter):
 documents_router.register("notes", NoteViewSet)
 documents_router.register("sections", SectionViewSet)
 documents_router.register("entities", EntityOccurrenceViewSet)
-documents_router.register("legacy_entities_2", LegacyEntity2ViewSet)
-documents_router.register("legacy_entities", LegacyEntityViewSet)
+documents_router.register(
+    "legacy_entities_2", LegacyEntity2ViewSet, basename="legacyentity2"
+)
+documents_router.register(
+    "legacy_entities", LegacyEntityViewSet, basename="legacyentity"
+)
 documents_router.register("dates", EntityDateViewSet)
 documents_router.register("errors", DocumentErrorViewSet)
 documents_router.register("data", DataViewSet, basename="data")
@@ -90,9 +92,6 @@ class BulkNestedDefaultRouter(BulkRouterMixin, NestedDefaultRouter):
 projects_router.register("documents", ProjectMembershipViewSet)
 projects_router.register("users", CollaborationViewSet)
 
-sidekick_router = SidekickRouter(router, "projects", lookup="project")
-sidekick_router.register("sidekick", SidekickViewSet)
-
 router.register("documents/search/saved", SavedSearchViewSet, basename="saved_search")
 
 urlpatterns = [
@@ -101,7 +100,6 @@ class BulkNestedDefaultRouter(BulkRouterMixin, NestedDefaultRouter):
     path("api/", include(router.urls)),
     path("api/", include(documents_router.urls)),
     path("api/", include(projects_router.urls)),
-    path("api/", include(sidekick_router.urls)),
     path("api/schema/", SpectacularAPIView.as_view(), name="schema"),
     path(
         "api/schema/redoc/",

documentcloud/addons/choices.py

@@ -1,14 +1,20 @@
 # Django
+from django.db import models
 from django.utils.translation import gettext_lazy as _
 
-# Third Party
-from djchoices import ChoiceItem, DjangoChoices
 
+class Event(models.IntegerChoices):
+    def __new__(cls, value, label=None, api=False):
+        obj = int.__new__(cls, value)
+        obj._value_ = value
+        if label is not None:
+            obj._label_ = label
+        obj.api = api
+        return obj
 
-class Event(DjangoChoices):
-    # `api` specifies if this attribute should be accessible via the API
-    disabled = ChoiceItem(0, _("Disabled"), api=True)
-    hourly = ChoiceItem(1, _("Hourly"), api=True)
-    daily = ChoiceItem(2, _("Daily"), api=True)
-    weekly = ChoiceItem(3, _("Weekly"), api=True)
-    upload = ChoiceItem(4, _("Upload"), api=True)
+    # pylint:disable = invalid-name
+    disabled = 0, _("Disabled"), True
+    hourly = 1, _("Hourly"), True
+    daily = 2, _("Daily"), True
+    weekly = 3, _("Weekly"), True
+    upload = 4, _("Upload"), True

documentcloud/addons/models.py

@@ -32,7 +32,7 @@
 
 logger = logging.getLogger(__name__)
 
-
+# pylint:disable=too-many-positional-arguments
 class AddOn(models.Model):
     objects = AddOnQuerySet.as_manager()
 

documentcloud/addons/tasks.py

@@ -61,7 +61,7 @@ def set_run_status(uuid):
         # if we are not in a completed status, continue polling for new status
         set_run_status.apply_async(args=[uuid], countdown=5)
 
-
+# pylint:disable=too-many-positional-arguments
 @shared_task
 def dispatch(addon_id, uuid, user_id, documents, query, parameters, event_id=None):
     # pylint: disable=too-many-arguments

documentcloud/addons/tests/test_views.py

@@ -262,9 +262,9 @@ def test_update_no_addon(self, client):
 
     def test_destroy(self, client, mocker):
         """Destroying an addon run cancels it"""
-        cancel = mocker.patch.object(AddOnRun, "cancel")
+        cancel = mocker.patch("documentcloud.addons.views.cancel.delay")
         run = AddOnRunFactory()
         client.force_authenticate(user=run.user)
         response = client.delete(f"/api/addon_runs/{run.uuid}/")
         assert response.status_code == status.HTTP_204_NO_CONTENT
-        assert cancel.called_once()
+        cancel.assert_called_once()

documentcloud/common/environment/local/pubsub.py

@@ -129,13 +129,6 @@ def finish_import_task(data):
     return finish_import_process.delay(data)
 
 
-def sidekick_preprocess_task(data):
-    # DocumentCloud
-    from documentcloud.sidekick.tasks import sidekick_preprocess
-
-    return sidekick_preprocess.delay(data)
-
-
 def retry_errors_task(data):
     # DocumentCloud
     from documentcloud.documents.tasks import retry_errors_local
@@ -191,13 +184,6 @@ def retry_errors_task(data):
     ("documentcloud", env.str("FINISH_IMPORT_TOPIC", default="finish-import")),
     finish_import_task,
 )
-publisher.register_internal_callback(
-    (
-        "documentcloud",
-        env.str("SIDEKICK_PREPROCESS_TOPIC", default="sidekick-preprocess-topic"),
-    ),
-    sidekick_preprocess_task,
-)
 publisher.register_internal_callback(
     ("documentcloud", env.str("RETRY_ERROR_TOPIC", default="retry-error-topic")),
     retry_errors_task,

documentcloud/common/serverless/tests/test_error_handling.py

@@ -11,7 +11,6 @@
 
 # Third Party
 import pytest
-from sharedmock.mock import SharedMock
 
 # DocumentCloud
 from documentcloud.common import redis_fields
@@ -27,6 +26,7 @@
     StorageHandler,
     Workspace,
 )
+from sharedmock.mock import SharedMock
 
 # Since redis is used in the SharedMock calls, it needs to be pickle-able
 # in order to be sent across the process boundary.  FakeRedis and Mock's both