Move public nightlies to PyPI and keep local installs stable

.github/workflows/huggingface-nightly.yml

@@ -7,13 +7,12 @@ on:
   workflow_dispatch:
     inputs:
       upload_to:
-        description: "Where to upload (none/testpypi/pypi)"
+        description: "Where to upload (none/pypi)"
         required: true
-        default: "testpypi"
+        default: "pypi"
         type: choice
         options:
           - none
-          - testpypi
           - pypi
       skip_existing:
         description: "Skip already-uploaded versions"
@@ -90,12 +89,15 @@ jobs:
           - id: nemotron-page-elements-v3
             url: https://huggingface.co/nvidia/nemotron-page-elements-v3
             project_subdir: ""
+            nightly_base_version: "3.0.2"
           - id: nemotron-table-structure-v1
             url: https://huggingface.co/nvidia/nemotron-table-structure-v1
             project_subdir: ""
+            nightly_base_version: "1.0.1"
           - id: nemotron-graphic-elements-v1
             url: https://huggingface.co/nvidia/nemotron-graphic-elements-v1
             project_subdir: ""
+            nightly_base_version: "1.0.1"
 
     steps:
       - name: Checkout orchestrator repo
@@ -117,8 +119,8 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          # Default for scheduled runs: testpypi
-          upload_to="testpypi"
+          # Default for scheduled runs: pypi
+          upload_to="pypi"
           skip_existing="true"
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             upload_to="${{ inputs.upload_to }}"
@@ -130,25 +132,19 @@ jobs:
       - name: Build (and maybe upload)
         env:
           NIGHTLY_DATE_SUFFIX: ${{ needs.nightly_coordinate.outputs.nightly_date_suffix }}
-          TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
           PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
         shell: bash
         run: |
           set -euo pipefail
 
           upload_flag=""
-          repo_url="https://test.pypi.org/legacy/"
-          token_env="TEST_PYPI_API_TOKEN"
+          repo_url="https://upload.pypi.org/legacy/"
+          token_env="PYPI_API_TOKEN"
           if [[ "${{ steps.target.outputs.upload_to }}" == "none" ]]; then
             upload_flag=""
           else
             upload_flag="--upload"
           fi
-          if [[ "${{ steps.target.outputs.upload_to }}" == "pypi" ]]; then
-            repo_url="https://upload.pypi.org/legacy/"
-            token_env="PYPI_API_TOKEN"
-          fi
-
           skip_existing_flag=""
           if [[ "${{ steps.target.outputs.skip_existing }}" == "true" ]]; then
             skip_existing_flag="--skip-existing"
@@ -160,6 +156,7 @@ jobs:
             --work-dir ".work" \
             --dist-dir "dist-out" \
             --project-subdir "${{ matrix.repo.project_subdir }}" \
+            --nightly-base-version "${{ matrix.repo.nightly_base_version }}" \
             ${upload_flag} \
             --repository-url "${repo_url}" \
             --token-env "${token_env}" \
@@ -228,8 +225,8 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          # Default for scheduled runs: testpypi
-          upload_to="testpypi"
+          # Default for scheduled runs: pypi
+          upload_to="pypi"
           skip_existing="true"
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             upload_to="${{ inputs.upload_to }}"
@@ -241,7 +238,6 @@ jobs:
       - name: Build ${{ matrix.ocr.id }} (and maybe upload)
         env:
           NIGHTLY_DATE_SUFFIX: ${{ needs.nightly_coordinate.outputs.nightly_date_suffix }}
-          TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
           PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
           CUDA_HOME: /usr/local/cuda
           BUILD_CPP_EXTENSION: "1"
@@ -257,18 +253,13 @@ jobs:
           set -euo pipefail
 
           upload_flag=""
-          repo_url="https://test.pypi.org/legacy/"
-          token_env="TEST_PYPI_API_TOKEN"
+          repo_url="https://upload.pypi.org/legacy/"
+          token_env="PYPI_API_TOKEN"
           if [[ "${{ steps.target.outputs.upload_to }}" == "none" ]]; then
             upload_flag=""
           else
             upload_flag="--upload"
           fi
-          if [[ "${{ steps.target.outputs.upload_to }}" == "pypi" ]]; then
-            repo_url="https://upload.pypi.org/legacy/"
-            token_env="PYPI_API_TOKEN"
-          fi
-
           skip_existing_flag=""
           if [[ "${{ steps.target.outputs.skip_existing }}" == "true" ]]; then
             skip_existing_flag="--skip-existing"
@@ -428,7 +419,7 @@ jobs:
                   "-linux_aarch64.whl"
               ):
                   raise SystemExit(
-                      "Wheel still has a bare linux_* tag; TestPyPI rejects these. "
+                      "Wheel still has a bare linux_* tag; PyPI rejects these. "
                       "auditwheel repair should emit manylinux_*. Got: "
                       f"{wheel.name}"
                   )

.github/workflows/pypi-nightly-publish.yml

@@ -25,20 +25,22 @@ on:
           - dev
           - release
       upload_to:
-        description: 'Where to upload (none/testpypi/pypi)'
+        description: 'Where to upload (none/pypi)'
         required: true
-        default: testpypi
+        default: pypi
         type: choice
         options:
           - none
-          - testpypi
           - pypi
       skip_existing:
         description: 'Skip already-uploaded versions'
         required: true
         default: true
         type: boolean
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: linux-large-disk
@@ -48,8 +50,8 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
-          # Default for scheduled runs: testpypi
-          upload_to="testpypi"
+          # Default for scheduled runs: pypi
+          upload_to="pypi"
           skip_existing="true"
           if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
             upload_to="${{ inputs.upload_to }}"
@@ -124,20 +126,13 @@ jobs:
 
       - name: Publish wheels
         env:
-          TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
-          PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
         shell: bash
         run: |
           set -euo pipefail
 
           upload_to="${{ steps.target.outputs.upload_to }}"
-          repository_url="https://test.pypi.org/legacy/"
-          token="${TEST_PYPI_API_TOKEN:-}"
-          if [[ "${upload_to}" == "pypi" ]]; then
-            repository_url="https://upload.pypi.org/legacy/"
-            token="${PYPI_API_TOKEN:-}"
-          fi
-
+          repository_url="https://upload.pypi.org/legacy/"
           if [[ "${upload_to}" == "none" ]]; then
             echo "upload_to=none; skipping package upload."
             exit 0
@@ -148,4 +143,4 @@ jobs:
             skip_existing_flag="--skip-existing"
           fi
 
-          twine upload ${skip_existing_flag} --repository-url "${repository_url}" -u __token__ -p "${token}" nemo_retriever/dist/*
+          twine upload ${skip_existing_flag} --repository-url "${repository_url}" -u __token__ nemo_retriever/dist/*

ci/scripts/nightly_build_publish.py

@@ -650,7 +650,7 @@ def _build(
 
 def _auditwheel_repair_dist_dir(dist_dir: Path, *, exclude_libs: list[str] | None = None) -> None:
     """
-    Rewrite linux_* wheels to manylinux_* so TestPyPI/PyPI accept the upload.
+    Rewrite linux_* wheels to manylinux_* so PyPI accepts the upload.
     Requires ``patchelf`` on PATH (e.g. apt install patchelf).
 
     *exclude_libs* is a list of shared library basenames (e.g. ``libtorch_cpu.so``)
@@ -804,8 +804,8 @@ def main() -> int:
         "before building (repeatable; useful for ABI-coupled deps like torch)",
     )
     ap.add_argument("--upload", action="store_true", help="Upload built dists via twine")
-    ap.add_argument("--repository-url", default="https://test.pypi.org/legacy/", help="Twine repository URL")
-    ap.add_argument("--token-env", default="TEST_PYPI_API_TOKEN", help="Env var containing API token")
+    ap.add_argument("--repository-url", default="https://upload.pypi.org/legacy/", help="Twine repository URL")
+    ap.add_argument("--token-env", default="PYPI_API_TOKEN", help="Env var containing API token")
     ap.add_argument("--skip-existing", action="store_true", help="Pass --skip-existing to twine")
     ap.add_argument(
         "--twine-verbose",
@@ -821,7 +821,7 @@ def main() -> int:
     ap.add_argument(
         "--auditwheel-repair",
         action="store_true",
-        help="Run auditwheel repair on built wheels (manylinux tag; needed for PyPI/TestPyPI)",
+        help="Run auditwheel repair on built wheels (manylinux tag; needed for PyPI)",
     )
     ap.add_argument(
         "--auditwheel-exclude",

ci/tests/test_huggingface_release_workflow.py

@@ -113,3 +113,21 @@ def test_huggingface_workflow_has_manual_stable_ocr_release_controls() -> None:
     assert "--release-version" in workflow
     assert 'expected_version="${INPUT_RELEASE_VERSION}"' in workflow
     assert "Built wheel metadata does not declare expected version" in workflow
+
+
+def test_huggingface_non_ocr_nightlies_are_versioned_after_current_stable() -> None:
+    workflow = (REPO_ROOT / ".github" / "workflows" / "huggingface-nightly.yml").read_text(encoding="utf-8")
+
+    assert '--nightly-base-version "${{ matrix.repo.nightly_base_version }}"' in workflow
+    assert "id: nemotron-page-elements-v3" in workflow
+    assert 'nightly_base_version: "3.0.2"' in workflow
+    assert "id: nemotron-table-structure-v1" in workflow
+    assert workflow.count('nightly_base_version: "1.0.1"') == 2
+    assert "id: nemotron-graphic-elements-v1" in workflow
+
+
+def test_huggingface_nightly_builder_defaults_to_public_pypi() -> None:
+    script = (REPO_ROOT / "ci" / "scripts" / "nightly_build_publish.py").read_text(encoding="utf-8")
+
+    assert 'default="https://upload.pypi.org/legacy/"' in script
+    assert 'default="PYPI_API_TOKEN"' in script

nemo_retriever/README.md

@@ -47,6 +47,14 @@ source retriever/bin/activate
 uv pip install "nemo-retriever[local]==26.05-RC1"
 ```
 
+The `[local]` extra resolves stable Nemotron extraction packages by default. To
+try prerelease/nightly Nemotron packages from PyPI within the same supported
+major-version windows, opt in with `--pre`:
+
+```bash
+uv pip install --pre "nemo-retriever[local]==26.05-RC1"
+```
+
 Install matching **ingestion client** and **ingestion runtime** wheels at the same version when your workflow expects them (see the [NeMo Retriever Library prerequisites](https://docs.nvidia.com/nemo/retriever/latest/extraction/overview/) for the exact PyPI coordinates for your release).
 
 For **remote NIM inference only** (no local GPU required), the base package is sufficient:

nemo_retriever/pyproject.toml

@@ -102,9 +102,8 @@ service = [
 ]
 
 # ── Local model inference (GPU assumed; torch resolves to CUDA on Linux) ─────
-# Adds HuggingFace transformers, torch, nemotron models, GPU monitoring, and vLLM.
+# Stable Nemotron extraction package selection for published local installs.
 local = [
-
   "glom",
   "transformers>=4.57.6,<5",
   "tokenizers>=0.21.1",
@@ -123,9 +122,9 @@ local = [
   "scikit-learn>=1.6.0",
   "timm==1.0.22",
   "albumentations==2.0.8",
-  "nemotron-page-elements-v3==3.0.1",
-  "nemotron-graphic-elements-v1==1.0.0",
-  "nemotron-table-structure-v1==1.0.0",
+  "nemotron-page-elements-v3>=3.0.1,<4",
+  "nemotron-graphic-elements-v1>=1.0.0,<2",
+  "nemotron-table-structure-v1>=1.0.0,<2",
   "nemotron-ocr>=2.0.0,<3; sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')",
   "nvidia-ml-py",
   "apscheduler>=3.10",
@@ -197,6 +196,19 @@ all = [
 retriever = "nemo_retriever.__main__:main"
 retriever-harness = "nemo_retriever.harness:main"
 
+# uv-only developer install: include every published extra plus the published
+# dev extra, then overlay the prerelease Nemotron selection without exposing it
+# in wheel metadata.
+[dependency-groups]
+dev = [
+  "nemo_retriever[all]",
+  "nemo_retriever[dev]",
+  "nemotron-page-elements-v3>=3.0.1.dev0,!=3.0.1,!=3.0.2,<4",
+  "nemotron-graphic-elements-v1>=1.0.1.dev0,!=1.0.1,<2",
+  "nemotron-table-structure-v1>=1.0.1.dev0,!=1.0.1,<2",
+  "nemotron-ocr>=2.0.1.dev0,!=2.0.1,<3; sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')",
+]
+
 [tool.setuptools.dynamic]
 version = {attr = "nemo_retriever.version.get_build_version"}
 
@@ -209,11 +221,6 @@ torch = [
 torchvision = [
   { index = "pytorch-cu130", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
 ]
-[[tool.uv.index]]
-name = "test-pypi"
-url = "https://test.pypi.org/simple/"
-explicit = true
-
 [[tool.uv.index]]
 name = "pytorch-cu130"
 url = "https://download.pytorch.org/whl/cu130"

nemo_retriever/tests/test_ci_workflows.py

@@ -127,6 +127,40 @@ def test_legacy_ghcr_push_publish_workflow_is_removed():
     assert not (WORKFLOWS / "docker-build-publish-retriever.yml").exists()
 
 
+@requires_workflows
+def test_public_nightly_python_publish_workflows_do_not_target_testpypi():
+    workflow_names = ("pypi-nightly-publish.yml", "huggingface-nightly.yml")
+
+    for workflow_name in workflow_names:
+        workflow = (WORKFLOWS / workflow_name).read_text(encoding="utf-8")
+
+        assert "testpypi" not in workflow.lower(), workflow_name
+        assert "test.pypi.org" not in workflow.lower(), workflow_name
+        assert "https://upload.pypi.org/legacy/" in workflow, workflow_name
+        assert "PYPI_API_TOKEN" in workflow, workflow_name
+
+
+@requires_workflows
+def test_public_nightly_python_publish_workflows_use_read_only_token_permissions():
+    workflow_names = ("pypi-nightly-publish.yml", "huggingface-nightly.yml")
+
+    for workflow_name in workflow_names:
+        workflow = _load_workflow(workflow_name)
+
+        assert workflow["permissions"] == {"contents": "read"}, workflow_name
+
+
+@requires_workflows
+def test_pypi_nightly_publish_uses_twine_password_env():
+    workflow = _load_workflow("pypi-nightly-publish.yml")
+    steps = workflow["jobs"]["build"]["steps"]
+    publish_step = next(step for step in steps if step.get("name") == "Publish wheels")
+
+    assert publish_step["env"] == {"TWINE_PASSWORD": "${{ secrets.PYPI_API_TOKEN }}"}
+    assert ' -p "${token}"' not in publish_step["run"]
+    assert "PYPI_API_TOKEN" not in publish_step["run"]
+
+
 def test_legacy_nv_ingest_root_compose_stack_is_removed():
     legacy_paths = (
         "docker-compose.yaml",