From 52eec09f620c50209f739c704c4ae8fa6363ae1b Mon Sep 17 00:00:00 2001 From: Nic Date: Thu, 6 Aug 2026 12:49:30 +0100 Subject: [PATCH] ci(ffi): cross-platform Python wheels + PyPI publishing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the release pipeline for the ant-ffi Python bindings: reproducible per-platform wheel builds and a publish workflow using PyPI Trusted Publishing (OIDC, no stored token). Build scripts (single source of truth — CI runs the same script a developer runs locally): - build-wheel-manylinux.sh compiles the native lib inside a manylinux_2_28 container (glibc 2.28) so the wheel installs on RHEL8 / Ubuntu 20.04+ / Debian 10+; auditwheel verifies + retags. Arg: x86_64 | aarch64. - build-wheel-macos.sh builds arm64 + x86_64, lipo-fuses to a universal2 wheel (deployment target 11.0); delocate verifies self-contained. - build-wheel-windows.ps1 native x86_64 DLL; delvewheel bundles non-system DLLs. Workflow (publish-python.yml): - Matrix: linux x86_64 (ubuntu-latest), linux aarch64 (ubuntu-24.04-arm, native — no QEMU), macOS universal2 (macos-latest), windows amd64. - Triggers: push tag `python-v` -> build+publish to PyPI; or manual dispatch with publish=none|testpypi|pypi (none = matrix smoke test). - Publishes via OIDC (id-token) into a GitHub environment (pypi/testpypi). Proven locally: linux-x86_64 and macOS-universal2 wheels build, are auditwheel/delocate-honest, and install + run in clean environments (incl. debian:11 / glibc 2.31). linux-aarch64 and Windows validate on first CI dispatch (no local hardware for either). Before the first publish, a maintainer must create the GitHub environments and register the PyPI/TestPyPI pending publisher (project ant-ffi, owner WithAutonomi, repo ant-sdk, workflow publish-python.yml). The workflow is inert until then (only runs on a python-v* tag or manual dispatch). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/publish-python.yml | 129 +++++++++++++++++++++++++++ ffi/scripts/build-wheel-macos.sh | 57 ++++++++++++ ffi/scripts/build-wheel-manylinux.sh | 85 ++++++++++++++++++ ffi/scripts/build-wheel-windows.ps1 | 51 +++++++++++ 4 files changed, 322 insertions(+) create mode 100644 .github/workflows/publish-python.yml create mode 100644 ffi/scripts/build-wheel-macos.sh create mode 100644 ffi/scripts/build-wheel-manylinux.sh create mode 100644 ffi/scripts/build-wheel-windows.ps1 diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml new file mode 100644 index 0000000..a7e26ae --- /dev/null +++ b/.github/workflows/publish-python.yml @@ -0,0 +1,129 @@ +name: Publish Python (ant-ffi) + +# Builds the ant-ffi Python wheel across the platform matrix and publishes to +# PyPI (or TestPyPI) via Trusted Publishing — no long-lived API token. +# +# Each platform reuses the same build script developers run locally, so CI and +# local builds are the single source of truth: +# linux -> ffi/scripts/build-wheel-manylinux.sh (manylinux_2_28 in Docker) +# macos -> ffi/scripts/build-wheel-macos.sh (universal2, lipo+delocate) +# windows-> ffi/scripts/build-wheel-windows.ps1 (delvewheel) +# +# Wheels are tagged py3-none-: one per OS/arch, valid for every +# Python 3 (the bindings are pure ctypes over a bundled native library). +# +# ── One-time setup required before the first publish ── +# 1. Create GitHub environments `pypi` and `testpypi` (Settings > Environments; +# optionally add reviewers as a release gate). +# 2. Register a PyPI/TestPyPI "pending publisher" (Trusted Publishing): +# PyPI project: ant-ffi owner: WithAutonomi repo: ant-sdk +# workflow: publish-python.yml environment: pypi (and again for testpypi) +# No secrets needed — OIDC via `id-token: write` below. +# +# ── How to run ── +# • Push a tag `python-v` (e.g. python-v0.0.8) -> builds + publishes to PyPI. +# • Or run manually (Actions > Run workflow) with `publish`: +# none -> build the full matrix only (matrix smoke test, no upload) +# testpypi -> build + upload to TestPyPI +# pypi -> build + upload to PyPI + +on: + workflow_dispatch: + inputs: + publish: + description: "Where to publish the built wheels" + type: choice + options: [none, testpypi, pypi] + default: none + push: + tags: + - "python-v*" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + name: Build wheel (${{ matrix.name }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + os: ubuntu-latest + script: bash ffi/scripts/build-wheel-manylinux.sh x86_64 + - name: linux-aarch64 + os: ubuntu-24.04-arm # native arm runner — no QEMU + script: bash ffi/scripts/build-wheel-manylinux.sh aarch64 + - name: macos-universal2 + os: macos-latest # arm64 host; lipo adds the x86_64 slice + script: bash ffi/scripts/build-wheel-macos.sh + - name: windows-amd64 + os: windows-latest + script: pwsh ffi/scripts/build-wheel-windows.ps1 + steps: + - uses: actions/checkout@v4 + + # Rust for the native macOS/Windows builds. The Linux jobs install Rust + # inside the manylinux container, so the host toolchain there is unused + # (harmless). + - uses: dtolnay/rust-toolchain@stable + + # ant-core's build touches protobuf on the native build paths; the Linux + # container build vendors its own, but installing here is harmless and + # covers macOS/Windows. + - uses: arduino/setup-protoc@v3 + with: + version: "25.x" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build wheel + run: ${{ matrix.script }} + + - uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.name }} + path: ffi/python/wheelhouse/*.whl + if-no-files-found: error + + publish: + name: Publish + needs: build + # Publish on a python-v* tag (-> PyPI), or when a manual run asks for it. + if: >- + startsWith(github.ref, 'refs/tags/python-v') || + (github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'none') + runs-on: ubuntu-latest + environment: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'testpypi') && 'testpypi' || 'pypi' }} + permissions: + id-token: write # OIDC token for Trusted Publishing + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + pattern: wheel-* + merge-multiple: true + + - name: List wheels to publish + run: ls -la dist + + - name: Publish to TestPyPI + if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'testpypi' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: dist + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags/python-v') || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'pypi') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist diff --git a/ffi/scripts/build-wheel-macos.sh b/ffi/scripts/build-wheel-macos.sh new file mode 100644 index 0000000..e2695af --- /dev/null +++ b/ffi/scripts/build-wheel-macos.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Build a universal2 macOS Python wheel for the ant-ffi bindings. +# +# Compiles the native lib for both arm64 (Apple Silicon) and x86_64 (Intel), +# lipo-fuses them into one fat dylib, and packages a single +# `macosx_11_0_universal2` wheel that installs on both Mac architectures. +# Deployment target is pinned to 11.0 (arm64's floor) so the tag is honest. +# `delocate` is the macOS analogue of auditwheel — it verifies the dylib is +# self-contained and carries both arches. +# +# Run on macOS with Xcode CLT + rustup. Output -> ffi/python/wheelhouse/. +set -euo pipefail + +export MACOSX_DEPLOYMENT_TARGET=11.0 +PLAT_TAG="macosx_11_0_universal2" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FFI_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RUST_DIR="$FFI_DIR/rust" +PY_PKG="$FFI_DIR/python/ant_ffi" + +echo "=== [1/6] add x86_64 target (arm64 is native here) ===" +rustup target add x86_64-apple-darwin aarch64-apple-darwin >/dev/null + +echo "=== [2/6] build both arches (deployment target $MACOSX_DEPLOYMENT_TARGET) ===" +cd "$RUST_DIR" +cargo build --release -p ant-ffi --target aarch64-apple-darwin +cargo build --release -p ant-ffi --target x86_64-apple-darwin +ARM=target/aarch64-apple-darwin/release/libant_ffi.dylib +X86=target/x86_64-apple-darwin/release/libant_ffi.dylib + +echo "=== [3/6] lipo -> universal2 dylib ===" +mkdir -p "$PY_PKG" +lipo -create -output "$PY_PKG/libant_ffi.dylib" "$ARM" "$X86" +lipo -info "$PY_PKG/libant_ffi.dylib" + +echo "=== [4/6] generate bindings (arch-independent) ===" +# The in-crate bindgen was built for the native (arm64) host by the build above. +BINDGEN=target/aarch64-apple-darwin/release/uniffi-bindgen +"$BINDGEN" generate --library "$ARM" --language python --out-dir "$PY_PKG" + +echo "=== [5/6] build universal2 wheel ===" +VENV="$(mktemp -d)/venv" +python3 -m venv "$VENV" +# shellcheck disable=SC1091 +source "$VENV/bin/activate" +pip install -q --upgrade pip setuptools wheel delocate +cd "$FFI_DIR/python" +rm -rf build dist ./*.egg-info +python setup.py -q bdist_wheel --plat-name "$PLAT_TAG" + +echo "=== [6/6] delocate: verify self-contained + both arches ===" +mkdir -p wheelhouse +delocate-listdeps --all dist/*.whl || true +delocate-wheel --require-archs x86_64,arm64 -w wheelhouse -v dist/*.whl +echo "=== done -> $FFI_DIR/python/wheelhouse/ ===" +ls -la wheelhouse/ diff --git a/ffi/scripts/build-wheel-manylinux.sh b/ffi/scripts/build-wheel-manylinux.sh new file mode 100644 index 0000000..060875d --- /dev/null +++ b/ffi/scripts/build-wheel-manylinux.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Build a manylinux-honest Python wheel for the ant-ffi bindings. +# +# Runs on any Linux host with Docker. The native library is compiled INSIDE a +# manylinux_2_28 container (glibc 2.28, AlmaLinux 8) — never against the host's +# glibc — so the wheel installs on any distro from ~2019 on (RHEL8, Ubuntu 20.04+, +# Debian 10+). auditwheel is the authority on the final tag. +# +# Usage (from anywhere): ffi/scripts/build-wheel-manylinux.sh [arch] +# arch: x86_64 (default) | aarch64 +# Output wheel lands in ffi/python/wheelhouse/. +set -euo pipefail + +ARCH="${1:-x86_64}" +IMAGE="quay.io/pypa/manylinux_2_28_${ARCH}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FFI_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Persist the cargo cache across runs so only the first build pays the full +# ant-core compile. +CARGO_CACHE="${HOME}/.cache/ant-ffi-cargo" +mkdir -p "$CARGO_CACHE/registry" "$CARGO_CACHE/git" + +echo "=== manylinux wheel build: $ARCH via $IMAGE ===" +# --network host: required when the Docker daemon itself runs inside an +# unprivileged incus/LXC container. Newer Docker applies the namespaced sysctl +# net.ipv4.ip_unprivileged_port_start on container init, which the nested +# container can't write ("permission denied"); host networking skips per-netns +# sysctls. The build only needs outbound internet (rustup/crates.io/pip/dnf). +docker run --rm --network host \ + -v "$FFI_DIR":/io \ + -v "$CARGO_CACHE/registry":/root/.cargo/registry \ + -v "$CARGO_CACHE/git":/root/.cargo/git \ + -e ARCH="$ARCH" \ + "$IMAGE" bash -euo pipefail -c ' + echo "--- host glibc floor: $(ldd --version | head -1) ---" + + # Build deps some crypto crates want (ring: perl/clang; aws-lc-sys: cmake/go). + dnf install -y -q cmake perl clang golang >/dev/null 2>&1 || \ + yum install -y -q cmake perl clang golang >/dev/null 2>&1 || true + + # Rust (crate needs 1.82+). + export RUSTUP_HOME=/root/.rustup CARGO_HOME=/root/.cargo + export PATH="/root/.cargo/bin:$PATH" + # Use latest stable: the ant-core graph (alloy 1.8.x) needs rustc >= 1.91. + if ! command -v cargo >/dev/null; then + curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --default-toolchain stable --profile minimal + fi + rustup update stable >/dev/null 2>&1 || true + echo "--- $(cargo --version) ---" + + # 1. Build the native lib + the in-crate uniffi-bindgen. + cd /io/rust + cargo build --release -p ant-ffi + LIB=/io/rust/target/release/libant_ffi.so + test -f "$LIB" + + # 2. Generate the pure-Python bindings and bundle the fresh .so. + OUT=/io/python/ant_ffi + mkdir -p "$OUT" + /io/rust/target/release/uniffi-bindgen generate \ + --library "$LIB" --language python --out-dir "$OUT" + cp "$LIB" "$OUT/" + + # 3. Build a platform-tagged wheel (setup.py forces py3-none-). + PY=/opt/python/cp312-cp312/bin/python + # setuptools+wheel are needed explicitly: modern CPython does not bundle + # setuptools, and we build with --no-isolation (setup.py imports it). + "$PY" -m pip install -q --upgrade pip build auditwheel setuptools wheel + cd /io/python + rm -rf build dist *.egg-info + "$PY" -m build --wheel --no-isolation + + # 4. auditwheel: verify glibc floor, bundle external libs, honest retag. + echo "=== auditwheel show (pre-repair) ===" + "$PY" -m auditwheel show dist/*.whl + "$PY" -m auditwheel repair dist/*.whl -w /io/python/wheelhouse/ + echo "=== auditwheel show (repaired) ===" + "$PY" -m auditwheel show /io/python/wheelhouse/*.whl + chown -R '"$(id -u)"':'"$(id -g)"' /io/python/wheelhouse /io/python/ant_ffi /io/python/dist 2>/dev/null || true + ' +echo "=== done -> $FFI_DIR/python/wheelhouse/ ===" +ls -la "$FFI_DIR/python/wheelhouse/" diff --git a/ffi/scripts/build-wheel-windows.ps1 b/ffi/scripts/build-wheel-windows.ps1 new file mode 100644 index 0000000..cca6a11 --- /dev/null +++ b/ffi/scripts/build-wheel-windows.ps1 @@ -0,0 +1,51 @@ +#!/usr/bin/env pwsh +# Build a Windows (win_amd64) Python wheel for the ant-ffi bindings. +# +# Compiles ant_ffi.dll natively (x86_64-pc-windows-msvc), generates the +# bindings, and packages a `win_amd64` wheel. delvewheel is the Windows +# analogue of auditwheel/delocate — it bundles any non-system DLL the native +# library needs (e.g. the VC runtime) so the wheel is self-contained. +# +# Run on Windows with the MSVC toolchain + rustup. Output -> ffi/python/wheelhouse/. +$ErrorActionPreference = "Stop" + +$FfiDir = Split-Path -Parent $PSScriptRoot # scripts/ -> ffi/ +$RustDir = Join-Path $FfiDir "rust" +$PyDir = Join-Path $FfiDir "python" +$PyPkg = Join-Path $PyDir "ant_ffi" + +Write-Host "=== [1/5] build ant-ffi + bindgen (x86_64-pc-windows-msvc) ===" +Push-Location $RustDir +cargo build --release -p ant-ffi +cargo build --release --bin uniffi-bindgen +$Dll = Join-Path $RustDir "target\release\ant_ffi.dll" +if (!(Test-Path $Dll)) { throw "missing native library: $Dll" } +Pop-Location + +Write-Host "=== [2/5] bundle DLL next to the module ===" +New-Item -ItemType Directory -Force -Path $PyPkg | Out-Null +Copy-Item $Dll $PyPkg -Force + +Write-Host "=== [3/5] generate bindings ===" +$Bindgen = Join-Path $RustDir "target\release\uniffi-bindgen.exe" +& $Bindgen generate --library $Dll --language python --out-dir $PyPkg +if ($LASTEXITCODE -ne 0) { throw "uniffi-bindgen failed" } + +Write-Host "=== [4/5] build wheel (setup.py forces py3-none-win_amd64) ===" +$Venv = Join-Path $env:TEMP "antffi-wheel-venv" +python -m venv $Venv +& (Join-Path $Venv "Scripts\python.exe") -m pip install -q --upgrade pip setuptools wheel delvewheel +$Py = Join-Path $Venv "Scripts\python.exe" +Push-Location $PyDir +Remove-Item -Recurse -Force build, dist, *.egg-info -ErrorAction SilentlyContinue +& $Py setup.py -q bdist_wheel --plat-name win_amd64 +if ($LASTEXITCODE -ne 0) { throw "wheel build failed" } + +Write-Host "=== [5/5] delvewheel repair: bundle non-system DLLs ===" +New-Item -ItemType Directory -Force -Path wheelhouse | Out-Null +$Whl = (Get-ChildItem dist\*.whl | Select-Object -First 1).FullName +& $Py -m delvewheel repair $Whl -w wheelhouse -v +if ($LASTEXITCODE -ne 0) { throw "delvewheel repair failed" } +Write-Host "=== done -> $PyDir\wheelhouse\ ===" +Get-ChildItem wheelhouse +Pop-Location