diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 968937e0c..c1dd01b94 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -48,8 +48,13 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - + # Must match the pyenv-installed version in dev/docker/livy-dev-base/Dockerfile. + # Per each Spark release's python/setup.py: Spark 4.1 supports 3.10..3.14 + # (python_requires=">=3.10"), Spark 3.5 supports 3.8..3.11 + # (python_requires=">=3.8"). The intersection is 3.10 and 3.11; 3.11.11 + # is picked as the shared version. name: Set Python 3 as default - run: pyenv global 3 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV" + run: pyenv global 3.11.11 && echo "PYSPARK_PYTHON=$(which python3)" >> "$GITHUB_ENV" - name: Set JDK version run: update-alternatives --set java ${{ matrix.jdk_path }} diff --git a/dev/docker/livy-dev-base/Dockerfile b/dev/docker/livy-dev-base/Dockerfile index 5686e61d1..b0d06b735 100644 --- a/dev/docker/livy-dev-base/Dockerfile +++ b/dev/docker/livy-dev-base/Dockerfile @@ -70,8 +70,17 @@ RUN git clone https://github.com/pyenv/pyenv.git $HOME/pyenv ENV PYENV_ROOT=$HOME/pyenv ENV PATH="$HOME/pyenv/shims:$HOME/pyenv/bin:$HOME/bin:$PATH" -RUN pyenv install -v 3.9.21 && \ - pyenv global 3.9.21 && \ +# Python 3.11.11 is chosen because it is in the officially supported range of +# BOTH matrix profiles, per each Spark release's python/setup.py: +# * -Pspark4 (Spark 4.1.2): python_requires=">=3.10", classifiers list +# 3.10 / 3.11 / 3.12 / 3.13 / 3.14. +# * -Pspark3 (Spark 3.5.6): python_requires=">=3.8", classifiers list +# 3.8 / 3.9 / 3.10 / 3.11. +# The intersection is 3.10 and 3.11; we pick the higher one (3.11.11) so the +# same image serves both profiles. When bumping this, keep +# .github/workflows/integration-tests.yaml's `pyenv global` in sync. +RUN pyenv install -v 3.11.11 && \ + pyenv global 3.11.11 && \ pyenv rehash # Install build dependencies for python3 diff --git a/python-api/setup.py b/python-api/setup.py index 4d7b9b5f2..df1a4de1e 100644 --- a/python-api/setup.py +++ b/python-api/setup.py @@ -37,7 +37,11 @@ setup( name='livy-python-api', - version="1.0.0-SNAPSHOT", + # PEP 440 disallows Maven-style `-SNAPSHOT` suffixes; use `.dev0` instead, + # which is the canonical Python equivalent for "pre-release under active + # development". Modern pip (>= 24) rejects `1.0.0-SNAPSHOT` outright with + # `Invalid version: '1.0.0-SNAPSHOT'` when parsing this package. + version="1.0.0.dev0", packages=["livy", "livy-tests"], package_dir={ "": "src/main/python",