diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 13509029..37655562 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -11,9 +11,6 @@ on: samples-revision: default: 11.0.0 description: khiops-samples repo revision - image-tag: - default: latest - description: Development Docker Image Tag pypi-target: type: choice default: None # no publishing to any Package Index by default @@ -30,10 +27,7 @@ jobs: build: runs-on: ubuntu-22.04 permissions: - checks: write contents: read - id-token: write - packages: read steps: - name: Checkout sources uses: actions/checkout@v4 @@ -53,30 +47,63 @@ jobs: pip install build # Build the source package (sdist) only - python3 -m build --sdist + python -m build --sdist - name: Upload the package as artifact uses: actions/upload-artifact@v4 with: name: pip-package path: ./dist/khiops*.tar.gz + # Test Pip package on brand new environments test: needs: build - runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: - container: [ubuntu22.04, rocky9, debian13] - container: - # 'latest' default image tag cannot be set as an environment variable, - # because the `env` context is only accessible at the step level; - # hence, it is hard-coded - image: |- - ghcr.io/khiopsml/khiops-python/khiopspydev-${{ matrix.container }}:${{ inputs.image-tag || 'latest' }} + env: + # Test without virtualenv only on platforms that support it out of the + # box (i.e. where passing `--break-system-packages` to Pip is not + # required) + - {os: ubuntu-24.04, json-image: '{"image": null}', use-virtualenv: true} + - {os: ubuntu-24.04, json-image: '{"image": null}', use-virtualenv: false} + - {os: ubuntu-22.04, json-image: '{"image": "debian:trixie"}', use-virtualenv: true} + - {os: ubuntu-22.04, json-image: '{"image": "rockylinux/rockylinux:10"}', use-virtualenv: true} + - {os: ubuntu-22.04, json-image: '{"image": "rockylinux/rockylinux:10"}', use-virtualenv: false} + - {os: windows-2025, json-image: '{"image": null}', use-virtualenv: true} + - {os: windows-2025, json-image: '{"image": null}', use-virtualenv: false} + - {os: macos-15, json-image: '{"image": null}', use-virtualenv: true} + runs-on: ${{ matrix.env.os }} + container: ${{ fromJSON(matrix.env.json-image) }} steps: + - name: Make sure Python is installed on Linux (might not be in containers) + if: fromJSON(matrix.env.json-image).image != null + shell: sh + run: | + if command -v apt-get; then + apt-get update -y + apt-get install -y python3 python3-pip python3-venv + elif command -v dnf; then + dnf install -y python3 python3-pip + # Install findutils on Rocky Linux when xargs cannot be found + if ! command -v xargs; then + dnf install -y findutils + fi + fi - name: Set parameters as env + shell: bash run: | SAMPLES_REVISION=${{ inputs.samples-revision || env.DEFAULT_SAMPLES_REVISION }} echo "SAMPLES_REVISION=$SAMPLES_REVISION" >> "$GITHUB_ENV" + if [[ $RUNNER_OS == 'Windows' ]]; then + VENV_ACTIVATE_SCRIPT="khiops-venv/Scripts/activate" + else + VENV_ACTIVATE_SCRIPT="khiops-venv/bin/activate" + fi + echo "VENV_ACTIVATE_SCRIPT=$VENV_ACTIVATE_SCRIPT" >> "$GITHUB_ENV" + # Setup Python (be portable wrt. python vs python3 in runners and + # containers; either python3 or python is present as per the Python + # installation step in the Docker container contexts) + PYTHON=$(basename $(command -v python3 || command -v python)) + echo "PYTHON=$PYTHON" >> "$GITHUB_ENV" - name: Checkout sources # Checkout the sources to be able to extract the dependencies list uses: actions/checkout@v4 with: @@ -97,30 +124,37 @@ jobs: - name: Install the package shell: bash run: | - # Allow Pip to write to its cache - mkdir -p /github/home/.cache/pip - chown -R $(whoami) /github/home/.cache/pip + # Allow Pip to write to its cache (only applies to Docker containers) + if [[ -n "${{ fromJSON(matrix.env.json-image).image }}" ]]; then + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip + fi # Install the Khiops Python library - # A virtual env is mandatory under debian - if [[ "${{ matrix.container }}" == "debian13" ]]; then - python -m venv khiops-debian-venv - source khiops-debian-venv/bin/activate + # Use a virtualenv if required + if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then + $PYTHON -m venv khiops-venv + source "$VENV_ACTIVATE_SCRIPT" fi - pip install --upgrade pip + $PYTHON -m pip install --upgrade pip # Add homogeneous TOML support (Python >= 3.12 has standard tomllib) - pip install tomli + $PYTHON -c "import tomllib" 2>/dev/null || $PYTHON -m pip install tomli # First, install all dependencies except khiops-core and khiops-drivers-* - python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --exclude-khiops-family > requires-no-khiops.txt - pip install `cat requires-no-khiops.txt` + $PYTHON scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --exclude-khiops-family > requires-no-khiops.txt + [ ! -s requires-no-khiops.txt ] || xargs $PYTHON -m pip install < requires-no-khiops.txt + # On Windows, the `impi-rt` MPI dependency must be installed + # separately, from PyPI, because it is not present on TestPyPI + if [[ $RUNNER_OS == 'Windows' ]]; then + $PYTHON -m pip install impi-rt + fi # khiops-core and khiops-drivers-* must always be installed from TestPyPI in order to avoid distorting usage statistics - python scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --khiops-family-only > requires-khiops.txt - pip install --index-url https://test.pypi.org/simple `cat requires-khiops.txt` + $PYTHON scripts/extract_dependencies_from_pyproject_toml.py -f "pyproject.toml" --khiops-family-only > requires-khiops.txt + [ ! -s requires-khiops.txt ] || xargs $PYTHON -m pip install --index-url https://test.pypi.org/simple < requires-khiops.txt rm -f requires-khiops.txt requires-no-khiops.txt # Lastly, install khiops-python - pip install $(ls khiops*.tar.gz) - if [[ "${{ matrix.container }}" == "debian13" ]]; then + $PYTHON -m pip install khiops*.tar.gz + if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then deactivate fi - name: Run tests @@ -140,12 +174,13 @@ jobs: # modules are currently not initializing the shell anyway if [ -n "$MODULESHOME" ]; then module unload mpi; fi - # A virtual env is mandatory under debian - if [[ "${{ matrix.container }}" == "debian13" ]]; then - source khiops-debian-venv/bin/activate + # Use a virtualenv if required + if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then + source "$VENV_ACTIVATE_SCRIPT" fi - if [[ $RUNNER_OS != 'Windows' ]]; then - # Set rights to OpenMPI-specific folder + if [[ -n "${{ fromJSON(matrix.env.json-image).image }}" ]]; then + # Set rights to OpenMPI-specific folder (only applies to Docker + # containers) mkdir -p /github/home/.pmix/components chown -R $(whoami) /github/home/.pmix/components fi @@ -163,7 +198,7 @@ jobs: # The MPI command is not always named mpiexec, but can be orterun etc # (as given by khiops_env) kh-status | grep "MPI command" | grep -vwq "" - if [[ "${{ matrix.container }}" == "debian13" ]]; then + if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then deactivate fi - name: Test package / Git tag version coherence @@ -172,19 +207,18 @@ jobs: run: | # Don't exit on first error: print relevant error message set +e - # A virtual env is mandatory under Debian 13 - if [[ "${{ matrix.container }}" == "debian13" ]]; then - python -m venv khiops-debian-venv - source khiops-debian-venv/bin/activate + # Use a virtualenv if required + if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then + source "$VENV_ACTIVATE_SCRIPT" fi # Convert pre-release version specification in the Git tag to the Pip # format and check that it matches the Pip package version - PACKAGE_VERSION=$(pip show khiops | awk 'BEGIN {FS = ": "} $1 ~ /^Version$/ {print $2}') - if [[ "${{ matrix.container }}" == "debian13" ]]; then + PACKAGE_VERSION=$($PYTHON -m pip show khiops | awk 'BEGIN {FS = ": "} $1 ~ /^Version$/ {print $2}') + if [[ "${{ matrix.env.use-virtualenv }}" == "true" ]]; then deactivate fi - echo ${{ github.ref_name }} | tr -d '-' | rev | sed -E 's/\.([^0-9].*)/\1/' | rev | \ - grep -wq $PACKAGE_VERSION + echo "${{ github.ref_name }}" | tr -d '-' | rev | sed -E 's/\.([^0-9].*)/\1/' | rev | \ + grep -wq "$PACKAGE_VERSION" if [[ $? -ne 0 ]] then echo "::error::Python package version $PACKAGE_VERSION does not match Git tag ${{ github.ref_name }}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 90c6ada8..6fea83c3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,6 @@ name: Tests env: DEFAULT_SAMPLES_REVISION: 11.0.0 - DEFAULT_KHIOPS_DESKTOP_REVISION: 11.0.0 on: workflow_dispatch: inputs: @@ -12,9 +11,6 @@ on: image-tag: default: latest description: Development Docker Image Tag - khiops-desktop-revision: - default: 11.0.0 - description: Khiops Windows Desktop Application Version run-expensive-tests: type: boolean required: false @@ -229,25 +225,6 @@ jobs: check-khiops-integration-on-windows: runs-on: windows-2022 steps: - - name: Download the Khiops Desktop NSIS Installer - shell: pwsh - run: | - $KHIOPS_DESKTOP_REVISION = '${{ inputs.khiops-desktop-revision || env.DEFAULT_KHIOPS_DESKTOP_REVISION }}' - $KHIOPS_DOWNLOAD_URL = "https://github.com/KhiopsML/khiops/releases/download/${KHIOPS_DESKTOP_REVISION}/khiops-${KHIOPS_DESKTOP_REVISION}-setup.exe" - Invoke-WebRequest "${KHIOPS_DOWNLOAD_URL}" ` - -OutFile .\khiops-setup.exe ` - -UseBasicParsing - Unblock-File .\khiops-setup.exe - - name: Install the Khiops Desktop Application - shell: pwsh - run: | - # Execute the installer - $ErrorActionPreference = 'Stop' - $ProgressPreference = 'SilentlyContinue' - Start-Process ` - -FilePath .\khiops-setup.exe ` - -ArgumentList '/S' ` - -Wait - name: Checkout sources uses: actions/checkout@v4 with: diff --git a/khiops/core/internals/runner.py b/khiops/core/internals/runner.py index b16a8acd..e9d7cb71 100644 --- a/khiops/core/internals/runner.py +++ b/khiops/core/internals/runner.py @@ -20,6 +20,7 @@ import site import subprocess import sys +import sysconfig import tempfile import uuid import warnings @@ -955,53 +956,59 @@ def __init__(self): def _initialize_khiops_environment(self): installation_method = _infer_khiops_installation_method() - # in a 'pip' installation method, - # the current Khiops Python library depends on the Khiops binary-only - # PyPI package. - # Let's ensure this dependency has not disappeared. - if installation_method == "pip": - try: - distribution("khiops-core") - except PackageNotFoundError as exc: - raise KhiopsEnvironmentError( - f"The Khiops binaries are not installed properly: {exc}. " - "Re-install the Khiops Python library to automatically install " - "Khiops. Go to https://khiops.org for more information.\n" + match installation_method: + # In conda-based environments, khiops_env is not in PATH; + # its location must be inferred from the conda env directory. + case "conda-based": + khiops_env_path = os.path.join( + _infer_env_bin_dir_for_conda_based_installations(), "khiops_env" ) - # Search for the `khiops_env` script location - if platform.system() == "Windows": - sys_executable_direct_parent = Path(sys.executable).parents[0] - probable_khiops_env = os.path.join( - sys_executable_direct_parent, "khiops_env.cmd" - ) - # The script is found in the current environment - if os.path.exists(probable_khiops_env): - khiops_env_path = probable_khiops_env - # Raise error otherwise - else: - raise KhiopsEnvironmentError( - "No 'khiops_env.cmd' found in the current environment. " - "Make sure you have installed Khiops properly. " - "Go to https://khiops.org for more information." - ) - # In Conda-based environments, `khiops_env` might not be in the PATH, - # hence its path must be inferred - elif installation_method == "conda-based": - khiops_env_path = os.path.join( - _infer_env_bin_dir_for_conda_based_installations(), "khiops_env" - ) - if platform.system() == "Windows": - khiops_env_path += ".cmd" - - # On UNIX or Conda, khiops_env is always in path for a proper installation - else: - khiops_env_path = shutil.which("khiops_env") - if khiops_env_path is None: + if platform.system() == "Windows": + khiops_env_path += ".cmd" + # In an activated conda environment, khiops_env is in PATH. + case "conda": + khiops_env_path = self._infer_khiops_env_from_path(installation_method) + case "pip": + # Ensure the binary dependency is still installed. + try: + distribution("khiops-core") + except PackageNotFoundError as exc: + raise KhiopsEnvironmentError( + f"The Khiops binaries are not installed properly: {exc}. " + "Re-install the Khiops Python library to automatically install " + "Khiops. Go to https://khiops.org for more information.\n" + ) from exc + # On Windows, determine the Scripts directory where pip placed + # khiops_env.cmd. + # If this library is installed under the user site-packages + # directory, khiops-core (and its khiops_env.cmd) was also installed + # there with `pip install --user`, so use the user Scripts directory. + # Otherwise use the standard Scripts directory (venv or system-wide + # install). This avoids an ambiguous search and mirrors how pip + # resolves scripts. + if platform.system() == "Windows": + library_root_dir_path = Path(__file__).parents[2] + user_site_packages_path = Path(site.getusersitepackages()) + if library_root_dir_path.is_relative_to(user_site_packages_path): + scripts_dir = sysconfig.get_path("scripts", "nt_user") + else: + scripts_dir = sysconfig.get_path("scripts") + khiops_env_path = os.path.join(scripts_dir, "khiops_env.cmd") + if not os.path.exists(khiops_env_path): + raise KhiopsEnvironmentError( + "No 'khiops_env.cmd' found in the current environment. " + "Make sure you have installed Khiops properly. " + "Go to https://khiops.org for more information." + ) + # On UNIX, pip places khiops_env in the bin directory, + # which is in PATH. + else: + khiops_env_path = self._infer_khiops_env_from_path( + installation_method + ) + case _: raise KhiopsEnvironmentError( - "The 'khiops_env' script not found for the current " - f"'{installation_method}' installation method. Make sure " - "you have installed Khiops properly. " - "Go to https://khiops.org for more information." + f"Unknown installation method '{installation_method}'." ) with subprocess.Popen( @@ -1063,6 +1070,17 @@ def _initialize_khiops_environment(self): # Initialize the default samples dir self._initialize_default_samples_dir() + def _infer_khiops_env_from_path(self, installation_method): + khiops_env_path = shutil.which("khiops_env") + if khiops_env_path is None: + raise KhiopsEnvironmentError( + "The 'khiops_env' script not found for the current " + f"'{installation_method}' installation method. Make sure " + "you have installed Khiops properly. " + "Go to https://khiops.org for more information." + ) + return khiops_env_path + def _initialize_default_samples_dir(self): """See class docstring""" samples_dir = get_default_samples_dir() @@ -1223,13 +1241,13 @@ def _detect_library_installation_incompatibilities(self, library_root_dir_path): platform.system() == "Windows" and # Under Windows, there are two cases : - ( - # for conda-based installations python is inside 'base_dir' - sys_executable_direct_parent != base_dir_path - and - # for 'pip' installations (within a virtual env) - # python is inside 'base_dir'/Scripts - sys_executable_grand_parent != base_dir_path + # for conda-based installations python is inside 'base_dir' + # for 'pip' installations (within a virtual env) + # python is inside 'base_dir'/Scripts + base_dir_path + not in ( + sys_executable_direct_parent, + sys_executable_grand_parent, ) # Under Linux or MacOS a bin/ folder exists or sys_executable_grand_parent != base_dir_path