diff --git a/README.md b/README.md index f5969a1e4..112d932ad 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,18 @@ Before you can run this tool, make sure the below requirements are set on your s source install requirements - [Other Odoo dependencies](https://www.odoo.com/documentation/19.0/administration/on_premise/source.html#dependencies) +Odoo builds part of its Python dependencies (`gevent`, `lxml`, `python-ldap`, …) from source, which needs a C compiler +and the matching development packages: the headers of the Python version Odoo runs on, the PostgreSQL client library +and the OpenLDAP and SASL headers. On Debian and Ubuntu, install them all from the Odoo sources Odev has cloned: + +```sh +sudo ~/odoo/repositories/odoo/odoo/setup/debinstall.sh +``` + +On Fedora, Arch, openSUSE, Alpine or macOS, install the equivalents with your own package manager. Odev checks +whenever it creates a virtual environment for a version of Odoo and, whatever the system, tells you what is missing +along with the command that installs it. It never installs anything itself. + Make sure `git` is properly setup with SSH key authentication before using commands, as Odev will try to connect to the Odoo [Community](https://github.com/odoo/odoo) and [Enterprise](https://github.com/odoo/enterprise) repositories to pull sources when required. diff --git a/odev/_version.py b/odev/_version.py index 552d766e1..3b405064c 100644 --- a/odev/_version.py +++ b/odev/_version.py @@ -22,4 +22,4 @@ # or merged change. # ------------------------------------------------------------------------------ -__version__ = "4.30.0" +__version__ = "4.30.1" diff --git a/odev/common/odoobin.py b/odev/common/odoobin.py index 8b0e881d0..a41993b41 100644 --- a/odev/common/odoobin.py +++ b/odev/common/odoobin.py @@ -2,6 +2,8 @@ import re import shlex +import shutil +import sys from ast import literal_eval from collections.abc import Callable, Generator, Mapping, Sequence from contextlib import nullcontext @@ -17,7 +19,7 @@ from packaging.version import Version -from odev.common import bash, string +from odev.common import bash, string, system from odev.common.cache import TTLCache from odev.common.connectors import GitConnector, GitWorktree from odev.common.databases import Branch, Repository @@ -29,6 +31,7 @@ from odev.common.progress import spinner from odev.common.python import PythonEnv from odev.common.signal_handling import capture_signals +from odev.common.system import SystemDependency from odev.common.version import OdooVersion @@ -51,6 +54,22 @@ ODOO_UPGRADE_REPOSITORY: str = "odoo/upgrade" +SETUPTOOLS_REQUIREMENT: str = "setuptools>=69.0.0,<82" +"""Version of setuptools to install in the virtual environments of Odoo installations. +At least 69 because setuptools 58 and 59 cannot serve as a PEP 517 backend for current pip, +which breaks installing gevent and other source distributions with `--no-build-isolation`. +Below 82 because it removed `pkg_resources`, which Odoo 15.0 and 16.0 import unconditionally. +""" + +DEBIAN_INSTALL_SCRIPT: str = "setup/debinstall.sh" +"""Path of the script listing and installing Odoo's system dependencies, relative to the Odoo sources.""" + +PROBED_DEPENDENCIES: Sequence[SystemDependency] = (system.C_COMPILER, system.POSTGRESQL_HEADERS) +"""System dependencies looked for on systems where Odoo does not list its own packages. Restricted +to the ones an executable on the `PATH` proves present, so that nothing is ever reported as missing +on a distribution that just names it differently. +""" + ODOO_PYTHON_VERSIONS: Mapping[int, str] = { 19: "3.12", @@ -711,6 +730,82 @@ def missing_npm_packages(self, packages: Sequence[str]) -> Generator[str, None, if f" {package}" not in installed_packages: yield package + def missing_debian_packages(self) -> list[str] | None: + """List the packages Odoo declares in `debian/control` and that are not installed. + + :return: The names of the missing packages, or None on a system where Odoo does not + describe them or where they cannot be queried. + :rtype: Optional[List[str]] + """ + script = self.odoo_path / DEBIAN_INSTALL_SCRIPT + + if not script.is_file() or not shutil.which("dpkg-query"): + return None + + # `--list` only prints the package names parsed out of `debian/control`, it needs no privileges + listed = bash.execute(f"sh {shlex.quote(script.as_posix())} --list", raise_on_error=False) + + if listed is None: + return None + + packages = listed.stdout.decode().split() + installed = bash.execute( + "dpkg-query --show --showformat '${Package} ${Status}\\n' " + " ".join(map(shlex.quote, packages)), + raise_on_error=False, + ) + satisfied = { + line.split(" ", 1)[0] + for line in (installed.stdout.decode().splitlines() if installed else []) + if line.endswith("install ok installed") + } + return [package for package in packages if package not in satisfied] + + def missing_system_dependencies(self) -> list[SystemDependency]: + """List what Odoo needs to build its python dependencies from source and that is missing. + + Odev runs on systems it does not control, so this never assumes a distribution: where Odoo + describes its own packages they are used, and anywhere else only what can be proven missing + is reported. + + :return: The missing dependencies. + :rtype: List[SystemDependency] + """ + if sys.platform not in {"linux", "darwin"}: + return [] + + debian_packages = self.missing_debian_packages() + + if debian_packages is None: + missing = [dependency for dependency in PROBED_DEPENDENCIES if not dependency.found] + else: + missing = [SystemDependency(name=package, packages={"apt-get": package}) for package in debian_packages] + + # Checked everywhere: `debian/control` asks for `python3-dev`, which is not necessarily the + # version of python the Odoo installation is built against + if self.venv.exists and not self.venv.has_development_headers: + missing.append(system.PYTHON_HEADERS) + + return missing + + def check_system_dependencies(self) -> None: + """Warn about what Odoo needs to build its python dependencies from source and that is + missing from the system. + + Building gevent, python-ldap or lxml from source fails with errors naming a missing header + rather than the package providing it, so this is checked when a virtual environment is + created. Nothing is installed: the machine odev runs on belongs to the user. + """ + missing = self.missing_system_dependencies() + + if not missing: + return + + logger.warning( + f"{len(missing)} system dependencies required by Odoo are missing, building its python " + "dependencies from source is likely to fail:\n" + + system.install_instructions(missing, version=self.venv.version) + ) + def prepare_venv(self): """Prepare the virtual environment of the Odoo installation.""" if not self.database.exists: @@ -718,7 +813,10 @@ def prepare_venv(self): if not self.venv.exists: self.venv.create() - self.venv.install_packages(["wheel", "setuptools", "pip", "cython<3.0.0"]) + # After creating the environment: the interpreter it was created from is the one whose + # development headers matter, and the warning still precedes the installs that need them + self.check_system_dependencies() + self.venv.install_packages(["wheel", SETUPTOOLS_REQUIREMENT, "pip", "cython<3.0.0"]) self.venv.install_packages(["pyyaml==5.4.1"], ["--no-build-isolation"]) for path in self.addons_requirements: @@ -728,6 +826,12 @@ def prepare_venv(self): ) if missing_gevent: + # `--no-build-isolation` builds against the setuptools of the virtual environment + # rather than a fresh one, so it has to be usable as a PEP 517 backend. Custom addons + # requirements are installed before odev's own, so this may still be the old one. + if not self.venv.satisfies(SETUPTOOLS_REQUIREMENT): + self.venv.install_packages([SETUPTOOLS_REQUIREMENT, "wheel"]) + self.venv.install_packages([missing_gevent.split(" ;")[0]], ["--no-build-isolation"]) if any(self.venv.missing_requirements(path)): diff --git a/odev/common/python.py b/odev/common/python.py index d67223f7b..94be1ab78 100644 --- a/odev/common/python.py +++ b/odev/common/python.py @@ -11,9 +11,11 @@ from typing import ClassVar import virtualenv +from packaging.markers import default_environment +from packaging.requirements import InvalidRequirement, Requirement from packaging.version import InvalidVersion, Version, parse as parse_version -from odev.common import bash, progress, string +from odev.common import bash, progress, string, system from odev.common.cache import TTLCache from odev.common.console import console from odev.common.errors import OdevError @@ -43,23 +45,11 @@ re.VERBOSE | re.IGNORECASE, ) -OS_PACKAGES = { - "dnf": [ - "gcc", - "libpq-devel", - "openldap-devel", - "python{version}-devel", - "python{version}", - ], - "apt": [ - "gcc", - "libldap2-dev", - "libpq-dev", - "libsasl2-dev", - "python{version}-dev", - "python{version}", - ], -} +INSTALLABLE_PACKAGE_MANAGERS: frozenset[str] = frozenset({"apt-get", "dnf"}) +"""Package managers odev installs packages with on its own. Anywhere else it only reports what is +missing and lets the user install it, as installing the wrong package under `sudo` on a system odev +has never been tested against is worse than saying nothing. +""" @lru_cache @@ -194,7 +184,11 @@ def create(self) -> None: ): raise OdevError("Failed to create virtual environment") from error - self.install_system_packages() + # Only retry if something was actually installed, otherwise the same question + # would be asked over and over on a system odev cannot install packages on + if not self.install_system_packages(): + raise OdevError("Failed to create virtual environment") from error + return self.create() raise OdevError("Failed to create virtual environment") from error @@ -213,21 +207,34 @@ def remove(self) -> None: return logger.info(f"Removed {venv_description}") - def install_system_packages(self) -> None: - """Install system packages for the current python version.""" + def install_system_packages(self) -> bool: + """Install the system packages needed to run and build against the current python version. + + Odev only installs packages on the systems it has been tested against; on any other one it + reports what is missing and how to install it, and leaves it to the user. + + :return: Whether packages were installed, so that callers know a retry is worth it. + :rtype: bool + """ if self._global: raise OdevError("Cannot install system packages for the global python interpreter") - with progress.spinner("Installing system packages"): - package_manager = next((pkg for pkg in OS_PACKAGES if shutil.which(pkg)), None) + package_manager = system.package_manager() - if not package_manager: - raise OdevError( - f"Neither {string.join_or(list(OS_PACKAGES.keys()))} package managers found on the system, " - "cannot install packages" - ) + if package_manager not in INSTALLABLE_PACKAGE_MANAGERS: + logger.warning( + f"Odev cannot install packages on this system, python {self.version} and the packages " + "needed to build Odoo's dependencies have to be installed manually:\n" + + system.install_instructions(system.ODOO_SYSTEM_DEPENDENCIES, version=self.version) + ) + return False - packages = " ".join([pkg.format(version=self.version) for pkg in OS_PACKAGES[package_manager]]) + with progress.spinner("Installing system packages"): + packages = " ".join( + package + for dependency in system.ODOO_SYSTEM_DEPENDENCIES + if (package := dependency.package(package_manager, version=self.version)) + ) logger.info( f"The following packages will be installed using {package_manager}:\n" + string.join_bullet(packages.split()) @@ -235,7 +242,7 @@ def install_system_packages(self) -> None: if not console.confirm("Continue?", default=True): logger.warning("Aborting system package installation") - return + return False try: bash.execute(f"{package_manager} install -y {packages}", sudo=True) @@ -255,6 +262,27 @@ def install_system_packages(self) -> None: bash.execute(f"ln -s {lldap} {lldap_r}", sudo=True) logger.info(f"Installed system packages for python {self.version}") + return True + + @property + def has_development_headers(self) -> bool: + """Whether the C headers needed to build python extensions are available for this interpreter. + + `INCLUDEPY` points to the headers of the interpreter this environment was created from, which + most distributions ship in a separate package (`python3.10-dev`, `python3.10-devel`). Building + `gevent` or `python-ldap` from source fails without them, with an error naming a missing + `Python.h` rather than the package that provides it. + """ + headers = bash.execute( + f"{self.python} -c 'import sysconfig; print(sysconfig.get_config_var(\"INCLUDEPY\"))'", + raise_on_error=False, + ) + + if headers is None: + # Never warn on a guess: an interpreter that cannot be questioned is assumed complete + return True + + return Path(headers.stdout.decode().strip(), "Python.h").is_file() def install_packages(self, packages: list[str], options: list[str] | None = None) -> None: """Install python packages. @@ -465,43 +493,32 @@ def missing_requirements(self, path: Path | str, raise_if_error: bool = True) -> continue - match = RE_PACKAGE.search(line) - - if match is None: + try: + requirement = Requirement(line) + except InvalidRequirement: + logger.debug(f"Ignoring unparsable requirement {line!r}") continue - if not self.__check_package_conditions(match.group("conditional")): + if requirement.marker is not None and not requirement.marker.evaluate(self.marker_environment): continue - installed_version = installed_packages.get(match.group("name").lower()) + installed_version = installed_packages.get(requirement.name.lower()) if installed_version is None: - logger.debug(f"Missing python package {match.group('name')}") + logger.debug(f"Missing python package {requirement.name}") yield line continue if not isinstance(installed_version, Version): - raise TypeError(f"Invalid version {installed_version!r} for python package {match.group('name')}") + raise TypeError(f"Invalid version {installed_version!r} for python package {requirement.name}") - if match.group("version") is None and match.group("op") is None: + if not requirement.specifier: continue - package_operator = match.group("op") - - if package_operator is None: - continue - - package_version = match.group("version").split("*", 1)[0].rstrip(".") - - version_locals = { - "installed_version": installed_version, - "package_version": parse_version(package_version), - } - - if not eval(f"installed_version {package_operator} package_version", version_locals): # noqa: S307 - known values + if not requirement.specifier.contains(installed_version, prereleases=True): logger.debug( - f"Incorrect python package version {match.group('name')} " - f"({installed_version} {package_operator} {package_version})" + f"Incorrect python package version {requirement.name} " + f"({installed_version} does not satisfy {requirement.specifier})" ) yield line @@ -516,24 +533,36 @@ def __check_requirements_path(self, path: Path | str) -> Path: return requirements_path - def __check_package_conditions(self, conditional: str | None) -> bool: - if conditional is None: - return True + def satisfies(self, specification: str) -> bool: + """Check whether a package installed in this environment satisfies a requirement. - if "python_version" in conditional: - conditional = re.sub( - r"(?:'|\")(3.\d+)(?:'|\")", - lambda m: m.group(1) and " {} ".format(int(m.group(1).replace(".", ""))), - conditional, - ) + :param specification: The requirement to check, e.g. `setuptools>=69.0.0,<82`. + :return: True if the package is installed and its version satisfies the requirement. + :rtype: bool + """ + requirement = Requirement(specification) + installed_version = self.installed_packages().get(requirement.name.lower()) - return eval( # noqa: S307 - known values and operations - conditional, - { - "sys_platform": sys.platform, - "python_version": int(self.version.replace(".", "")), - }, - ) + if not isinstance(installed_version, Version): + return False + + return requirement.specifier.contains(installed_version, prereleases=True) + + @property + def marker_environment(self) -> MutableMapping[str, str]: + """Environment against which the markers of a requirement are evaluated. + + Requirements are evaluated for the python of *this* environment, not for the one odev + itself runs under, so that a requirement conditioned on the python version is resolved + for the Odoo installation it is going to be installed in. + """ + version = self.version + return { + **default_environment(), + "python_version": version, + "python_full_version": version, + "implementation_version": version, + } def run_script( self, diff --git a/odev/common/system.py b/odev/common/system.py new file mode 100644 index 000000000..ea21de19b --- /dev/null +++ b/odev/common/system.py @@ -0,0 +1,174 @@ +"""Description of the operating system odev runs on and of the packages it can install.""" + +import shutil +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field + +from odev.common import string + + +__all__ = [ + "ODOO_SYSTEM_DEPENDENCIES", + "SystemDependency", + "install_instructions", + "package_manager", +] + + +PACKAGE_MANAGERS: Mapping[str, str] = { + "apt-get": "sudo apt-get install -y {packages}", + "dnf": "sudo dnf install -y {packages}", + "zypper": "sudo zypper install -y {packages}", + "pacman": "sudo pacman -S --needed {packages}", + "apk": "sudo apk add {packages}", + "brew": "brew install {packages}", +} +"""Command installing packages with each of the package managers odev knows about, keyed by the +executable to look for on the `PATH`. Package managers shipped by the distribution come first so +that a Linux machine on which Homebrew is also installed is told to use its own. +""" + + +@dataclass(frozen=True) +class SystemDependency: + """A part of the operating system Odoo needs to build its python dependencies from source.""" + + name: str + """Human-readable description, displayed as-is when no package can be named for the current + system. May contain placeholders, `{version}` being the version of python being installed. + """ + + packages: Mapping[str, str] = field(default_factory=dict) + """Package providing this dependency, per package manager. A package manager absent from this + mapping has nothing to install: Arch Linux ships the python headers within `python` itself and + the compiler of macOS comes from `xcode-select --install`, not from Homebrew. + """ + + commands: Sequence[str] = () + """Executables proving, if any of them is found on the `PATH`, that this dependency is + installed. Dependencies detected some other way leave this empty and are never found. + """ + + @property + def found(self) -> bool: + """Whether the dependency can be proven to be installed on the current system.""" + return any(shutil.which(command) for command in self.commands) + + def package(self, manager: str, **placeholders: str) -> str | None: + """Package to install to provide this dependency with a given package manager. + + :param manager: The package manager to install the dependency with. + :param placeholders: Values for the placeholders in the name of the package. + :return: The name of the package, or None if the package manager does not ship one. + :rtype: Optional[str] + """ + package = self.packages.get(manager) + return package.format(**placeholders) if package is not None else None + + +PYTHON_INTERPRETER = SystemDependency( + name="the python {version} interpreter", + packages={ + "apt-get": "python{version}", + "dnf": "python{version}", + "zypper": "python{version}", + "pacman": "python", + "apk": "python3", + "brew": "python@{version}", + }, +) + +PYTHON_HEADERS = SystemDependency( + name="the development headers of python {version} (Python.h)", + packages={ + "apt-get": "python{version}-dev", + "dnf": "python{version}-devel", + "zypper": "python{version}-devel", + "apk": "python3-dev", + }, +) + +C_COMPILER = SystemDependency( + name="a C compiler", + packages={ + "apt-get": "build-essential", + "dnf": "gcc", + "zypper": "gcc", + "pacman": "base-devel", + "apk": "build-base", + }, + commands=("cc", "gcc", "clang"), +) + +POSTGRESQL_HEADERS = SystemDependency( + name="the PostgreSQL client headers (pg_config)", + packages={ + "apt-get": "libpq-dev", + "dnf": "libpq-devel", + "zypper": "postgresql-devel", + "pacman": "postgresql-libs", + "apk": "postgresql-dev", + "brew": "libpq", + }, + commands=("pg_config",), +) + +LDAP_HEADERS = SystemDependency( + name="the OpenLDAP and SASL headers", + packages={ + "apt-get": "libldap2-dev libsasl2-dev", + "dnf": "openldap-devel", + "zypper": "openldap2-devel cyrus-sasl-devel", + "pacman": "libldap libsasl", + "apk": "openldap-dev", + "brew": "openldap", + }, +) + +ODOO_SYSTEM_DEPENDENCIES: Sequence[SystemDependency] = ( + PYTHON_INTERPRETER, + PYTHON_HEADERS, + C_COMPILER, + POSTGRESQL_HEADERS, + LDAP_HEADERS, +) +"""Everything Odoo needs to build its python dependencies from source, in the order in which it is +worth installing it. +""" + + +def package_manager() -> str | None: + """Find the package manager of the current system. + + :return: The name of the package manager, or None if odev does not know the one in use. + :rtype: Optional[str] + """ + return next((manager for manager in PACKAGE_MANAGERS if shutil.which(manager)), None) + + +def install_instructions(dependencies: Sequence[SystemDependency], **placeholders: str) -> str: + """Describe what is missing from the current system and how to install it. + + Odev runs on systems it does not control, and cannot name a package for all of them, so the + dependencies are always described in plain words; the command is only added on top when the + package manager in use is one odev knows. + + :param dependencies: The dependencies missing from the current system. + :param placeholders: Values for the placeholders in the names of the dependencies and of their + packages, `version` being the version of python being installed. + :return: A bullet list of the missing dependencies, followed by the command installing them. + :rtype: str + """ + instructions = string.join_bullet([dependency.name.format(**placeholders) for dependency in dependencies]) + manager = package_manager() + + if manager is None: + return instructions + + packages = [package for dependency in dependencies if (package := dependency.package(manager, **placeholders))] + + if not packages: + return instructions + + command = PACKAGE_MANAGERS[manager].format(packages=" ".join(packages)) + return f"{instructions}\n\nInstall them by running:\n{string.stylize(command, 'color.cyan')}" diff --git a/odev/static/requirements.txt b/odev/static/requirements.txt index c5bbc4f71..7c832434f 100644 --- a/odev/static/requirements.txt +++ b/odev/static/requirements.txt @@ -2,7 +2,9 @@ ipdb phonenumbers pudb pydevd-odoo +# setuptools 58-59 cannot serve as a PEP 517 backend for current pip, which breaks +# `pip install --no-build-isolation` for gevent and other source distributions. +# Capped below 82, which removed `pkg_resources`; Odoo 15.0 and 16.0 import it unconditionally. +setuptools>=69.0.0,<82; python_version >= '3.8' setuptools<58.0.0; python_version < '3.8' -setuptools>=58.0.0, <59.0.0; python_version >= '3.8' and python_version < '3.12' -setuptools>59.0.0; python_version >= '3.12' websocket-client diff --git a/tests/tests/common/test_odoobin_prepare_venv.py b/tests/tests/common/test_odoobin_prepare_venv.py new file mode 100644 index 000000000..501b46330 --- /dev/null +++ b/tests/tests/common/test_odoobin_prepare_venv.py @@ -0,0 +1,233 @@ +"""Tests for the python and system packages odev needs to prepare an Odoo installation.""" + +from collections.abc import Sequence +from pathlib import Path +from unittest.mock import MagicMock, patch + +from packaging.requirements import Requirement +from packaging.version import Version + +from odev.common import system +from odev.common.odoobin import DEBIAN_INSTALL_SCRIPT, SETUPTOOLS_REQUIREMENT, OdoobinProcess +from odev.common.python import PythonEnv + +from tests.fixtures import OdevTestCase + + +class TestSetuptoolsRequirement(OdevTestCase): + """Guardrails for #93 and for the Odoo versions that cannot run on a recent setuptools.""" + + def setuptools_requirement(self, python_version: str) -> Requirement: + """Return the setuptools requirement that applies to a given python version.""" + text = (self.odev.static_path / "requirements.txt").read_text(encoding="utf-8") + requirements = [ + Requirement(stripped) + for line in text.splitlines() + if (stripped := line.split("#", 1)[0].strip()) and Requirement(stripped).name == "setuptools" + ] + return next( + requirement + for requirement in requirements + if requirement.marker is None or requirement.marker.evaluate({"python_version": python_version}) + ) + + def test_modern_python_requires_a_usable_pep517_backend(self): + """Setuptools 58 and 59 cannot load `setuptools.build_meta` for current pip, which is what + breaks installing gevent with `--no-build-isolation`. + """ + specifier = self.setuptools_requirement("3.10").specifier + self.assertFalse(specifier.contains(Version("58.0.0")), "regression of #93") + self.assertFalse(specifier.contains(Version("59.0.0")), "regression of #93") + self.assertTrue(specifier.contains(Version("69.0.0"))) + + def test_modern_python_excludes_setuptools_without_pkg_resources(self): + """Setuptools 82 removed `pkg_resources`, which Odoo 15.0 and 16.0 import unconditionally + in `odoo/modules/module.py`; both run on python 3.10. + """ + specifier = self.setuptools_requirement("3.10").specifier + self.assertTrue(specifier.contains(Version("81.2.0"))) + self.assertFalse(specifier.contains(Version("82.0.0"))) + self.assertFalse(specifier.contains(Version("83.0.0"))) + + def test_legacy_python_keeps_the_legacy_cap(self): + """Odoo 13.0 and older run on python 3.7 or 2.7, where setuptools must stay old.""" + specifier = self.setuptools_requirement("3.7").specifier + self.assertTrue(specifier.contains(Version("57.5.0"))) + self.assertFalse(specifier.contains(Version("69.0.0"))) + + def test_constant_matches_the_static_requirements(self): + self.assertEqual( + Requirement(SETUPTOOLS_REQUIREMENT).specifier, + self.setuptools_requirement("3.10").specifier, + ) + + +class TestMissingRequirements(OdevTestCase): + """`missing_requirements` must honour the full specifier and the markers of a requirement.""" + + def write_requirements(self, *lines: str) -> Path: + path = self.run_path / "requirements.txt" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + return path + + def missing(self, requirement: str, installed: dict[str, Version], python_version: str = "3.10") -> list[str]: + venv = PythonEnv(str(self.run_path / "venv")) + path = self.write_requirements(requirement) + + with ( + self.patch(PythonEnv, "installed_packages", return_value=installed), + self.patch_property(PythonEnv, "version", python_version), + ): + return list(venv.missing_requirements(path)) + + def test_upper_bound_is_honoured(self): + """The requirement parser only kept the first operator of a specifier, so an upper bound + was silently dropped and an over-new package was considered satisfied. + """ + requirement = "setuptools>=69.0.0,<82; python_version >= '3.8'" + self.assertEqual(self.missing(requirement, {"setuptools": Version("83.0.0")}), [requirement]) + self.assertEqual(self.missing(requirement, {"setuptools": Version("75.0.0")}), []) + + def test_lower_bound_is_honoured(self): + requirement = "setuptools>=69.0.0,<82; python_version >= '3.8'" + self.assertEqual(self.missing(requirement, {"setuptools": Version("58.0.0")}), [requirement]) + + def test_marker_is_evaluated_for_the_environment_python(self): + """Markers apply to the python of the virtual environment, not to the one odev runs under.""" + requirement = "setuptools<58.0.0; python_version < '3.8'" + self.assertEqual(self.missing(requirement, {"setuptools": Version("69.0.0")}, python_version="3.10"), []) + self.assertEqual( + self.missing(requirement, {"setuptools": Version("69.0.0")}, python_version="3.7"), + [requirement], + ) + + def test_missing_package_is_reported(self): + self.assertEqual(self.missing("phonenumbers", {}), ["phonenumbers"]) + + def test_unpinned_installed_package_is_satisfied(self): + self.assertEqual(self.missing("phonenumbers", {"phonenumbers": Version("8.13.0")}), []) + + def test_comments_and_blank_lines_are_skipped(self): + self.assertEqual(self.missing("# just a comment", {}), []) + + +class TestSatisfies(OdevTestCase): + """`PythonEnv.satisfies` answers whether an installed package matches a specification.""" + + def satisfies(self, specification: str, installed: dict[str, Version]) -> bool: + venv = PythonEnv(str(self.run_path / "venv")) + + with self.patch(PythonEnv, "installed_packages", return_value=installed): + return venv.satisfies(specification) + + def test_within_bounds(self): + self.assertTrue(self.satisfies(SETUPTOOLS_REQUIREMENT, {"setuptools": Version("75.0.0")})) + + def test_below_lower_bound(self): + self.assertFalse(self.satisfies(SETUPTOOLS_REQUIREMENT, {"setuptools": Version("58.0.0")})) + + def test_above_upper_bound(self): + self.assertFalse(self.satisfies(SETUPTOOLS_REQUIREMENT, {"setuptools": Version("83.0.0")})) + + def test_not_installed(self): + self.assertFalse(self.satisfies(SETUPTOOLS_REQUIREMENT, {})) + + +class TestSystemDependencies(OdevTestCase): + """The system dependencies check must work on any unix and never install anything itself.""" + + def setUp(self): + super().setUp() + self.odoo_path = self.run_path / "odoo" + (self.odoo_path / "setup").mkdir(parents=True, exist_ok=True) + self.process = OdoobinProcess.__new__(OdoobinProcess) + self.process._framework = self.odev + + def write_script(self): + (self.odoo_path / DEBIAN_INSTALL_SCRIPT).write_text("#!/bin/sh\n", encoding="utf-8") + + def missing( + self, + platform: str = "linux", + on_path: Sequence[str] = (), + debian_packages: list[str] | None = None, + headers: bool = True, + ) -> list[str]: + """Return the names of the dependencies reported as missing on a simulated system. + + :param platform: The value of `sys.platform` to simulate. + :param on_path: The executables available on the `PATH`. + :param debian_packages: The packages Odoo declares and that are missing, None on a system + where they cannot be listed. + :param headers: Whether the interpreter ships its development headers. + """ + venv = MagicMock(exists=True, version="3.10", has_development_headers=headers) + + with ( + self.patch_property(OdoobinProcess, "odoo_path", self.odoo_path), + self.patch_property(OdoobinProcess, "venv", venv), + self.patch(OdoobinProcess, "missing_debian_packages", return_value=debian_packages), + patch("odev.common.odoobin.sys.platform", platform), + patch("odev.common.system.shutil.which", side_effect=lambda command: command in on_path or None), + ): + return [dependency.name for dependency in self.process.missing_system_dependencies()] + + def test_probes_run_where_odoo_lists_no_package(self): + """Regression guard: a system that is not Debian-based used to be reported as complete, and + the user only found out when the build of gevent failed in the compiler. + """ + self.assertEqual( + self.missing(platform="darwin"), + [system.C_COMPILER.name, system.POSTGRESQL_HEADERS.name], + ) + + def test_probes_are_satisfied_by_any_of_their_commands(self): + self.assertEqual(self.missing(platform="darwin", on_path=["clang"]), [system.POSTGRESQL_HEADERS.name]) + self.assertEqual(self.missing(platform="darwin", on_path=["clang", "pg_config"]), []) + + def test_packages_declared_by_odoo_take_over_the_probes(self): + self.assertEqual(self.missing(debian_packages=["libpq-dev", "libsasl2-dev"]), ["libpq-dev", "libsasl2-dev"]) + + def test_headers_are_checked_even_where_odoo_lists_its_packages(self): + """`debian/control` asks for `python3-dev`, not for the headers of the version of python the + Odoo installation is actually built against. + """ + self.assertEqual(self.missing(debian_packages=[], headers=False), [system.PYTHON_HEADERS.name]) + + def test_no_check_outside_unix(self): + self.assertEqual(self.missing(platform="win32", headers=False), []) + + def test_nothing_reported_on_a_complete_system(self): + self.assertEqual(self.missing(debian_packages=[]), []) + + def test_never_installs_anything(self): + """The machine odev runs on belongs to the user: the check reports and never escalates.""" + run, execute = MagicMock(), MagicMock() + venv = MagicMock(exists=True, version="3.10", has_development_headers=False) + + with ( + self.patch_property(OdoobinProcess, "venv", venv), + self.patch( + OdoobinProcess, + "missing_system_dependencies", + return_value=[system.C_COMPILER, system.POSTGRESQL_HEADERS], + ), + patch("odev.common.odoobin.bash.run", run), + patch("odev.common.odoobin.bash.execute", execute), + ): + self.process.check_system_dependencies() + + run.assert_not_called() + execute.assert_not_called() + + def test_nothing_happens_when_no_dependency_is_missing(self): + run = MagicMock() + + with ( + self.patch(OdoobinProcess, "missing_system_dependencies", return_value=[]), + patch("odev.common.odoobin.bash.run", run), + ): + self.process.check_system_dependencies() + + run.assert_not_called() diff --git a/tests/tests/common/test_python_env.py b/tests/tests/common/test_python_env.py index 0ca92fd12..f6a7d8348 100644 --- a/tests/tests/common/test_python_env.py +++ b/tests/tests/common/test_python_env.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch from odev.common import bash +from odev.common.errors import OdevError from odev.common.python import PythonEnv from tests.fixtures import OdevTestCase @@ -55,3 +56,75 @@ def streaming_failure(command, **_kwargs): self.assertIsInstance(result, CompletedProcess) self.assertEqual(result.returncode, 1) stream_filter_mock.assert_called_once_with("line 1") + + +class TestSystemPackages(OdevTestCase): + """Odev runs on systems whose package manager it does not know, and must say what is missing + rather than dead-end on them. + """ + + @classmethod + def setUpClass(cls): + with patch("odev.common.odev.Odev.start", return_value=None): + super().setUpClass() + + def setUp(self): + super().setUp() + self.env = PythonEnv(path=self.run_path / "venv", version="3.10") + + def on_path(self, *commands: str): + return patch("odev.common.system.shutil.which", side_effect=lambda command: command in commands or None) + + def test_reports_without_a_known_package_manager(self): + """Used to raise `Neither dnf or apt package managers found on the system`, which named + neither the packages to install nor a way to move forward. + """ + with self.on_path(): + self.assertFalse(self.env.install_system_packages()) + + def test_reports_on_a_package_manager_odev_does_not_install_with(self): + """Arch is understood well enough to name its packages, not well enough to run `sudo` on.""" + with self.on_path("pacman"): + self.assertFalse(self.env.install_system_packages()) + + def test_creation_gives_up_once_when_nothing_can_be_installed(self): + """`create` used to retry unconditionally, asking the same question again on a system where + the answer could not change anything. + """ + error = RuntimeError("failed to find interpreter for Builtin discover of python_spec='3.10'") + + with ( + patch("odev.common.python.virtualenv.cli_run", side_effect=error), + patch("odev.common.python.console.confirm", return_value=True), + self.patch(PythonEnv, "install_system_packages", return_value=False) as install, + self.assertRaises(OdevError), + ): + self.env.create() + + install.assert_called_once() + + +class TestDevelopmentHeaders(OdevTestCase): + """Missing python headers are the most common reason for a source build to fail.""" + + def has_headers(self, include_path: Path | None) -> bool: + env = PythonEnv(path=self.run_path / "venv", version="3.10") + result = None if include_path is None else CompletedProcess("", 0, stdout=f"{include_path}\n".encode()) + + with self.patch(bash, "execute", return_value=result): + return env.has_development_headers + + def test_headers_present(self): + include_path = self.run_path / "include" / "python3.10" + include_path.mkdir(parents=True, exist_ok=True) + (include_path / "Python.h").write_text("", encoding="utf-8") + self.assertTrue(self.has_headers(include_path)) + + def test_headers_missing(self): + self.assertFalse(self.has_headers(self.run_path / "include" / "python3.10")) + + def test_never_warns_on_a_guess(self): + """An interpreter that cannot be questioned is assumed complete: a false alarm on every + `odev run` would be worse than staying silent. + """ + self.assertTrue(self.has_headers(None)) diff --git a/tests/tests/common/test_system.py b/tests/tests/common/test_system.py new file mode 100644 index 000000000..6e85f2456 --- /dev/null +++ b/tests/tests/common/test_system.py @@ -0,0 +1,87 @@ +"""Tests for the description of the operating system odev runs on.""" + +from collections.abc import Sequence +from unittest.mock import patch + +from odev.common import system +from odev.common.system import SystemDependency + +from tests.fixtures import OdevTestCase + + +class TestPackageManager(OdevTestCase): + """Odev has to recognize the package manager of the system it runs on, whichever it is.""" + + def package_manager(self, on_path: Sequence[str]) -> str | None: + with patch("odev.common.system.shutil.which", side_effect=lambda command: command in on_path or None): + return system.package_manager() + + def test_every_known_package_manager_is_detected(self): + for manager in system.PACKAGE_MANAGERS: + self.assertEqual(self.package_manager([manager]), manager) + + def test_the_package_manager_of_the_distribution_wins_over_homebrew(self): + """Homebrew installs on Linux too, but a Fedora machine has to be told about `dnf`.""" + self.assertEqual(self.package_manager(["dnf", "brew"]), "dnf") + + def test_unknown_package_manager(self): + self.assertIsNone(self.package_manager([])) + + +class TestInstallInstructions(OdevTestCase): + """Whatever the system, the user must be told what is missing; the command is a bonus.""" + + def instructions(self, dependencies: Sequence[SystemDependency], on_path: Sequence[str] = ()) -> str: + with patch("odev.common.system.shutil.which", side_effect=lambda command: command in on_path or None): + return system.install_instructions(dependencies, version="3.10") + + def test_names_and_command(self): + instructions = self.instructions([system.C_COMPILER, system.POSTGRESQL_HEADERS], on_path=["dnf"]) + self.assertIn("• a C compiler", instructions) + self.assertIn("• the PostgreSQL client headers (pg_config)", instructions) + self.assertIn("sudo dnf install -y gcc libpq-devel", instructions) + + def test_placeholders_are_expanded(self): + instructions = self.instructions([system.PYTHON_HEADERS], on_path=["apt-get"]) + self.assertIn("• the development headers of python 3.10 (Python.h)", instructions) + self.assertIn("sudo apt-get install -y python3.10-dev", instructions) + + def test_names_only_without_a_known_package_manager(self): + """On a system odev knows nothing about, saying what is missing is still useful.""" + instructions = self.instructions([system.C_COMPILER]) + self.assertEqual(instructions, "• a C compiler") + + def test_dependencies_without_a_package_are_still_named(self): + """Arch Linux ships the python headers within `python` itself: there is nothing to install, + but the user still has to know they are what is missing. + """ + instructions = self.instructions([system.PYTHON_HEADERS, system.C_COMPILER], on_path=["pacman"]) + self.assertIn("• the development headers of python 3.10 (Python.h)", instructions) + self.assertIn("sudo pacman -S --needed base-devel", instructions) + self.assertNotIn("python3.10", instructions.rsplit("\n", 1)[-1]) + + def test_no_command_when_nothing_can_be_named(self): + instructions = self.instructions([system.PYTHON_HEADERS], on_path=["pacman"]) + self.assertEqual(instructions, "• the development headers of python 3.10 (Python.h)") + + +class TestSystemDependencyProbes(OdevTestCase): + """A dependency is only ever reported as missing when it can be proven to be.""" + + def found(self, dependency: SystemDependency, on_path: Sequence[str]) -> bool: + with patch("odev.common.system.shutil.which", side_effect=lambda command: command in on_path or None): + return dependency.found + + def test_any_command_proves_the_dependency(self): + self.assertTrue(self.found(system.C_COMPILER, ["clang"])) + self.assertTrue(self.found(system.C_COMPILER, ["gcc"])) + self.assertFalse(self.found(system.C_COMPILER, ["pg_config"])) + + def test_dependencies_without_a_command_are_never_found(self): + """Those are detected some other way and must not be probed on the `PATH` by mistake.""" + self.assertFalse(self.found(system.PYTHON_HEADERS, ["python3.10", "cc"])) + + def test_odoo_dependencies_name_a_package_for_every_manager_that_ships_one(self): + for dependency in system.ODOO_SYSTEM_DEPENDENCIES: + for manager in dependency.packages: + self.assertIn(manager, system.PACKAGE_MANAGERS, f"{dependency.name} names an unknown {manager!r}")