From 955ddade4bf9785fb82336f0aba7c653ef085526 Mon Sep 17 00:00:00 2001 From: Mark Harris Date: Tue, 1 Sep 2026 08:03:08 +0000 Subject: [PATCH] Enforce agreeing toolchain pins across the conda envs env/ holds several conda environments that each pin python, cuda-version and pytorch-gpu independently, with nothing keeping them in agreement. They agree today (3.12 / 13.0 / 2.13.0), which is coincidence rather than a guarantee. A disagreement is not cosmetic. build_environment.yml is the environment the wheel is compiled in, so it fixes the ABI: a developer working from a different env file with a different PyTorch would build or load a wheel against a mismatched libtorch and get an undefined-symbol ImportError at `import fvdb`. fvdb-reality-capture also derives its benchmark environment from build_environment.yml, so a disagreement here leaves "which PyTorch does fvdb-core use?" without a single answer. Add a codestyle job enforcing the rule, following the existing check_runner_token_policy.py precedent for CI policy scripts: If two environment files pin the same key, they must pin the same value. The rule is permissive about presence and strict about agreement, so a file may omit a key -- release_base_environment.yml pins none of them -- and adding a slimmer environment does not require touching the script. --fix propagates build_environment.yml's values to the others, so bumping a version stays a one-file edit plus one command. This is entirely in-repo, so unlike a cross-repo gate it cannot deadlock: the fix belongs in the same PR that breaks it. Nine unit tests cover agreement, omitted keys, disagreement on each key, surgical --fix, and the error paths. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Mark Harris --- .github/scripts/check_env_pin_consistency.py | 145 ++++++++++++++++++ .../scripts/test_check_env_pin_consistency.py | 94 ++++++++++++ .github/workflows/codestyle.yml | 26 ++++ 3 files changed, 265 insertions(+) create mode 100755 .github/scripts/check_env_pin_consistency.py create mode 100644 .github/scripts/test_check_env_pin_consistency.py diff --git a/.github/scripts/check_env_pin_consistency.py b/.github/scripts/check_env_pin_consistency.py new file mode 100755 index 000000000..ce55cd873 --- /dev/null +++ b/.github/scripts/check_env_pin_consistency.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright Contributors to the OpenVDB Project +# SPDX-License-Identifier: Apache-2.0 +"""Repo-specific CI policy for toolchain pins shared across the conda envs. + +``env/`` holds several conda environments (build, dev, test, learn, ...) that +each pin the toolchain independently. Nothing has kept them in agreement, yet a +disagreement is not cosmetic: + + * ``build_environment.yml`` is the environment the wheel is compiled in, so it + determines the resulting ABI. + * A developer working from ``dev_environment.yml`` with a different PyTorch + would build or load a wheel against a mismatched libtorch and hit an + undefined-symbol ImportError. + * fvdb-reality-capture derives its benchmark environment from + ``build_environment.yml``; if the env files here disagree, "which PyTorch + does fvdb-core use?" has no single answer. + +The rule enforced here is deliberately permissive about *presence* and strict +about *agreement*: + + If two environment files pin the same key, they must pin the same value. + +A file that omits a key is fine (``release_base_environment.yml`` pins none of +them), so adding a slimmer environment does not require touching this script. + +Run ``--fix`` to propagate the canonical file's values to the others. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import defaultdict +from pathlib import Path + +# Keys that affect the compiled ABI or the interpreter the wheel targets. +PINNED_KEYS = ("python", "cuda-version", "pytorch-gpu") + +# The environment the wheel is actually built in, hence the source of truth for +# --fix and the file downstream repositories derive from. +CANONICAL_ENV = "build_environment.yml" + + +def pin_pattern(key: str) -> re.Pattern[str]: + return re.compile(rf"^(?P\s*-\s*{re.escape(key)}=)(?P\S+)\s*$", re.MULTILINE) + + +def read_pins(text: str) -> dict[str, str]: + """Return only the keys this file actually pins.""" + pins = {} + for key in PINNED_KEYS: + match = pin_pattern(key).search(text) + if match is not None: + pins[key] = match.group("value") + return pins + + +def collect(env_dir: Path) -> dict[Path, dict[str, str]]: + paths = sorted(p for p in env_dir.glob("*.yml")) + if not paths: + raise SystemExit(f"error: no environment files found in {env_dir}") + return {path: read_pins(path.read_text(encoding="utf-8")) for path in paths} + + +def find_disagreements(pins_by_file: dict[Path, dict[str, str]]) -> dict[str, dict[str, list[Path]]]: + """Map key -> {value -> files pinning it}, for keys pinned inconsistently.""" + disagreements: dict[str, dict[str, list[Path]]] = {} + for key in PINNED_KEYS: + by_value: dict[str, list[Path]] = defaultdict(list) + for path, pins in pins_by_file.items(): + if key in pins: + by_value[pins[key]].append(path) + if len(by_value) > 1: + disagreements[key] = dict(by_value) + return disagreements + + +def apply_fix(pins_by_file: dict[Path, dict[str, str]], canonical: Path) -> list[Path]: + canonical_pins = pins_by_file[canonical] + changed = [] + for path, pins in pins_by_file.items(): + if path == canonical: + continue + text = original = path.read_text(encoding="utf-8") + for key, value in canonical_pins.items(): + if key in pins and pins[key] != value: + text = pin_pattern(key).sub(lambda m: f"{m.group('prefix')}{value}", text) + if text != original: + path.write_text(text, encoding="utf-8") + changed.append(path) + return changed + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("env_dir", nargs="?", default="env", type=Path, help="directory of conda env files") + parser.add_argument("--fix", action="store_true", help=f"propagate {CANONICAL_ENV}'s values to the other files") + args = parser.parse_args(argv) + + env_dir = args.env_dir + if not env_dir.is_dir(): + raise SystemExit(f"error: {env_dir} is not a directory") + + pins_by_file = collect(env_dir) + canonical = env_dir / CANONICAL_ENV + if canonical not in pins_by_file: + raise SystemExit(f"error: canonical environment {canonical} not found") + + for path in sorted(pins_by_file): + pins = pins_by_file[path] + rendered = " ".join(f"{key}={pins[key]}" for key in PINNED_KEYS if key in pins) or "(pins none)" + print(f" {path.name:<32} {rendered}") + + if args.fix: + changed = apply_fix(pins_by_file, canonical) + if not changed: + print(f"\nAll environment files already agree with {CANONICAL_ENV}.") + return 0 + print(f"\nUpdated from {CANONICAL_ENV}: {', '.join(p.name for p in changed)}") + return 0 + + disagreements = find_disagreements(pins_by_file) + if not disagreements: + print("\nAll environment files agree on the shared toolchain pins.") + return 0 + + print("", file=sys.stderr) + for key, by_value in disagreements.items(): + print(f"error: '{key}' is pinned inconsistently across env/:", file=sys.stderr) + for value, paths in sorted(by_value.items()): + print(f" {value:<12} {', '.join(p.name for p in paths)}", file=sys.stderr) + print( + "\nThese environments share a toolchain: a mismatch means a wheel built in one\n" + "cannot be loaded in another (undefined libtorch symbols at `import fvdb`).\n" + f"Fix by editing {CANONICAL_ENV} and running:\n" + " python3 .github/scripts/check_env_pin_consistency.py --fix", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_check_env_pin_consistency.py b/.github/scripts/test_check_env_pin_consistency.py new file mode 100644 index 000000000..c1a1e7d15 --- /dev/null +++ b/.github/scripts/test_check_env_pin_consistency.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# Copyright Contributors to the OpenVDB Project +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the shared-toolchain-pin policy in check_env_pin_consistency.py.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import check_env_pin_consistency as policy # noqa: E402 + +BUILD = """name: fvdb_build +dependencies: + - cuda-version=13.0 + - python=3.12 + - pytorch-gpu=2.13.0 +""" + +DEV = """name: fvdb_dev +dependencies: + - cuda-version=13.0 + - python=3.12 + - pytorch-gpu=2.13.0 + - ipython +""" + +# Pins none of the shared keys, like release_base_environment.yml. +SLIM = """name: fvdb_release_base +dependencies: + - cmake +""" + + +def write_env(tmp_path: Path, **files: str) -> Path: + env_dir = tmp_path / "env" + env_dir.mkdir() + for name, text in files.items(): + (env_dir / f"{name}.yml").write_text(text, encoding="utf-8") + return env_dir + + +def test_agreeing_envs_pass(tmp_path): + env_dir = write_env(tmp_path, build_environment=BUILD, dev_environment=DEV) + assert policy.main([str(env_dir)]) == 0 + + +def test_file_pinning_nothing_is_allowed(tmp_path): + env_dir = write_env(tmp_path, build_environment=BUILD, release_base_environment=SLIM) + assert policy.main([str(env_dir)]) == 0 + + +def test_disagreement_fails(tmp_path): + env_dir = write_env(tmp_path, build_environment=BUILD, dev_environment=DEV.replace("2.13.0", "2.11.0")) + assert policy.main([str(env_dir)]) == 1 + + +@pytest.mark.parametrize( + ("old", "new"), + [ + ("python=3.12", "python=3.11"), + ("cuda-version=13.0", "cuda-version=12.8"), + ("pytorch-gpu=2.13.0", "pytorch-gpu=2.11.0"), + ], +) +def test_disagreement_on_any_key_fails(tmp_path, old, new): + env_dir = write_env(tmp_path, build_environment=BUILD, dev_environment=DEV.replace(old, new)) + assert policy.main([str(env_dir)]) == 1 + + +def test_fix_propagates_from_canonical(tmp_path): + env_dir = write_env(tmp_path, build_environment=BUILD, dev_environment=DEV.replace("2.13.0", "2.11.0")) + assert policy.main([str(env_dir), "--fix"]) == 0 + assert "pytorch-gpu=2.13.0" in (env_dir / "dev_environment.yml").read_text(encoding="utf-8") + # The fix must be surgical: unrelated content is preserved. + assert "ipython" in (env_dir / "dev_environment.yml").read_text(encoding="utf-8") + assert policy.main([str(env_dir)]) == 0 + + +def test_missing_canonical_env_is_an_error(tmp_path): + env_dir = write_env(tmp_path, dev_environment=DEV) + with pytest.raises(SystemExit): + policy.main([str(env_dir)]) + + +def test_empty_env_dir_is_an_error(tmp_path): + env_dir = tmp_path / "env" + env_dir.mkdir() + with pytest.raises(SystemExit): + policy.main([str(env_dir)]) diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml index 88111d493..155ac8ed1 100644 --- a/.github/workflows/codestyle.yml +++ b/.github/workflows/codestyle.yml @@ -92,3 +92,29 @@ jobs: set +e git grep -n " " -- ':!*/codestyle.yml' ':!*.svg' ':!*.webp' ':!*.cmd' ':!*.png' ':!*.wlt' ':!*.jpg' ':!*.gif' ':!*.mp4' ':!*.pt' ':!*.pth' ':!*.nvdb' ':!*.npz' ':!*.gitmodules' ':!*/wip/*' test $? -eq 1 + + # env/ holds several conda environments that each pin python, cuda-version and + # pytorch-gpu independently. build_environment.yml is the environment the wheel + # is compiled in, so it fixes the ABI; a developer working from a different env + # file with a different PyTorch hits an undefined-symbol ImportError at + # `import fvdb`. fvdb-reality-capture also derives its benchmark environment + # from build_environment.yml, so a disagreement here leaves "which PyTorch does + # fvdb-core use?" with no single answer. + # + # The rule is permissive about presence and strict about agreement: a file may + # omit a key (release_base_environment.yml pins none), but two files that pin + # the same key must pin the same value. + env-pin-consistency: + name: Shared toolchain pins agree + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Enforce shared toolchain pins + run: | + set -euo pipefail + python3 -m pip install --quiet "pytest==9.0.3" + python3 -m pytest .github/scripts/test_check_env_pin_consistency.py -q + python3 .github/scripts/check_env_pin_consistency.py env +