From 8879b80ef64ac954731f7dd4effa0dcefc350dc0 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Sun, 26 Apr 2026 18:00:23 +0200 Subject: [PATCH 01/37] Add ASV backend benchmarks --- .github/workflows/asv-benchmarks.yml | 48 ++++++++ .gitignore | 3 + asv.conf.json | 23 ++++ benchmarks/README.md | 64 ++++++++++ benchmarks/__init__.py | 1 + benchmarks/benchmark_backends.py | 147 +++++++++++++++++++++++ tools/asv_speedup_summary.py | 173 +++++++++++++++++++++++++++ 7 files changed, 459 insertions(+) create mode 100644 .github/workflows/asv-benchmarks.yml create mode 100644 asv.conf.json create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/benchmark_backends.py create mode 100644 tools/asv_speedup_summary.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml new file mode 100644 index 000000000..294dbff42 --- /dev/null +++ b/.github/workflows/asv-benchmarks.yml @@ -0,0 +1,48 @@ +name: ASV Benchmarks + +on: + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: asv + python-version: "3.12" + channels: conda-forge + auto-activate-base: false + + - name: Install ASV + shell: bash -l {0} + run: | + conda install -y -c conda-forge asv + + - name: Configure ASV machine + shell: bash -l {0} + run: | + asv machine --yes + + - name: Run ASV benchmarks + shell: bash -l {0} + run: | + asv run + + - name: Publish ASV report + shell: bash -l {0} + run: | + asv publish + + - name: Upload ASV results + uses: actions/upload-artifact@v4 + with: + name: asv-results + path: | + .asv/results/ + .asv/html/ diff --git a/.gitignore b/.gitignore index bcdc980be..aa7b7f9fd 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,9 @@ htmlcov/ .coverage .coverage.* .cache +.asv/env/ +.asv/results/ +.asv/html/ nosetests.xml coverage.xml *.cover diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 000000000..211c4f51c --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,23 @@ +{ + "version": 1, + "project": "GSTools", + "project_url": "https://github.com/jeilealr/GSTools", + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "pythons": ["3.12"], + "matrix": { + "req": { + "numpy": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python -m pip install {build_dir}[rust]" + ] +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..1c36b5be3 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,64 @@ +# GSTools ASV Benchmarks + +This directory contains performance benchmarks for GSTools. Unit tests in +`tests/` should remain focused on correctness; ASV benchmarks in this +directory measure runtime and peak memory only. + +For a beginner-friendly explanation of ASV and every file added here, read +[`ASV_TUTORIAL.md`](ASV_TUTORIAL.md). + +## Setup + +Run ASV commands from the GSTools repository root, where `asv.conf.json` +is located: + +```bash +cd /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools +conda install -c conda-forge asv +asv machine +``` + +## Common Commands + +```bash +asv run --quick +asv run HEAD^! +asv run +asv publish +asv preview +asv compare HEAD~1 HEAD +``` + +`asv run --quick` is the quick development check for ASV 0.6.x. It runs each +benchmark only once and does not save useful performance results. + +`asv run HEAD^!` benchmarks only the current commit. Plain `asv run` follows +the branches configured in `asv.conf.json`. + +If you need to run ASV from another directory, pass the config explicitly: + +```bash +asv --config /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools/asv.conf.json run --quick +``` + +## Backend Comparison + +Benchmarks are parameterized with readable backend labels: + +- `cython_fallback` +- `rust_core` + +ASV tracks each backend separately. Interpret Rust speedup on the same machine +and same benchmark as: + +```text +speedup = cython_fallback_time / rust_core_time +``` + +So: + +- `speedup > 1.0` means Rust is faster +- `speedup = 1.0` means similar performance +- `speedup < 1.0` means Rust is slower + +Do not compare absolute benchmark times across different machines. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 000000000..c94e5d288 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""ASV benchmarks for GSTools.""" diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py new file mode 100644 index 000000000..2469fbd0c --- /dev/null +++ b/benchmarks/benchmark_backends.py @@ -0,0 +1,147 @@ +"""Backend benchmarks for GSTools. + +Usage: + cd /my/path/to/GSTools + conda install -c conda-forge asv + asv machine + asv run --quick + asv run HEAD^! + asv run + asv publish + asv preview + asv compare HEAD~1 HEAD + +Backend speedup should be interpreted as: + speedup = cython_fallback_time / rust_core_time + +Values greater than 1.0 mean the Rust backend is faster on the same machine +for the same benchmark and commit. +""" + +from __future__ import annotations + +import contextlib + +import numpy as np + +import gstools as gs + + +BACKENDS = ("cython_fallback", "rust_core") + + +@contextlib.contextmanager +def gstools_backend(use_core): + """Temporarily force either gstools-core or the Cython fallback.""" + previous = (gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE) + try: + if use_core: + if not previous[0]: + raise NotImplementedError("gstools_core is not available") + gs.config._GSTOOLS_CORE_AVAIL = True + gs.config.USE_GSTOOLS_CORE = True + else: + gs.config._GSTOOLS_CORE_AVAIL = False + gs.config.USE_GSTOOLS_CORE = False + yield + finally: + gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE = previous + + +def _use_core(backend): + if backend == "rust_core": + return True + if backend == "cython_fallback": + return False + raise ValueError(f"Unknown backend: {backend}") + + +class BackendBenchmarks: + """Runtime and peak-memory benchmarks for backend-dispatched operations.""" + + params = BACKENDS + param_names = ["backend"] + + def setup_cache(self): + """Create deterministic data once per benchmark environment.""" + srf_x = np.random.RandomState(20220425).rand(2000) * 100.0 + srf_y = np.random.RandomState(20220426).rand(2000) * 100.0 + + vario_x = np.random.RandomState(20220427).rand(900) * 100.0 + vario_y = np.random.RandomState(20220428).rand(900) * 100.0 + vario_field = np.sin(vario_x / 10.0) + np.cos(vario_y / 15.0) + vario_bins = np.linspace(0.0, 60.0, 16) + + rng = np.random.RandomState(20220429) + cond_x = rng.rand(40) * 50.0 + cond_y = rng.rand(40) * 50.0 + cond_val = np.sin(cond_x / 8.0) + np.cos(cond_y / 9.0) + target_pos = (rng.rand(1000) * 50.0, rng.rand(1000) * 50.0) + + return { + "srf": (srf_x, srf_y), + "variogram": ((vario_x, vario_y), vario_field, vario_bins), + "krige": ((cond_x, cond_y), cond_val, target_pos), + } + + def setup(self, data, backend): + """Skip only the Rust parameter when gstools-core is unavailable.""" + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + def time_srf(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_srf(data) + + def peakmem_srf(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_srf(data) + + def time_variogram(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_variogram(data) + + def peakmem_variogram(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_variogram(data) + + def time_krige(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_krige(data) + + def peakmem_krige(self, data, backend): + with gstools_backend(_use_core(backend)): + self._run_krige(data) + + def _run_srf(self, data): + x, y = data["srf"] + model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) + srf = gs.SRF(model, mean=1.0, seed=20220425, mode_no=512) + return srf((x, y), mesh_type="unstructured") + + def _run_variogram(self, data): + pos, field, bins = data["variogram"] + return gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + ) + + def _run_krige(self, data): + cond_pos, cond_val, target_pos = data["krige"] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + return krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) diff --git a/tools/asv_speedup_summary.py b/tools/asv_speedup_summary.py new file mode 100644 index 000000000..01ebf3700 --- /dev/null +++ b/tools/asv_speedup_summary.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +"""Print Rust-vs-Cython speedups from local ASV result files. + +The summary is optional. ASV itself remains the source of truth for benchmark +storage and visualization. + +Usage: + python tools/asv_speedup_summary.py + python tools/asv_speedup_summary.py --results-dir .asv/results + +Speedup is calculated as: + cython_fallback_time / rust_core_time + +Values greater than 1.0 mean Rust was faster on the same machine, commit, +environment, and benchmark. +""" + +from __future__ import annotations + +import argparse +import itertools +import json +import math +from pathlib import Path + + +BACKENDS = ("cython_fallback", "rust_core") + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--results-dir", + default=".asv/results", + type=Path, + help="Path to the ASV results directory.", + ) + parser.add_argument( + "--all", + action="store_true", + help="Include non-time benchmarks as ratios too.", + ) + return parser.parse_args() + + +def iter_result_files(results_dir): + for path in sorted(results_dir.glob("**/*.json")): + if path.name in {"benchmarks.json", "machine.json"}: + continue + yield path + + +def load_json(path): + try: + with path.open(encoding="utf8") as handle: + return json.load(handle) + except json.JSONDecodeError: + return None + + +def result_entry(raw_result, result_columns): + if isinstance(raw_result, dict): + return raw_result + if isinstance(raw_result, list) and result_columns: + return dict(zip(result_columns, raw_result)) + return {"result": raw_result, "params": []} + + +def is_number(value): + return isinstance(value, (int, float)) and not math.isnan(value) + + +def backend_values(entry): + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list) or not params: + return {} + + values = {} + combinations = itertools.product(*params) + for combo, value in zip(combinations, result): + if not is_number(value): + continue + combo_values = [str(item).strip("'\"") for item in combo] + for backend in BACKENDS: + if backend in combo_values: + values[backend] = float(value) + return values + + +def short_benchmark_name(name): + return name.rsplit(".", maxsplit=1)[-1] + + +def collect_speedups(results_dir, include_all): + rows = [] + for path in iter_result_files(results_dir): + data = load_json(path) + if not data: + continue + result_columns = data.get("result_columns", []) + commit = data.get("commit_hash", "unknown")[:8] + env_name = data.get("env_name", path.stem) + results = data.get("results", {}) + for benchmark, raw_result in results.items(): + if not include_all and ".time_" not in benchmark: + continue + values = backend_values(result_entry(raw_result, result_columns)) + cython = values.get("cython_fallback") + rust = values.get("rust_core") + if not is_number(cython) or not is_number(rust) or rust == 0: + continue + rows.append( + { + "commit": commit, + "env": env_name, + "benchmark": short_benchmark_name(benchmark), + "cython": cython, + "rust": rust, + "speedup": cython / rust, + } + ) + return rows + + +def print_table(rows): + if not rows: + print("No matching Rust-vs-Cython ASV results found.") + return + + headers = [ + "commit", + "env", + "benchmark", + "cython", + "rust", + "speedup", + ] + table = [ + [ + row["commit"], + row["env"], + row["benchmark"], + f"{row['cython']:.6g}", + f"{row['rust']:.6g}", + f"{row['speedup']:.3f}x", + ] + for row in rows + ] + widths = [ + max(len(str(item)) for item in column) + for column in zip(headers, *table) + ] + + def fmt(row): + return " ".join( + str(item).ljust(width) for item, width in zip(row, widths) + ) + + print(fmt(headers)) + print(fmt(["-" * width for width in widths])) + for row in table: + print(fmt(row)) + + +def main(): + args = parse_args() + rows = collect_speedups(args.results_dir, args.all) + print_table(rows) + + +if __name__ == "__main__": + main() From bfba184514a13fdcad8037b1948afd88dc26fbef Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 7 May 2026 16:36:15 +0200 Subject: [PATCH 02/37] First Bench marking: current status, both backends, ASV and cProfile --- benchmarks/README.md | 451 ++++++++++++++++-- benchmarks/benchmark_backends.py | 226 ++++++--- .../tools}/asv_speedup_summary.py | 104 +++- .../tools/profile_benchmark_workflows.py | 192 ++++++++ pyproject.toml | 1 + 5 files changed, 867 insertions(+), 107 deletions(-) rename {tools => benchmarks/tools}/asv_speedup_summary.py (56%) create mode 100644 benchmarks/tools/profile_benchmark_workflows.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 1c36b5be3..061062be4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,64 +1,455 @@ -# GSTools ASV Benchmarks +# GSTools Benchmark Guide -This directory contains performance benchmarks for GSTools. Unit tests in -`tests/` should remain focused on correctness; ASV benchmarks in this -directory measure runtime and peak memory only. +This directory contains the Airspeed Velocity ([ASV](https://github.com/airspeed-velocity/asv/)) benchmark suite for GSTools and a complementary profiling helper implemented with cProfile (part of the Python standard library). -For a beginner-friendly explanation of ASV and every file added here, read -[`ASV_TUTORIAL.md`](ASV_TUTORIAL.md). +This is a measurement-first guide: benchmark real workflows, inspect the +results, profile the slow paths, and then decide what to optimize. + +Unit tests in `tests/` answer "is the code correct?". The ASV benchmarks in +`benchmarks/` answer "how fast is this workflow, how much memory does it use, +and did that change across commits?". The complementary cProfile helper +answers "inside this workflow, which Python functions are taking most of the +time right now?". + +The benchmarks compare two GSTools backends, which gives more context for +deciding where optimization work should go: + +- `cython_fallback`: the default Cython-backed fallback implementation from + [gstools-cython](https://github.com/GeoStat-Framework/GSTools-Cython). +- `rust_core`: the Rust-backed implementation from + [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core). + +## Index + +- [Setup](#setup) +- [Benchmarking Scripts](#benchmarking-scripts) +- [ASV Configuration](#asv-configuration) +- [Benchmark Naming](#benchmark-naming) +- [Benchmark Coverage](#benchmark-coverage) +- [Benchmark Classes](#benchmark-classes) +- [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) +- [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) +- [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) +- [Running The Benchmarks](#running-the-benchmarks) +- [Profiling With cProfile](#profiling-with-cprofile) +- [More ASV Commands](#more-asv-commands) +- [External Reference](#external-reference) ## Setup -Run ASV commands from the GSTools repository root, where `asv.conf.json` -is located: +The regular installation commands in the main `README.md` install GSTools for +normal use. For benchmark work, install this local checkout with the optional +benchmark dependencies. + +1. Move to the GSTools repository root: ```bash -cd /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools -conda install -c conda-forge asv -asv machine +cd /path/to/GSTools ``` -## Common Commands +2. Install GSTools in editable mode with the benchmark tooling and Rust backend: ```bash -asv run --quick -asv run HEAD^! -asv run -asv publish -asv preview -asv compare HEAD~1 HEAD +python -m pip install -e ".[benchmark,rust]" +``` + +3. Create a machine profile once per computer: + +```bash +asv machine --yes +``` + +Notes: + +- The machine profile records local hardware information so ASV can label + results correctly. Do not compare absolute times across different machines. +- You can also install ASV with conda or pip, and you can install the Rust + backend package from + [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core) directly. + +## Benchmarking Scripts + +The benchmarking setup currently consists of: + +- `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where + to store results, and which Python/environment matrix to use. +- `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. +- `benchmarks/README.md`: this practical guide. +- `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints + Rust-vs-Cython speedup ratios. +- `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative + workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so + you can see which functions take time in the current checkout. + +Do not run `benchmarks/benchmark_backends.py` directly with Python. ASV loads +that file, discovers benchmark classes and methods, and runs them inside +isolated benchmark environments. The scripts in `benchmarks/tools/` are +different: run them directly with Python. The profiling helper can run against +the current checkout at any time; the speedup-summary helper needs saved ASV +results in `.asv/results/`. + +### ASV Configuration + +The repo root `asv.conf.json` is tailored to this GSTools checkout: + +```json +{ + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "environment_type": "conda", + "pythons": ["3.12"], + "install_command": [ + "in-dir={env_dir} python -m pip install {build_dir}[rust]" + ] +} +``` + +Important details: + +- `install_command` installs the checked-out GSTools revision with the `[rust]` + extra, so `gstools_core` should be available for Rust backend measurements. + ASV still needs its own `install_command` because it creates isolated + environments for the commits it benchmarks. +- This is separate from any editable install in your active development + environment, such as `python -m pip install -e ".[benchmark,rust]"`. + The editable install is only needed when you want your active environment to + import the current checkout directly, for example when running + `benchmarks/tools/profile_benchmark_workflows.py` with `--backend rust_core`. +- ASV and the cProfile helper use different environments. ASV runs + `benchmarks/benchmark_backends.py` inside `.asv/env/`; the cProfile helper + imports the same benchmark classes but runs them in your active Python + environment. + +ASV creates these generated directories: + +```text +.asv/env/ benchmark environments +.asv/results/ local benchmark result JSON files +.asv/html/ generated local benchmark website ``` -`asv run --quick` is the quick development check for ASV 0.6.x. It runs each -benchmark only once and does not save useful performance results. +Those directories are machine-specific generated artifacts. They should +normally stay out of git. + +If needed, users can list more than one branch, Python version, benchmark +directory, and so on. For example: -`asv run HEAD^!` benchmarks only the current commit. Plain `asv run` follows -the branches configured in `asv.conf.json`. +```json +"branches": ["main", "my-feature-branch"] +``` -If you need to run ASV from another directory, pass the config explicitly: +Users can also benchmark any explicit branch, commit, tag, or range without +changing `asv.conf.json`: ```bash -asv --config /Users/lealroja/Documents/UFZ/MPS-Tools/GSTools/asv.conf.json run --quick +asv run my-feature-branch^! --bench benchmark_backends +asv run main..my-feature-branch --bench benchmark_backends +``` + +ASV checks out package code at each git commit being benchmarked. Commit source +changes before benchmarking them with ASV. Otherwise ASV may benchmark the last +committed package code rather than your uncommitted source changes. + + +### Benchmark Naming + +ASV recognizes benchmark methods by name: + +- methods starting with `time_` measure runtime +- methods starting with `peakmem_` measure peak memory +- `setup_cache()` creates reusable data once per benchmark environment +- `setup()` can skip or prepare individual parameter combinations + +## Benchmark Coverage + +### Shared Constants + +```python +BACKENDS = ("cython_fallback", "rust_core") +VARIOGRAM_CASES = ( + "full_900", + "sampled_5000_to_1500", + "sampled_15000_to_4500", +) +KRIGE_CASES = ("small_30x500", "large_120x2000", "extra_large_360x6000") +FIELD_CASES = ( + "srf_unstructured_randmeth", + "srf_structured_randmeth", + "srf_structured_fourier", + "condsrf_unstructured", +) ``` -## Backend Comparison +These constants define parameter labels shown in ASV results. -Benchmarks are parameterized with readable backend labels: +`BACKENDS` compares: - `cython_fallback` - `rust_core` -ASV tracks each backend separately. Interpret Rust speedup on the same machine -and same benchmark as: +### Shared Helpers + +`gstools_backend(use_core)` temporarily forces GSTools to use either the Cython +fallback backend or the Rust `gstools_core` backend. + +`_random_points(seed, count, scale)` creates deterministic 2D point clouds. + +`_smooth_field(x, y)` creates deterministic synthetic values: + +```python +np.sin(x / 10.0) + np.cos(y / 15.0) +``` + +`_make_variogram_data(...)` creates positions, field values, and bins for +variogram estimation. + +`_make_krige_data(...)` creates conditioning points, conditioning values, and +target points for kriging and conditioned random fields. + +The fixed random seeds are intentional. They keep benchmark inputs stable so +changes in results are more likely to come from code changes, not new random +data. + +### Benchmark Classes + +The ASV benchmarking is organized around workflow classes. Each workflow class +compares `cython_fallback` and `rust_core`, and each class includes both +runtime and peak-memory methods. + +The suite currently measures: + +- `VariogramWorkflowBenchmarks`: full pairwise work vs sampled large work +- `KrigingWorkflowBenchmarks`: small vs larger global kriging systems +- `RandomFieldWorkflowBenchmarks`: unstructured SRF, structured SRF, Fourier + SRF, and conditioned SRF + +This keeps the ASV suite focused on representative workflows rather than +separate duplicate backend checks. + +#### VariogramWorkflowBenchmarks + +This class measures variogram estimation cases: + +```text +full_900 +sampled_5000_to_1500 +sampled_15000_to_4500 +``` + +The labels mean: + +- `full_900`: create 900 scattered points and use all 900 points for the + variogram calculation. +- `sampled_5000_to_1500`: create 5,000 scattered points, then randomly select + 1,500 of those points for the variogram calculation. +- `sampled_15000_to_4500`: create 15,000 scattered points, then randomly select + 4,500 of those points for the variogram calculation. + +The sampled cases still represent larger input datasets, but the variogram +calculation is done on the randomly selected subset so the pairwise work stays +practical. + +#### KrigingWorkflowBenchmarks + +This class measures global kriging at three scales: + +```text +small_30x500 +large_120x2000 +extra_large_360x6000 +``` + +The labels mean: + +- `small_30x500`: 30 conditioning points, 500 target points +- `large_120x2000`: 120 conditioning points, 2,000 target points +- `extra_large_360x6000`: 360 conditioning points, 6,000 target points + +#### RandomFieldWorkflowBenchmarks + +This class measures SRF and CondSRF generation workflows: + +```text +srf_unstructured_randmeth +srf_structured_randmeth +srf_structured_fourier +condsrf_unstructured +``` + +The cases are: + +- `srf_unstructured_randmeth`: SRF using RandMeth on 2,000 unstructured points +- `srf_structured_randmeth`: SRF using RandMeth on a 64 by 64 structured grid +- `srf_structured_fourier`: SRF using the Fourier generator on a 64 by 64 + structured grid +- `condsrf_unstructured`: conditioned SRF with 40 conditioning points and 1,000 + target points + +## Running The Benchmarks + +Check that the benchmark module imports and runs: + +```bash +asv run --quick --show-stderr --bench benchmark_backends +``` + +Save a baseline for the current commit: + +```bash +asv run HEAD^! --bench benchmark_backends +``` + +Run the last five commits on a linear branch: + +```bash +asv run HEAD~5..HEAD --bench benchmark_backends +``` + +Build and open the local website: + +```bash +asv publish +asv preview +``` + +Then open the printed local URL, for example: + +```text +http://127.0.0.1:8082/#/ +``` +(or any other `http://127.0.0.1:/#/` URL shown by the running preview). + +After ASV has saved results, print explicit Rust-vs-Cython speedup ratios: + +```bash +python benchmarks/tools/asv_speedup_summary.py +``` + +The helper reads `.asv/results/` and reports: ```text speedup = cython_fallback_time / rust_core_time ``` -So: +Interpret the ratio as: - `speedup > 1.0` means Rust is faster - `speedup = 1.0` means similar performance - `speedup < 1.0` means Rust is slower -Do not compare absolute benchmark times across different machines. +The browser report shows ASV plots and trends. The speedup helper prints the +backend ratio explicitly in the terminal. By default, the helper skips removed +legacy duplicate rows from older saved results. + +## Profiling With cProfile + +`cProfile` is useful for the current checkout. It does not update the ASV +browser report. Instead, it prints a table in the terminal showing which Python +functions consumed time while one workflow ran. + +The helper script is: + +```text +benchmarks/tools/profile_benchmark_workflows.py +``` + +It imports the ASV benchmark classes from `benchmark_backends.py`, selects one +case, forces one backend, and runs that case under `cProfile`. + +List available cases: + +```bash +python benchmarks/tools/profile_benchmark_workflows.py --list +``` + +Profile selected cases: + +```bash +python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --limit 10 +python benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --limit 10 +``` + +Useful options: + +- `--case`: choose one workflow, or use `all` +- `--backend`: choose `cython_fallback` or `rust_core` +- `--limit`: number of function rows to print from the cProfile table +- `--sort cumtime`: sort by cumulative time, usually the best first view +- `--sort tottime`: sort by time spent directly in each function +- `--repeat`: repeat a workflow inside the profiler + +For example, `--limit 10` means "print the top 10 function rows after sorting". + +## More ASV Commands + +Save results for only the current commit: + +```bash +asv run HEAD^! --bench benchmark_backends +``` + +Compare current commit with previous commit: + +```bash +asv run HEAD~1^! --bench benchmark_backends +asv run HEAD^! --bench benchmark_backends +asv compare HEAD~1 HEAD +``` + +Compare local `main` with the current branch tip: + +```bash +asv run main^! --bench benchmark_backends +asv run HEAD^! --bench benchmark_backends +asv compare main HEAD +``` + +Compare remote `main` with the current branch tip: + +```bash +git fetch origin main +asv run origin/main^! --bench benchmark_backends +asv run HEAD^! --bench benchmark_backends +asv compare origin/main HEAD +``` + +On a linear branch, `HEAD~5..HEAD` benchmarks: + +```text +HEAD~4 +HEAD~3 +HEAD~2 +HEAD~1 +HEAD +``` + +Run a selected list of commits: + +```bash +git rev-parse HEAD HEAD~3 main e20c88f7 > /tmp/gstools-asv-commits.txt +asv run HASHFILE:/tmp/gstools-asv-commits.txt --bench benchmark_backends +``` + +Use full commit hashes when sharing results. Short hashes and branch names are +fine locally but can become ambiguous later. + +If running ASV from outside the repo root, pass the config explicitly: + +```bash +asv --config /path/to/MPS-Tools/GSTools/asv.conf.json run --quick --bench benchmark_backends +``` + +## External Reference + +For complete ASV command syntax, see: + +```text +https://asv.readthedocs.io/en/stable/commands.html +``` diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index 2469fbd0c..517c61893 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -1,11 +1,11 @@ -"""Backend benchmarks for GSTools. +"""Workflow benchmarks for GSTools backends. Usage: - cd /my/path/to/GSTools - conda install -c conda-forge asv - asv machine - asv run --quick - asv run HEAD^! + cd /path/to/MPS-Tools/GSTools + python -m pip install -e ".[benchmark]" + asv machine --yes + asv run --quick --show-stderr --bench benchmark_backends + asv run HEAD^! --bench benchmark_backends asv run asv publish asv preview @@ -28,6 +28,18 @@ BACKENDS = ("cython_fallback", "rust_core") +VARIOGRAM_CASES = ( + "full_900", + "sampled_5000_to_1500", + "sampled_15000_to_4500", +) +KRIGE_CASES = ("small_30x500", "large_120x2000", "extra_large_360x6000") +FIELD_CASES = ( + "srf_unstructured_randmeth", + "srf_structured_randmeth", + "srf_structured_fourier", + "condsrf_unstructured", +) @contextlib.contextmanager @@ -56,81 +68,181 @@ def _use_core(backend): raise ValueError(f"Unknown backend: {backend}") -class BackendBenchmarks: - """Runtime and peak-memory benchmarks for backend-dispatched operations.""" +def _random_points(seed, count, scale): + rng = np.random.RandomState(seed) + return rng.rand(count) * scale, rng.rand(count) * scale - params = BACKENDS - param_names = ["backend"] - def setup_cache(self): - """Create deterministic data once per benchmark environment.""" - srf_x = np.random.RandomState(20220425).rand(2000) * 100.0 - srf_y = np.random.RandomState(20220426).rand(2000) * 100.0 +def _smooth_field(x, y): + return np.sin(x / 10.0) + np.cos(y / 15.0) + + +def _make_variogram_data(seed, count, scale=100.0): + x, y = _random_points(seed, count, scale) + field = _smooth_field(x, y) + bins = np.linspace(0.0, scale * 0.6, 16) + return (x, y), field, bins + - vario_x = np.random.RandomState(20220427).rand(900) * 100.0 - vario_y = np.random.RandomState(20220428).rand(900) * 100.0 - vario_field = np.sin(vario_x / 10.0) + np.cos(vario_y / 15.0) - vario_bins = np.linspace(0.0, 60.0, 16) +def _make_krige_data(seed, cond_count, target_count, scale=50.0): + rng = np.random.RandomState(seed) + cond_x = rng.rand(cond_count) * scale + cond_y = rng.rand(cond_count) * scale + cond_val = _smooth_field(cond_x, cond_y) + target_pos = ( + rng.rand(target_count) * scale, + rng.rand(target_count) * scale, + ) + return (cond_x, cond_y), cond_val, target_pos - rng = np.random.RandomState(20220429) - cond_x = rng.rand(40) * 50.0 - cond_y = rng.rand(40) * 50.0 - cond_val = np.sin(cond_x / 8.0) + np.cos(cond_y / 9.0) - target_pos = (rng.rand(1000) * 50.0, rng.rand(1000) * 50.0) +class VariogramWorkflowBenchmarks: + """Variogram workflow benchmarks by case and backend.""" + + params = [VARIOGRAM_CASES, BACKENDS] + param_names = ["case", "backend"] + + def setup_cache(self): return { - "srf": (srf_x, srf_y), - "variogram": ((vario_x, vario_y), vario_field, vario_bins), - "krige": ((cond_x, cond_y), cond_val, target_pos), + "full_900": _make_variogram_data(20220501, 900), + "sampled_5000_to_1500": _make_variogram_data(20220502, 5000), + "sampled_15000_to_4500": _make_variogram_data(20220503, 15000), } - def setup(self, data, backend): - """Skip only the Rust parameter when gstools-core is unavailable.""" + def setup(self, data, case, backend): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") - def time_srf(self, data, backend): + def time_variogram_estimate(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_srf(data) + self._run_variogram(data, case) - def peakmem_srf(self, data, backend): + def peakmem_variogram_estimate(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_srf(data) + self._run_variogram(data, case) + + def _run_variogram(self, data, case): + pos, field, bins = data[case] + kwargs = {} + if case == "sampled_5000_to_1500": + kwargs = {"sampling_size": 1500, "sampling_seed": 20220504} + if case == "sampled_15000_to_4500": + kwargs = {"sampling_size": 4500, "sampling_seed": 20220505} + return gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + **kwargs, + ) + - def time_variogram(self, data, backend): +class KrigingWorkflowBenchmarks: + """Global kriging workflow benchmarks by case and backend.""" + + params = [KRIGE_CASES, BACKENDS] + param_names = ["case", "backend"] + + def setup_cache(self): + return { + "small_30x500": _make_krige_data(20220506, 30, 500), + "large_120x2000": _make_krige_data(20220507, 120, 2000), + "extra_large_360x6000": _make_krige_data(20220508, 360, 6000), + } + + def setup(self, data, case, backend): + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + def time_global_krige(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_variogram(data) + self._run_krige(data, case) - def peakmem_variogram(self, data, backend): + def peakmem_global_krige(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_variogram(data) + self._run_krige(data, case) + + def _run_krige(self, data, case): + cond_pos, cond_val, target_pos = data[case] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + return krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) + + +class RandomFieldWorkflowBenchmarks: + """SRF and CondSRF workflow benchmarks by case and backend.""" + + params = [FIELD_CASES, BACKENDS] + param_names = ["case", "backend"] + + def setup_cache(self): + return { + "unstructured_pos": _random_points(20220509, 2000, 100.0), + "structured_pos": ( + np.linspace(0.0, 100.0, 64), + np.linspace(0.0, 100.0, 64), + ), + "condsrf": _make_krige_data(20220510, 40, 1000), + } - def time_krige(self, data, backend): + def setup(self, data, case, backend): + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + def time_field_generation(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_krige(data) + self._run_field(data, case) - def peakmem_krige(self, data, backend): + def peakmem_field_generation(self, data, case, backend): with gstools_backend(_use_core(backend)): - self._run_krige(data) + self._run_field(data, case) + + def _run_field(self, data, case): + if case == "srf_unstructured_randmeth": + return self._run_srf_unstructured(data) + if case == "srf_structured_randmeth": + return self._run_srf_structured(data) + if case == "srf_structured_fourier": + return self._run_srf_fourier(data) + if case == "condsrf_unstructured": + return self._run_condsrf(data) + raise ValueError(f"Unknown field benchmark case: {case}") - def _run_srf(self, data): - x, y = data["srf"] + def _run_srf_unstructured(self, data): model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) - srf = gs.SRF(model, mean=1.0, seed=20220425, mode_no=512) - return srf((x, y), mesh_type="unstructured") + srf = gs.SRF(model, mean=1.0, seed=20220508, mode_no=512) + return srf(data["unstructured_pos"], mesh_type="unstructured") - def _run_variogram(self, data): - pos, field, bins = data["variogram"] - return gs.vario_estimate( - pos, - field, - bins, - mesh_type="unstructured", - return_counts=True, + def _run_srf_structured(self, data): + model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) + srf = gs.SRF(model, mean=1.0, seed=20220509, mode_no=512) + return srf(data["structured_pos"], mesh_type="structured") + + def _run_srf_fourier(self, data): + model = gs.Gaussian(dim=2, var=2.0, len_scale=30.0) + srf = gs.SRF( + model, + generator="Fourier", + period=[100.0, 100.0], + mode_no=[32, 32], + seed=20220510, ) + return srf(data["structured_pos"], mesh_type="structured") - def _run_krige(self, data): - cond_pos, cond_val, target_pos = data["krige"] + def _run_condsrf(self, data): + cond_pos, cond_val, target_pos = data["condsrf"] model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) krige = gs.Krige( model, @@ -139,9 +251,11 @@ def _run_krige(self, data): exact=False, cond_err=0.05, ) - return krige( + cond_srf = gs.CondSRF(krige, seed=20220511, mode_no=512) + return cond_srf( target_pos, mesh_type="unstructured", - return_var=True, + seed=20220512, store=False, + krige_store=False, ) diff --git a/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py similarity index 56% rename from tools/asv_speedup_summary.py rename to benchmarks/tools/asv_speedup_summary.py index 01ebf3700..d341d2178 100644 --- a/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -5,14 +5,15 @@ storage and visualization. Usage: - python tools/asv_speedup_summary.py - python tools/asv_speedup_summary.py --results-dir .asv/results + python benchmarks/tools/asv_speedup_summary.py + python benchmarks/tools/asv_speedup_summary.py --results-dir .asv/results + python benchmarks/tools/asv_speedup_summary.py --include-legacy Speedup is calculated as: cython_fallback_time / rust_core_time Values greater than 1.0 mean Rust was faster on the same machine, commit, -environment, and benchmark. +environment, benchmark, and non-backend parameter combination. """ from __future__ import annotations @@ -25,6 +26,14 @@ BACKENDS = ("cython_fallback", "rust_core") +LEGACY_BENCHMARKS = { + "time_srf", + "peakmem_srf", + "time_variogram", + "peakmem_variogram", + "time_krige", + "peakmem_krige", +} def parse_args(): @@ -40,6 +49,11 @@ def parse_args(): action="store_true", help="Include non-time benchmarks as ratios too.", ) + parser.add_argument( + "--include-legacy", + action="store_true", + help="Include removed BackendBenchmarks rows from older saved results.", + ) return parser.parse_args() @@ -70,6 +84,14 @@ def is_number(value): return isinstance(value, (int, float)) and not math.isnan(value) +def flatten_values(values): + if isinstance(values, list): + for value in values: + yield from flatten_values(value) + return + yield values + + def backend_values(entry): result = entry.get("result") params = entry.get("params") or [] @@ -78,7 +100,7 @@ def backend_values(entry): values = {} combinations = itertools.product(*params) - for combo, value in zip(combinations, result): + for combo, value in zip(combinations, flatten_values(result)): if not is_number(value): continue combo_values = [str(item).strip("'\"") for item in combo] @@ -88,11 +110,40 @@ def backend_values(entry): return values +def backend_rows(entry): + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list) or not params: + return [] + + rows = [] + combinations = itertools.product(*params) + for combo, value in zip(combinations, flatten_values(result)): + if not is_number(value): + continue + combo_values = [str(item).strip("'\"") for item in combo] + backend = next( + (candidate for candidate in BACKENDS if candidate in combo_values), + None, + ) + if backend is None: + continue + case_values = [item for item in combo_values if item not in BACKENDS] + rows.append( + { + "backend": backend, + "case": "/".join(case_values) if case_values else "-", + "value": float(value), + } + ) + return rows + + def short_benchmark_name(name): return name.rsplit(".", maxsplit=1)[-1] -def collect_speedups(results_dir, include_all): +def collect_speedups(results_dir, include_all, include_legacy): rows = [] for path in iter_result_files(results_dir): data = load_json(path) @@ -103,23 +154,32 @@ def collect_speedups(results_dir, include_all): env_name = data.get("env_name", path.stem) results = data.get("results", {}) for benchmark, raw_result in results.items(): - if not include_all and ".time_" not in benchmark: + benchmark_name = short_benchmark_name(benchmark) + if not include_legacy and benchmark_name in LEGACY_BENCHMARKS: continue - values = backend_values(result_entry(raw_result, result_columns)) - cython = values.get("cython_fallback") - rust = values.get("rust_core") - if not is_number(cython) or not is_number(rust) or rust == 0: + if not include_all and ".time_" not in benchmark: continue - rows.append( - { - "commit": commit, - "env": env_name, - "benchmark": short_benchmark_name(benchmark), - "cython": cython, - "rust": rust, - "speedup": cython / rust, - } - ) + by_case = {} + for row in backend_rows(result_entry(raw_result, result_columns)): + by_case.setdefault(row["case"], {})[row["backend"]] = row[ + "value" + ] + for case, values in by_case.items(): + cython = values.get("cython_fallback") + rust = values.get("rust_core") + if not is_number(cython) or not is_number(rust) or rust == 0: + continue + rows.append( + { + "commit": commit, + "env": env_name, + "benchmark": benchmark_name, + "case": case, + "cython": cython, + "rust": rust, + "speedup": cython / rust, + } + ) return rows @@ -132,6 +192,7 @@ def print_table(rows): "commit", "env", "benchmark", + "case", "cython", "rust", "speedup", @@ -141,6 +202,7 @@ def print_table(rows): row["commit"], row["env"], row["benchmark"], + row["case"], f"{row['cython']:.6g}", f"{row['rust']:.6g}", f"{row['speedup']:.3f}x", @@ -165,7 +227,7 @@ def fmt(row): def main(): args = parse_args() - rows = collect_speedups(args.results_dir, args.all) + rows = collect_speedups(args.results_dir, args.all, args.include_legacy) print_table(rows) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py new file mode 100644 index 000000000..7f953e0dd --- /dev/null +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +"""Profile the representative GSTools benchmark workflows with cProfile. + +This is a quick measurement helper. ASV remains the source of truth for saved +benchmark results, while this script helps identify the top cumulative Python +call sites before making algorithmic changes. + +Usage: + cd /path/to/MPS-Tools/GSTools + python benchmarks/tools/profile_benchmark_workflows.py --list + python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled + python benchmarks/tools/profile_benchmark_workflows.py --case krige-large \ + --backend rust_core --limit 30 +""" + +from __future__ import annotations + +import argparse +import cProfile +from pathlib import Path +import pstats +import sys + + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "src")) + + +CASES = { + "variogram-full": ( + "VariogramWorkflowBenchmarks", + "time_variogram_estimate", + ("full_900",), + ), + "variogram-sampled": ( + "VariogramWorkflowBenchmarks", + "time_variogram_estimate", + ("sampled_5000_to_1500",), + ), + "variogram-extra-large": ( + "VariogramWorkflowBenchmarks", + "time_variogram_estimate", + ("sampled_15000_to_4500",), + ), + "krige-small": ( + "KrigingWorkflowBenchmarks", + "time_global_krige", + ("small_30x500",), + ), + "krige-large": ( + "KrigingWorkflowBenchmarks", + "time_global_krige", + ("large_120x2000",), + ), + "krige-extra-large": ( + "KrigingWorkflowBenchmarks", + "time_global_krige", + ("extra_large_360x6000",), + ), + "srf-unstructured": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("srf_unstructured_randmeth",), + ), + "srf-structured": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("srf_structured_randmeth",), + ), + "srf-fourier": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("srf_structured_fourier",), + ), + "condsrf": ( + "RandomFieldWorkflowBenchmarks", + "time_field_generation", + ("condsrf_unstructured",), + ), +} + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--case", + default="all", + choices=["all", *CASES], + help="Workflow to profile. Defaults to all workflows.", + ) + parser.add_argument( + "--repeat", + default=1, + type=int, + help="Number of times to run each selected workflow.", + ) + parser.add_argument( + "--limit", + default=25, + type=int, + help="Number of cProfile rows to print per workflow.", + ) + parser.add_argument( + "--sort", + default="cumtime", + choices=["cumtime", "tottime", "calls"], + help="pstats sort key.", + ) + parser.add_argument( + "--backend", + default="rust_core", + choices=["cython_fallback", "rust_core"], + help="Backend label to force while profiling.", + ) + parser.add_argument( + "--list", + action="store_true", + help="List available workflow cases and exit.", + ) + return parser.parse_args() + + +def iter_selected(case): + if case == "all": + yield from CASES.items() + return + yield case, CASES[case] + + +def load_suite_class(class_name): + try: + from benchmarks import benchmark_backends + except ModuleNotFoundError as err: + print( + "Could not import GSTools benchmark dependencies. Activate the " + "GSTools benchmark environment or install the project dependencies " + f"first. Original error: {err}", + file=sys.stderr, + ) + raise SystemExit(1) from err + return getattr(benchmark_backends, class_name) + + +def run_case( + name, + class_name, + method_name, + params, + repeat, + limit, + sort, + backend, +): + suite_cls = load_suite_class(class_name) + suite = suite_cls() + data = suite.setup_cache() + method = getattr(suite, method_name) + + profiler = cProfile.Profile() + profiler.enable() + for _ in range(repeat): + method(data, *params, backend) + profiler.disable() + + print(f"\n== {name} [{backend}] ==") + stats = pstats.Stats(profiler, stream=sys.stdout) + stats.strip_dirs().sort_stats(sort).print_stats(limit) + + +def main(): + args = parse_args() + if args.list: + for name in CASES: + print(name) + return + + for name, (suite_cls, method_name, params) in iter_selected(args.case): + run_case( + name, + suite_cls, + method_name, + params, + args.repeat, + args.limit, + args.sort, + args.backend, + ) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 885bcd774..f39ea2eab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ doc = [ ] plotting = ["matplotlib>=3.7", "pyvista>=0.40"] rust = ["gstools_core>=1.0.0"] +benchmark = ["asv"] test = ["pytest-cov>=3"] lint = ["ruff"] From f8870650ec99193498736a3a5061a28920b0d726 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Fri, 15 May 2026 16:53:34 +0200 Subject: [PATCH 03/37] fix parallelisation in benchmarking, adding macos parallelisation --- .gitignore | 5 +- asv.conf.json | 11 +- asv.macos-openmp.conf.json | 36 ++ benchmarks/README.md | 462 +++++++++++++++--- benchmarks/benchmark_backends.py | 100 +++- benchmarks/tools/asv_speedup_summary.py | 28 +- benchmarks/tools/check_cython_openmp.py | 103 ++++ .../tools/install_macos_openmp_cython.py | 139 ++++++ .../tools/profile_benchmark_workflows.py | 36 +- 9 files changed, 798 insertions(+), 122 deletions(-) create mode 100644 asv.macos-openmp.conf.json create mode 100644 benchmarks/tools/check_cython_openmp.py create mode 100644 benchmarks/tools/install_macos_openmp_cython.py diff --git a/.gitignore b/.gitignore index aa7b7f9fd..1cec6e510 100644 --- a/.gitignore +++ b/.gitignore @@ -41,9 +41,8 @@ htmlcov/ .coverage .coverage.* .cache -.asv/env/ -.asv/results/ -.asv/html/ +.asv/* +.asv-openmp/* nosetests.xml coverage.xml *.cover diff --git a/asv.conf.json b/asv.conf.json index 211c4f51c..289cc426a 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -14,10 +14,17 @@ "pythons": ["3.12"], "matrix": { "req": { - "numpy": [""] + "emcee": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "gstools-cython": [""] } }, "install_command": [ - "in-dir={env_dir} python -m pip install {build_dir}[rust]" + "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ] } diff --git a/asv.macos-openmp.conf.json b/asv.macos-openmp.conf.json new file mode 100644 index 000000000..9df8067ab --- /dev/null +++ b/asv.macos-openmp.conf.json @@ -0,0 +1,36 @@ +{ + "version": 1, + "project": "GSTools", + "project_url": "https://github.com/jeilealr/GSTools", + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv-openmp/env", + "results_dir": ".asv-openmp/results", + "html_dir": ".asv-openmp/html", + "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "pythons": ["3.12"], + "matrix": { + "req": { + "cython": [""], + "emcee": [""], + "extension-helpers": [""], + "hankel": [""], + "llvm-openmp": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "setuptools": [""], + "wheel": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_macos_openmp_cython.py {env_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" + ] +} diff --git a/benchmarks/README.md b/benchmarks/README.md index 061062be4..b4df15b6a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,8 +2,8 @@ This directory contains the Airspeed Velocity ([ASV](https://github.com/airspeed-velocity/asv/)) benchmark suite for GSTools and a complementary profiling helper implemented with cProfile (part of the Python standard library). -This is a measurement-first guide: benchmark real workflows, inspect the -results, profile the slow paths, and then decide what to optimize. +This guide benchmarks GSTools, inspects the +results, profiles where runtime is spent, and then decides what to optimize. Unit tests in `tests/` answer "is the code correct?". The ASV benchmarks in `benchmarks/` answer "how fast is this workflow, how much memory does it use, @@ -23,23 +23,54 @@ deciding where optimization work should go: - [Setup](#setup) - [Benchmarking Scripts](#benchmarking-scripts) -- [ASV Configuration](#asv-configuration) -- [Benchmark Naming](#benchmark-naming) + - [ASV Configuration](#asv-configuration) + - [Benchmark Naming](#benchmark-naming) - [Benchmark Coverage](#benchmark-coverage) -- [Benchmark Classes](#benchmark-classes) -- [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) -- [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) -- [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) + - [Shared Constants](#shared-constants) + - [Shared Helpers](#shared-helpers) + - [Benchmark Classes](#benchmark-classes) + - [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) + - [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) + - [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) - [Running The Benchmarks](#running-the-benchmarks) -- [Profiling With cProfile](#profiling-with-cprofile) + - [Baseline Benchmark](#baseline-benchmark) + - [Current Commit Baseline](#current-commit-baseline) + - [Several Commits Baseline](#several-commits-baseline) + - [Summary of Results](#summary-of-results) + - [Visualization of Results](#visualization-of-results) + - [Profiling With cProfile](#profiling-with-cprofile) +- [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) + - [Shared OpenMP Rule](#shared-openmp-rule) + - [macOS Example](#macos-example) + - [What The macOS OpenMP Config Does](#what-the-macos-openmp-config-does) + - [Run On macOS](#run-on-macos) + - [Interpreting The macOS OpenMP Run](#interpreting-the-macos-openmp-run) + - [Windows Example](#windows-example) + - [Linux Example](#linux-example) + - [HPC Example](#hpc-example) + - [Profiling With cProfile for Multiple Threads](#profiling-with-cprofile-for-multiple-threads) - [More ASV Commands](#more-asv-commands) - [External Reference](#external-reference) ## Setup The regular installation commands in the main `README.md` install GSTools for -normal use. For benchmark work, install this local checkout with the optional -benchmark dependencies. +normal use. This benchmark guide uses conda because ASV creates isolated +benchmark environments for the commits it measures. + +The default benchmark configuration intentionally compares both backends with +one GSTools thread: + +```text +gstools.config.NUM_THREADS = 1 +``` + +That keeps the first comparison simple: Cython fallback vs Rust core without +parallelism as a confounding factor. Parallel/OpenMP scaling is treated as a +separate optional experiment because the correct Cython OpenMP build depends on +the user's operating system, compiler, and runtime environment. + +To run the benchmark and the optional cProfile helper, follow these steps: 1. Move to the GSTools repository root: @@ -47,25 +78,28 @@ benchmark dependencies. cd /path/to/GSTools ``` -2. Install GSTools in editable mode with the benchmark tooling and Rust backend: +2. Create and activate a conda environment for local benchmark work: ```bash -python -m pip install -e ".[benchmark,rust]" +conda create -n gstools-benchmark -c conda-forge python=3.12 asv packaging +conda activate gstools-benchmark ``` -3. Create a machine profile once per computer: +If you already have a suitable conda environment, activate that instead. + +3. If you use an existing environment, make sure ASV is installed: ```bash -asv machine --yes +conda install -c conda-forge asv ``` -Notes: +4. Create a machine profile once per computer: -- The machine profile records local hardware information so ASV can label - results correctly. Do not compare absolute times across different machines. -- You can also install ASV with conda or pip, and you can install the Rust - backend package from - [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core) directly. +```bash +asv machine --yes +``` + +The machine profile records local hardware information so ASV can label results correctly. Do not compare absolute times across different machines. ## Benchmarking Scripts @@ -73,6 +107,8 @@ The benchmarking setup currently consists of: - `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where to store results, and which Python/environment matrix to use. +- `asv.macos-openmp.conf.json`: optional macOS-specific ASV configuration that + builds `gstools-cython` from source with OpenMP inside ASV's own environment. - `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. - `benchmarks/README.md`: this practical guide. - `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints @@ -80,13 +116,13 @@ The benchmarking setup currently consists of: - `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so you can see which functions take time in the current checkout. +- `benchmarks/tools/check_cython_openmp.py`: optional helper for checking + whether the active Python environment's GSTools-Cython extensions detect + OpenMP parallel support. +- `benchmarks/tools/install_macos_openmp_cython.py`: helper used only by + `asv.macos-openmp.conf.json` to compile `gstools-cython` with `llvm-openmp` + on macOS. -Do not run `benchmarks/benchmark_backends.py` directly with Python. ASV loads -that file, discovers benchmark classes and methods, and runs them inside -isolated benchmark environments. The scripts in `benchmarks/tools/` are -different: run them directly with Python. The profiling helper can run against -the current checkout at any time; the speedup-summary helper needs saved ASV -results in `.asv/results/`. ### ASV Configuration @@ -102,27 +138,46 @@ The repo root `asv.conf.json` is tailored to this GSTools checkout: "html_dir": ".asv/html", "environment_type": "conda", "pythons": ["3.12"], + "matrix": { + "req": { + "emcee": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "gstools-cython": [""] + } + }, "install_command": [ - "in-dir={env_dir} python -m pip install {build_dir}[rust]" + "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ] } ``` Important details: -- `install_command` installs the checked-out GSTools revision with the `[rust]` - extra, so `gstools_core` should be available for Rust backend measurements. - ASV still needs its own `install_command` because it creates isolated +- `environment_type: "conda"` means conda is required for the ASV workflow in + this guide. ASV creates isolated conda environments for the commits it + benchmarks. +- `pythons: ["3.12"]` means ASV creates Python 3.12 benchmark environments. + Keep this pinned unless you intentionally validate a newer Python/GSTools + backend stack. +- `matrix.req` asks ASV to install GSTools runtime dependencies before + installing the checked-out GSTools source. It includes `gstools-cython` + explicitly because the GSTools commit is installed with `--no-deps`. +- `{build_dir}` is ASV's temporary checkout/build directory for the exact + GSTools commit being benchmarked. +- `install_command` installs the checked-out GSTools revision with `--no-deps`. + It also installs `gstools_core` with pip because `gstools-core` is not + available as a conda package in every solver/platform combination. +- ASV still needs its own `install_command` because it creates isolated environments for the commits it benchmarks. -- This is separate from any editable install in your active development - environment, such as `python -m pip install -e ".[benchmark,rust]"`. - The editable install is only needed when you want your active environment to - import the current checkout directly, for example when running - `benchmarks/tools/profile_benchmark_workflows.py` with `--backend rust_core`. -- ASV and the cProfile helper use different environments. ASV runs - `benchmarks/benchmark_backends.py` inside `.asv/env/`; the cProfile helper - imports the same benchmark classes but runs them in your active Python - environment. +- Run the cProfile helper with the Python executable from ASV's isolated + environment, for example `.asv/env//bin/python`. In that mode, the + ASV environment provides dependencies while the helper imports the current + checkout through the repo `src/` path. ASV creates these generated directories: @@ -154,7 +209,6 @@ ASV checks out package code at each git commit being benchmarked. Commit source changes before benchmarking them with ASV. Otherwise ASV may benchmark the last committed package code rather than your uncommitted source changes. - ### Benchmark Naming ASV recognizes benchmark methods by name: @@ -166,10 +220,21 @@ ASV recognizes benchmark methods by name: ## Benchmark Coverage +This section describes what is measured by the ASV suite and how the benchmark +labels map to real GSTools workflows. The goal is to cover representative +operations that are relevant for geostatistical work, not isolated +micro-functions. + +The current suite measures runtime and peak memory for variogram estimation, +global kriging, spatial random field generation, and conditioned random field +generation. Each workflow is run with both backends so the results can show +both absolute performance and Rust-vs-Cython differences. + ### Shared Constants ```python BACKENDS = ("cython_fallback", "rust_core") +THREAD_COUNTS = _configured_thread_counts() VARIOGRAM_CASES = ( "full_900", "sampled_5000_to_1500", @@ -191,10 +256,18 @@ These constants define parameter labels shown in ASV results. - `cython_fallback` - `rust_core` +`THREAD_COUNTS` defaults to: + +- `threads_1`: force `gstools.config.NUM_THREADS = 1` + +That is the default because the first benchmark target is a clean Cython-vs-Rust +backend comparison without parallelism. + ### Shared Helpers -`gstools_backend(use_core)` temporarily forces GSTools to use either the Cython -fallback backend or the Rust `gstools_core` backend. +`gstools_backend(use_core, num_threads)` temporarily forces GSTools to use +either the Cython fallback backend or the Rust `gstools_core` backend, and +sets `gstools.config.NUM_THREADS` for that benchmark run. `_random_points(seed, count, scale)` creates deterministic 2D point clouds. @@ -291,45 +364,39 @@ The cases are: ## Running The Benchmarks -Check that the benchmark module imports and runs: +### Baseline Benchmark -```bash -asv run --quick --show-stderr --bench benchmark_backends -``` +The baseline benchmark is the first result set to create before doing any +optimization work. It uses the default ASV configuration, so each workflow is +measured with `threads_1` for both `cython_fallback` and `rust_core`. + +#### Current Commit Baseline -Save a baseline for the current commit: +- Save a baseline for the current commit: ```bash asv run HEAD^! --bench benchmark_backends ``` -Run the last five commits on a linear branch: +#### Several Commits Baseline -```bash -asv run HEAD~5..HEAD --bench benchmark_backends -``` +As mentioned previously, ASV can also compare several commits, here we will run the last five commits: -Build and open the local website: +- Run the last five commits on main branch: ```bash -asv publish -asv preview +asv run HEAD~5..HEAD --bench benchmark_backends ``` -Then open the printed local URL, for example: - -```text -http://127.0.0.1:8082/#/ -``` -(or any other `http://127.0.0.1:/#/` URL shown by the running preview). +#### Summary of Results -After ASV has saved results, print explicit Rust-vs-Cython speedup ratios: +After running ASV, inspect the explicit Rust-vs-Cython speedup ratios: ```bash python benchmarks/tools/asv_speedup_summary.py ``` -The helper reads `.asv/results/` and reports: +The helper reads `.asv/results/` and reports ratios per case and thread label: ```text speedup = cython_fallback_time / rust_core_time @@ -341,14 +408,39 @@ Interpret the ratio as: - `speedup = 1.0` means similar performance - `speedup < 1.0` means Rust is slower -The browser report shows ASV plots and trends. The speedup helper prints the -backend ratio explicitly in the terminal. By default, the helper skips removed -legacy duplicate rows from older saved results. +The speedup helper prints the backend ratio explicitly in the terminal. By +default, the helper skips removed legacy duplicate rows from older saved +results. -## Profiling With cProfile +#### Visualization of Results -`cProfile` is useful for the current checkout. It does not update the ASV -browser report. Instead, it prints a table in the terminal showing which Python +You can inspect the results in the ASV browser report by building and opening +the local website: + +```bash +asv publish +asv preview +``` + +Then open the printed local URL, for example: + +```text +http://127.0.0.1:8082/#/ +``` +(or any other `http://127.0.0.1:/#/` URL shown by the running preview). + +The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run HEAD^! --bench benchmark_backends` will most likely not load any graphs. + +For the default benchmark run, the `threads` column should show `threads_1`. +If you later run the +[optional OpenMP scaling experiment](#optional-parallelisation-with-openmp), +the same column can be used to compare several threads. + + +### Profiling With cProfile + +`cProfile` does not update the ASV results shown in the browser report. +Instead, it prints a table in the terminal showing which Python functions consumed time while one workflow ran. The helper script is: @@ -360,26 +452,244 @@ benchmarks/tools/profile_benchmark_workflows.py It imports the ASV benchmark classes from `benchmark_backends.py`, selects one case, forces one backend, and runs that case under `cProfile`. +Since ASV has already created an isolated Python environment, select that +environment to execute the profiling helper: + +```bash +ASV_ENV="$(ls -td .asv/env/* | head -n 1)" +ASV_PYTHON="$ASV_ENV/bin/python" +``` + +The helper still profiles the current checkout because +`profile_benchmark_workflows.py` adds the repository `src/` directory to +`sys.path`. The ASV environment provides the installed dependencies, including +`gstools-cython` and `gstools_core`. + List available cases: ```bash -python benchmarks/tools/profile_benchmark_workflows.py --list +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --list ``` -Profile selected cases: +Possible profile selected cases: ```bash -python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --limit 10 -python benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --threads threads_1 --limit 10 +``` + +## Optional Parallelisation with OpenMP + +This section collects optional workflows for testing Cython and Rust with +several thread counts. OpenMP setup is platform-dependent, so each operating +system should have its own tested instructions. + +The default setup above remains the recommended baseline: one thread, normal +ASV environment, and no extra OpenMP build steps. Use this section only when +you explicitly want to measure backend scaling with multiple thread counts. + +### Shared OpenMP Rule + +The benchmark code can be run with several thread labels by setting for example +`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. That only passes different +`gstools.config.NUM_THREADS` values to GSTools. It does not, by itself, make +the Cython backend parallel. + +For Cython OpenMP scaling, the Cython extension must be compiled with OpenMP +support inside the same ASV environment that runs the benchmark. Always verify +that environment before interpreting Cython scaling results: + +```bash +ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +``` + +If the check fails, the benchmark may still run, but the Cython backend should +not be interpreted as an OpenMP-enabled Cython run. + +### macOS Example + +This is the currently tested OpenMP workflow. It is separate from the +default setup above. + +The default ASV configuration, `asv.conf.json`, stays conservative: it is the +one-thread baseline and uses the normal conda-forge `gstools-cython` package. +The default `.asv/env/` environment does not provide Cython OpenMP support. That is why this section uses a second ASV configuration: + +```text +asv.macos-openmp.conf.json +``` + +This OpenMP config creates separate generated directories: + +```text +.asv-openmp/env/ +.asv-openmp/results/ +.asv-openmp/html/ +``` + +That keeps the OpenMP experiment separate from the default `.asv/` baseline. + +#### What The macOS OpenMP Config Does + +`asv.macos-openmp.conf.json` asks conda to install the build/runtime pieces +needed for the macOS OpenMP experiment: + +```text +llvm-openmp +cython +extension-helpers +setuptools +wheel +``` + +During ASV installation, it runs: + +```bash +benchmarks/tools/install_macos_openmp_cython.py +``` + +That helper compiles `gstools-cython` from source inside ASV's own environment, +not inside your active conda environment. This matters because ASV benchmarks +the packages installed under `.asv-openmp/env/`. + +Internally, the helper sets: + +```text +GSTOOLS_BUILD_PARALLEL=1 +CC=/bin/gstools-asv-clang-openmp +CXX=/bin/gstools-asv-clang-openmp++ +``` + +The wrapper translates the plain `-fopenmp` flag used by the Cython build into +Apple-clang-compatible compiler and linker arguments that use conda's +`llvm-openmp`. + +#### Run On macOS + +In the previous section, the default config gives a quick overview for both +backends with `threads_1`. In this section, the OpenMP config runs several +thread labels: `threads_1`, `threads_2`, `threads_4`, `threads_8`, and +`threads_16`. + +Start from the GSTools repository root: + +```bash +cd /path/to/GSTools +``` + +Create a clean driver environment. This environment only runs ASV; ASV will +create the real benchmark environment under `.asv-openmp/env/`. + +```bash +conda create -n gstools-benchmark -c conda-forge python=3.12 asv +conda activate gstools-benchmark +``` + +Create the ASV machine profile once: + +```bash +asv --config asv.macos-openmp.conf.json machine --yes +``` + +Run a quick current-commit OpenMP check. This builds the OpenMP-enabled +`gstools-cython` package inside `.asv-openmp/env/` and runs the benchmark suite: + +```bash +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.macos-openmp.conf.json run HEAD^! --quick --bench benchmark_backends --show-stderr +``` + +Verify that the ASV OpenMP environment really uses Cython OpenMP: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +``` + +Expected result on the tested Mac M2 setup: + +```text +variogram default None -> 10 +field default None -> 10 +krige default None -> 10 +OpenMP check: PASS +``` + +If that check passes, run the last-five-commits OpenMP benchmark: + +```bash +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.macos-openmp.conf.json run HEAD~5..HEAD --bench benchmark_backends --show-stderr +``` + +Print Rust-vs-Cython ratios from the OpenMP result folder: + +```bash +python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results +``` + +Build and preview the OpenMP browser report: + +```bash +asv --config asv.macos-openmp.conf.json publish +asv --config asv.macos-openmp.conf.json preview +``` + +#### Interpreting The macOS OpenMP Run + +- Use default `asv.conf.json` for the reproducible one-thread baseline. +- Use `asv.macos-openmp.conf.json` for the macOS OpenMP experiment. +- Only claim Cython OpenMP scaling if `check_cython_openmp.py` passes inside + `.asv-openmp/env/...`. +- The active `gstools-benchmark` conda environment does not need `gstools` + installed. It only needs ASV. The benchmarked GSTools packages live inside + `.asv-openmp/env/...`. + +This workflow is intended for macOS systems that use Apple clang with conda's +`llvm-openmp`. It should be portable across many macOS machines, including +Apple Silicon and Intel Macs, but it is not guaranteed for every macOS setup. + +It is not guaranteed to run without local changes on: + +- older macOS versions +- systems missing Xcode command-line tools +- systems with a nonstandard compiler setup +- HPC or managed macOS environments +- unusual conda installations + +Do not assume this exact OpenMP setup applies to Linux, Windows, or HPC systems. + +### Windows Example + +### Linux Example + +### HPC Example + +### Profiling With cProfile for Multiple Threads + +To profile how a workflow changes across configured thread counts, run the +same cProfile case several times with the OpenMP ASV environment: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +ASV_OPENMP_PYTHON="$ASV_OPENMP_ENV/bin/python" + +for threads in threads_1 threads_2 threads_4 threads_8 threads_16; do + "$ASV_OPENMP_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads "$threads" --limit 10 +done ``` Useful options: - `--case`: choose one workflow, or use `all` - `--backend`: choose `cython_fallback` or `rust_core` +- `--threads`: choose `threads_1`, `threads_2`, `threads_4`, `threads_8`, + or `threads_16` - `--limit`: number of function rows to print from the cProfile table - `--sort cumtime`: sort by cumulative time, usually the best first view - `--sort tottime`: sort by time spent directly in each function diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index 517c61893..c9526eb79 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -2,7 +2,7 @@ Usage: cd /path/to/MPS-Tools/GSTools - python -m pip install -e ".[benchmark]" + # See benchmarks/README.md for ASV and optional cProfile setup. asv machine --yes asv run --quick --show-stderr --bench benchmark_backends asv run HEAD^! --bench benchmark_backends @@ -15,12 +15,17 @@ speedup = cython_fallback_time / rust_core_time Values greater than 1.0 mean the Rust backend is faster on the same machine -for the same benchmark and commit. +for the same benchmark, commit, and thread label. + +By default the suite uses one GSTools thread. For local OpenMP scaling +experiments, set GSTOOLS_BENCHMARK_THREADS, for example: + GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 asv run HEAD^! """ from __future__ import annotations import contextlib +import os import numpy as np @@ -28,6 +33,29 @@ BACKENDS = ("cython_fallback", "rust_core") + + +def _configured_thread_counts(): + raw = os.environ.get("GSTOOLS_BENCHMARK_THREADS", "1") + thread_counts = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + if item.startswith("threads_"): + label = item + value = item.removeprefix("threads_") + else: + label = f"threads_{item}" + value = item + int(value) + thread_counts.append(label) + if not thread_counts: + raise ValueError("GSTOOLS_BENCHMARK_THREADS did not define threads") + return tuple(thread_counts) + + +THREAD_COUNTS = _configured_thread_counts() VARIOGRAM_CASES = ( "full_900", "sampled_5000_to_1500", @@ -43,9 +71,13 @@ @contextlib.contextmanager -def gstools_backend(use_core): - """Temporarily force either gstools-core or the Cython fallback.""" - previous = (gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE) +def gstools_backend(use_core, num_threads): + """Temporarily force backend and GSTools thread count.""" + previous = ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) try: if use_core: if not previous[0]: @@ -55,9 +87,14 @@ def gstools_backend(use_core): else: gs.config._GSTOOLS_CORE_AVAIL = False gs.config.USE_GSTOOLS_CORE = False + gs.config.NUM_THREADS = num_threads yield finally: - gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE = previous + ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) = previous def _use_core(backend): @@ -68,6 +105,12 @@ def _use_core(backend): raise ValueError(f"Unknown backend: {backend}") +def _num_threads(thread_count): + if thread_count.startswith("threads_"): + return int(thread_count.removeprefix("threads_")) + raise ValueError(f"Unknown thread count: {thread_count}") + + def _random_points(seed, count, scale): rng = np.random.RandomState(seed) return rng.rand(count) * scale, rng.rand(count) * scale @@ -99,8 +142,8 @@ def _make_krige_data(seed, cond_count, target_count, scale=50.0): class VariogramWorkflowBenchmarks: """Variogram workflow benchmarks by case and backend.""" - params = [VARIOGRAM_CASES, BACKENDS] - param_names = ["case", "backend"] + params = [VARIOGRAM_CASES, BACKENDS, THREAD_COUNTS] + param_names = ["case", "backend", "threads"] def setup_cache(self): return { @@ -109,16 +152,17 @@ def setup_cache(self): "sampled_15000_to_4500": _make_variogram_data(20220503, 15000), } - def setup(self, data, case, backend): + def setup(self, data, case, backend, threads): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") + _num_threads(threads) - def time_variogram_estimate(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def time_variogram_estimate(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_variogram(data, case) - def peakmem_variogram_estimate(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def peakmem_variogram_estimate(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_variogram(data, case) def _run_variogram(self, data, case): @@ -141,8 +185,8 @@ def _run_variogram(self, data, case): class KrigingWorkflowBenchmarks: """Global kriging workflow benchmarks by case and backend.""" - params = [KRIGE_CASES, BACKENDS] - param_names = ["case", "backend"] + params = [KRIGE_CASES, BACKENDS, THREAD_COUNTS] + param_names = ["case", "backend", "threads"] def setup_cache(self): return { @@ -151,16 +195,17 @@ def setup_cache(self): "extra_large_360x6000": _make_krige_data(20220508, 360, 6000), } - def setup(self, data, case, backend): + def setup(self, data, case, backend, threads): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") + _num_threads(threads) - def time_global_krige(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def time_global_krige(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_krige(data, case) - def peakmem_global_krige(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def peakmem_global_krige(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_krige(data, case) def _run_krige(self, data, case): @@ -184,8 +229,8 @@ def _run_krige(self, data, case): class RandomFieldWorkflowBenchmarks: """SRF and CondSRF workflow benchmarks by case and backend.""" - params = [FIELD_CASES, BACKENDS] - param_names = ["case", "backend"] + params = [FIELD_CASES, BACKENDS, THREAD_COUNTS] + param_names = ["case", "backend", "threads"] def setup_cache(self): return { @@ -197,16 +242,17 @@ def setup_cache(self): "condsrf": _make_krige_data(20220510, 40, 1000), } - def setup(self, data, case, backend): + def setup(self, data, case, backend, threads): if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: raise NotImplementedError("gstools_core is not available") + _num_threads(threads) - def time_field_generation(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def time_field_generation(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_field(data, case) - def peakmem_field_generation(self, data, case, backend): - with gstools_backend(_use_core(backend)): + def peakmem_field_generation(self, data, case, backend, threads): + with gstools_backend(_use_core(backend), _num_threads(threads)): self._run_field(data, case) def _run_field(self, data, case): diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index d341d2178..b3239d702 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -13,7 +13,7 @@ cython_fallback_time / rust_core_time Values greater than 1.0 mean Rust was faster on the same machine, commit, -environment, benchmark, and non-backend parameter combination. +environment, benchmark, case, and thread-count combination. """ from __future__ import annotations @@ -26,6 +26,7 @@ BACKENDS = ("cython_fallback", "rust_core") +THREAD_PREFIX = "threads_" LEGACY_BENCHMARKS = { "time_srf", "peakmem_srf", @@ -128,11 +129,24 @@ def backend_rows(entry): ) if backend is None: continue - case_values = [item for item in combo_values if item not in BACKENDS] + case_values = [ + item + for item in combo_values + if item not in BACKENDS and not item.startswith(THREAD_PREFIX) + ] + threads = next( + ( + item + for item in combo_values + if item.startswith(THREAD_PREFIX) + ), + "-", + ) rows.append( { "backend": backend, "case": "/".join(case_values) if case_values else "-", + "threads": threads, "value": float(value), } ) @@ -161,10 +175,9 @@ def collect_speedups(results_dir, include_all, include_legacy): continue by_case = {} for row in backend_rows(result_entry(raw_result, result_columns)): - by_case.setdefault(row["case"], {})[row["backend"]] = row[ - "value" - ] - for case, values in by_case.items(): + key = (row["case"], row["threads"]) + by_case.setdefault(key, {})[row["backend"]] = row["value"] + for (case, threads), values in by_case.items(): cython = values.get("cython_fallback") rust = values.get("rust_core") if not is_number(cython) or not is_number(rust) or rust == 0: @@ -175,6 +188,7 @@ def collect_speedups(results_dir, include_all, include_legacy): "env": env_name, "benchmark": benchmark_name, "case": case, + "threads": threads, "cython": cython, "rust": rust, "speedup": cython / rust, @@ -193,6 +207,7 @@ def print_table(rows): "env", "benchmark", "case", + "threads", "cython", "rust", "speedup", @@ -203,6 +218,7 @@ def print_table(rows): row["env"], row["benchmark"], row["case"], + row["threads"], f"{row['cython']:.6g}", f"{row['rust']:.6g}", f"{row['speedup']:.3f}x", diff --git a/benchmarks/tools/check_cython_openmp.py b/benchmarks/tools/check_cython_openmp.py new file mode 100644 index 000000000..02fe73db3 --- /dev/null +++ b/benchmarks/tools/check_cython_openmp.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +"""Check whether GSTools-Cython detects OpenMP parallel support. + +This script verifies the active Python environment. Use it with the editable +development environment or with an ASV-created environment. + +Examples: + python benchmarks/tools/check_cython_openmp.py + python benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp + python benchmarks/tools/check_cython_openmp.py --verbose + .asv/env//bin/python3 benchmarks/tools/check_cython_openmp.py +""" + +from __future__ import annotations + +import argparse +import importlib +import sys + + +MODULES = { + "variogram": "gstools_cython.variogram", + "field": "gstools_cython.field", + "krige": "gstools_cython.krige", +} +EXPLICIT_THREAD_COUNTS = (1, 2, 4, 8, 16) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fail-if-no-openmp", + action="store_true", + help="Exit with status 1 if OpenMP thread detection reports <= 1.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-module default and explicit thread-count values.", + ) + return parser.parse_args() + + +def package_version(package_name): + try: + package = importlib.import_module(package_name) + except ModuleNotFoundError: + return "not installed" + return getattr(package, "__version__", "unknown") + + +def check_module(label, module_name): + module = importlib.import_module(module_name) + default_threads = module.set_num_threads(None) + explicit = { + count: module.set_num_threads(count) + for count in EXPLICIT_THREAD_COUNTS + } + return label, default_threads, explicit + + +def main(): + args = parse_args() + + print(f"python: {sys.executable}") + print(f"gstools: {package_version('gstools')}") + print(f"gstools_cython: {package_version('gstools_cython')}") + print(f"gstools_core: {package_version('gstools_core')}") + if args.verbose: + print( + "OpenMP evidence: default None should be >1. " + "Explicit values only prove the wrapper accepts the requested count." + ) + + default_values = [] + for label, module_name in MODULES.items(): + try: + label, default_threads, explicit = check_module(label, module_name) + except ModuleNotFoundError as err: + print(f"OpenMP check: FAIL. Missing module: {err.name}") + return 1 + default_values.append(default_threads) + if args.verbose: + explicit_text = ", ".join( + f"{request}->{actual}" for request, actual in explicit.items() + ) + print(f"{label} default None -> {default_threads}") + print(f"{label} explicit -> {explicit_text}") + + if min(default_values) > 1: + print("OpenMP check: PASS") + return 0 + + print( + "OpenMP check: FAIL. GSTools-Cython reports one default thread. " + "Explicit thread values may be accepted by the wrapper, but this does " + "not prove that the compiled extension is using OpenMP." + ) + return 1 if args.fail_if_no_openmp else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/install_macos_openmp_cython.py b/benchmarks/tools/install_macos_openmp_cython.py new file mode 100644 index 000000000..09f0840cc --- /dev/null +++ b/benchmarks/tools/install_macos_openmp_cython.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +"""Install GSTools-Cython with OpenMP inside a macOS ASV environment. + +This helper is intentionally macOS-specific. It is called from +``asv.macos-openmp.conf.json`` after ASV has created a conda environment that +contains ``llvm-openmp``. +""" + +from __future__ import annotations + +import os +import platform +import stat +import subprocess +import sys +from pathlib import Path + + +def run(command, env=None, check=True): + print("+ " + " ".join(str(part) for part in command), flush=True) + return subprocess.run(command, check=check, env=env) + + +def write_wrapper(path, force_cxx=False): + text = """#!/bin/bash +set -e +prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" +name="$(basename "$0")" +if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then + real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" +else + real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" +fi +is_compile=0 +for arg in "$@"; do + [[ "$arg" == "-c" ]] && is_compile=1 +done +args=() +for arg in "$@"; do + if [[ "$arg" == "-fopenmp" ]]; then + if [[ "$is_compile" == "1" ]]; then + args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") + else + args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") + fi + else + args+=("$arg") + fi +done +exec "$real" "${args[@]}" +""" + if force_cxx: + text = """#!/bin/bash +GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" +""" + path.write_text(text, encoding="utf8") + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def main(): + if len(sys.argv) != 2: + print( + "Usage: install_macos_openmp_cython.py ", + file=sys.stderr, + ) + return 2 + + if platform.system() != "Darwin": + print( + "This helper is macOS-specific. Use the default ASV config or " + "write an OpenMP setup for this platform.", + file=sys.stderr, + ) + return 2 + + env_dir = Path(sys.argv[1]).resolve() + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + omp_header = include_dir / "omp.h" + omp_lib = lib_dir / "libomp.dylib" + + if not omp_header.exists() or not omp_lib.exists(): + print( + "llvm-openmp was not found in the ASV environment. Expected " + f"{omp_header} and {omp_lib}.", + file=sys.stderr, + ) + return 2 + + cc_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp" + cxx_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp++" + write_wrapper(cc_wrapper) + write_wrapper(cxx_wrapper, force_cxx=True) + + build_env = os.environ.copy() + build_env.update( + { + "GSTOOLS_BUILD_PARALLEL": "1", + "GSTOOLS_OPENMP_PREFIX": str(env_dir), + "CC": str(cc_wrapper), + "CXX": str(cxx_wrapper), + "CFLAGS": f"-I{include_dir}", + "LDFLAGS": f"-L{lib_dir}", + } + ) + + run( + [ + sys.executable, + "-m", + "pip", + "uninstall", + "-y", + "gstools-cython", + "gstools_cython", + ], + env=build_env, + check=False, + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-build-isolation", + "--no-cache-dir", + "--force-reinstall", + "--no-binary=gstools-cython", + "--no-deps", + "gstools-cython", + ], + env=build_env, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index 7f953e0dd..2fd2fdff5 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -7,10 +7,13 @@ Usage: cd /path/to/MPS-Tools/GSTools - python benchmarks/tools/profile_benchmark_workflows.py --list - python benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled - python benchmarks/tools/profile_benchmark_workflows.py --case krige-large \ - --backend rust_core --limit 30 + ASV_ENV="$(ls -td .asv/env/* | head -n 1)" + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py --list + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py \ + --case variogram-sampled + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py \ + --case krige-large \ + --backend rust_core --threads threads_1 --limit 30 """ from __future__ import annotations @@ -80,6 +83,14 @@ ), } +THREAD_COUNTS = ( + "threads_1", + "threads_2", + "threads_4", + "threads_8", + "threads_16", +) + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) @@ -113,6 +124,12 @@ def parse_args(): choices=["cython_fallback", "rust_core"], help="Backend label to force while profiling.", ) + parser.add_argument( + "--threads", + default="threads_1", + choices=THREAD_COUNTS, + help="GSTools thread count label.", + ) parser.add_argument( "--list", action="store_true", @@ -134,8 +151,9 @@ def load_suite_class(class_name): except ModuleNotFoundError as err: print( "Could not import GSTools benchmark dependencies. Activate the " - "GSTools benchmark environment or install the project dependencies " - f"first. Original error: {err}", + "GSTools benchmark environment, run this script with an ASV env " + "Python from .asv/env//bin/python, or install the project " + f"dependencies first. Original error: {err}", file=sys.stderr, ) raise SystemExit(1) from err @@ -151,6 +169,7 @@ def run_case( limit, sort, backend, + threads, ): suite_cls = load_suite_class(class_name) suite = suite_cls() @@ -160,10 +179,10 @@ def run_case( profiler = cProfile.Profile() profiler.enable() for _ in range(repeat): - method(data, *params, backend) + method(data, *params, backend, threads) profiler.disable() - print(f"\n== {name} [{backend}] ==") + print(f"\n== {name} [{backend}, {threads}] ==") stats = pstats.Stats(profiler, stream=sys.stdout) stats.strip_dirs().sort_stats(sort).print_stats(limit) @@ -185,6 +204,7 @@ def main(): args.limit, args.sort, args.backend, + args.threads, ) From 806e3868c3599f680ed91105059b7f745af196ee Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 27 May 2026 15:16:08 +0200 Subject: [PATCH 04/37] Add cross-platform benchmark availability CI --- .github/dependabot.yml | 20 + .github/linters/.markdown-lint.yml | 2 + .github/linters/.yaml-lint.yml | 10 + .github/workflows/asv-benchmarks.yml | 262 +++++++++++++- .github/workflows/super-linter.yml | 44 +++ .gitignore | 1 + ...s-openmp.conf.json => asv.openmp.conf.json | 7 +- benchmarks/README.md | 341 ++++++++++++------ benchmarks/benchmark_backends.py | 7 +- .../tools/check_backend_parallel_ready.py | 130 +++++++ benchmarks/tools/check_cython_openmp.py | 1 - .../tools/install_macos_openmp_cython.py | 139 ------- benchmarks/tools/install_openmp_cython.py | 298 +++++++++++++++ benchmarks/tools/write_asv_ci_config.py | 52 +++ 14 files changed, 1027 insertions(+), 287 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/linters/.markdown-lint.yml create mode 100644 .github/linters/.yaml-lint.yml create mode 100644 .github/workflows/super-linter.yml rename asv.macos-openmp.conf.json => asv.openmp.conf.json (89%) create mode 100644 benchmarks/tools/check_backend_parallel_ready.py delete mode 100644 benchmarks/tools/install_macos_openmp_cython.py create mode 100644 benchmarks/tools/install_openmp_cython.py create mode 100644 benchmarks/tools/write_asv_ci_config.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..4dfddfd9b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +version: 2 +updates: + # Dependabot version updates do not run benchmark CI every week. They only + # check whether workflow actions such as actions/checkout or Super-Linter + # have newer versions, then open PRs for review. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Weekly is a low-noise maintenance cadence. Change to monthly if the + # update PRs become distracting. + interval: "weekly" + day: "monday" + time: "05:00" + groups: + # Group action bumps into one PR where possible, instead of one PR per + # action version. + github-actions: + patterns: + - "*" + open-pull-requests-limit: 5 diff --git a/.github/linters/.markdown-lint.yml b/.github/linters/.markdown-lint.yml new file mode 100644 index 000000000..ee0ceb79a --- /dev/null +++ b/.github/linters/.markdown-lint.yml @@ -0,0 +1,2 @@ +MD013: false +MD033: false diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml new file mode 100644 index 000000000..b55f2aa14 --- /dev/null +++ b/.github/linters/.yaml-lint.yml @@ -0,0 +1,10 @@ +extends: default + +rules: + document-start: disable + line-length: disable + truthy: + allowed-values: + - "true" + - "false" + - "on" diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 294dbff42..63c495504 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -1,48 +1,270 @@ -name: ASV Benchmarks +name: Benchmark Availability Checks on: + # This workflow checks benchmark installability, not package correctness. + # It is intentionally path-filtered because ASV creates conda environments + # and the OpenMP job compiles extensions. Normal source/test validation is + # handled by main.yml; this workflow runs when benchmark-related files or + # package dependency metadata change. + pull_request: + branches: + - "main" + paths: + - ".github/workflows/asv-benchmarks.yml" + - "asv*.json" + - "benchmarks/**" + - "pyproject.toml" + push: + branches: + - "main" + paths: + - ".github/workflows/asv-benchmarks.yml" + - "asv*.json" + - "benchmarks/**" + - "pyproject.toml" workflow_dispatch: +# The jobs only need to read the repository. Keep the default token minimal. +permissions: + contents: read + +defaults: + run: + # setup-miniconda initializes conda for bash login shells on all runners. + shell: bash -el {0} + jobs: - benchmark: - runs-on: ubuntu-latest + benchmark_1_thread_availability: + # First stage: prove the normal one-thread ASV benchmark can be installed + # and started across the supported runner/dependency combinations. + name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + timeout-minutes: 75 + strategy: + # Keep the rest of the OS/dependency matrix running after one failure. + # That gives a full compatibility picture from a single workflow run. + fail-fast: false + matrix: + # This list is hand-picked, not imported from main.yml or GSTools-Core. + # Each entry below becomes one independent GitHub Actions job. The goal + # is representative coverage: each OS runner gets older/current/latest + # Python/NumPy/SciPy stacks without running every possible cross-product. + include: + - case-id: ubuntu-py38-oldest + os: ubuntu-latest + python-version: "3.8" + numpy: "==1.20.0" + scipy: "==1.5.4" + - case-id: ubuntu-py312-current + os: ubuntu-latest + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: ubuntu-py313-latest + os: ubuntu-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: windows-py38-oldest + os: windows-latest + python-version: "3.8" + numpy: "==1.20.0" + scipy: "==1.5.4" + - case-id: windows-py312-current + os: windows-latest + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: windows-py313-latest + os: windows-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-py311-older + os: macos-15 + python-version: "3.11" + numpy: "==1.23.2" + scipy: "==1.9.2" + - case-id: macos15-py312-current + os: macos-15 + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: macos15-py313-latest + os: macos-15 + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-intel-py38-oldest + os: macos-15-intel + python-version: "3.8" + numpy: "==1.20.0" + scipy: "==1.5.4" + - case-id: macos15-intel-py312-current + os: macos-15-intel + python-version: "3.12" + numpy: "==1.26.2" + scipy: "==1.11.2" + - case-id: macos15-intel-py313-latest + os: macos-15-intel + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" steps: - uses: actions/checkout@v4 with: + # ASV needs git history for revision expressions such as HEAD^!. fetch-depth: 0 - uses: conda-incubator/setup-miniconda@v3 with: - activate-environment: asv + # Driver environment: only runs ASV and helper scripts. The packages + # being benchmarked are installed inside ASV-created environments. + activate-environment: gstools-benchmark-driver python-version: "3.12" channels: conda-forge auto-activate-base: false - - name: Install ASV - shell: bash -l {0} + - name: Install ASV driver run: | - conda install -y -c conda-forge asv + conda install -n gstools-benchmark-driver -y -c conda-forge asv - - name: Configure ASV machine - shell: bash -l {0} + - name: Write per-job ASV config run: | - asv machine --yes + # The committed asv.conf.json stays simple for local users. CI writes + # a temporary config so each matrix child can pin one dependency set. + conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ + --base-config asv.conf.json \ + --output ".asv-ci/config/${{ matrix.case-id }}.json" \ + --python-version "${{ matrix.python-version }}" \ + --numpy "${{ matrix.numpy }}" \ + --scipy "${{ matrix.scipy }}" \ + --env-dir ".asv-ci/env/${{ matrix.case-id }}" \ + --results-dir ".asv-ci/results/${{ matrix.case-id }}" \ + --html-dir ".asv-ci/html/${{ matrix.case-id }}" - - name: Run ASV benchmarks - shell: bash -l {0} + - name: Configure ASV machine run: | - asv run + conda run -n gstools-benchmark-driver asv \ + --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + machine --yes - - name: Publish ASV report - shell: bash -l {0} + - name: Run 1-thread ASV availability check + env: + # Public benchmark-thread interface. Baseline CI intentionally checks + # the non-parallel path first. + GSTOOLS_BENCHMARK_THREADS: "1" run: | - asv publish + conda run -n gstools-benchmark-driver asv \ + --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + run 'HEAD^!' --quick --bench benchmark_backends --show-stderr - - name: Upload ASV results + - name: Upload ASV availability results + # Upload even on failure so environment/config details are available + # when debugging resolver, compiler, or benchmark discovery problems. + if: always() uses: actions/upload-artifact@v4 with: - name: asv-results + name: asv-1-thread-${{ matrix.case-id }} path: | - .asv/results/ - .asv/html/ + .asv-ci/config/${{ matrix.case-id }}.json + .asv-ci/results/${{ matrix.case-id }} + .asv-ci/html/${{ matrix.case-id }} + if-no-files-found: ignore + + benchmark_parallel_backend_availability: + # Second stage: after every 1-thread ASV matrix child succeeds, prove that + # the parallel backends can be installed and used on each OS runner. + name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + # For a matrix job, this waits for all children of the upstream matrix. + needs: benchmark_1_thread_availability + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + # OpenMP builds are compiler-heavy. Check them on every OS, but only + # with the latest dependency stack to avoid multiplying CI cost. + include: + - case-id: ubuntu-py313-latest + os: ubuntu-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: windows-py313-latest + os: windows-latest + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-py313-latest + os: macos-15 + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + - case-id: macos15-intel-py313-latest + os: macos-15-intel + python-version: "3.13" + numpy: ">=2.1.0" + scipy: ">=1.14.1" + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@v3 + with: + # OpenMP environment: unlike gstools-benchmark-driver, this env gets + # GSTools, gstools-cython, gstools_core, compilers, and runtime deps. + activate-environment: gstools-benchmark-openmp + python-version: ${{ matrix.python-version }} + channels: conda-forge + auto-activate-base: false + + - name: Install benchmark and build requirements + run: | + # Install runtime deps plus compilers/build tools before compiling + # gstools-cython from source with GSTOOLS_BUILD_PARALLEL=1. + conda install -n gstools-benchmark-openmp -y -c conda-forge \ + "numpy${{ matrix.numpy }}" \ + "scipy${{ matrix.scipy }}" \ + c-compiler \ + cxx-compiler \ + cython \ + emcee \ + extension-helpers \ + hankel \ + meshio \ + pip \ + pyevtk \ + "setuptools>=77" \ + wheel + + - name: Install Rust backend package + run: | + # gstools_core is installed with pip because conda availability is not + # uniform across every runner/platform combination. + conda run -n gstools-benchmark-openmp python -m pip install \ + "gstools_core>=1.0.0" + + - name: Build GSTools-Cython with OpenMP + run: | + # The helper dispatches by platform: llvm-openmp wrapper on macOS, + # conda compiler toolchain on Linux, and native MSVC on Windows. + conda run -n gstools-benchmark-openmp python \ + benchmarks/tools/install_openmp_cython.py + + - name: Install local GSTools checkout + run: | + # Dependencies are already pinned by the matrix. Install the checkout + # without letting pip resolve or replace them. + conda run -n gstools-benchmark-openmp python -m pip install \ + --no-deps --editable . + + - name: Check Cython OpenMP and Rust backend readiness + run: | + # Fast readiness check, not a performance benchmark. It fails if + # Cython does not report OpenMP or Rust cannot run with NUM_THREADS=2. + conda run -n gstools-benchmark-openmp python \ + benchmarks/tools/check_backend_parallel_ready.py --verbose diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml new file mode 100644 index 000000000..17ab77687 --- /dev/null +++ b/.github/workflows/super-linter.yml @@ -0,0 +1,44 @@ +name: Super-Linter + +on: + # This workflow is cheap compared with ASV/compiler checks, so run it on + # every PR and push to main to catch metadata/documentation issues early. + pull_request: + push: + branches: + - "main" + workflow_dispatch: + +# Super-Linter needs read access and a status check permission so it can report +# results back to the commit/PR. It does not need write access to repository +# contents. +permissions: + contents: read + packages: read + statuses: write + +jobs: + lint_repository_metadata: + name: Lint GitHub Actions, YAML, JSON, and Markdown + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Run Super-Linter + uses: super-linter/super-linter@v8.6.0 + env: + DEFAULT_BRANCH: main + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Lint changed files only. This avoids introducing unrelated failures + # from old Markdown/YAML while still checking new PR changes. + VALIDATE_ALL_CODEBASE: false + # Opt-in mode: these true values make Super-Linter skip Python. Ruff + # already owns Python linting in main.yml. + VALIDATE_GITHUB_ACTIONS: true + VALIDATE_JSON: true + VALIDATE_MARKDOWN: true + VALIDATE_YAML: true diff --git a/.gitignore b/.gitignore index 1cec6e510..b154c9fac 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ htmlcov/ .cache .asv/* .asv-openmp/* +.asv-ci/* nosetests.xml coverage.xml *.cover diff --git a/asv.macos-openmp.conf.json b/asv.openmp.conf.json similarity index 89% rename from asv.macos-openmp.conf.json rename to asv.openmp.conf.json index 9df8067ab..274e2d3cd 100644 --- a/asv.macos-openmp.conf.json +++ b/asv.openmp.conf.json @@ -14,22 +14,23 @@ "pythons": ["3.12"], "matrix": { "req": { + "c-compiler": [""], + "cxx-compiler": [""], "cython": [""], "emcee": [""], "extension-helpers": [""], "hankel": [""], - "llvm-openmp": [""], "meshio": [""], "numpy": [""], "pyevtk": [""], "scipy": [""], - "setuptools": [""], + "setuptools>=77": [""], "wheel": [""] } }, "install_command": [ "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", - "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_macos_openmp_cython.py {env_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp", "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ] diff --git a/benchmarks/README.md b/benchmarks/README.md index b4df15b6a..013077232 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,14 +41,13 @@ deciding where optimization work should go: - [Profiling With cProfile](#profiling-with-cprofile) - [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) - [Shared OpenMP Rule](#shared-openmp-rule) - - [macOS Example](#macos-example) - - [What The macOS OpenMP Config Does](#what-the-macos-openmp-config-does) - - [Run On macOS](#run-on-macos) - - [Interpreting The macOS OpenMP Run](#interpreting-the-macos-openmp-run) - - [Windows Example](#windows-example) - - [Linux Example](#linux-example) - - [HPC Example](#hpc-example) + - [OpenMP ASV Configuration](#openmp-asv-configuration) + - [Verify Cython OpenMP](#verify-cython-openmp) + - [Run On macOS And Linux](#run-on-macos-and-linux) + - [Run On Windows](#run-on-windows) + - [HPC Notes](#hpc-notes) - [Profiling With cProfile for Multiple Threads](#profiling-with-cprofile-for-multiple-threads) +- [GitHub Action Benchmark Availability Checks](#github-action-benchmark-availability-checks) - [More ASV Commands](#more-asv-commands) - [External Reference](#external-reference) @@ -81,7 +80,7 @@ cd /path/to/GSTools 2. Create and activate a conda environment for local benchmark work: ```bash -conda create -n gstools-benchmark -c conda-forge python=3.12 asv packaging +conda create -n gstools-benchmark -c conda-forge python=3.12 asv conda activate gstools-benchmark ``` @@ -107,8 +106,8 @@ The benchmarking setup currently consists of: - `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where to store results, and which Python/environment matrix to use. -- `asv.macos-openmp.conf.json`: optional macOS-specific ASV configuration that - builds `gstools-cython` from source with OpenMP inside ASV's own environment. +- `asv.openmp.conf.json`: optional cross-platform ASV configuration that builds + `gstools-cython` from source with OpenMP inside ASV's own environment. - `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. - `benchmarks/README.md`: this practical guide. - `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints @@ -119,10 +118,14 @@ The benchmarking setup currently consists of: - `benchmarks/tools/check_cython_openmp.py`: optional helper for checking whether the active Python environment's GSTools-Cython extensions detect OpenMP parallel support. -- `benchmarks/tools/install_macos_openmp_cython.py`: helper used only by - `asv.macos-openmp.conf.json` to compile `gstools-cython` with `llvm-openmp` - on macOS. - +- `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies + Cython OpenMP detection and Rust backend execution with more than one GSTools + thread. +- `benchmarks/tools/write_asv_ci_config.py`: CI helper that writes a temporary + per-job ASV config for one Python/NumPy/SciPy combination. +- `benchmarks/tools/install_openmp_cython.py`: helper used by + `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, + Linux, and native Windows. ### ASV Configuration @@ -190,6 +193,22 @@ ASV creates these generated directories: Those directories are machine-specific generated artifacts. They should normally stay out of git. +The optional `asv.openmp.conf.json` uses the same benchmark suite, but stores +its generated files separately: + +```text +.asv-openmp/env/ OpenMP benchmark environments +.asv-openmp/results/ OpenMP result JSON files +.asv-openmp/html/ OpenMP benchmark website +``` + +It also installs build tools needed to compile `gstools-cython` from source +with `GSTOOLS_BUILD_PARALLEL=1`. The platform-specific OpenMP setup is handled +by `benchmarks/tools/install_openmp_cython.py`: macOS uses conda's +`llvm-openmp` with an Apple-clang wrapper, Linux uses the compiler toolchain +from the ASV conda environment when available, and native Windows uses the +installed MSVC Build Tools. + If needed, users can list more than one branch, Python version, benchmark directory, and so on. For example: @@ -201,7 +220,7 @@ Users can also benchmark any explicit branch, commit, tag, or range without changing `asv.conf.json`: ```bash -asv run my-feature-branch^! --bench benchmark_backends +asv run 'my-feature-branch^!' --bench benchmark_backends asv run main..my-feature-branch --bench benchmark_backends ``` @@ -375,7 +394,7 @@ measured with `threads_1` for both `cython_fallback` and `rust_core`. - Save a baseline for the current commit: ```bash -asv run HEAD^! --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends ``` #### Several Commits Baseline @@ -429,14 +448,13 @@ http://127.0.0.1:8082/#/ ``` (or any other `http://127.0.0.1:/#/` URL shown by the running preview). -The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run HEAD^! --bench benchmark_backends` will most likely not load any graphs. +The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run 'HEAD^!' --bench benchmark_backends` will most likely not load any graphs. For the default benchmark run, the `threads` column should show `threads_1`. If you later run the [optional OpenMP scaling experiment](#optional-parallelisation-with-openmp), the same column can be used to compare several threads. - ### Profiling With cProfile `cProfile` does not update the ASV results shown in the browser report. @@ -460,6 +478,15 @@ ASV_ENV="$(ls -td .asv/env/* | head -n 1)" ASV_PYTHON="$ASV_ENV/bin/python" ``` +On Windows PowerShell, use: + +```powershell +$asvEnv = Get-ChildItem .asv\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvPython = Join-Path $asvEnv.FullName 'python.exe' +``` + The helper still profiles the current checkout because `profile_benchmark_workflows.py` adds the repository `src/` directory to `sys.path`. The ASV environment provides the installed dependencies, including @@ -471,6 +498,8 @@ List available cases: "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --list ``` +On Windows PowerShell, replace `"$ASV_PYTHON"` with `& $asvPython`. + Possible profile selected cases: ```bash @@ -485,7 +514,7 @@ Possible profile selected cases: This section collects optional workflows for testing Cython and Rust with several thread counts. OpenMP setup is platform-dependent, so each operating -system should have its own tested instructions. +system must be verified on the machine that produces the results. The default setup above remains the recommended baseline: one thread, normal ASV environment, and no extra OpenMP build steps. Use this section only when @@ -510,20 +539,15 @@ ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" If the check fails, the benchmark may still run, but the Cython backend should not be interpreted as an OpenMP-enabled Cython run. -### macOS Example - -This is the currently tested OpenMP workflow. It is separate from the -default setup above. +### OpenMP ASV Configuration -The default ASV configuration, `asv.conf.json`, stays conservative: it is the -one-thread baseline and uses the normal conda-forge `gstools-cython` package. -The default `.asv/env/` environment does not provide Cython OpenMP support. That is why this section uses a second ASV configuration: +Use one OpenMP ASV config on all supported desktop platforms: ```text -asv.macos-openmp.conf.json +asv.openmp.conf.json ``` -This OpenMP config creates separate generated directories: +This config keeps the OpenMP experiment separate from the default baseline: ```text .asv-openmp/env/ @@ -531,144 +555,157 @@ This OpenMP config creates separate generated directories: .asv-openmp/html/ ``` -That keeps the OpenMP experiment separate from the default `.asv/` baseline. +During ASV installation, it runs: + +```bash +benchmarks/tools/install_openmp_cython.py +``` + +That helper compiles `gstools-cython` from source inside ASV's own +environment, not inside your active `gstools-benchmark` driver environment. +The helper sets `GSTOOLS_BUILD_PARALLEL=1` and then uses platform-specific +compiler handling: -#### What The macOS OpenMP Config Does +- macOS: uses Apple clang through wrapper scripts and conda's `llvm-openmp`. +- Linux: uses the ASV conda compiler toolchain when available. +- Windows: uses native MSVC Build Tools. -`asv.macos-openmp.conf.json` asks conda to install the build/runtime pieces -needed for the macOS OpenMP experiment: +### Verify Cython OpenMP -```text -llvm-openmp -cython -extension-helpers -setuptools -wheel -``` +Only interpret the Cython rows as OpenMP-enabled after this check passes inside +the `.asv-openmp/env/...` environment. The active `gstools-benchmark` conda +environment is only the ASV driver environment; it is normal for +`python benchmarks/tools/check_backend_parallel_ready.py --verbose` to fail +there with `gstools: not installed`, `gstools_cython: not installed`, or +`gstools_core: not installed`. -During ASV installation, it runs: +On macOS and Linux: ```bash -benchmarks/tools/install_macos_openmp_cython.py +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose ``` -That helper compiles `gstools-cython` from source inside ASV's own environment, -not inside your active conda environment. This matters because ASV benchmarks -the packages installed under `.asv-openmp/env/`. +On Windows PowerShell: -Internally, the helper sets: +```powershell +$asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' +& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --verbose +& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --fail-if-no-openmp +& $asvOpenmpPython benchmarks\tools\check_backend_parallel_ready.py --verbose +``` + +Expected passing output contains: ```text -GSTOOLS_BUILD_PARALLEL=1 -CC=/bin/gstools-asv-clang-openmp -CXX=/bin/gstools-asv-clang-openmp++ +OpenMP check: PASS +Cython OpenMP readiness: PASS +Rust backend readiness: PASS with NUM_THREADS=2 ``` -The wrapper translates the plain `-fopenmp` flag used by the Cython build into -Apple-clang-compatible compiler and linker arguments that use conda's -`llvm-openmp`. +### Run On macOS And Linux -#### Run On macOS +Use these commands from a POSIX shell on macOS or Linux. On macOS, install +Xcode command-line tools first. On Linux, make sure the conda compiler packages +from `asv.openmp.conf.json` solve for your platform. -In the previous section, the default config gives a quick overview for both -backends with `threads_1`. In this section, the OpenMP config runs several -thread labels: `threads_1`, `threads_2`, `threads_4`, `threads_8`, and -`threads_16`. - -Start from the GSTools repository root: +Create the ASV machine profile once: ```bash -cd /path/to/GSTools +asv --config asv.openmp.conf.json machine --yes ``` -Create a clean driver environment. This environment only runs ASV; ASV will -create the real benchmark environment under `.asv-openmp/env/`. +Run a quick current-commit OpenMP smoke run: ```bash -conda create -n gstools-benchmark -c conda-forge python=3.12 asv -conda activate gstools-benchmark +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Create the ASV machine profile once: +Verify Cython OpenMP with the commands in +[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +commits: ```bash -asv --config asv.macos-openmp.conf.json machine --yes +GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` -Run a quick current-commit OpenMP check. This builds the OpenMP-enabled -`gstools-cython` package inside `.asv-openmp/env/` and runs the benchmark suite: +Print Rust-vs-Cython ratios from the OpenMP result folder: ```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ -asv --config asv.macos-openmp.conf.json run HEAD^! --quick --bench benchmark_backends --show-stderr +python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results ``` -Verify that the ASV OpenMP environment really uses Cython OpenMP: +Build and preview the OpenMP browser report: ```bash -ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +asv --config asv.openmp.conf.json publish +asv --config asv.openmp.conf.json preview ``` -Expected result on the tested Mac M2 setup: +### Run On Windows -```text -variogram default None -> 10 -field default None -> 10 -krige default None -> 10 -OpenMP check: PASS -``` +Use native Windows, not WSL, when you want Windows benchmark results. Install +Microsoft C++ Build Tools first, including the C++ build tools workload and a +Windows SDK. Run the commands from PowerShell or Anaconda Prompt after +activating the `gstools-benchmark` conda environment. -If that check passes, run the last-five-commits OpenMP benchmark: +Create the ASV machine profile once: -```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ -asv --config asv.macos-openmp.conf.json run HEAD~5..HEAD --bench benchmark_backends --show-stderr +```powershell +asv --config asv.openmp.conf.json machine --yes ``` -Print Rust-vs-Cython ratios from the OpenMP result folder: +Run a quick current-commit OpenMP smoke run: -```bash -python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results +```powershell +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Build and preview the OpenMP browser report: +Verify Cython OpenMP with the PowerShell commands in +[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +commits: -```bash -asv --config asv.macos-openmp.conf.json publish -asv --config asv.macos-openmp.conf.json preview +```powershell +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` -#### Interpreting The macOS OpenMP Run - -- Use default `asv.conf.json` for the reproducible one-thread baseline. -- Use `asv.macos-openmp.conf.json` for the macOS OpenMP experiment. -- Only claim Cython OpenMP scaling if `check_cython_openmp.py` passes inside - `.asv-openmp/env/...`. -- The active `gstools-benchmark` conda environment does not need `gstools` - installed. It only needs ASV. The benchmarked GSTools packages live inside - `.asv-openmp/env/...`. +Print Rust-vs-Cython ratios from the OpenMP result folder: -This workflow is intended for macOS systems that use Apple clang with conda's -`llvm-openmp`. It should be portable across many macOS machines, including -Apple Silicon and Intel Macs, but it is not guaranteed for every macOS setup. +```powershell +python benchmarks\tools\asv_speedup_summary.py --results-dir .asv-openmp\results +``` -It is not guaranteed to run without local changes on: +Build and preview the OpenMP browser report: -- older macOS versions -- systems missing Xcode command-line tools -- systems with a nonstandard compiler setup -- HPC or managed macOS environments -- unusual conda installations +```powershell +asv --config asv.openmp.conf.json publish +asv --config asv.openmp.conf.json preview +``` -Do not assume this exact OpenMP setup applies to Linux, Windows, or HPC systems. +When finished, clear the thread-count override if the PowerShell session will +be reused: -### Windows Example +```powershell +Remove-Item Env:\GSTOOLS_BENCHMARK_THREADS +``` -### Linux Example +### HPC Notes -### HPC Example +The `asv.openmp.conf.json` workflow is intended for local macOS, Linux, and +native Windows machines. Managed HPC systems often use custom compiler modules, +MPI/OpenMP runtimes, scheduler pinning, and CPU affinity rules. Use the same +validation rule there: only interpret Cython as OpenMP-enabled if +`check_cython_openmp.py --fail-if-no-openmp` passes inside the exact ASV +environment used for the benchmark run. ### Profiling With cProfile for Multiple Threads @@ -684,6 +721,19 @@ for threads in threads_1 threads_2 threads_4 threads_8 threads_16; do done ``` +On Windows PowerShell: + +```powershell +$asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' + +foreach ($threads in 'threads_1', 'threads_2', 'threads_4', 'threads_8', 'threads_16') { + & $asvOpenmpPython benchmarks\tools\profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads $threads --limit 10 +} +``` + Useful options: - `--case`: choose one workflow, or use `all` @@ -697,27 +747,78 @@ Useful options: For example, `--limit 10` means "print the top 10 function rows after sorting". +## GitHub Action Benchmark Availability Checks + +The repository includes `.github/workflows/asv-benchmarks.yml` to check that +the benchmark tooling can be installed and started on GitHub-hosted Linux, +Windows, macOS Apple Silicon, and macOS Intel runners. This workflow is an +availability check, not a performance benchmark run. + +The workflow has two dependent stages: + +- `benchmark_1_thread_availability`: runs `asv.conf.json` with + `GSTOOLS_BENCHMARK_THREADS=1` on a representative Python/NumPy/SciPy matrix. +- `benchmark_parallel_backend_availability`: waits for the 1-thread stage, then + verifies Cython OpenMP and Rust backend readiness on the latest dependency + combo for each OS runner. + +The 1-thread stage runs a quick ASV command: + +```bash +asv run "HEAD^!" --quick --bench benchmark_backends --show-stderr +``` + +The parallel-readiness stage intentionally does not run a full ASV OpenMP +benchmark. Instead, it builds `gstools-cython` with OpenMP, installs +`gstools_core`, installs the local GSTools checkout, and runs: + +```bash +conda run -n gstools-benchmark-openmp python \ + benchmarks/tools/check_backend_parallel_ready.py --verbose +``` + +Locally, run the same helper with the Python executable from the ASV OpenMP +environment: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose +``` + +That fast check proves that Cython reports OpenMP support and that the Rust +backend can run a small workflow with `gstools.config.NUM_THREADS=2`. For real +OpenMP scaling measurements, use `asv.openmp.conf.json` locally with +`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. + +The workflow runs automatically on pull requests and pushes that change ASV +configs, benchmark files, package metadata, or the workflow itself. It can also +be started from the GitHub Actions tab with `workflow_dispatch`. + +For a short explanation of GitHub Actions contexts, matrix jobs, `needs`, +runner labels, secrets, variables, and why this workflow is path-filtered, see +[`benchmarks/github_actions_guide.md`](github_actions_guide.md). + ## More ASV Commands Save results for only the current commit: ```bash -asv run HEAD^! --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends ``` Compare current commit with previous commit: ```bash -asv run HEAD~1^! --bench benchmark_backends -asv run HEAD^! --bench benchmark_backends +asv run 'HEAD~1^!' --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends asv compare HEAD~1 HEAD ``` Compare local `main` with the current branch tip: ```bash -asv run main^! --bench benchmark_backends -asv run HEAD^! --bench benchmark_backends +asv run 'main^!' --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends asv compare main HEAD ``` @@ -725,8 +826,8 @@ Compare remote `main` with the current branch tip: ```bash git fetch origin main -asv run origin/main^! --bench benchmark_backends -asv run HEAD^! --bench benchmark_backends +asv run 'origin/main^!' --bench benchmark_backends +asv run 'HEAD^!' --bench benchmark_backends asv compare origin/main HEAD ``` diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index c9526eb79..cc588c1e7 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -19,7 +19,8 @@ By default the suite uses one GSTools thread. For local OpenMP scaling experiments, set GSTOOLS_BENCHMARK_THREADS, for example: - GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 asv run HEAD^! + GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ + asv --config asv.openmp.conf.json run 'HEAD^!' """ from __future__ import annotations @@ -27,10 +28,8 @@ import contextlib import os -import numpy as np - import gstools as gs - +import numpy as np BACKENDS = ("cython_fallback", "rust_core") diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py new file mode 100644 index 000000000..8a6c7aa38 --- /dev/null +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python +"""Check that GSTools benchmark backends are ready for parallel runs. + +This is a fast CI probe. It verifies that GSTools-Cython reports OpenMP support +and that the Rust backend can run a small workflow while GSTools is configured +with more than one thread. +""" + +from __future__ import annotations + +import argparse +import importlib +import sys + +import numpy as np +from check_cython_openmp import MODULES, check_module, package_version + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--threads", + default=2, + type=int, + help="Thread count to request for the Rust backend smoke workflow.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-module Cython OpenMP thread details.", + ) + return parser.parse_args() + + +def check_cython_openmp(verbose=False): + default_values = [] + for label, module_name in MODULES.items(): + try: + label, default_threads, explicit = check_module(label, module_name) + except ModuleNotFoundError as err: + print(f"Cython OpenMP readiness: FAIL. Missing module: {err.name}") + return False + default_values.append(default_threads) + if verbose: + explicit_text = ", ".join( + f"{request}->{actual}" for request, actual in explicit.items() + ) + print(f"{label} default None -> {default_threads}") + print(f"{label} explicit -> {explicit_text}") + if explicit.get(2) != 2: + print( + "Cython OpenMP readiness: FAIL. " + f"{label} did not accept an explicit 2-thread request." + ) + return False + + if min(default_values) <= 1: + print( + "Cython OpenMP readiness: FAIL. GSTools-Cython reports one " + "default thread." + ) + return False + + print("Cython OpenMP readiness: PASS") + return True + + +def check_rust_backend(threads): + try: + import gstools as gs + except ModuleNotFoundError as err: + print(f"Rust backend readiness: FAIL. Missing module: {err.name}") + return False + + try: + importlib.import_module("gstools_core") + except ModuleNotFoundError as err: + print(f"Rust backend readiness: FAIL. Missing module: {err.name}") + return False + + if not gs.config._GSTOOLS_CORE_AVAIL: + print("Rust backend readiness: FAIL. GSTools did not detect gstools_core.") + return False + + previous = ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) + try: + gs.config._GSTOOLS_CORE_AVAIL = True + gs.config.USE_GSTOOLS_CORE = True + gs.config.NUM_THREADS = threads + + x = np.linspace(0.0, 10.0, 12) + y = np.linspace(0.0, 5.0, 12) + field = np.sin(x) + np.cos(y) + bins = np.linspace(0.0, 8.0, 6) + gs.vario_estimate( + (x, y), + field, + bins, + mesh_type="unstructured", + return_counts=True, + ) + finally: + ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) = previous + + print(f"Rust backend readiness: PASS with NUM_THREADS={threads}") + return True + + +def main(): + args = parse_args() + print(f"python: {sys.executable}") + print(f"gstools: {package_version('gstools')}") + print(f"gstools_cython: {package_version('gstools_cython')}") + print(f"gstools_core: {package_version('gstools_core')}") + + cython_ready = check_cython_openmp(verbose=args.verbose) + rust_ready = check_rust_backend(args.threads) + return 0 if cython_ready and rust_ready else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/check_cython_openmp.py b/benchmarks/tools/check_cython_openmp.py index 02fe73db3..b3a410cc9 100644 --- a/benchmarks/tools/check_cython_openmp.py +++ b/benchmarks/tools/check_cython_openmp.py @@ -17,7 +17,6 @@ import importlib import sys - MODULES = { "variogram": "gstools_cython.variogram", "field": "gstools_cython.field", diff --git a/benchmarks/tools/install_macos_openmp_cython.py b/benchmarks/tools/install_macos_openmp_cython.py deleted file mode 100644 index 09f0840cc..000000000 --- a/benchmarks/tools/install_macos_openmp_cython.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -"""Install GSTools-Cython with OpenMP inside a macOS ASV environment. - -This helper is intentionally macOS-specific. It is called from -``asv.macos-openmp.conf.json`` after ASV has created a conda environment that -contains ``llvm-openmp``. -""" - -from __future__ import annotations - -import os -import platform -import stat -import subprocess -import sys -from pathlib import Path - - -def run(command, env=None, check=True): - print("+ " + " ".join(str(part) for part in command), flush=True) - return subprocess.run(command, check=check, env=env) - - -def write_wrapper(path, force_cxx=False): - text = """#!/bin/bash -set -e -prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" -name="$(basename "$0")" -if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then - real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" -else - real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" -fi -is_compile=0 -for arg in "$@"; do - [[ "$arg" == "-c" ]] && is_compile=1 -done -args=() -for arg in "$@"; do - if [[ "$arg" == "-fopenmp" ]]; then - if [[ "$is_compile" == "1" ]]; then - args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") - else - args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") - fi - else - args+=("$arg") - fi -done -exec "$real" "${args[@]}" -""" - if force_cxx: - text = """#!/bin/bash -GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" -""" - path.write_text(text, encoding="utf8") - path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - -def main(): - if len(sys.argv) != 2: - print( - "Usage: install_macos_openmp_cython.py ", - file=sys.stderr, - ) - return 2 - - if platform.system() != "Darwin": - print( - "This helper is macOS-specific. Use the default ASV config or " - "write an OpenMP setup for this platform.", - file=sys.stderr, - ) - return 2 - - env_dir = Path(sys.argv[1]).resolve() - include_dir = env_dir / "include" - lib_dir = env_dir / "lib" - omp_header = include_dir / "omp.h" - omp_lib = lib_dir / "libomp.dylib" - - if not omp_header.exists() or not omp_lib.exists(): - print( - "llvm-openmp was not found in the ASV environment. Expected " - f"{omp_header} and {omp_lib}.", - file=sys.stderr, - ) - return 2 - - cc_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp" - cxx_wrapper = env_dir / "bin" / "gstools-asv-clang-openmp++" - write_wrapper(cc_wrapper) - write_wrapper(cxx_wrapper, force_cxx=True) - - build_env = os.environ.copy() - build_env.update( - { - "GSTOOLS_BUILD_PARALLEL": "1", - "GSTOOLS_OPENMP_PREFIX": str(env_dir), - "CC": str(cc_wrapper), - "CXX": str(cxx_wrapper), - "CFLAGS": f"-I{include_dir}", - "LDFLAGS": f"-L{lib_dir}", - } - ) - - run( - [ - sys.executable, - "-m", - "pip", - "uninstall", - "-y", - "gstools-cython", - "gstools_cython", - ], - env=build_env, - check=False, - ) - run( - [ - sys.executable, - "-m", - "pip", - "install", - "--no-build-isolation", - "--no-cache-dir", - "--force-reinstall", - "--no-binary=gstools-cython", - "--no-deps", - "gstools-cython", - ], - env=build_env, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/tools/install_openmp_cython.py b/benchmarks/tools/install_openmp_cython.py new file mode 100644 index 000000000..2883fe0ca --- /dev/null +++ b/benchmarks/tools/install_openmp_cython.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python +"""Install GSTools-Cython with OpenMP inside an ASV environment. + +This helper is called by ``asv.openmp.conf.json`` after ASV has created a +conda environment with the common build requirements. GitHub Actions can also +call it inside an already-active conda environment. It keeps the ASV config +portable and handles platform-specific compiler/OpenMP details here. +""" + +from __future__ import annotations + +import argparse +import os +import platform +import shutil +import stat +import subprocess +import sys +from pathlib import Path + + +def run(command, env=None, check=True): + print("+ " + " ".join(str(part) for part in command), flush=True) + return subprocess.run(command, check=check, env=env) + + +def prepend_path(build_env, *paths): + current = build_env.get("PATH", "") + existing = [str(path) for path in paths if path.exists()] + if existing: + build_env["PATH"] = os.pathsep.join(existing + [current]) + + +def first_match(directory, patterns): + for pattern in patterns: + matches = sorted(path for path in directory.glob(pattern) if path.is_file()) + if matches: + return matches[0] + return None + + +def require_file(path, message): + if not path.exists(): + print(message, file=sys.stderr) + return False + return True + + +def conda_executable(): + conda = os.environ.get("CONDA_EXE") or shutil.which("conda") + if conda is None: + print("No conda executable was found.", file=sys.stderr) + return conda + + +def conda_install(env_dir, *packages): + conda = conda_executable() + if conda is None: + return False + run( + [ + conda, + "install", + "-y", + "-p", + str(env_dir), + "-c", + "conda-forge", + *packages, + ] + ) + return True + + +def ensure_macos_llvm_openmp(env_dir): + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + omp_header = include_dir / "omp.h" + omp_lib = lib_dir / "libomp.dylib" + + if omp_header.exists() and omp_lib.exists(): + return True + + if not conda_install(env_dir, "llvm-openmp"): + print( + "llvm-openmp is required on macOS but was not found in the ASV " + "environment, and it could not be installed with conda.", + file=sys.stderr, + ) + return False + + return require_file( + omp_header, + f"llvm-openmp install did not create expected header: {omp_header}", + ) and require_file( + omp_lib, + f"llvm-openmp install did not create expected library: {omp_lib}", + ) + + +def ensure_linux_libgomp(env_dir): + lib_dir = env_dir / "lib" + if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: + return True + + if not conda_install(env_dir, "libgomp"): + print( + "libgomp is required on Linux but could not be installed with conda.", + file=sys.stderr, + ) + return False + + if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: + return True + + print( + f"libgomp install did not create an OpenMP runtime under {lib_dir}.", + file=sys.stderr, + ) + return False + + +def write_macos_wrapper(path, force_cxx=False): + text = """#!/bin/bash +set -e +prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" +name="$(basename "$0")" +if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then + real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" +else + real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" +fi +is_compile=0 +for arg in "$@"; do + [[ "$arg" == "-c" ]] && is_compile=1 +done +args=() +for arg in "$@"; do + if [[ "$arg" == "-fopenmp" ]]; then + if [[ "$is_compile" == "1" ]]; then + args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") + else + args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") + fi + else + args+=("$arg") + fi +done +exec "$real" "${args[@]}" +""" + if force_cxx: + text = """#!/bin/bash +GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" +""" + path.write_text(text, encoding="utf8") + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def macos_build_env(env_dir): + if not ensure_macos_llvm_openmp(env_dir): + return None + + bin_dir = env_dir / "bin" + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + cc_wrapper = bin_dir / "gstools-asv-clang-openmp" + cxx_wrapper = bin_dir / "gstools-asv-clang-openmp++" + write_macos_wrapper(cc_wrapper) + write_macos_wrapper(cxx_wrapper, force_cxx=True) + + build_env = os.environ.copy() + prepend_path(build_env, bin_dir) + build_env.update( + { + "GSTOOLS_BUILD_PARALLEL": "1", + "GSTOOLS_OPENMP_PREFIX": str(env_dir), + "CC": str(cc_wrapper), + "CXX": str(cxx_wrapper), + "CFLAGS": f"-I{include_dir}", + "LDFLAGS": f"-L{lib_dir}", + } + ) + return build_env + + +def linux_build_env(env_dir): + if not ensure_linux_libgomp(env_dir): + return None + + bin_dir = env_dir / "bin" + cc = first_match( + bin_dir, + [ + "*-conda-linux-gnu-cc", + "*-conda-linux-gnu-gcc", + "gcc", + "cc", + ], + ) + cxx = first_match( + bin_dir, + [ + "*-conda-linux-gnu-c++", + "*-conda-linux-gnu-g++", + "g++", + "c++", + ], + ) + + build_env = os.environ.copy() + prepend_path(build_env, bin_dir) + build_env["GSTOOLS_BUILD_PARALLEL"] = "1" + if cc is not None: + build_env["CC"] = str(cc) + if cxx is not None: + build_env["CXX"] = str(cxx) + return build_env + + +def windows_build_env(env_dir): + build_env = os.environ.copy() + prepend_path( + build_env, + env_dir, + env_dir / "Scripts", + env_dir / "Library" / "bin", + ) + build_env["GSTOOLS_BUILD_PARALLEL"] = "1" + return build_env + + +def install_gstools_cython(build_env): + run( + [ + sys.executable, + "-m", + "pip", + "uninstall", + "-y", + "gstools-cython", + "gstools_cython", + ], + env=build_env, + check=False, + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-build-isolation", + "--no-cache-dir", + "--force-reinstall", + "--no-binary=gstools-cython", + "--no-deps", + "gstools-cython", + ], + env=build_env, + ) + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "env_dir", + nargs="?", + default=sys.prefix, + help="ASV or active conda environment directory. Defaults to sys.prefix.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + env_dir = Path(args.env_dir).resolve() + system = platform.system() + print(f"Preparing GSTools-Cython OpenMP build for {system}") + + if system == "Darwin": + build_env = macos_build_env(env_dir) + elif system == "Linux": + build_env = linux_build_env(env_dir) + elif system == "Windows": + build_env = windows_build_env(env_dir) + else: + print(f"Unsupported platform for OpenMP ASV build: {system}", file=sys.stderr) + return 2 + + if build_env is None: + return 2 + + install_gstools_cython(build_env) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py new file mode 100644 index 000000000..9ec5b2b8a --- /dev/null +++ b/benchmarks/tools/write_asv_ci_config.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +"""Write a per-job ASV configuration for GitHub Actions. + +The committed ASV configs stay convenient for local benchmarking. CI uses this +helper to pin one Python/NumPy/SciPy combination per matrix job without turning +normal local ``asv run`` commands into a large dependency matrix. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-config", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--python-version", required=True) + parser.add_argument("--numpy", required=True) + parser.add_argument("--scipy", required=True) + parser.add_argument("--env-dir", required=True) + parser.add_argument("--results-dir", required=True) + parser.add_argument("--html-dir", required=True) + return parser.parse_args() + + +def main(): + args = parse_args() + with args.base_config.open(encoding="utf8") as config_file: + config = json.load(config_file) + + config["pythons"] = [args.python_version] + config["env_dir"] = args.env_dir + config["results_dir"] = args.results_dir + config["html_dir"] = args.html_dir + + req = config.setdefault("matrix", {}).setdefault("req", {}) + req["numpy"] = [args.numpy] + req["scipy"] = [args.scipy] + + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf8") as config_file: + json.dump(config, config_file, indent=2) + config_file.write("\n") + + print(f"Wrote {args.output}") + + +if __name__ == "__main__": + main() From 253114655aea41d57aa6add2b57b39fe638e7771 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 27 May 2026 22:55:05 +0200 Subject: [PATCH 05/37] Fix benchmark CI config resolution and linter scope --- .github/linters/.markdown-lint.yml | 2 -- .github/workflows/asv-benchmarks.yml | 12 ++++++------ .github/workflows/super-linter.yml | 10 +++++----- .gitignore | 1 + 4 files changed, 12 insertions(+), 13 deletions(-) delete mode 100644 .github/linters/.markdown-lint.yml diff --git a/.github/linters/.markdown-lint.yml b/.github/linters/.markdown-lint.yml deleted file mode 100644 index ee0ceb79a..000000000 --- a/.github/linters/.markdown-lint.yml +++ /dev/null @@ -1,2 +0,0 @@ -MD013: false -MD033: false diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 63c495504..e03f1bde3 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -124,7 +124,7 @@ jobs: activate-environment: gstools-benchmark-driver python-version: "3.12" channels: conda-forge - auto-activate-base: false + conda-remove-defaults: true - name: Install ASV driver run: | @@ -136,7 +136,7 @@ jobs: # a temporary config so each matrix child can pin one dependency set. conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ --base-config asv.conf.json \ - --output ".asv-ci/config/${{ matrix.case-id }}.json" \ + --output "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ --python-version "${{ matrix.python-version }}" \ --numpy "${{ matrix.numpy }}" \ --scipy "${{ matrix.scipy }}" \ @@ -147,7 +147,7 @@ jobs: - name: Configure ASV machine run: | conda run -n gstools-benchmark-driver asv \ - --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ machine --yes - name: Run 1-thread ASV availability check @@ -157,7 +157,7 @@ jobs: GSTOOLS_BENCHMARK_THREADS: "1" run: | conda run -n gstools-benchmark-driver asv \ - --config ".asv-ci/config/${{ matrix.case-id }}.json" \ + --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ run 'HEAD^!' --quick --bench benchmark_backends --show-stderr - name: Upload ASV availability results @@ -168,7 +168,7 @@ jobs: with: name: asv-1-thread-${{ matrix.case-id }} path: | - .asv-ci/config/${{ matrix.case-id }}.json + asv.ci.${{ matrix.case-id }}.json .asv-ci/results/${{ matrix.case-id }} .asv-ci/html/${{ matrix.case-id }} if-no-files-found: ignore @@ -220,7 +220,7 @@ jobs: activate-environment: gstools-benchmark-openmp python-version: ${{ matrix.python-version }} channels: conda-forge - auto-activate-base: false + conda-remove-defaults: true - name: Install benchmark and build requirements run: | diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 17ab77687..9bac92ae9 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -2,7 +2,7 @@ name: Super-Linter on: # This workflow is cheap compared with ASV/compiler checks, so run it on - # every PR and push to main to catch metadata/documentation issues early. + # every PR and push to main to catch workflow/configuration issues early. pull_request: push: branches: @@ -19,7 +19,7 @@ permissions: jobs: lint_repository_metadata: - name: Lint GitHub Actions, YAML, JSON, and Markdown + name: Lint GitHub Actions, YAML, and JSON runs-on: ubuntu-latest timeout-minutes: 15 @@ -36,9 +36,9 @@ jobs: # Lint changed files only. This avoids introducing unrelated failures # from old Markdown/YAML while still checking new PR changes. VALIDATE_ALL_CODEBASE: false - # Opt-in mode: these true values make Super-Linter skip Python. Ruff - # already owns Python linting in main.yml. + # Opt-in mode: these true values make Super-Linter skip Python and + # Markdown. Ruff owns Python linting in main.yml. Prose/style checks + # should use a dedicated tool such as Vale or textlint if added later. VALIDATE_GITHUB_ACTIONS: true VALIDATE_JSON: true - VALIDATE_MARKDOWN: true VALIDATE_YAML: true diff --git a/.gitignore b/.gitignore index b154c9fac..460490cd6 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ htmlcov/ .asv/* .asv-openmp/* .asv-ci/* +asv.ci.*.json nosetests.xml coverage.xml *.cover From 99e53286bc17da5a1ff807d93c1c1154595a53bc Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 08:43:33 +0200 Subject: [PATCH 06/37] Add Vale Markdown terminology checks --- .github/styles/GSTools/Terms.yml | 14 ++++++ .../config/vocabularies/GSTools/accept.txt | 11 ++++ .../config/vocabularies/GSTools/reject.txt | 1 + .github/workflows/super-linter.yml | 4 +- .github/workflows/vale.yml | 50 +++++++++++++++++++ .vale.ini | 6 +++ README.md | 4 +- 7 files changed, 86 insertions(+), 4 deletions(-) create mode 100644 .github/styles/GSTools/Terms.yml create mode 100644 .github/styles/config/vocabularies/GSTools/accept.txt create mode 100644 .github/styles/config/vocabularies/GSTools/reject.txt create mode 100644 .github/workflows/vale.yml create mode 100644 .vale.ini diff --git a/.github/styles/GSTools/Terms.yml b/.github/styles/GSTools/Terms.yml new file mode 100644 index 000000000..9d528e238 --- /dev/null +++ b/.github/styles/GSTools/Terms.yml @@ -0,0 +1,14 @@ +extends: substitution +message: "Use '%s' instead of '%s'." +level: error +ignorecase: false +swap: + "Github": "GitHub" + "MacOS": "macOS" + "MacOs": "macOS" + "Macos": "macOS" + "OpenMp": "OpenMP" + "openMP": "OpenMP" + "Powershell": "PowerShell" + "Power Shell": "PowerShell" + "conda forge": "conda-forge" diff --git a/.github/styles/config/vocabularies/GSTools/accept.txt b/.github/styles/config/vocabularies/GSTools/accept.txt new file mode 100644 index 000000000..581358563 --- /dev/null +++ b/.github/styles/config/vocabularies/GSTools/accept.txt @@ -0,0 +1,11 @@ +ASV +Cython +GeoStat +GSTools +OpenMP +PowerShell +NumPy +SciPy +conda-forge +gstools-cython +gstools_core diff --git a/.github/styles/config/vocabularies/GSTools/reject.txt b/.github/styles/config/vocabularies/GSTools/reject.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/.github/styles/config/vocabularies/GSTools/reject.txt @@ -0,0 +1 @@ + diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 9bac92ae9..83eab46eb 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -37,8 +37,8 @@ jobs: # from old Markdown/YAML while still checking new PR changes. VALIDATE_ALL_CODEBASE: false # Opt-in mode: these true values make Super-Linter skip Python and - # Markdown. Ruff owns Python linting in main.yml. Prose/style checks - # should use a dedicated tool such as Vale or textlint if added later. + # Markdown. Ruff owns Python linting in main.yml, and vale.yml owns + # Markdown prose/terminology checks. VALIDATE_GITHUB_ACTIONS: true VALIDATE_JSON: true VALIDATE_YAML: true diff --git a/.github/workflows/vale.yml b/.github/workflows/vale.yml new file mode 100644 index 000000000..b0da9d3e5 --- /dev/null +++ b/.github/workflows/vale.yml @@ -0,0 +1,50 @@ +name: Vale + +on: + # Vale is for documentation/prose quality. It runs only when Markdown or Vale + # configuration changes, so it complements Super-Linter without adding noise + # to source-only changes. + pull_request: + paths: + - "**/*.md" + - ".vale.ini" + - ".github/styles/**" + - ".github/workflows/vale.yml" + push: + branches: + - "main" + paths: + - "**/*.md" + - ".vale.ini" + - ".github/styles/**" + - ".github/workflows/vale.yml" + workflow_dispatch: + +# The Vale action reads repository content and reports findings back to the PR +# check. It does not need permission to modify files. +permissions: + contents: read + pull-requests: read + checks: write + +jobs: + lint_markdown_prose: + name: Lint Markdown prose and terminology + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Run Vale + uses: vale-cli/vale-action@v2.1.2 + with: + # Pin the Vale CLI separately from the GitHub Action wrapper so prose + # checks do not change behavior unexpectedly between runs. + version: "3.14.1" + # Check tracked documentation paths, not generated local ASV + # environments such as .asv/ or .asv-openmp/. + files: '["README.md", "CONTRIBUTING.md", "AUTHORS.md", "CHANGELOG.md", "benchmarks"]' + filter_mode: nofilter + reporter: github-check + fail_on_error: true diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 000000000..045ef69f8 --- /dev/null +++ b/.vale.ini @@ -0,0 +1,6 @@ +StylesPath = .github/styles +MinAlertLevel = warning +Vocab = GSTools + +[*.md] +BasedOnStyles = GSTools diff --git a/README.md b/README.md index 1947df0bf..237f7587a 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ Install the package by typing the following command in a command terminal: conda install gstools -In case conda forge is not set up for your system yet, see the easy to follow -instructions on [conda forge][conda_forge_link]. Using conda, the parallelized +In case conda-forge is not set up for your system yet, see the easy to follow +instructions on [conda-forge][conda_forge_link]. Using conda, the parallelized version of GSTools should be installed. From bddc2dffeba6c1f4efd044c76dd8e92cd8381285 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 09:24:31 +0200 Subject: [PATCH 07/37] Second iteration to fix GitHub actions + pin GitHub actions to commit SHAs --- .github/workflows/asv-benchmarks.yml | 119 ++++++++---------- .github/workflows/main.yml | 34 +++-- .github/workflows/super-linter.yml | 4 +- .github/workflows/vale.yml | 4 +- .../tools/check_backend_parallel_ready.py | 4 +- benchmarks/tools/install_openmp_cython.py | 13 +- pyproject.toml | 1 + 7 files changed, 93 insertions(+), 86 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index e03f1bde3..97ae3e9be 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -37,7 +37,7 @@ jobs: benchmark_1_thread_availability: # First stage: prove the normal one-thread ASV benchmark can be installed # and started across the supported runner/dependency combinations. - name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.python-version }} + name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} timeout-minutes: 75 strategy: @@ -52,72 +52,60 @@ jobs: include: - case-id: ubuntu-py38-oldest os: ubuntu-latest - python-version: "3.8" - numpy: "==1.20.0" - scipy: "==1.5.4" + ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - case-id: ubuntu-py312-current os: ubuntu-latest - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: ubuntu-py313-latest os: ubuntu-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: ubuntu-py314-latest + os: ubuntu-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: windows-py38-oldest os: windows-latest - python-version: "3.8" - numpy: "==1.20.0" - scipy: "==1.5.4" + ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - case-id: windows-py312-current os: windows-latest - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: windows-py313-latest os: windows-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: windows-py314-latest + os: windows-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: macos15-py311-older os: macos-15 - python-version: "3.11" - numpy: "==1.23.2" - scipy: "==1.9.2" + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - case-id: macos15-py312-current os: macos-15 - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: macos15-py313-latest os: macos-15 - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: macos15-py314-latest + os: macos-15 + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: macos15-intel-py38-oldest os: macos-15-intel - python-version: "3.8" - numpy: "==1.20.0" - scipy: "==1.5.4" + ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - case-id: macos15-intel-py312-current os: macos-15-intel - python-version: "3.12" - numpy: "==1.26.2" - scipy: "==1.11.2" + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - case-id: macos15-intel-py313-latest os: macos-15-intel - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - case-id: macos15-intel-py314-latest + os: macos-15-intel + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: # ASV needs git history for revision expressions such as HEAD^!. fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: # Driver environment: only runs ASV and helper scripts. The packages # being benchmarked are installed inside ASV-created environments. @@ -128,7 +116,14 @@ jobs: - name: Install ASV driver run: | - conda install -n gstools-benchmark-driver -y -c conda-forge asv + # ASV loads environment backends as optional plugins. Installing + # packaging/virtualenv explicitly prevents CI from falling back to + # the "existing" backend only, which would make environment_type: + # conda unavailable. + conda install -n gstools-benchmark-driver -y -c conda-forge \ + asv \ + packaging \ + virtualenv - name: Write per-job ASV config run: | @@ -137,9 +132,9 @@ jobs: conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ --base-config asv.conf.json \ --output "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - --python-version "${{ matrix.python-version }}" \ - --numpy "${{ matrix.numpy }}" \ - --scipy "${{ matrix.scipy }}" \ + --python-version "${{ matrix.ver.py }}" \ + --numpy "${{ matrix.ver.np }}" \ + --scipy "${{ matrix.ver.sp }}" \ --env-dir ".asv-ci/env/${{ matrix.case-id }}" \ --results-dir ".asv-ci/results/${{ matrix.case-id }}" \ --html-dir ".asv-ci/html/${{ matrix.case-id }}" @@ -164,7 +159,7 @@ jobs: # Upload even on failure so environment/config details are available # when debugging resolver, compiler, or benchmark discovery problems. if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: asv-1-thread-${{ matrix.case-id }} path: | @@ -176,7 +171,7 @@ jobs: benchmark_parallel_backend_availability: # Second stage: after every 1-thread ASV matrix child succeeds, prove that # the parallel backends can be installed and used on each OS runner. - name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.python-version }} + name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} # For a matrix job, this waits for all children of the upstream matrix. needs: benchmark_1_thread_availability @@ -187,38 +182,30 @@ jobs: # OpenMP builds are compiler-heavy. Check them on every OS, but only # with the latest dependency stack to avoid multiplying CI cost. include: - - case-id: ubuntu-py313-latest + - case-id: ubuntu-py314-latest os: ubuntu-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" - - case-id: windows-py313-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - case-id: windows-py314-latest os: windows-latest - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" - - case-id: macos15-py313-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - case-id: macos15-py314-latest os: macos-15 - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" - - case-id: macos15-intel-py313-latest + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - case-id: macos15-intel-py314-latest os: macos-15-intel - python-version: "3.13" - numpy: ">=2.1.0" - scipy: ">=1.14.1" + ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3 + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: # OpenMP environment: unlike gstools-benchmark-driver, this env gets # GSTools, gstools-cython, gstools_core, compilers, and runtime deps. activate-environment: gstools-benchmark-openmp - python-version: ${{ matrix.python-version }} + python-version: ${{ matrix.ver.py }} channels: conda-forge conda-remove-defaults: true @@ -227,8 +214,8 @@ jobs: # Install runtime deps plus compilers/build tools before compiling # gstools-cython from source with GSTOOLS_BUILD_PARALLEL=1. conda install -n gstools-benchmark-openmp -y -c conda-forge \ - "numpy${{ matrix.numpy }}" \ - "scipy${{ matrix.scipy }}" \ + "numpy${{ matrix.ver.np }}" \ + "scipy${{ matrix.ver.sp }}" \ c-compiler \ cxx-compiler \ cython \ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 79993388a..6e4683e4f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,12 +18,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: "0" - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: 3.11 @@ -55,7 +55,13 @@ jobs: fail-fast: false matrix: os: - [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] + [ + ubuntu-latest, + ubuntu-24.04-arm, + windows-latest, + macos-15, + macos-15-intel, + ] # https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg ver: - { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } @@ -65,22 +71,26 @@ jobs: - { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + - { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } exclude: - os: ubuntu-24.04-arm ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-14 + # macos-15 is the Apple Silicon runner. Old Python/NumPy/SciPy + # combinations do not have reliable arm64 wheels there, so keep + # legacy dependency coverage on Linux, Windows, and macOS Intel. + - os: macos-15 ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-14 + - os: macos-15 ver: { py: "3.9", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-14 + - os: macos-15 ver: { py: "3.10", np: "==1.21.6", sp: "==1.7.2" } steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: "0" - name: Set up Python ${{ matrix.ver.py }} - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: ${{ matrix.ver.py }} @@ -103,7 +113,7 @@ jobs: # PEP 517 package builder from pypa python -m build . - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 if: matrix.os == 'ubuntu-latest' && matrix.ver.py == '3.11' with: name: dist @@ -114,7 +124,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 with: name: dist path: dist @@ -122,7 +132,7 @@ jobs: - name: Publish to Test PyPI # only if working on main if: github.ref == 'refs/heads/main' - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: user: __token__ password: ${{ secrets.test_pypi_password }} @@ -132,7 +142,7 @@ jobs: - name: Publish to PyPI # only if tagged if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml index 83eab46eb..044f4251b 100644 --- a/.github/workflows/super-linter.yml +++ b/.github/workflows/super-linter.yml @@ -24,12 +24,12 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 - name: Run Super-Linter - uses: super-linter/super-linter@v8.6.0 + uses: super-linter/super-linter@9e863354e3ff62e0727d37183162c4a88873df41 # v8.6.0 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/vale.yml b/.github/workflows/vale.yml index b0da9d3e5..606028117 100644 --- a/.github/workflows/vale.yml +++ b/.github/workflows/vale.yml @@ -34,10 +34,10 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Run Vale - uses: vale-cli/vale-action@v2.1.2 + uses: vale-cli/vale-action@d89dee975228ae261d22c15adcd03578634d429c # v2 with: # Pin the Vale CLI separately from the GitHub Action wrapper so prose # checks do not change behavior unexpectedly between runs. diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index 8a6c7aa38..cf56845be 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -79,7 +79,9 @@ def check_rust_backend(threads): return False if not gs.config._GSTOOLS_CORE_AVAIL: - print("Rust backend readiness: FAIL. GSTools did not detect gstools_core.") + print( + "Rust backend readiness: FAIL. GSTools did not detect gstools_core." + ) return False previous = ( diff --git a/benchmarks/tools/install_openmp_cython.py b/benchmarks/tools/install_openmp_cython.py index 2883fe0ca..6c720218d 100644 --- a/benchmarks/tools/install_openmp_cython.py +++ b/benchmarks/tools/install_openmp_cython.py @@ -33,7 +33,9 @@ def prepend_path(build_env, *paths): def first_match(directory, patterns): for pattern in patterns: - matches = sorted(path for path in directory.glob(pattern) if path.is_file()) + matches = sorted( + path for path in directory.glob(pattern) if path.is_file() + ) if matches: return matches[0] return None @@ -153,7 +155,9 @@ def write_macos_wrapper(path, force_cxx=False): GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" """ path.write_text(text, encoding="utf8") - path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + path.chmod( + path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) def macos_build_env(env_dir): @@ -284,7 +288,10 @@ def main(): elif system == "Windows": build_env = windows_build_env(env_dir) else: - print(f"Unsupported platform for OpenMP ASV build: {system}", file=sys.stderr) + print( + f"Unsupported platform for OpenMP ASV build: {system}", + file=sys.stderr, + ) return 2 if build_env is None: diff --git a/pyproject.toml b/pyproject.toml index f39ea2eab..ce43bdb71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: GIS", "Topic :: Scientific/Engineering :: Hydrology", From fca014eab7b31d7bba4babd27ff01af30d0bf828 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 10:16:34 +0200 Subject: [PATCH 08/37] Fix ASV benchmark CI matrix and lint config --- .github/linters/.yaml-lint.yml | 5 +++++ .github/workflows/asv-benchmarks.yml | 25 +++++++++++++------------ .github/workflows/main.yml | 7 +++++-- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml index b55f2aa14..c9872e0bd 100644 --- a/.github/linters/.yaml-lint.yml +++ b/.github/linters/.yaml-lint.yml @@ -1,6 +1,11 @@ extends: default rules: + braces: + max-spaces-inside: 1 + min-spaces-inside: 0 + comments: + min-spaces-from-content: 1 document-start: disable line-length: disable truthy: diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 97ae3e9be..9e24f6fe1 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -48,26 +48,27 @@ jobs: # This list is hand-picked, not imported from main.yml or GSTools-Core. # Each entry below becomes one independent GitHub Actions job. The goal # is representative coverage: each OS runner gets older/current/latest - # Python/NumPy/SciPy stacks without running every possible cross-product. + # conda-solvable Python/NumPy/SciPy stacks without running every + # possible cross-product. include: - - case-id: ubuntu-py38-oldest + - case-id: ubuntu-py311-older os: ubuntu-latest - ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: ubuntu-py312-current os: ubuntu-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: ubuntu-py313-latest os: ubuntu-latest ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } - case-id: ubuntu-py314-latest os: ubuntu-latest ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: windows-py38-oldest + - case-id: windows-py311-older os: windows-latest - ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: windows-py312-current os: windows-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: windows-py313-latest os: windows-latest ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } @@ -76,22 +77,22 @@ jobs: ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - case-id: macos15-py311-older os: macos-15 - ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: macos15-py312-current os: macos-15 - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: macos15-py313-latest os: macos-15 ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } - case-id: macos15-py314-latest os: macos-15 ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: macos15-intel-py38-oldest + - case-id: macos15-intel-py311-older os: macos-15-intel - ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } + ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } - case-id: macos15-intel-py312-current os: macos-15-intel - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } - case-id: macos15-intel-py313-latest os: macos-15-intel ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6e4683e4f..b09c61712 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -101,11 +101,14 @@ jobs: - name: Install GSTools run: | - pip install -v --editable .[test] + pip install -v --editable '.[test]' - name: Run tests + env: + NUMPY_SPEC: ${{ matrix.ver.np }} + SCIPY_SPEC: ${{ matrix.ver.sp }} run: | - pip install "numpy${{ matrix.ver.np }}" "scipy${{ matrix.ver.sp }}" + pip install "numpy${NUMPY_SPEC}" "scipy${SCIPY_SPEC}" python -m pytest -v tests/ - name: Build dist From 00d19093a21a29605965fb86c3568271dd42d8e5 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 13:27:55 +0200 Subject: [PATCH 09/37] third iteration GitHub actions --- .github/workflows/asv-benchmarks.yml | 127 +++++++++++++----------- benchmarks/README.md | 26 +++-- benchmarks/tools/write_asv_ci_config.py | 16 ++- 3 files changed, 99 insertions(+), 70 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 9e24f6fe1..356764762 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -34,10 +34,13 @@ defaults: shell: bash -el {0} jobs: - benchmark_1_thread_availability: - # First stage: prove the normal one-thread ASV benchmark can be installed - # and started across the supported runner/dependency combinations. - name: 1-thread ASV on ${{ matrix.os }} py${{ matrix.ver.py }} + benchmark_availability: + # Each matrix child is its own small chain: first run the normal one-thread + # ASV smoke check, then run parallel readiness immediately for entries + # marked with parallel_check. GitHub Actions cannot make one matrix child + # depend only on the matching child of another matrix job, so keeping both + # phases in one job avoids waiting for unrelated OS/Python combinations. + name: Benchmark availability on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} timeout-minutes: 75 strategy: @@ -49,56 +52,73 @@ jobs: # Each entry below becomes one independent GitHub Actions job. The goal # is representative coverage: each OS runner gets older/current/latest # conda-solvable Python/NumPy/SciPy stacks without running every - # possible cross-product. + # possible cross-product. Use exact package pins here because ASV's + # conda backend does not handle open-ended version ranges reliably. include: - case-id: ubuntu-py311-older os: ubuntu-latest ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: ubuntu-py312-current os: ubuntu-latest ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: ubuntu-py313-latest os: ubuntu-latest - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: ubuntu-py314-latest os: ubuntu-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" - case-id: windows-py311-older os: windows-latest ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: windows-py312-current os: windows-latest ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: windows-py313-latest os: windows-latest - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: windows-py314-latest os: windows-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" - case-id: macos15-py311-older os: macos-15 ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: macos15-py312-current os: macos-15 ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: macos15-py313-latest os: macos-15 - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: macos15-py314-latest os: macos-15 - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" - case-id: macos15-intel-py311-older os: macos-15-intel ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + parallel_check: "false" - case-id: macos15-intel-py312-current os: macos-15-intel ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + parallel_check: "false" - case-id: macos15-intel-py313-latest os: macos-15-intel - ver: { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } + ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } + parallel_check: "false" - case-id: macos15-intel-py314-latest os: macos-15-intel - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } + parallel_check: "true" steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -142,9 +162,20 @@ jobs: - name: Configure ASV machine run: | + # Supply explicit machine metadata for CI. ASV can infer this on + # Linux/macOS ARM, but Windows runner detection is less reliable. + # The case-id is also passed to asv run below so both commands use + # the same machine entry. conda run -n gstools-benchmark-driver asv \ --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - machine --yes + machine \ + --machine "${{ matrix.case-id }}" \ + --os "${{ runner.os }}" \ + --arch "${{ runner.arch }}" \ + --cpu "GitHub Actions ${{ matrix.os }}" \ + --num_cpu "2" \ + --ram "unknown" \ + --yes - name: Run 1-thread ASV availability check env: @@ -152,9 +183,18 @@ jobs: # the non-parallel path first. GSTOOLS_BENCHMARK_THREADS: "1" run: | + # Run one lightweight ASV benchmark method. Its parameters still + # exercise both backend values, but CI avoids running the full + # benchmark suite because this workflow checks availability, not + # performance quality. conda run -n gstools-benchmark-driver asv \ --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - run 'HEAD^!' --quick --bench benchmark_backends --show-stderr + run \ + --machine "${{ matrix.case-id }}" \ + --quick \ + --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ + --show-stderr \ + 'HEAD^!' - name: Upload ASV availability results # Upload even on failure so environment/config details are available @@ -169,52 +209,15 @@ jobs: .asv-ci/html/${{ matrix.case-id }} if-no-files-found: ignore - benchmark_parallel_backend_availability: - # Second stage: after every 1-thread ASV matrix child succeeds, prove that - # the parallel backends can be installed and used on each OS runner. - name: Parallel backend readiness on ${{ matrix.os }} py${{ matrix.ver.py }} - runs-on: ${{ matrix.os }} - # For a matrix job, this waits for all children of the upstream matrix. - needs: benchmark_1_thread_availability - timeout-minutes: 75 - strategy: - fail-fast: false - matrix: - # OpenMP builds are compiler-heavy. Check them on every OS, but only - # with the latest dependency stack to avoid multiplying CI cost. - include: - - case-id: ubuntu-py314-latest - os: ubuntu-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: windows-py314-latest - os: windows-latest - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: macos15-py314-latest - os: macos-15 - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - case-id: macos15-intel-py314-latest - os: macos-15-intel - ver: { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 - with: - # OpenMP environment: unlike gstools-benchmark-driver, this env gets - # GSTools, gstools-cython, gstools_core, compilers, and runtime deps. - activate-environment: gstools-benchmark-openmp - python-version: ${{ matrix.ver.py }} - channels: conda-forge - conda-remove-defaults: true - - - name: Install benchmark and build requirements + - name: Create OpenMP readiness environment + if: matrix.parallel_check == 'true' run: | - # Install runtime deps plus compilers/build tools before compiling - # gstools-cython from source with GSTOOLS_BUILD_PARALLEL=1. - conda install -n gstools-benchmark-openmp -y -c conda-forge \ + # OpenMP builds are compiler-heavy. Check them on every OS, but only + # with the latest dependency stack to avoid multiplying CI cost. This + # step runs only after this exact matrix child's 1-thread ASV check + # has passed. + conda create -n gstools-benchmark-openmp -y -c conda-forge \ + "python=${{ matrix.ver.py }}" \ "numpy${{ matrix.ver.np }}" \ "scipy${{ matrix.ver.sp }}" \ c-compiler \ @@ -230,6 +233,7 @@ jobs: wheel - name: Install Rust backend package + if: matrix.parallel_check == 'true' run: | # gstools_core is installed with pip because conda availability is not # uniform across every runner/platform combination. @@ -237,6 +241,7 @@ jobs: "gstools_core>=1.0.0" - name: Build GSTools-Cython with OpenMP + if: matrix.parallel_check == 'true' run: | # The helper dispatches by platform: llvm-openmp wrapper on macOS, # conda compiler toolchain on Linux, and native MSVC on Windows. @@ -244,6 +249,7 @@ jobs: benchmarks/tools/install_openmp_cython.py - name: Install local GSTools checkout + if: matrix.parallel_check == 'true' run: | # Dependencies are already pinned by the matrix. Install the checkout # without letting pip resolve or replace them. @@ -251,6 +257,7 @@ jobs: --no-deps --editable . - name: Check Cython OpenMP and Rust backend readiness + if: matrix.parallel_check == 'true' run: | # Fast readiness check, not a performance benchmark. It fails if # Cython does not report OpenMP or Rust cannot run with NUM_THREADS=2. diff --git a/benchmarks/README.md b/benchmarks/README.md index 013077232..0e323e977 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -754,21 +754,31 @@ the benchmark tooling can be installed and started on GitHub-hosted Linux, Windows, macOS Apple Silicon, and macOS Intel runners. This workflow is an availability check, not a performance benchmark run. -The workflow has two dependent stages: +Each matrix job is its own small chain: -- `benchmark_1_thread_availability`: runs `asv.conf.json` with - `GSTOOLS_BENCHMARK_THREADS=1` on a representative Python/NumPy/SciPy matrix. -- `benchmark_parallel_backend_availability`: waits for the 1-thread stage, then - verifies Cython OpenMP and Rust backend readiness on the latest dependency - combo for each OS runner. +- first, it runs `asv.conf.json` with `GSTOOLS_BENCHMARK_THREADS=1` on one + representative Python/NumPy/SciPy combination. +- then, for entries marked as the latest dependency combo for an OS runner, it + builds the OpenMP environment and verifies Cython OpenMP plus Rust backend + readiness. + +This layout keeps the required order within each runner: the parallel check for +one OS/dependency setup starts only after that setup's 1-thread ASV check has +passed. It does not wait for unrelated matrix entries on other operating +systems. The 1-thread stage runs a quick ASV command: ```bash -asv run "HEAD^!" --quick --bench benchmark_backends --show-stderr +asv run "HEAD^!" --quick \ + --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ + --show-stderr ``` -The parallel-readiness stage intentionally does not run a full ASV OpenMP +That benchmark method still exercises both configured backend values through +ASV parameters, but avoids running the full performance suite in CI. + +The parallel-readiness steps intentionally do not run a full ASV OpenMP benchmark. Instead, it builds `gstools-cython` with OpenMP, installs `gstools_core`, installs the local GSTools checkout, and runs: diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py index 9ec5b2b8a..0999ada01 100644 --- a/benchmarks/tools/write_asv_ci_config.py +++ b/benchmarks/tools/write_asv_ci_config.py @@ -13,6 +13,15 @@ from pathlib import Path +def validate_asv_req_spec(package, spec): + if spec.startswith((">", "<", "~", "!=")) or "," in spec: + raise ValueError( + f"{package} uses open-ended CI spec {spec!r}. ASV's conda " + "backend expects exact matrix pins here, for example '==2.1.3'." + ) + return spec + + def parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-config", required=True, type=Path) @@ -37,8 +46,11 @@ def main(): config["html_dir"] = args.html_dir req = config.setdefault("matrix", {}).setdefault("req", {}) - req["numpy"] = [args.numpy] - req["scipy"] = [args.scipy] + try: + req["numpy"] = [validate_asv_req_spec("numpy", args.numpy)] + req["scipy"] = [validate_asv_req_spec("scipy", args.scipy)] + except ValueError as err: + raise SystemExit(str(err)) from err args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("w", encoding="utf8") as config_file: From db86d44d3914fb861acde67c15f84c81e235b110 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 15:30:23 +0200 Subject: [PATCH 10/37] removing not needed dependencies --- .github/dependabot.yml | 20 -------- .github/linters/.yaml-lint.yml | 15 ------ .github/styles/GSTools/Terms.yml | 14 ------ .../config/vocabularies/GSTools/accept.txt | 11 ---- .../config/vocabularies/GSTools/reject.txt | 1 - .github/workflows/asv-benchmarks.yml | 16 +++--- .github/workflows/main.yml | 44 ++++++---------- .github/workflows/super-linter.yml | 44 ---------------- .github/workflows/vale.yml | 50 ------------------- .vale.ini | 6 --- benchmarks/README.md | 6 +-- 11 files changed, 25 insertions(+), 202 deletions(-) delete mode 100644 .github/dependabot.yml delete mode 100644 .github/linters/.yaml-lint.yml delete mode 100644 .github/styles/GSTools/Terms.yml delete mode 100644 .github/styles/config/vocabularies/GSTools/accept.txt delete mode 100644 .github/styles/config/vocabularies/GSTools/reject.txt delete mode 100644 .github/workflows/super-linter.yml delete mode 100644 .github/workflows/vale.yml delete mode 100644 .vale.ini diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 4dfddfd9b..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: 2 -updates: - # Dependabot version updates do not run benchmark CI every week. They only - # check whether workflow actions such as actions/checkout or Super-Linter - # have newer versions, then open PRs for review. - - package-ecosystem: "github-actions" - directory: "/" - schedule: - # Weekly is a low-noise maintenance cadence. Change to monthly if the - # update PRs become distracting. - interval: "weekly" - day: "monday" - time: "05:00" - groups: - # Group action bumps into one PR where possible, instead of one PR per - # action version. - github-actions: - patterns: - - "*" - open-pull-requests-limit: 5 diff --git a/.github/linters/.yaml-lint.yml b/.github/linters/.yaml-lint.yml deleted file mode 100644 index c9872e0bd..000000000 --- a/.github/linters/.yaml-lint.yml +++ /dev/null @@ -1,15 +0,0 @@ -extends: default - -rules: - braces: - max-spaces-inside: 1 - min-spaces-inside: 0 - comments: - min-spaces-from-content: 1 - document-start: disable - line-length: disable - truthy: - allowed-values: - - "true" - - "false" - - "on" diff --git a/.github/styles/GSTools/Terms.yml b/.github/styles/GSTools/Terms.yml deleted file mode 100644 index 9d528e238..000000000 --- a/.github/styles/GSTools/Terms.yml +++ /dev/null @@ -1,14 +0,0 @@ -extends: substitution -message: "Use '%s' instead of '%s'." -level: error -ignorecase: false -swap: - "Github": "GitHub" - "MacOS": "macOS" - "MacOs": "macOS" - "Macos": "macOS" - "OpenMp": "OpenMP" - "openMP": "OpenMP" - "Powershell": "PowerShell" - "Power Shell": "PowerShell" - "conda forge": "conda-forge" diff --git a/.github/styles/config/vocabularies/GSTools/accept.txt b/.github/styles/config/vocabularies/GSTools/accept.txt deleted file mode 100644 index 581358563..000000000 --- a/.github/styles/config/vocabularies/GSTools/accept.txt +++ /dev/null @@ -1,11 +0,0 @@ -ASV -Cython -GeoStat -GSTools -OpenMP -PowerShell -NumPy -SciPy -conda-forge -gstools-cython -gstools_core diff --git a/.github/styles/config/vocabularies/GSTools/reject.txt b/.github/styles/config/vocabularies/GSTools/reject.txt deleted file mode 100644 index 8b1378917..000000000 --- a/.github/styles/config/vocabularies/GSTools/reject.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 356764762..757a331a2 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -87,20 +87,20 @@ jobs: os: windows-latest ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } parallel_check: "true" - - case-id: macos15-py311-older - os: macos-15 + - case-id: macos-latest-py311-older + os: macos-latest ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } parallel_check: "false" - - case-id: macos15-py312-current - os: macos-15 + - case-id: macos-latest-py312-current + os: macos-latest ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } parallel_check: "false" - - case-id: macos15-py313-latest - os: macos-15 + - case-id: macos-latest-py313-latest + os: macos-latest ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } parallel_check: "false" - - case-id: macos15-py314-latest - os: macos-15 + - case-id: macos-latest-py314-latest + os: macos-latest ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } parallel_check: "true" - case-id: macos15-intel-py311-older diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b09c61712..da78b6457 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,12 +18,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@v4 with: fetch-depth: "0" - name: Set up Python 3.11 - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@v5 with: python-version: 3.11 @@ -55,13 +55,7 @@ jobs: fail-fast: false matrix: os: - [ - ubuntu-latest, - ubuntu-24.04-arm, - windows-latest, - macos-15, - macos-15-intel, - ] + [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] # https://github.com/scipy/oldest-supported-numpy/blob/main/setup.cfg ver: - { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } @@ -70,27 +64,24 @@ jobs: - { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - - { py: "3.13", np: ">=2.1.0", sp: ">=1.14.1" } - - { py: "3.14", np: ">=2.3.3", sp: ">=1.16.1" } + - { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } + - { py: "3.14", np: ">=2.3.2", sp: ">=1.16.1" } exclude: - os: ubuntu-24.04-arm ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - # macos-15 is the Apple Silicon runner. Old Python/NumPy/SciPy - # combinations do not have reliable arm64 wheels there, so keep - # legacy dependency coverage on Linux, Windows, and macOS Intel. - - os: macos-15 + - os: macos-latest ver: { py: "3.8", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-15 + - os: macos-latest ver: { py: "3.9", np: "==1.20.0", sp: "==1.5.4" } - - os: macos-15 + - os: macos-latest ver: { py: "3.10", np: "==1.21.6", sp: "==1.7.2" } steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@v4 with: fetch-depth: "0" - name: Set up Python ${{ matrix.ver.py }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.ver.py }} @@ -101,14 +92,11 @@ jobs: - name: Install GSTools run: | - pip install -v --editable '.[test]' + pip install -v --editable .[test] - name: Run tests - env: - NUMPY_SPEC: ${{ matrix.ver.np }} - SCIPY_SPEC: ${{ matrix.ver.sp }} run: | - pip install "numpy${NUMPY_SPEC}" "scipy${SCIPY_SPEC}" + pip install "numpy${{ matrix.ver.np }}" "scipy${{ matrix.ver.sp }}" python -m pytest -v tests/ - name: Build dist @@ -116,7 +104,7 @@ jobs: # PEP 517 package builder from pypa python -m build . - - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + - uses: actions/upload-artifact@v4 if: matrix.os == 'ubuntu-latest' && matrix.ver.py == '3.11' with: name: dist @@ -127,7 +115,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + - uses: actions/download-artifact@v4 with: name: dist path: dist @@ -135,7 +123,7 @@ jobs: - name: Publish to Test PyPI # only if working on main if: github.ref == 'refs/heads/main' - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.test_pypi_password }} @@ -145,7 +133,7 @@ jobs: - name: Publish to PyPI # only if tagged if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/super-linter.yml b/.github/workflows/super-linter.yml deleted file mode 100644 index 044f4251b..000000000 --- a/.github/workflows/super-linter.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Super-Linter - -on: - # This workflow is cheap compared with ASV/compiler checks, so run it on - # every PR and push to main to catch workflow/configuration issues early. - pull_request: - push: - branches: - - "main" - workflow_dispatch: - -# Super-Linter needs read access and a status check permission so it can report -# results back to the commit/PR. It does not need write access to repository -# contents. -permissions: - contents: read - packages: read - statuses: write - -jobs: - lint_repository_metadata: - name: Lint GitHub Actions, YAML, and JSON - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - name: Run Super-Linter - uses: super-linter/super-linter@9e863354e3ff62e0727d37183162c4a88873df41 # v8.6.0 - env: - DEFAULT_BRANCH: main - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Lint changed files only. This avoids introducing unrelated failures - # from old Markdown/YAML while still checking new PR changes. - VALIDATE_ALL_CODEBASE: false - # Opt-in mode: these true values make Super-Linter skip Python and - # Markdown. Ruff owns Python linting in main.yml, and vale.yml owns - # Markdown prose/terminology checks. - VALIDATE_GITHUB_ACTIONS: true - VALIDATE_JSON: true - VALIDATE_YAML: true diff --git a/.github/workflows/vale.yml b/.github/workflows/vale.yml deleted file mode 100644 index 606028117..000000000 --- a/.github/workflows/vale.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Vale - -on: - # Vale is for documentation/prose quality. It runs only when Markdown or Vale - # configuration changes, so it complements Super-Linter without adding noise - # to source-only changes. - pull_request: - paths: - - "**/*.md" - - ".vale.ini" - - ".github/styles/**" - - ".github/workflows/vale.yml" - push: - branches: - - "main" - paths: - - "**/*.md" - - ".vale.ini" - - ".github/styles/**" - - ".github/workflows/vale.yml" - workflow_dispatch: - -# The Vale action reads repository content and reports findings back to the PR -# check. It does not need permission to modify files. -permissions: - contents: read - pull-requests: read - checks: write - -jobs: - lint_markdown_prose: - name: Lint Markdown prose and terminology - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - - name: Run Vale - uses: vale-cli/vale-action@d89dee975228ae261d22c15adcd03578634d429c # v2 - with: - # Pin the Vale CLI separately from the GitHub Action wrapper so prose - # checks do not change behavior unexpectedly between runs. - version: "3.14.1" - # Check tracked documentation paths, not generated local ASV - # environments such as .asv/ or .asv-openmp/. - files: '["README.md", "CONTRIBUTING.md", "AUTHORS.md", "CHANGELOG.md", "benchmarks"]' - filter_mode: nofilter - reporter: github-check - fail_on_error: true diff --git a/.vale.ini b/.vale.ini deleted file mode 100644 index 045ef69f8..000000000 --- a/.vale.ini +++ /dev/null @@ -1,6 +0,0 @@ -StylesPath = .github/styles -MinAlertLevel = warning -Vocab = GSTools - -[*.md] -BasedOnStyles = GSTools diff --git a/benchmarks/README.md b/benchmarks/README.md index 0e323e977..382b87c83 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -751,7 +751,7 @@ For example, `--limit 10` means "print the top 10 function rows after sorting". The repository includes `.github/workflows/asv-benchmarks.yml` to check that the benchmark tooling can be installed and started on GitHub-hosted Linux, -Windows, macOS Apple Silicon, and macOS Intel runners. This workflow is an +Windows, `macos-latest`, and `macos-15-intel` runners. This workflow is an availability check, not a performance benchmark run. Each matrix job is its own small chain: @@ -804,10 +804,6 @@ The workflow runs automatically on pull requests and pushes that change ASV configs, benchmark files, package metadata, or the workflow itself. It can also be started from the GitHub Actions tab with `workflow_dispatch`. -For a short explanation of GitHub Actions contexts, matrix jobs, `needs`, -runner labels, secrets, variables, and why this workflow is path-filtered, see -[`benchmarks/github_actions_guide.md`](github_actions_guide.md). - ## More ASV Commands Save results for only the current commit: From bd58de319362a8d4e95688bdc915a0571a974992 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 22:42:43 +0200 Subject: [PATCH 11/37] 4th iteration benchmarking GitHub Actions --- .github/workflows/asv-benchmarks.yml | 96 +++++++++-------- asv.openmp.conf.json | 4 +- benchmarks/README.md | 85 ++++++++------- benchmarks/benchmark_backends.py | 23 +++- .../tools/check_backend_parallel_ready.py | 51 +++++++-- benchmarks/tools/check_cython_openmp.py | 102 ------------------ .../tools/profile_benchmark_workflows.py | 4 +- benchmarks/tools/write_asv_ci_config.py | 20 ++-- benchmarks/tools/write_asv_ci_machine.py | 55 ++++++++++ 9 files changed, 237 insertions(+), 203 deletions(-) delete mode 100644 benchmarks/tools/check_cython_openmp.py create mode 100644 benchmarks/tools/write_asv_ci_machine.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 757a331a2..3987bbd85 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -52,73 +52,77 @@ jobs: # Each entry below becomes one independent GitHub Actions job. The goal # is representative coverage: each OS runner gets older/current/latest # conda-solvable Python/NumPy/SciPy stacks without running every - # possible cross-product. Use exact package pins here because ASV's - # conda backend does not handle open-ended version ranges reliably. + # possible cross-product. Use bare exact package versions here; the + # helper writes conda-style pins into ASV's generated config. include: - case-id: ubuntu-py311-older os: ubuntu-latest - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: ubuntu-py312-current os: ubuntu-latest - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: ubuntu-py313-latest os: ubuntu-latest - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: ubuntu-py314-latest os: ubuntu-latest - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" - case-id: windows-py311-older os: windows-latest - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: windows-py312-current os: windows-latest - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: windows-py313-latest os: windows-latest - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: windows-py314-latest os: windows-latest - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" - case-id: macos-latest-py311-older os: macos-latest - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: macos-latest-py312-current os: macos-latest - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: macos-latest-py313-latest os: macos-latest - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: macos-latest-py314-latest os: macos-latest - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" - case-id: macos15-intel-py311-older os: macos-15-intel - ver: { py: "3.11", np: "==1.23.5", sp: "==1.9.3" } + ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } parallel_check: "false" - case-id: macos15-intel-py312-current os: macos-15-intel - ver: { py: "3.12", np: "==1.26.4", sp: "==1.11.4" } + ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } parallel_check: "false" - case-id: macos15-intel-py313-latest os: macos-15-intel - ver: { py: "3.13", np: "==2.1.3", sp: "==1.14.1" } - parallel_check: "false" + ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } + parallel_check: "true" - case-id: macos15-intel-py314-latest os: macos-15-intel - ver: { py: "3.14", np: "==2.3.3", sp: "==1.16.1" } - parallel_check: "true" + ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + asv_backends: "cython_fallback" + parallel_check: "false" steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -162,31 +166,33 @@ jobs: - name: Configure ASV machine run: | - # Supply explicit machine metadata for CI. ASV can infer this on - # Linux/macOS ARM, but Windows runner detection is less reliable. - # The case-id is also passed to asv run below so both commands use - # the same machine entry. - conda run -n gstools-benchmark-driver asv \ - --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - machine \ + # Supply explicit machine metadata for CI. Writing the ASV machine + # file directly avoids platform-specific issues with `asv machine` + # under the Windows conda/bash wrapper. + conda run -n gstools-benchmark-driver python \ + benchmarks/tools/write_asv_ci_machine.py \ --machine "${{ matrix.case-id }}" \ --os "${{ runner.os }}" \ --arch "${{ runner.arch }}" \ --cpu "GitHub Actions ${{ matrix.os }}" \ - --num_cpu "2" \ - --ram "unknown" \ - --yes + --num-cpu "2" \ + --ram "unknown" - name: Run 1-thread ASV availability check env: # Public benchmark-thread interface. Baseline CI intentionally checks # the non-parallel path first. GSTOOLS_BENCHMARK_THREADS: "1" + # Keep the default benchmark behavior unless a matrix entry narrows + # the smoke run. Python 3.14 currently runs Cython-only here because + # gstools_core 1.2.0 can segfault during the variogram smoke case. + GSTOOLS_BENCHMARK_BACKENDS: ${{ matrix.asv_backends || '' }} run: | - # Run one lightweight ASV benchmark method. Its parameters still - # exercise both backend values, but CI avoids running the full - # benchmark suite because this workflow checks availability, not - # performance quality. + # Run one lightweight ASV benchmark method. Most matrix entries + # exercise both backend values; Python 3.14 is limited to Cython + # until gstools_core is stable there. CI avoids the full benchmark + # suite because this workflow checks availability, not performance + # quality. conda run -n gstools-benchmark-driver asv \ --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ run \ @@ -213,13 +219,13 @@ jobs: if: matrix.parallel_check == 'true' run: | # OpenMP builds are compiler-heavy. Check them on every OS, but only - # with the latest dependency stack to avoid multiplying CI cost. This - # step runs only after this exact matrix child's 1-thread ASV check - # has passed. + # with the newest Rust-compatible dependency stack to avoid + # multiplying CI cost. This step runs only after this exact matrix + # child's 1-thread ASV check has passed. conda create -n gstools-benchmark-openmp -y -c conda-forge \ "python=${{ matrix.ver.py }}" \ - "numpy${{ matrix.ver.np }}" \ - "scipy${{ matrix.ver.sp }}" \ + "numpy=${{ matrix.ver.np }}" \ + "scipy=${{ matrix.ver.sp }}" \ c-compiler \ cxx-compiler \ cython \ diff --git a/asv.openmp.conf.json b/asv.openmp.conf.json index 274e2d3cd..6d707b994 100644 --- a/asv.openmp.conf.json +++ b/asv.openmp.conf.json @@ -31,7 +31,7 @@ "install_command": [ "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", - "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp", - "in-dir={env_dir} python -m pip install --no-deps {build_dir}" + "in-dir={env_dir} python -m pip install --no-deps {build_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_backend_parallel_ready.py --verbose" ] } diff --git a/benchmarks/README.md b/benchmarks/README.md index 382b87c83..04d095e04 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -42,7 +42,7 @@ deciding where optimization work should go: - [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) - [Shared OpenMP Rule](#shared-openmp-rule) - [OpenMP ASV Configuration](#openmp-asv-configuration) - - [Verify Cython OpenMP](#verify-cython-openmp) + - [Verify Parallel Backends](#verify-parallel-backends) - [Run On macOS And Linux](#run-on-macos-and-linux) - [Run On Windows](#run-on-windows) - [HPC Notes](#hpc-notes) @@ -115,14 +115,13 @@ The benchmarking setup currently consists of: - `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so you can see which functions take time in the current checkout. -- `benchmarks/tools/check_cython_openmp.py`: optional helper for checking - whether the active Python environment's GSTools-Cython extensions detect - OpenMP parallel support. - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies - Cython OpenMP detection and Rust backend execution with more than one GSTools - thread. + Cython OpenMP detection and Rust backend execution with more than one + GSTools thread. - `benchmarks/tools/write_asv_ci_config.py`: CI helper that writes a temporary per-job ASV config for one Python/NumPy/SciPy combination. +- `benchmarks/tools/write_asv_ci_machine.py`: CI helper that writes ASV machine + metadata non-interactively for GitHub-hosted runners. - `benchmarks/tools/install_openmp_cython.py`: helper used by `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, Linux, and native Windows. @@ -522,8 +521,8 @@ you explicitly want to measure backend scaling with multiple thread counts. ### Shared OpenMP Rule -The benchmark code can be run with several thread labels by setting for example -`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. That only passes different +The benchmark code can be run with several thread labels by setting, for +example, `GSTOOLS_BENCHMARK_THREADS=2,4,8`. That only passes different `gstools.config.NUM_THREADS` values to GSTools. It does not, by itself, make the Cython backend parallel. @@ -533,7 +532,7 @@ that environment before interpreting Cython scaling results: ```bash ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp +"$ASV_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose ``` If the check fails, the benchmark may still run, but the Cython backend should @@ -570,11 +569,12 @@ compiler handling: - Linux: uses the ASV conda compiler toolchain when available. - Windows: uses native MSVC Build Tools. -### Verify Cython OpenMP +### Verify Parallel Backends -Only interpret the Cython rows as OpenMP-enabled after this check passes inside -the `.asv-openmp/env/...` environment. The active `gstools-benchmark` conda -environment is only the ASV driver environment; it is normal for +Only interpret the Cython rows as OpenMP-enabled, and the Rust rows as +parallel-ready, after this check passes inside the `.asv-openmp/env/...` +environment. The active `gstools-benchmark` conda environment is only the ASV +driver environment; it is normal for `python benchmarks/tools/check_backend_parallel_ready.py --verbose` to fail there with `gstools: not installed`, `gstools_cython: not installed`, or `gstools_core: not installed`. @@ -583,8 +583,6 @@ On macOS and Linux: ```bash ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --verbose -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp "$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose ``` @@ -595,15 +593,12 @@ $asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | Sort-Object LastWriteTime -Descending | Select-Object -First 1 $asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' -& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --verbose -& $asvOpenmpPython benchmarks\tools\check_cython_openmp.py --fail-if-no-openmp & $asvOpenmpPython benchmarks\tools\check_backend_parallel_ready.py --verbose ``` Expected passing output contains: ```text -OpenMP check: PASS Cython OpenMP readiness: PASS Rust backend readiness: PASS with NUM_THREADS=2 ``` @@ -623,16 +618,16 @@ asv --config asv.openmp.conf.json machine --yes Run a quick current-commit OpenMP smoke run: ```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +GSTOOLS_BENCHMARK_THREADS=2,4,8 \ asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Verify Cython OpenMP with the commands in -[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +Verify the parallel backends with the commands in +[Verify Parallel Backends](#verify-parallel-backends). If it passes, run the last-five commits: ```bash -GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ +GSTOOLS_BENCHMARK_THREADS=2,4,8 \ asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` @@ -665,16 +660,16 @@ asv --config asv.openmp.conf.json machine --yes Run a quick current-commit OpenMP smoke run: ```powershell -$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +$env:GSTOOLS_BENCHMARK_THREADS = '2,4,8' asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr ``` -Verify Cython OpenMP with the PowerShell commands in -[Verify Cython OpenMP](#verify-cython-openmp). If it passes, run the last-five +Verify the parallel backends with the PowerShell commands in +[Verify Parallel Backends](#verify-parallel-backends). If it passes, run the last-five commits: ```powershell -$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8,16' +$env:GSTOOLS_BENCHMARK_THREADS = '2,4,8' asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr ``` @@ -704,7 +699,7 @@ The `asv.openmp.conf.json` workflow is intended for local macOS, Linux, and native Windows machines. Managed HPC systems often use custom compiler modules, MPI/OpenMP runtimes, scheduler pinning, and CPU affinity rules. Use the same validation rule there: only interpret Cython as OpenMP-enabled if -`check_cython_openmp.py --fail-if-no-openmp` passes inside the exact ASV +`check_backend_parallel_ready.py --verbose` passes inside the exact ASV environment used for the benchmark run. ### Profiling With cProfile for Multiple Threads @@ -716,7 +711,7 @@ same cProfile case several times with the OpenMP ASV environment: ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" ASV_OPENMP_PYTHON="$ASV_OPENMP_ENV/bin/python" -for threads in threads_1 threads_2 threads_4 threads_8 threads_16; do +for threads in threads_2 threads_4 threads_8; do "$ASV_OPENMP_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads "$threads" --limit 10 done ``` @@ -729,7 +724,7 @@ $asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | Select-Object -First 1 $asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' -foreach ($threads in 'threads_1', 'threads_2', 'threads_4', 'threads_8', 'threads_16') { +foreach ($threads in 'threads_2', 'threads_4', 'threads_8') { & $asvOpenmpPython benchmarks\tools\profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads $threads --limit 10 } ``` @@ -738,8 +733,8 @@ Useful options: - `--case`: choose one workflow, or use `all` - `--backend`: choose `cython_fallback` or `rust_core` -- `--threads`: choose `threads_1`, `threads_2`, `threads_4`, `threads_8`, - or `threads_16` +- `--threads`: choose `threads_1` for a baseline profile, or `threads_2`, + `threads_4`, or `threads_8` for parallel profiles - `--limit`: number of function rows to print from the cProfile table - `--sort cumtime`: sort by cumulative time, usually the best first view - `--sort tottime`: sort by time spent directly in each function @@ -758,9 +753,9 @@ Each matrix job is its own small chain: - first, it runs `asv.conf.json` with `GSTOOLS_BENCHMARK_THREADS=1` on one representative Python/NumPy/SciPy combination. -- then, for entries marked as the latest dependency combo for an OS runner, it - builds the OpenMP environment and verifies Cython OpenMP plus Rust backend - readiness. +- then, for entries marked as the newest Rust-compatible dependency combo for + an OS runner, it builds the OpenMP environment and verifies Cython OpenMP + plus Rust backend readiness. This layout keeps the required order within each runner: the parallel check for one OS/dependency setup starts only after that setup's 1-thread ASV check has @@ -775,8 +770,11 @@ asv run "HEAD^!" --quick \ --show-stderr ``` -That benchmark method still exercises both configured backend values through -ASV parameters, but avoids running the full performance suite in CI. +That benchmark method exercises both configured backend values through ASV +parameters, but avoids running the full performance suite in CI. Python 3.14 +currently runs this ASV smoke check with the Cython fallback backend only, +because `gstools_core` 1.2.0 can fail during the Rust variogram smoke case on +that interpreter. The parallel-readiness steps intentionally do not run a full ASV OpenMP benchmark. Instead, it builds `gstools-cython` with OpenMP, installs @@ -798,7 +796,7 @@ ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" That fast check proves that Cython reports OpenMP support and that the Rust backend can run a small workflow with `gstools.config.NUM_THREADS=2`. For real OpenMP scaling measurements, use `asv.openmp.conf.json` locally with -`GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16`. +`GSTOOLS_BENCHMARK_THREADS=2,4,8`. The workflow runs automatically on pull requests and pushes that change ASV configs, benchmark files, package metadata, or the workflow itself. It can also @@ -837,6 +835,19 @@ asv run 'HEAD^!' --bench benchmark_backends asv compare origin/main HEAD ``` +Run the last three commits on local `main`: + +```bash +asv run 'main~3..main' --bench benchmark_backends --show-stderr +``` + +Run the last three commits on the latest remote `main`: + +```bash +git fetch origin main +asv run 'origin/main~3..origin/main' --bench benchmark_backends --show-stderr +``` + On a linear branch, `HEAD~5..HEAD` benchmarks: ```text diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_backends.py index cc588c1e7..f4c853f39 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_backends.py @@ -19,7 +19,7 @@ By default the suite uses one GSTools thread. For local OpenMP scaling experiments, set GSTOOLS_BENCHMARK_THREADS, for example: - GSTOOLS_BENCHMARK_THREADS=1,2,4,8,16 \ + GSTOOLS_BENCHMARK_THREADS=2,4,8 \ asv --config asv.openmp.conf.json run 'HEAD^!' """ @@ -31,7 +31,26 @@ import gstools as gs import numpy as np -BACKENDS = ("cython_fallback", "rust_core") +AVAILABLE_BACKENDS = ("cython_fallback", "rust_core") + + +def _configured_backends(): + raw = os.environ.get("GSTOOLS_BENCHMARK_BACKENDS", "") + if not raw.strip(): + return AVAILABLE_BACKENDS + backends = tuple(item.strip() for item in raw.split(",") if item.strip()) + unknown = sorted(set(backends) - set(AVAILABLE_BACKENDS)) + if unknown: + raise ValueError( + "GSTOOLS_BENCHMARK_BACKENDS contains unknown backends: " + + ", ".join(unknown) + ) + if not backends: + raise ValueError("GSTOOLS_BENCHMARK_BACKENDS did not define backends") + return backends + + +BACKENDS = _configured_backends() def _configured_thread_counts(): diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index cf56845be..01ad8f374 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -13,7 +13,13 @@ import sys import numpy as np -from check_cython_openmp import MODULES, check_module, package_version + +MODULES = { + "variogram": "gstools_cython.variogram", + "field": "gstools_cython.field", + "krige": "gstools_cython.krige", +} +EXPLICIT_THREAD_COUNTS = (2, 4, 8) def parse_args(): @@ -29,10 +35,33 @@ def parse_args(): action="store_true", help="Print per-module Cython OpenMP thread details.", ) + parser.add_argument( + "--cython-only", + action="store_true", + help="Only check GSTools-Cython OpenMP readiness.", + ) return parser.parse_args() -def check_cython_openmp(verbose=False): +def package_version(package_name): + try: + package = importlib.import_module(package_name) + except ModuleNotFoundError: + return "not installed" + return getattr(package, "__version__", "unknown") + + +def check_module(label, module_name): + module = importlib.import_module(module_name) + default_threads = module.set_num_threads(None) + explicit = { + count: module.set_num_threads(count) + for count in EXPLICIT_THREAD_COUNTS + } + return label, default_threads, explicit + + +def check_cython_parallel(verbose=False): default_values = [] for label, module_name in MODULES.items(): try: @@ -47,10 +76,20 @@ def check_cython_openmp(verbose=False): ) print(f"{label} default None -> {default_threads}") print(f"{label} explicit -> {explicit_text}") - if explicit.get(2) != 2: + mismatches = { + request: actual + for request, actual in explicit.items() + if actual != request + } + if mismatches: + mismatch_text = ", ".join( + f"{request}->{actual}" + for request, actual in mismatches.items() + ) print( "Cython OpenMP readiness: FAIL. " - f"{label} did not accept an explicit 2-thread request." + f"{label} did not accept requested thread counts: " + f"{mismatch_text}." ) return False @@ -123,8 +162,8 @@ def main(): print(f"gstools_cython: {package_version('gstools_cython')}") print(f"gstools_core: {package_version('gstools_core')}") - cython_ready = check_cython_openmp(verbose=args.verbose) - rust_ready = check_rust_backend(args.threads) + cython_ready = check_cython_parallel(verbose=args.verbose) + rust_ready = True if args.cython_only else check_rust_backend(args.threads) return 0 if cython_ready and rust_ready else 1 diff --git a/benchmarks/tools/check_cython_openmp.py b/benchmarks/tools/check_cython_openmp.py deleted file mode 100644 index b3a410cc9..000000000 --- a/benchmarks/tools/check_cython_openmp.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -"""Check whether GSTools-Cython detects OpenMP parallel support. - -This script verifies the active Python environment. Use it with the editable -development environment or with an ASV-created environment. - -Examples: - python benchmarks/tools/check_cython_openmp.py - python benchmarks/tools/check_cython_openmp.py --fail-if-no-openmp - python benchmarks/tools/check_cython_openmp.py --verbose - .asv/env//bin/python3 benchmarks/tools/check_cython_openmp.py -""" - -from __future__ import annotations - -import argparse -import importlib -import sys - -MODULES = { - "variogram": "gstools_cython.variogram", - "field": "gstools_cython.field", - "krige": "gstools_cython.krige", -} -EXPLICIT_THREAD_COUNTS = (1, 2, 4, 8, 16) - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--fail-if-no-openmp", - action="store_true", - help="Exit with status 1 if OpenMP thread detection reports <= 1.", - ) - parser.add_argument( - "--verbose", - action="store_true", - help="Print per-module default and explicit thread-count values.", - ) - return parser.parse_args() - - -def package_version(package_name): - try: - package = importlib.import_module(package_name) - except ModuleNotFoundError: - return "not installed" - return getattr(package, "__version__", "unknown") - - -def check_module(label, module_name): - module = importlib.import_module(module_name) - default_threads = module.set_num_threads(None) - explicit = { - count: module.set_num_threads(count) - for count in EXPLICIT_THREAD_COUNTS - } - return label, default_threads, explicit - - -def main(): - args = parse_args() - - print(f"python: {sys.executable}") - print(f"gstools: {package_version('gstools')}") - print(f"gstools_cython: {package_version('gstools_cython')}") - print(f"gstools_core: {package_version('gstools_core')}") - if args.verbose: - print( - "OpenMP evidence: default None should be >1. " - "Explicit values only prove the wrapper accepts the requested count." - ) - - default_values = [] - for label, module_name in MODULES.items(): - try: - label, default_threads, explicit = check_module(label, module_name) - except ModuleNotFoundError as err: - print(f"OpenMP check: FAIL. Missing module: {err.name}") - return 1 - default_values.append(default_threads) - if args.verbose: - explicit_text = ", ".join( - f"{request}->{actual}" for request, actual in explicit.items() - ) - print(f"{label} default None -> {default_threads}") - print(f"{label} explicit -> {explicit_text}") - - if min(default_values) > 1: - print("OpenMP check: PASS") - return 0 - - print( - "OpenMP check: FAIL. GSTools-Cython reports one default thread. " - "Explicit thread values may be accepted by the wrapper, but this does " - "not prove that the compiled extension is using OpenMP." - ) - return 1 if args.fail_if_no_openmp else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index 2fd2fdff5..ebd8b925d 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -20,10 +20,9 @@ import argparse import cProfile -from pathlib import Path import pstats import sys - +from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT)) @@ -88,7 +87,6 @@ "threads_2", "threads_4", "threads_8", - "threads_16", ) diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py index 0999ada01..edf6f5206 100644 --- a/benchmarks/tools/write_asv_ci_config.py +++ b/benchmarks/tools/write_asv_ci_config.py @@ -13,13 +13,21 @@ from pathlib import Path -def validate_asv_req_spec(package, spec): - if spec.startswith((">", "<", "~", "!=")) or "," in spec: +def conda_exact_pin(package, spec): + """Return an ASV/conda exact package pin from a CI matrix value.""" + spec = spec.strip() + if spec.startswith((">", "<", "~", "!")) or "," in spec: raise ValueError( f"{package} uses open-ended CI spec {spec!r}. ASV's conda " - "backend expects exact matrix pins here, for example '==2.1.3'." + "backend expects exact matrix pins here, for example '2.1.3'." ) - return spec + if spec.startswith("=="): + spec = spec[2:] + elif spec.startswith("="): + spec = spec[1:] + if not spec: + raise ValueError(f"{package} has an empty CI version pin.") + return f"={spec}" def parse_args(): @@ -47,8 +55,8 @@ def main(): req = config.setdefault("matrix", {}).setdefault("req", {}) try: - req["numpy"] = [validate_asv_req_spec("numpy", args.numpy)] - req["scipy"] = [validate_asv_req_spec("scipy", args.scipy)] + req["numpy"] = [conda_exact_pin("numpy", args.numpy)] + req["scipy"] = [conda_exact_pin("scipy", args.scipy)] except ValueError as err: raise SystemExit(str(err)) from err diff --git a/benchmarks/tools/write_asv_ci_machine.py b/benchmarks/tools/write_asv_ci_machine.py new file mode 100644 index 000000000..fd6ef25a5 --- /dev/null +++ b/benchmarks/tools/write_asv_ci_machine.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +"""Write ASV machine metadata for a non-interactive CI runner.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--machine", required=True) + parser.add_argument("--os", required=True) + parser.add_argument("--arch", required=True) + parser.add_argument("--cpu", required=True) + parser.add_argument("--num-cpu", required=True) + parser.add_argument("--ram", required=True) + parser.add_argument( + "--path", + type=Path, + default=Path.home() / ".asv-machine.json", + help="Machine metadata file written by 'asv machine'.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + if args.path.exists(): + with args.path.open(encoding="utf8") as machine_file: + machines = json.load(machine_file) + else: + machines = {"version": 1} + + machines["version"] = 1 + machines[args.machine] = { + "machine": args.machine, + "os": args.os, + "arch": args.arch, + "cpu": args.cpu, + "num_cpu": str(args.num_cpu), + "ram": args.ram, + } + + args.path.parent.mkdir(parents=True, exist_ok=True) + with args.path.open("w", encoding="utf8") as machine_file: + json.dump(machines, machine_file, indent=2) + machine_file.write("\n") + + print(f"Wrote ASV machine metadata for {args.machine} to {args.path}") + + +if __name__ == "__main__": + main() From 9bdb2c1c1c38e24528cb6d2e903ada4d8ddc119e Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Thu, 28 May 2026 23:54:16 +0200 Subject: [PATCH 12/37] simplified version GitHub actions for a pracmatic approach --- .github/workflows/asv-benchmarks.yml | 290 +++++++++-------------- benchmarks/README.md | 37 ++- benchmarks/tools/asv_speedup_summary.py | 1 - benchmarks/tools/write_asv_ci_config.py | 72 ------ benchmarks/tools/write_asv_ci_machine.py | 55 ----- 5 files changed, 121 insertions(+), 334 deletions(-) delete mode 100644 benchmarks/tools/write_asv_ci_config.py delete mode 100644 benchmarks/tools/write_asv_ci_machine.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 3987bbd85..6a15c0977 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -1,11 +1,9 @@ name: Benchmark Availability Checks on: - # This workflow checks benchmark installability, not package correctness. - # It is intentionally path-filtered because ASV creates conda environments - # and the OpenMP job compiles extensions. Normal source/test validation is - # handled by main.yml; this workflow runs when benchmark-related files or - # package dependency metadata change. + # This workflow checks whether the benchmark code can be installed and + # started on the supported GitHub-hosted runners. It is a smoke check, not a + # performance benchmark suite. pull_request: branches: - "main" @@ -24,248 +22,172 @@ on: - "pyproject.toml" workflow_dispatch: -# The jobs only need to read the repository. Keep the default token minimal. permissions: contents: read -defaults: - run: - # setup-miniconda initializes conda for bash login shells on all runners. - shell: bash -el {0} - jobs: - benchmark_availability: - # Each matrix child is its own small chain: first run the normal one-thread - # ASV smoke check, then run parallel readiness immediately for entries - # marked with parallel_check. GitHub Actions cannot make one matrix child - # depend only on the matching child of another matrix job, so keeping both - # phases in one job avoids waiting for unrelated OS/Python combinations. - name: Benchmark availability on ${{ matrix.os }} py${{ matrix.ver.py }} + benchmark_asv_existing_env: + # Use the same dependency combinations as main.yml, but run only one + # lightweight benchmark method in the already-created Python environment. + # That avoids ASV's conda environment management in CI while still proving + # that users can start the benchmark suite on each runner family. + name: ASV smoke on ${{ matrix.os }} py${{ matrix.ver.py }} runs-on: ${{ matrix.os }} - timeout-minutes: 75 + timeout-minutes: 30 strategy: - # Keep the rest of the OS/dependency matrix running after one failure. - # That gives a full compatibility picture from a single workflow run. fail-fast: false matrix: - # This list is hand-picked, not imported from main.yml or GSTools-Core. - # Each entry below becomes one independent GitHub Actions job. The goal - # is representative coverage: each OS runner gets older/current/latest - # conda-solvable Python/NumPy/SciPy stacks without running every - # possible cross-product. Use bare exact package versions here; the - # helper writes conda-style pins into ASV's generated config. include: - - case-id: ubuntu-py311-older + - case-id: ubuntu-py311 os: ubuntu-latest - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: ubuntu-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: ubuntu-py312 os: ubuntu-latest - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: ubuntu-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: ubuntu-py313 os: ubuntu-latest - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: ubuntu-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: ubuntu-py314 os: ubuntu-latest - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" - - case-id: windows-py311-older + - case-id: windows-py311 os: windows-latest - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: windows-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: windows-py312 os: windows-latest - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: windows-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: windows-py313 os: windows-latest - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: windows-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: windows-py314 os: windows-latest - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" - - case-id: macos-latest-py311-older + - case-id: macos-latest-py311 os: macos-latest - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: macos-latest-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: macos-latest-py312 os: macos-latest - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: macos-latest-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: macos-latest-py313 os: macos-latest - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: macos-latest-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: macos-latest-py314 os: macos-latest - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" - - case-id: macos15-intel-py311-older + - case-id: macos15-intel-py311 os: macos-15-intel - ver: { py: "3.11", np: "1.23.5", sp: "1.9.3" } - parallel_check: "false" - - case-id: macos15-intel-py312-current + ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } + - case-id: macos15-intel-py312 os: macos-15-intel - ver: { py: "3.12", np: "1.26.4", sp: "1.11.4" } - parallel_check: "false" - - case-id: macos15-intel-py313-latest + ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } + - case-id: macos15-intel-py313 os: macos-15-intel - ver: { py: "3.13", np: "2.1.3", sp: "1.14.1" } - parallel_check: "true" - - case-id: macos15-intel-py314-latest + ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } + - case-id: macos15-intel-py314 os: macos-15-intel - ver: { py: "3.14", np: "2.3.2", sp: "1.16.1" } + ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } asv_backends: "cython_fallback" - parallel_check: "false" + + defaults: + run: + # Use one shell syntax for Linux, Windows, and macOS smoke commands. + shell: bash steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: - # ASV needs git history for revision expressions such as HEAD^!. fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - # Driver environment: only runs ASV and helper scripts. The packages - # being benchmarked are installed inside ASV-created environments. - activate-environment: gstools-benchmark-driver - python-version: "3.12" - channels: conda-forge - conda-remove-defaults: true + python-version: ${{ matrix.ver.py }} - - name: Install ASV driver + - name: Install benchmark smoke dependencies run: | - # ASV loads environment backends as optional plugins. Installing - # packaging/virtualenv explicitly prevents CI from falling back to - # the "existing" backend only, which would make environment_type: - # conda unavailable. - conda install -n gstools-benchmark-driver -y -c conda-forge \ + python -m pip install --upgrade pip + python -m pip install \ asv \ - packaging \ - virtualenv - - - name: Write per-job ASV config - run: | - # The committed asv.conf.json stays simple for local users. CI writes - # a temporary config so each matrix child can pin one dependency set. - conda run -n gstools-benchmark-driver python benchmarks/tools/write_asv_ci_config.py \ - --base-config asv.conf.json \ - --output "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - --python-version "${{ matrix.ver.py }}" \ - --numpy "${{ matrix.ver.np }}" \ - --scipy "${{ matrix.ver.sp }}" \ - --env-dir ".asv-ci/env/${{ matrix.case-id }}" \ - --results-dir ".asv-ci/results/${{ matrix.case-id }}" \ - --html-dir ".asv-ci/html/${{ matrix.case-id }}" + emcee \ + "gstools-cython>=1,<2" \ + "gstools_core>=1.0.0" \ + hankel \ + meshio \ + "numpy${{ matrix.ver.np }}" \ + pyevtk \ + "scipy${{ matrix.ver.sp }}" + python -m pip install --no-deps --editable . - name: Configure ASV machine run: | - # Supply explicit machine metadata for CI. Writing the ASV machine - # file directly avoids platform-specific issues with `asv machine` - # under the Windows conda/bash wrapper. - conda run -n gstools-benchmark-driver python \ - benchmarks/tools/write_asv_ci_machine.py \ - --machine "${{ matrix.case-id }}" \ - --os "${{ runner.os }}" \ - --arch "${{ runner.arch }}" \ - --cpu "GitHub Actions ${{ matrix.os }}" \ - --num-cpu "2" \ - --ram "unknown" + # existing environments still need a machine profile for ASV result + # metadata. --yes keeps the command non-interactive in CI. + asv machine --yes - - name: Run 1-thread ASV availability check + - name: Run ASV smoke check in existing environment env: - # Public benchmark-thread interface. Baseline CI intentionally checks - # the non-parallel path first. GSTOOLS_BENCHMARK_THREADS: "1" - # Keep the default benchmark behavior unless a matrix entry narrows - # the smoke run. Python 3.14 currently runs Cython-only here because - # gstools_core 1.2.0 can segfault during the variogram smoke case. GSTOOLS_BENCHMARK_BACKENDS: ${{ matrix.asv_backends || '' }} run: | - # Run one lightweight ASV benchmark method. Most matrix entries - # exercise both backend values; Python 3.14 is limited to Cython - # until gstools_core is stable there. CI avoids the full benchmark - # suite because this workflow checks availability, not performance - # quality. - conda run -n gstools-benchmark-driver asv \ - --config "${GITHUB_WORKSPACE}/asv.ci.${{ matrix.case-id }}.json" \ - run \ - --machine "${{ matrix.case-id }}" \ + # -E existing uses the Python environment prepared above. Python 3.14 + # is currently Cython-only because gstools_core 1.2.0 can fail during + # the Rust variogram smoke case. + asv run \ + -E existing \ --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr \ 'HEAD^!' - - name: Upload ASV availability results - # Upload even on failure so environment/config details are available - # when debugging resolver, compiler, or benchmark discovery problems. - if: always() - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + benchmark_parallel_readiness: + # Keep this separate from the ASV smoke job. It checks the smallest useful + # runtime proof for parallel backends without running a full ASV benchmark + # and without building gstools-cython from the PyPI source distribution. + name: Parallel readiness on ${{ matrix.os }} py3.13 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + - macos-15-intel + + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: - name: asv-1-thread-${{ matrix.case-id }} - path: | - asv.ci.${{ matrix.case-id }}.json - .asv-ci/results/${{ matrix.case-id }} - .asv-ci/html/${{ matrix.case-id }} - if-no-files-found: ignore + activate-environment: gstools-benchmark-readiness + python-version: "3.13" + channels: conda-forge + conda-remove-defaults: true - - name: Create OpenMP readiness environment - if: matrix.parallel_check == 'true' + - name: Install parallel readiness dependencies run: | - # OpenMP builds are compiler-heavy. Check them on every OS, but only - # with the newest Rust-compatible dependency stack to avoid - # multiplying CI cost. This step runs only after this exact matrix - # child's 1-thread ASV check has passed. - conda create -n gstools-benchmark-openmp -y -c conda-forge \ - "python=${{ matrix.ver.py }}" \ - "numpy=${{ matrix.ver.np }}" \ - "scipy=${{ matrix.ver.sp }}" \ - c-compiler \ - cxx-compiler \ - cython \ + conda install -n gstools-benchmark-readiness -y -c conda-forge \ emcee \ - extension-helpers \ + gstools-cython \ hankel \ meshio \ + "numpy=2.1.0" \ pip \ pyevtk \ - "setuptools>=77" \ - wheel - - - name: Install Rust backend package - if: matrix.parallel_check == 'true' - run: | - # gstools_core is installed with pip because conda availability is not - # uniform across every runner/platform combination. - conda run -n gstools-benchmark-openmp python -m pip install \ + "scipy=1.14.1" + conda run -n gstools-benchmark-readiness python -m pip install \ "gstools_core>=1.0.0" - - - name: Build GSTools-Cython with OpenMP - if: matrix.parallel_check == 'true' - run: | - # The helper dispatches by platform: llvm-openmp wrapper on macOS, - # conda compiler toolchain on Linux, and native MSVC on Windows. - conda run -n gstools-benchmark-openmp python \ - benchmarks/tools/install_openmp_cython.py - - - name: Install local GSTools checkout - if: matrix.parallel_check == 'true' - run: | - # Dependencies are already pinned by the matrix. Install the checkout - # without letting pip resolve or replace them. - conda run -n gstools-benchmark-openmp python -m pip install \ + conda run -n gstools-benchmark-readiness python -m pip install \ --no-deps --editable . - name: Check Cython OpenMP and Rust backend readiness - if: matrix.parallel_check == 'true' run: | - # Fast readiness check, not a performance benchmark. It fails if - # Cython does not report OpenMP or Rust cannot run with NUM_THREADS=2. - conda run -n gstools-benchmark-openmp python \ + conda run -n gstools-benchmark-readiness python \ benchmarks/tools/check_backend_parallel_ready.py --verbose diff --git a/benchmarks/README.md b/benchmarks/README.md index 04d095e04..3651dad0c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -118,10 +118,6 @@ The benchmarking setup currently consists of: - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies Cython OpenMP detection and Rust backend execution with more than one GSTools thread. -- `benchmarks/tools/write_asv_ci_config.py`: CI helper that writes a temporary - per-job ASV config for one Python/NumPy/SciPy combination. -- `benchmarks/tools/write_asv_ci_machine.py`: CI helper that writes ASV machine - metadata non-interactively for GitHub-hosted runners. - `benchmarks/tools/install_openmp_cython.py`: helper used by `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, Linux, and native Windows. @@ -749,23 +745,18 @@ the benchmark tooling can be installed and started on GitHub-hosted Linux, Windows, `macos-latest`, and `macos-15-intel` runners. This workflow is an availability check, not a performance benchmark run. -Each matrix job is its own small chain: +The workflow has two independent matrix jobs: -- first, it runs `asv.conf.json` with `GSTOOLS_BENCHMARK_THREADS=1` on one - representative Python/NumPy/SciPy combination. -- then, for entries marked as the newest Rust-compatible dependency combo for - an OS runner, it builds the OpenMP environment and verifies Cython OpenMP - plus Rust backend readiness. +- `benchmark_asv_existing_env` installs GSTools and the benchmark + dependencies with pip, then runs one ASV benchmark method with + `-E existing`. +- `benchmark_parallel_readiness` creates a small conda-forge environment and + runs `check_backend_parallel_ready.py --verbose`. -This layout keeps the required order within each runner: the parallel check for -one OS/dependency setup starts only after that setup's 1-thread ASV check has -passed. It does not wait for unrelated matrix entries on other operating -systems. - -The 1-thread stage runs a quick ASV command: +The ASV smoke stage runs a quick command in the existing Python environment: ```bash -asv run "HEAD^!" --quick \ +asv run -E existing "HEAD^!" --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr ``` @@ -777,16 +768,18 @@ because `gstools_core` 1.2.0 can fail during the Rust variogram smoke case on that interpreter. The parallel-readiness steps intentionally do not run a full ASV OpenMP -benchmark. Instead, it builds `gstools-cython` with OpenMP, installs -`gstools_core`, installs the local GSTools checkout, and runs: +benchmark. They install `gstools-cython` from conda-forge, install +`gstools_core`, install the local GSTools checkout, and run: ```bash -conda run -n gstools-benchmark-openmp python \ +conda run -n gstools-benchmark-readiness python \ benchmarks/tools/check_backend_parallel_ready.py --verbose ``` -Locally, run the same helper with the Python executable from the ASV OpenMP -environment: +CI does not call `benchmarks/tools/install_openmp_cython.py`; that helper is +kept for local/manual OpenMP ASV builds through `asv.openmp.conf.json`. +Locally, run the readiness helper with the Python executable from the ASV +OpenMP environment: ```bash ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index b3239d702..f50e7a542 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -24,7 +24,6 @@ import math from pathlib import Path - BACKENDS = ("cython_fallback", "rust_core") THREAD_PREFIX = "threads_" LEGACY_BENCHMARKS = { diff --git a/benchmarks/tools/write_asv_ci_config.py b/benchmarks/tools/write_asv_ci_config.py deleted file mode 100644 index edf6f5206..000000000 --- a/benchmarks/tools/write_asv_ci_config.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -"""Write a per-job ASV configuration for GitHub Actions. - -The committed ASV configs stay convenient for local benchmarking. CI uses this -helper to pin one Python/NumPy/SciPy combination per matrix job without turning -normal local ``asv run`` commands into a large dependency matrix. -""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - - -def conda_exact_pin(package, spec): - """Return an ASV/conda exact package pin from a CI matrix value.""" - spec = spec.strip() - if spec.startswith((">", "<", "~", "!")) or "," in spec: - raise ValueError( - f"{package} uses open-ended CI spec {spec!r}. ASV's conda " - "backend expects exact matrix pins here, for example '2.1.3'." - ) - if spec.startswith("=="): - spec = spec[2:] - elif spec.startswith("="): - spec = spec[1:] - if not spec: - raise ValueError(f"{package} has an empty CI version pin.") - return f"={spec}" - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--base-config", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--python-version", required=True) - parser.add_argument("--numpy", required=True) - parser.add_argument("--scipy", required=True) - parser.add_argument("--env-dir", required=True) - parser.add_argument("--results-dir", required=True) - parser.add_argument("--html-dir", required=True) - return parser.parse_args() - - -def main(): - args = parse_args() - with args.base_config.open(encoding="utf8") as config_file: - config = json.load(config_file) - - config["pythons"] = [args.python_version] - config["env_dir"] = args.env_dir - config["results_dir"] = args.results_dir - config["html_dir"] = args.html_dir - - req = config.setdefault("matrix", {}).setdefault("req", {}) - try: - req["numpy"] = [conda_exact_pin("numpy", args.numpy)] - req["scipy"] = [conda_exact_pin("scipy", args.scipy)] - except ValueError as err: - raise SystemExit(str(err)) from err - - args.output.parent.mkdir(parents=True, exist_ok=True) - with args.output.open("w", encoding="utf8") as config_file: - json.dump(config, config_file, indent=2) - config_file.write("\n") - - print(f"Wrote {args.output}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/tools/write_asv_ci_machine.py b/benchmarks/tools/write_asv_ci_machine.py deleted file mode 100644 index fd6ef25a5..000000000 --- a/benchmarks/tools/write_asv_ci_machine.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -"""Write ASV machine metadata for a non-interactive CI runner.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - - -def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--machine", required=True) - parser.add_argument("--os", required=True) - parser.add_argument("--arch", required=True) - parser.add_argument("--cpu", required=True) - parser.add_argument("--num-cpu", required=True) - parser.add_argument("--ram", required=True) - parser.add_argument( - "--path", - type=Path, - default=Path.home() / ".asv-machine.json", - help="Machine metadata file written by 'asv machine'.", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - if args.path.exists(): - with args.path.open(encoding="utf8") as machine_file: - machines = json.load(machine_file) - else: - machines = {"version": 1} - - machines["version"] = 1 - machines[args.machine] = { - "machine": args.machine, - "os": args.os, - "arch": args.arch, - "cpu": args.cpu, - "num_cpu": str(args.num_cpu), - "ram": args.ram, - } - - args.path.parent.mkdir(parents=True, exist_ok=True) - with args.path.open("w", encoding="utf8") as machine_file: - json.dump(machines, machine_file, indent=2) - machine_file.write("\n") - - print(f"Wrote ASV machine metadata for {args.machine} to {args.path}") - - -if __name__ == "__main__": - main() From 9c4f46b0ee137ec99a2a309b690245ddd5fedf02 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Fri, 29 May 2026 07:02:23 +0200 Subject: [PATCH 13/37] fifth iteration GitHub Actions for benchmarking --- .github/workflows/asv-benchmarks.yml | 7 ++++--- benchmarks/README.md | 7 ++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 6a15c0977..e469b1d29 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -133,13 +133,14 @@ jobs: run: | # -E existing uses the Python environment prepared above. Python 3.14 # is currently Cython-only because gstools_core 1.2.0 can fail during - # the Rust variogram smoke case. + # the Rust variogram smoke case. Existing environments cannot accept + # a commit range, so the checked-out commit is recorded explicitly. asv run \ -E existing \ + --set-commit-hash "$GITHUB_SHA" \ --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ - --show-stderr \ - 'HEAD^!' + --show-stderr benchmark_parallel_readiness: # Keep this separate from the ASV smoke job. It checks the smallest useful diff --git a/benchmarks/README.md b/benchmarks/README.md index 3651dad0c..a1a891ae7 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -756,11 +756,16 @@ The workflow has two independent matrix jobs: The ASV smoke stage runs a quick command in the existing Python environment: ```bash -asv run -E existing "HEAD^!" --quick \ +asv run -E existing --set-commit-hash "$(git rev-parse HEAD)" --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr ``` +Do not pass a range such as `HEAD^!` with `-E existing`: ASV cannot checkout +revisions inside an existing environment. The workflow uses +`--set-commit-hash "$GITHUB_SHA"` to label the result with the commit already +checked out by GitHub Actions. + That benchmark method exercises both configured backend values through ASV parameters, but avoids running the full performance suite in CI. Python 3.14 currently runs this ASV smoke check with the Cython fallback backend only, From c8947091b89dfa23ffcd75b8426ff9340f963e05 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Fri, 29 May 2026 07:20:14 +0200 Subject: [PATCH 14/37] Simplify benchmark CI smoke checks --- .github/workflows/asv-benchmarks.yml | 18 ++++++++++++ benchmarks/README.md | 28 +++++++++++++++---- .../tools/check_backend_parallel_ready.py | 24 ++++++++++++---- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index e469b1d29..e6a8d1e98 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -126,6 +126,22 @@ jobs: # metadata. --yes keeps the command non-interactive in CI. asv machine --yes + - name: Write ASV smoke config for checked-out commit + run: | + # With -E existing, ASV cannot receive a range like HEAD^!. If no + # range is supplied, ASV resolves the configured branches. PR + # checkouts do not necessarily have a local "main" branch, so this + # temporary config points ASV at the exact checked-out commit. + python - <<'PY' + import json + import os + from pathlib import Path + + config = json.loads(Path("asv.conf.json").read_text()) + config["branches"] = [os.environ["GITHUB_SHA"]] + Path("asv.ci-existing.json").write_text(json.dumps(config, indent=2)) + PY + - name: Run ASV smoke check in existing environment env: GSTOOLS_BENCHMARK_THREADS: "1" @@ -136,8 +152,10 @@ jobs: # the Rust variogram smoke case. Existing environments cannot accept # a commit range, so the checked-out commit is recorded explicitly. asv run \ + --config asv.ci-existing.json \ -E existing \ --set-commit-hash "$GITHUB_SHA" \ + --no-pull \ --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr diff --git a/benchmarks/README.md b/benchmarks/README.md index a1a891ae7..2187eeb07 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -756,15 +756,29 @@ The workflow has two independent matrix jobs: The ASV smoke stage runs a quick command in the existing Python environment: ```bash -asv run -E existing --set-commit-hash "$(git rev-parse HEAD)" --quick \ +python - <<'PY' +import json +from pathlib import Path + +config = json.loads(Path("asv.conf.json").read_text()) +config["branches"] = ["HEAD"] +Path("asv.ci-existing.json").write_text(json.dumps(config, indent=2)) +PY + +asv run --config asv.ci-existing.json \ + -E existing \ + --set-commit-hash "$(git rev-parse HEAD)" \ + --no-pull \ + --quick \ --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ --show-stderr ``` Do not pass a range such as `HEAD^!` with `-E existing`: ASV cannot checkout -revisions inside an existing environment. The workflow uses -`--set-commit-hash "$GITHUB_SHA"` to label the result with the commit already -checked out by GitHub Actions. +revisions inside an existing environment. If no range is supplied, ASV resolves +the branches from the config file. The workflow writes a temporary config that +points `branches` at `$GITHUB_SHA`, then uses `--set-commit-hash "$GITHUB_SHA"` +to label the result with the commit already checked out by GitHub Actions. That benchmark method exercises both configured backend values through ASV parameters, but avoids running the full performance suite in CI. Python 3.14 @@ -792,8 +806,10 @@ ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" ``` That fast check proves that Cython reports OpenMP support and that the Rust -backend can run a small workflow with `gstools.config.NUM_THREADS=2`. For real -OpenMP scaling measurements, use `asv.openmp.conf.json` locally with +backend can run a small workflow with `gstools.config.NUM_THREADS=2`. The CI +probe requires Cython to accept explicit thread counts `2,4,8`; it prints the +default thread count but does not fail when a runner defaults to one thread. For +real OpenMP scaling measurements, use `asv.openmp.conf.json` locally with `GSTOOLS_BENCHMARK_THREADS=2,4,8`. The workflow runs automatically on pull requests and pushes that change ASV diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index 01ad8f374..2542702f4 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -1,9 +1,9 @@ #!/usr/bin/env python """Check that GSTools benchmark backends are ready for parallel runs. -This is a fast CI probe. It verifies that GSTools-Cython reports OpenMP support -and that the Rust backend can run a small workflow while GSTools is configured -with more than one thread. +This is a fast CI probe. It verifies that GSTools-Cython accepts explicit +OpenMP thread counts and that the Rust backend can run a small workflow while +GSTools is configured with more than one thread. """ from __future__ import annotations @@ -40,6 +40,15 @@ def parse_args(): action="store_true", help="Only check GSTools-Cython OpenMP readiness.", ) + parser.add_argument( + "--require-default-openmp", + action="store_true", + help=( + "Also fail when GSTools-Cython reports only one default thread. " + "CI does not use this because some runners default to one thread " + "while still accepting explicit OpenMP thread counts." + ), + ) return parser.parse_args() @@ -61,7 +70,7 @@ def check_module(label, module_name): return label, default_threads, explicit -def check_cython_parallel(verbose=False): +def check_cython_parallel(verbose=False, require_default_openmp=False): default_values = [] for label, module_name in MODULES.items(): try: @@ -93,7 +102,7 @@ def check_cython_parallel(verbose=False): ) return False - if min(default_values) <= 1: + if require_default_openmp and min(default_values) <= 1: print( "Cython OpenMP readiness: FAIL. GSTools-Cython reports one " "default thread." @@ -162,7 +171,10 @@ def main(): print(f"gstools_cython: {package_version('gstools_cython')}") print(f"gstools_core: {package_version('gstools_core')}") - cython_ready = check_cython_parallel(verbose=args.verbose) + cython_ready = check_cython_parallel( + verbose=args.verbose, + require_default_openmp=args.require_default_openmp, + ) rust_ready = True if args.cython_only else check_rust_backend(args.threads) return 0 if cython_ready and rust_ready else 1 From 2d4f157804a44e85728c7fcdc38645b8dbd993f3 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 1 Jun 2026 15:13:40 +0200 Subject: [PATCH 15/37] final implementation of benchmarking, adding isolated plotting results from ASV in a python script --- .github/workflows/asv-benchmarks.yml | 3 +- asv.conf.json | 4 +- asv.openmp.conf.json | 4 +- benchmarks/README.md | 402 +++--- benchmarks/__init__.py | 2 +- ...s.py => benchmark_two_point_statistics.py} | 228 +-- benchmarks/tools/asv_speedup_summary.py | 421 +++++- .../tools/check_backend_parallel_ready.py | 13 +- benchmarks/tools/install_openmp_cython.py | 28 +- .../tools/plot_case_backend_comparison.py | 1259 +++++++++++++++++ .../tools/profile_benchmark_workflows.py | 73 +- 11 files changed, 2039 insertions(+), 398 deletions(-) rename benchmarks/{benchmark_backends.py => benchmark_two_point_statistics.py} (51%) create mode 100644 benchmarks/tools/plot_case_backend_comparison.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index e6a8d1e98..cb3e522cd 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -151,13 +151,14 @@ jobs: # is currently Cython-only because gstools_core 1.2.0 can fail during # the Rust variogram smoke case. Existing environments cannot accept # a commit range, so the checked-out commit is recorded explicitly. + ASV_SMOKE_BENCHMARK="benchmark_two_point_statistics.VariogramBenchmarks.time_variogram_estimate_full_900_threads_1" asv run \ --config asv.ci-existing.json \ -E existing \ --set-commit-hash "$GITHUB_SHA" \ --no-pull \ --quick \ - --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ + --bench "$ASV_SMOKE_BENCHMARK" \ --show-stderr benchmark_parallel_readiness: diff --git a/asv.conf.json b/asv.conf.json index 289cc426a..87a278e13 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -26,5 +26,7 @@ "install_command": [ "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", "in-dir={env_dir} python -m pip install --no-deps {build_dir}" - ] + ], + "number": 1, + "repeat": 20 } diff --git a/asv.openmp.conf.json b/asv.openmp.conf.json index 6d707b994..22f2bb6fa 100644 --- a/asv.openmp.conf.json +++ b/asv.openmp.conf.json @@ -33,5 +33,7 @@ "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", "in-dir={env_dir} python -m pip install --no-deps {build_dir}", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_backend_parallel_ready.py --verbose" - ] + ], + "number": 1, + "repeat": 20 } diff --git a/benchmarks/README.md b/benchmarks/README.md index 2187eeb07..237e3a35f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -29,25 +29,24 @@ deciding where optimization work should go: - [Shared Constants](#shared-constants) - [Shared Helpers](#shared-helpers) - [Benchmark Classes](#benchmark-classes) - - [VariogramWorkflowBenchmarks](#variogramworkflowbenchmarks) - - [KrigingWorkflowBenchmarks](#krigingworkflowbenchmarks) - - [RandomFieldWorkflowBenchmarks](#randomfieldworkflowbenchmarks) + - [VariogramBenchmarks](#variogrambenchmarks) + - [KrigingBenchmarks](#krigingbenchmarks) + - [RandomFieldBenchmarks](#randomfieldbenchmarks) - [Running The Benchmarks](#running-the-benchmarks) - [Baseline Benchmark](#baseline-benchmark) - - [Current Commit Baseline](#current-commit-baseline) + - [Main Branch Baseline](#main-branch-baseline) - [Several Commits Baseline](#several-commits-baseline) - [Summary of Results](#summary-of-results) - [Visualization of Results](#visualization-of-results) - [Profiling With cProfile](#profiling-with-cprofile) -- [Optional Parallelisation with OpenMP](#optional-parallelisation-with-openmp) - - [Shared OpenMP Rule](#shared-openmp-rule) +- [Optional Parallelization with OpenMP](#optional-parallelization-with-openmp) - [OpenMP ASV Configuration](#openmp-asv-configuration) + - [Run on macOS and Linux](#run-on-macos-and-linux) - [Verify Parallel Backends](#verify-parallel-backends) - - [Run On macOS And Linux](#run-on-macos-and-linux) - - [Run On Windows](#run-on-windows) + - [Run on Windows](#run-on-windows) + - [OpenMP Thread Rule](#openmp-thread-rule) - [HPC Notes](#hpc-notes) - [Profiling With cProfile for Multiple Threads](#profiling-with-cprofile-for-multiple-threads) -- [GitHub Action Benchmark Availability Checks](#github-action-benchmark-availability-checks) - [More ASV Commands](#more-asv-commands) - [External Reference](#external-reference) @@ -69,7 +68,7 @@ parallelism as a confounding factor. Parallel/OpenMP scaling is treated as a separate optional experiment because the correct Cython OpenMP build depends on the user's operating system, compiler, and runtime environment. -To run the benchmark and the optional cProfile helper, follow these steps: +To run the benchmark and the optional cProfile helper, follow these steps (this guide will use python version 3.12. However, users can decide which version to use): 1. Move to the GSTools repository root: @@ -98,7 +97,8 @@ conda install -c conda-forge asv asv machine --yes ``` -The machine profile records local hardware information so ASV can label results correctly. Do not compare absolute times across different machines. +The machine profile records local hardware information so ASV can label +results correctly. Do not compare absolute times across different machines. ## Benchmarking Scripts @@ -108,13 +108,19 @@ The benchmarking setup currently consists of: to store results, and which Python/environment matrix to use. - `asv.openmp.conf.json`: optional cross-platform ASV configuration that builds `gstools-cython` from source with OpenMP inside ASV's own environment. -- `benchmarks/benchmark_backends.py`: contains the ASV benchmark classes. +- `benchmarks/benchmark_two_point_statistics.py`: contains the ASV benchmark + classes. - `benchmarks/README.md`: this practical guide. -- `benchmarks/tools/asv_speedup_summary.py`: reads `.asv/results/` and prints - Rust-vs-Cython speedup ratios. +- `benchmarks/tools/asv_speedup_summary.py`: reads ASV result JSON files and + prints Rust-vs-Cython speedup ratios or writes a curated Markdown/HTML + report. +- `benchmarks/tools/plot_case_backend_comparison.py`: reads ASV result JSON + files and writes a self-contained HTML report with grouped backend bars and + commit trends for selected cases, thread counts, and metrics. - `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative - workflow from `benchmark_backends.py` under Python's built-in `cProfile`, so - you can see which functions take time in the current checkout. + workflow from `benchmark_two_point_statistics.py` under Python's built-in + `cProfile`, so you can see which functions take time in the current + checkout. - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies Cython OpenMP detection and Rust backend execution with more than one GSTools thread. @@ -150,7 +156,9 @@ The repo root `asv.conf.json` is tailored to this GSTools checkout: "install_command": [ "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", "in-dir={env_dir} python -m pip install --no-deps {build_dir}" - ] + ], + "number": 1, + "repeat": 20 } ``` @@ -160,8 +168,12 @@ Important details: this guide. ASV creates isolated conda environments for the commits it benchmarks. - `pythons: ["3.12"]` means ASV creates Python 3.12 benchmark environments. - Keep this pinned unless you intentionally validate a newer Python/GSTools - backend stack. + Both ASV configs use Python 3.12 by default; change `pythons` in + `asv.conf.json` and `asv.openmp.conf.json` only when intentionally + validating another Python/GSTools backend stack. +- `number: 1` means ASV runs the benchmark code once per iteration. +- `repeat: 20` means the default baseline records 20 independent measurements + per benchmark case. - `matrix.req` asks ASV to install GSTools runtime dependencies before installing the checked-out GSTools source. It includes `gstools-cython` explicitly because the GSTools commit is installed with `--no-deps`. @@ -188,22 +200,6 @@ ASV creates these generated directories: Those directories are machine-specific generated artifacts. They should normally stay out of git. -The optional `asv.openmp.conf.json` uses the same benchmark suite, but stores -its generated files separately: - -```text -.asv-openmp/env/ OpenMP benchmark environments -.asv-openmp/results/ OpenMP result JSON files -.asv-openmp/html/ OpenMP benchmark website -``` - -It also installs build tools needed to compile `gstools-cython` from source -with `GSTOOLS_BUILD_PARALLEL=1`. The platform-specific OpenMP setup is handled -by `benchmarks/tools/install_openmp_cython.py`: macOS uses conda's -`llvm-openmp` with an Apple-clang wrapper, Linux uses the compiler toolchain -from the ASV conda environment when available, and native Windows uses the -installed MSVC Build Tools. - If needed, users can list more than one branch, Python version, benchmark directory, and so on. For example: @@ -215,8 +211,8 @@ Users can also benchmark any explicit branch, commit, tag, or range without changing `asv.conf.json`: ```bash -asv run 'my-feature-branch^!' --bench benchmark_backends -asv run main..my-feature-branch --bench benchmark_backends +asv run 'my-feature-branch^!' --bench benchmark_two_point_statistics +asv run main..my-feature-branch --bench benchmark_two_point_statistics ``` ASV checks out package code at each git commit being benchmarked. Commit source @@ -263,7 +259,8 @@ FIELD_CASES = ( ) ``` -These constants define parameter labels shown in ASV results. +These constants define backend parameter values and the case/thread labels used +in generated benchmark method names. `BACKENDS` compares: @@ -274,8 +271,8 @@ These constants define parameter labels shown in ASV results. - `threads_1`: force `gstools.config.NUM_THREADS = 1` -That is the default because the first benchmark target is a clean Cython-vs-Rust -backend comparison without parallelism. +That is the default because the first benchmark target is a clean +Cython-vs-Rust backend comparison without parallelism. ### Shared Helpers @@ -309,15 +306,15 @@ runtime and peak-memory methods. The suite currently measures: -- `VariogramWorkflowBenchmarks`: full pairwise work vs sampled large work -- `KrigingWorkflowBenchmarks`: small vs larger global kriging systems -- `RandomFieldWorkflowBenchmarks`: unstructured SRF, structured SRF, Fourier +- `VariogramBenchmarks`: full pairwise work vs sampled large work +- `KrigingBenchmarks`: small vs larger global kriging systems +- `RandomFieldBenchmarks`: unstructured SRF, structured SRF, Fourier SRF, and conditioned SRF This keeps the ASV suite focused on representative workflows rather than separate duplicate backend checks. -#### VariogramWorkflowBenchmarks +#### VariogramBenchmarks This class measures variogram estimation cases: @@ -340,7 +337,7 @@ The sampled cases still represent larger input datasets, but the variogram calculation is done on the randomly selected subset so the pairwise work stays practical. -#### KrigingWorkflowBenchmarks +#### KrigingBenchmarks This class measures global kriging at three scales: @@ -356,7 +353,7 @@ The labels mean: - `large_120x2000`: 120 conditioning points, 2,000 target points - `extra_large_360x6000`: 360 conditioning points, 6,000 target points -#### RandomFieldWorkflowBenchmarks +#### RandomFieldBenchmarks This class measures SRF and CondSRF generation workflows: @@ -384,22 +381,24 @@ The baseline benchmark is the first result set to create before doing any optimization work. It uses the default ASV configuration, so each workflow is measured with `threads_1` for both `cython_fallback` and `rust_core`. -#### Current Commit Baseline +#### Main Branch Baseline -- Save a baseline for the current commit: +- Save a baseline for the latest local `main` commit: ```bash -asv run 'HEAD^!' --bench benchmark_backends +asv run 'main^!' --bench benchmark_two_point_statistics ``` #### Several Commits Baseline -As mentioned previously, ASV can also compare several commits, here we will run the last five commits: +ASV can also compare several commits. This example runs the last three commits +on local `main`; choose a different range when that better matches your +branch. -- Run the last five commits on main branch: +- Run the last three commits on the local `main` branch: ```bash -asv run HEAD~5..HEAD --bench benchmark_backends +asv run 'main~3..main' --bench benchmark_two_point_statistics ``` #### Summary of Results @@ -426,6 +425,18 @@ The speedup helper prints the backend ratio explicitly in the terminal. By default, the helper skips removed legacy duplicate rows from older saved results. +For a report that is easier to paste into a PR or issue: + +```bash +python benchmarks/tools/asv_speedup_summary.py --format markdown +``` + +For a small standalone HTML report: + +```bash +python benchmarks/tools/asv_speedup_summary.py --format html --output .asv/backend-report-overview.html +``` + #### Visualization of Results You can inspect the results in the ASV browser report by building and opening @@ -436,19 +447,17 @@ asv publish asv preview ``` -Then open the printed local URL, for example: +Then open the printed local URL, for example: ```text -http://127.0.0.1:8082/#/ +http://127.0.0.1:8080/#/ ``` (or any other `http://127.0.0.1:/#/` URL shown by the running preview). -The browser report shows ASV plots and trends. ASV plot views do not draw a line/graph when there is only one x-axis point, therefore running `asv run 'HEAD^!' --bench benchmark_backends` will most likely not load any graphs. - -For the default benchmark run, the `threads` column should show `threads_1`. -If you later run the -[optional OpenMP scaling experiment](#optional-parallelisation-with-openmp), -the same column can be used to compare several threads. +The browser report shows raw ASV plots and trends. ASV plot views do not draw a +line when there is only one x-axis point, so a single-commit run such as +`asv run 'main^!' --bench benchmark_two_point_statistics` may show results +without useful commit-trend graphs. ### Profiling With cProfile @@ -462,8 +471,8 @@ The helper script is: benchmarks/tools/profile_benchmark_workflows.py ``` -It imports the ASV benchmark classes from `benchmark_backends.py`, selects one -case, forces one backend, and runs that case under `cProfile`. +It imports the ASV benchmark classes from `benchmark_two_point_statistics.py`, +selects one case, forces one backend, and runs that case under `cProfile`. Since ASV has already created an isolated Python environment, select that environment to execute the profiling helper: @@ -505,7 +514,7 @@ Possible profile selected cases: "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --threads threads_1 --limit 10 ``` -## Optional Parallelisation with OpenMP +## Optional Parallelization with OpenMP This section collects optional workflows for testing Cython and Rust with several thread counts. OpenMP setup is platform-dependent, so each operating @@ -515,25 +524,6 @@ The default setup above remains the recommended baseline: one thread, normal ASV environment, and no extra OpenMP build steps. Use this section only when you explicitly want to measure backend scaling with multiple thread counts. -### Shared OpenMP Rule - -The benchmark code can be run with several thread labels by setting, for -example, `GSTOOLS_BENCHMARK_THREADS=2,4,8`. That only passes different -`gstools.config.NUM_THREADS` values to GSTools. It does not, by itself, make -the Cython backend parallel. - -For Cython OpenMP scaling, the Cython extension must be compiled with OpenMP -support inside the same ASV environment that runs the benchmark. Always verify -that environment before interpreting Cython scaling results: - -```bash -ASV_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose -``` - -If the check fails, the benchmark may still run, but the Cython backend should -not be interpreted as an OpenMP-enabled Cython run. - ### OpenMP ASV Configuration Use one OpenMP ASV config on all supported desktop platforms: @@ -550,6 +540,16 @@ This config keeps the OpenMP experiment separate from the default baseline: .asv-openmp/html/ ``` +It also uses fewer repeats than the baseline config: + +```json +"number": 1, +"repeat": 20 +``` + +That means each OpenMP benchmark case records 10 independent measurements by +default, while still running the benchmark code once per measurement. + During ASV installation, it runs: ```bash @@ -565,15 +565,23 @@ compiler handling: - Linux: uses the ASV conda compiler toolchain when available. - Windows: uses native MSVC Build Tools. -### Verify Parallel Backends +### Run on macOS and Linux + +Use these commands from a POSIX shell on macOS or Linux. On macOS, install +Xcode command-line tools first. On Linux, make sure the conda compiler packages +from `asv.openmp.conf.json` solve for your platform. + +Create the ASV machine profile once: + +```bash +asv --config asv.openmp.conf.json machine --yes +``` + +#### Verify Parallel Backends Only interpret the Cython rows as OpenMP-enabled, and the Rust rows as parallel-ready, after this check passes inside the `.asv-openmp/env/...` -environment. The active `gstools-benchmark` conda environment is only the ASV -driver environment; it is normal for -`python benchmarks/tools/check_backend_parallel_ready.py --verbose` to fail -there with `gstools: not installed`, `gstools_cython: not installed`, or -`gstools_core: not installed`. +environment. On macOS and Linux: @@ -599,40 +607,39 @@ Cython OpenMP readiness: PASS Rust backend readiness: PASS with NUM_THREADS=2 ``` -### Run On macOS And Linux - -Use these commands from a POSIX shell on macOS or Linux. On macOS, install -Xcode command-line tools first. On Linux, make sure the conda compiler packages -from `asv.openmp.conf.json` solve for your platform. - -Create the ASV machine profile once: +If the check passes, run the last five commits: ```bash -asv --config asv.openmp.conf.json machine --yes +GSTOOLS_BENCHMARK_THREADS=1,2,4,8 \ +asv --config asv.openmp.conf.json run 'main~5..main' --bench benchmark_two_point_statistics --show-stderr ``` -Run a quick current-commit OpenMP smoke run: +Print Rust-vs-Cython ratios from the OpenMP result folder: ```bash -GSTOOLS_BENCHMARK_THREADS=2,4,8 \ -asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr +python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results ``` -Verify the parallel backends with the commands in -[Verify Parallel Backends](#verify-parallel-backends). If it passes, run the last-five -commits: +Build a curated OpenMP HTML report: ```bash -GSTOOLS_BENCHMARK_THREADS=2,4,8 \ -asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr +python benchmarks/tools/asv_speedup_summary.py \ + --results-dir .asv-openmp/results \ + --format html \ + --output .asv-openmp/backend-report-openmp-overview.html ``` -Print Rust-vs-Cython ratios from the OpenMP result folder: +Build the case/backend comparison report: ```bash -python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results +python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --output .asv-openmp/case-backend-comparison.html ``` +By default this report includes both time and peak-memory benchmarks. Use +`--metric time` or `--metric memory` when you want only one metric. + Build and preview the OpenMP browser report: ```bash @@ -640,7 +647,7 @@ asv --config asv.openmp.conf.json publish asv --config asv.openmp.conf.json preview ``` -### Run On Windows +### Run on Windows Use native Windows, not WSL, when you want Windows benchmark results. Install Microsoft C++ Build Tools first, including the C++ build tools workload and a @@ -653,20 +660,20 @@ Create the ASV machine profile once: asv --config asv.openmp.conf.json machine --yes ``` -Run a quick current-commit OpenMP smoke run: +Run a quick OpenMP smoke run for the latest local `main` commit: ```powershell -$env:GSTOOLS_BENCHMARK_THREADS = '2,4,8' -asv --config asv.openmp.conf.json run 'HEAD^!' --quick --bench benchmark_backends --show-stderr +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8' +asv --config asv.openmp.conf.json run 'main^!' --quick --bench benchmark_two_point_statistics --show-stderr ``` Verify the parallel backends with the PowerShell commands in -[Verify Parallel Backends](#verify-parallel-backends). If it passes, run the last-five -commits: +[Verify Parallel Backends](#verify-parallel-backends). If the check passes, +run the last five commits: ```powershell -$env:GSTOOLS_BENCHMARK_THREADS = '2,4,8' -asv --config asv.openmp.conf.json run 'HEAD~5..HEAD' --bench benchmark_backends --show-stderr +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8' +asv --config asv.openmp.conf.json run 'main~5..main' --bench benchmark_two_point_statistics --show-stderr ``` Print Rust-vs-Cython ratios from the OpenMP result folder: @@ -675,6 +682,26 @@ Print Rust-vs-Cython ratios from the OpenMP result folder: python benchmarks\tools\asv_speedup_summary.py --results-dir .asv-openmp\results ``` +Build a curated OpenMP HTML report: + +```powershell +python benchmarks\tools\asv_speedup_summary.py ` + --results-dir .asv-openmp\results ` + --format html ` + --output .asv-openmp\backend-report-openmp-overview.html +``` + +Build the case/backend comparison report: + +```powershell +python benchmarks\tools\plot_case_backend_comparison.py ` + --results-dir .asv-openmp\results ` + --output .asv-openmp\case-backend-comparison.html +``` + +By default this report includes both time and peak-memory benchmarks. Use +`--metric time` or `--metric memory` when you want only one metric. + Build and preview the OpenMP browser report: ```powershell @@ -689,6 +716,20 @@ be reused: Remove-Item Env:\GSTOOLS_BENCHMARK_THREADS ``` +### OpenMP Thread Rule + +The `GSTOOLS_BENCHMARK_THREADS=1,2,4,8` setting only tells the benchmark code to +run the same workflows with different `gstools.config.NUM_THREADS` values. It +does not, by itself, make the Cython backend parallel. + +For Cython OpenMP scaling, `gstools-cython` must be compiled with OpenMP inside +the same `.asv-openmp/env/...` environment that runs the benchmark. The +`asv.openmp.conf.json` install command does that through +`benchmarks/tools/install_openmp_cython.py`; the commands in +[Verify Parallel Backends](#verify-parallel-backends) are the proof that the +environment is ready. If that check fails, the benchmark may still run, but the +Cython rows should not be interpreted as OpenMP-enabled Cython results. + ### HPC Notes The `asv.openmp.conf.json` workflow is intended for local macOS, Linux, and @@ -707,7 +748,7 @@ same cProfile case several times with the OpenMP ASV environment: ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" ASV_OPENMP_PYTHON="$ASV_OPENMP_ENV/bin/python" -for threads in threads_2 threads_4 threads_8; do +for threads in threads_1 threads_2 threads_4 threads_8; do "$ASV_OPENMP_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads "$threads" --limit 10 done ``` @@ -720,7 +761,7 @@ $asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | Select-Object -First 1 $asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' -foreach ($threads in 'threads_2', 'threads_4', 'threads_8') { +foreach ($threads in 'threads_1', 'threads_2', 'threads_4', 'threads_8') { & $asvOpenmpPython benchmarks\tools\profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads $threads --limit 10 } ``` @@ -738,145 +779,68 @@ Useful options: For example, `--limit 10` means "print the top 10 function rows after sorting". -## GitHub Action Benchmark Availability Checks - -The repository includes `.github/workflows/asv-benchmarks.yml` to check that -the benchmark tooling can be installed and started on GitHub-hosted Linux, -Windows, `macos-latest`, and `macos-15-intel` runners. This workflow is an -availability check, not a performance benchmark run. - -The workflow has two independent matrix jobs: - -- `benchmark_asv_existing_env` installs GSTools and the benchmark - dependencies with pip, then runs one ASV benchmark method with - `-E existing`. -- `benchmark_parallel_readiness` creates a small conda-forge environment and - runs `check_backend_parallel_ready.py --verbose`. - -The ASV smoke stage runs a quick command in the existing Python environment: - -```bash -python - <<'PY' -import json -from pathlib import Path - -config = json.loads(Path("asv.conf.json").read_text()) -config["branches"] = ["HEAD"] -Path("asv.ci-existing.json").write_text(json.dumps(config, indent=2)) -PY - -asv run --config asv.ci-existing.json \ - -E existing \ - --set-commit-hash "$(git rev-parse HEAD)" \ - --no-pull \ - --quick \ - --bench benchmark_backends.VariogramWorkflowBenchmarks.time_variogram_estimate \ - --show-stderr -``` - -Do not pass a range such as `HEAD^!` with `-E existing`: ASV cannot checkout -revisions inside an existing environment. If no range is supplied, ASV resolves -the branches from the config file. The workflow writes a temporary config that -points `branches` at `$GITHUB_SHA`, then uses `--set-commit-hash "$GITHUB_SHA"` -to label the result with the commit already checked out by GitHub Actions. - -That benchmark method exercises both configured backend values through ASV -parameters, but avoids running the full performance suite in CI. Python 3.14 -currently runs this ASV smoke check with the Cython fallback backend only, -because `gstools_core` 1.2.0 can fail during the Rust variogram smoke case on -that interpreter. - -The parallel-readiness steps intentionally do not run a full ASV OpenMP -benchmark. They install `gstools-cython` from conda-forge, install -`gstools_core`, install the local GSTools checkout, and run: - -```bash -conda run -n gstools-benchmark-readiness python \ - benchmarks/tools/check_backend_parallel_ready.py --verbose -``` - -CI does not call `benchmarks/tools/install_openmp_cython.py`; that helper is -kept for local/manual OpenMP ASV builds through `asv.openmp.conf.json`. -Locally, run the readiness helper with the Python executable from the ASV -OpenMP environment: - -```bash -ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" -"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose -``` - -That fast check proves that Cython reports OpenMP support and that the Rust -backend can run a small workflow with `gstools.config.NUM_THREADS=2`. The CI -probe requires Cython to accept explicit thread counts `2,4,8`; it prints the -default thread count but does not fail when a runner defaults to one thread. For -real OpenMP scaling measurements, use `asv.openmp.conf.json` locally with -`GSTOOLS_BENCHMARK_THREADS=2,4,8`. - -The workflow runs automatically on pull requests and pushes that change ASV -configs, benchmark files, package metadata, or the workflow itself. It can also -be started from the GitHub Actions tab with `workflow_dispatch`. - ## More ASV Commands -Save results for only the current commit: +Save results for only the latest local `main` commit: ```bash -asv run 'HEAD^!' --bench benchmark_backends +asv run 'main^!' --bench benchmark_two_point_statistics ``` -Compare current commit with previous commit: +Compare the latest local `main` commit with the previous local `main` commit: ```bash -asv run 'HEAD~1^!' --bench benchmark_backends -asv run 'HEAD^!' --bench benchmark_backends -asv compare HEAD~1 HEAD +asv run 'main~1^!' --bench benchmark_two_point_statistics +asv run 'main^!' --bench benchmark_two_point_statistics +asv compare main~1 main ``` -Compare local `main` with the current branch tip: +Compare local `main` with the latest remote `main`: ```bash -asv run 'main^!' --bench benchmark_backends -asv run 'HEAD^!' --bench benchmark_backends -asv compare main HEAD +asv run 'main^!' --bench benchmark_two_point_statistics +git fetch origin main +asv run 'origin/main^!' --bench benchmark_two_point_statistics +asv compare origin/main main ``` -Compare remote `main` with the current branch tip: +Compare the previous local `main` commit with the latest remote `main`: ```bash git fetch origin main -asv run 'origin/main^!' --bench benchmark_backends -asv run 'HEAD^!' --bench benchmark_backends -asv compare origin/main HEAD +asv run 'main~1^!' --bench benchmark_two_point_statistics +asv run 'origin/main^!' --bench benchmark_two_point_statistics +asv compare main~1 origin/main ``` Run the last three commits on local `main`: ```bash -asv run 'main~3..main' --bench benchmark_backends --show-stderr +asv run 'main~3..main' --bench benchmark_two_point_statistics --show-stderr ``` Run the last three commits on the latest remote `main`: ```bash git fetch origin main -asv run 'origin/main~3..origin/main' --bench benchmark_backends --show-stderr +asv run 'origin/main~3..origin/main' --bench benchmark_two_point_statistics --show-stderr ``` -On a linear branch, `HEAD~5..HEAD` benchmarks: +On a linear branch, `main~5..main` benchmarks: ```text -HEAD~4 -HEAD~3 -HEAD~2 -HEAD~1 -HEAD +main~4 +main~3 +main~2 +main~1 +main ``` Run a selected list of commits: ```bash -git rev-parse HEAD HEAD~3 main e20c88f7 > /tmp/gstools-asv-commits.txt -asv run HASHFILE:/tmp/gstools-asv-commits.txt --bench benchmark_backends +git rev-parse main main~3 origin/main e20c88f7 > /tmp/gstools-asv-commits.txt +asv run HASHFILE:/tmp/gstools-asv-commits.txt --bench benchmark_two_point_statistics ``` Use full commit hashes when sharing results. Short hashes and branch names are @@ -885,7 +849,7 @@ fine locally but can become ambiguous later. If running ASV from outside the repo root, pass the config explicitly: ```bash -asv --config /path/to/MPS-Tools/GSTools/asv.conf.json run --quick --bench benchmark_backends +asv --config /path/to/MPS-Tools/GSTools/asv.conf.json run --quick --bench benchmark_two_point_statistics ``` ## External Reference diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py index c94e5d288..09113f745 100644 --- a/benchmarks/__init__.py +++ b/benchmarks/__init__.py @@ -1 +1 @@ -"""ASV benchmarks for GSTools.""" +"""ASV benchmark package for GSTools workflow performance checks.""" diff --git a/benchmarks/benchmark_backends.py b/benchmarks/benchmark_two_point_statistics.py similarity index 51% rename from benchmarks/benchmark_backends.py rename to benchmarks/benchmark_two_point_statistics.py index f4c853f39..91c98d2a7 100644 --- a/benchmarks/benchmark_backends.py +++ b/benchmarks/benchmark_two_point_statistics.py @@ -1,26 +1,27 @@ -"""Workflow benchmarks for GSTools backends. +"""Benchmark representative two-point statistics workflows. Usage: cd /path/to/MPS-Tools/GSTools # See benchmarks/README.md for ASV and optional cProfile setup. asv machine --yes - asv run --quick --show-stderr --bench benchmark_backends - asv run HEAD^! --bench benchmark_backends + asv run --quick --show-stderr --bench benchmark_two_point_statistics + asv run main^! --bench benchmark_two_point_statistics asv run asv publish asv preview - asv compare HEAD~1 HEAD + asv compare main~1 main Backend speedup should be interpreted as: speedup = cython_fallback_time / rust_core_time Values greater than 1.0 mean the Rust backend is faster on the same machine -for the same benchmark, commit, and thread label. +for the same benchmark, case, commit, and thread count. By default the suite uses one GSTools thread. For local OpenMP scaling -experiments, set GSTOOLS_BENCHMARK_THREADS, for example: - GSTOOLS_BENCHMARK_THREADS=2,4,8 \ - asv --config asv.openmp.conf.json run 'HEAD^!' +experiments, set GSTOOLS_BENCHMARK_THREADS before ASV discovers the +benchmarks, for example: + GSTOOLS_BENCHMARK_THREADS=1,2,4,8 \ + asv --config asv.openmp.conf.json run 'main^!' """ from __future__ import annotations @@ -35,6 +36,7 @@ def _configured_backends(): + """Return backend labels requested through the environment.""" raw = os.environ.get("GSTOOLS_BENCHMARK_BACKENDS", "") if not raw.strip(): return AVAILABLE_BACKENDS @@ -54,6 +56,7 @@ def _configured_backends(): def _configured_thread_counts(): + """Return thread count integers requested through the environment.""" raw = os.environ.get("GSTOOLS_BENCHMARK_THREADS", "1") thread_counts = [] for item in raw.split(","): @@ -61,13 +64,10 @@ def _configured_thread_counts(): if not item: continue if item.startswith("threads_"): - label = item value = item.removeprefix("threads_") else: - label = f"threads_{item}" value = item - int(value) - thread_counts.append(label) + thread_counts.append(int(value)) if not thread_counts: raise ValueError("GSTOOLS_BENCHMARK_THREADS did not define threads") return tuple(thread_counts) @@ -90,7 +90,7 @@ def _configured_thread_counts(): @contextlib.contextmanager def gstools_backend(use_core, num_threads): - """Temporarily force backend and GSTools thread count.""" + """Temporarily select the GSTools backend and thread count.""" previous = ( gs.config._GSTOOLS_CORE_AVAIL, gs.config.USE_GSTOOLS_CORE, @@ -116,6 +116,7 @@ def gstools_backend(use_core, num_threads): def _use_core(backend): + """Map a benchmark backend label to the GSTools core flag.""" if backend == "rust_core": return True if backend == "cython_fallback": @@ -124,21 +125,25 @@ def _use_core(backend): def _num_threads(thread_count): - if thread_count.startswith("threads_"): - return int(thread_count.removeprefix("threads_")) + """Convert a thread count value to an integer.""" + if isinstance(thread_count, int): + return thread_count raise ValueError(f"Unknown thread count: {thread_count}") def _random_points(seed, count, scale): + """Create a deterministic two-dimensional point cloud.""" rng = np.random.RandomState(seed) return rng.rand(count) * scale, rng.rand(count) * scale def _smooth_field(x, y): + """Evaluate a smooth deterministic field on matching coordinates.""" return np.sin(x / 10.0) + np.cos(y / 15.0) def _make_variogram_data(seed, count, scale=100.0): + """Create positions, values, and lag bins for variogram estimation.""" x, y = _random_points(seed, count, scale) field = _smooth_field(x, y) bins = np.linspace(0.0, scale * 0.6, 16) @@ -146,6 +151,7 @@ def _make_variogram_data(seed, count, scale=100.0): def _make_krige_data(seed, cond_count, target_count, scale=50.0): + """Create conditioning and target data for kriging workflows.""" rng = np.random.RandomState(seed) cond_x = rng.rand(cond_count) * scale cond_y = rng.rand(cond_count) * scale @@ -157,76 +163,87 @@ def _make_krige_data(seed, cond_count, target_count, scale=50.0): return (cond_x, cond_y), cond_val, target_pos -class VariogramWorkflowBenchmarks: - """Variogram workflow benchmarks by case and backend.""" +def _check_backend_available(backend): + """Skip Rust benchmark rows when ``gstools_core`` is unavailable.""" + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") - params = [VARIOGRAM_CASES, BACKENDS, THREAD_COUNTS] - param_names = ["case", "backend", "threads"] + +class VariogramBenchmarks: + """Measure variogram estimation cases for each backend, case, and thread count.""" + + params = [BACKENDS, VARIOGRAM_CASES, THREAD_COUNTS] + param_names = ["backend", "case", "threads"] def setup_cache(self): + """Create deterministic variogram inputs once per ASV environment.""" return { "full_900": _make_variogram_data(20220501, 900), "sampled_5000_to_1500": _make_variogram_data(20220502, 5000), "sampled_15000_to_4500": _make_variogram_data(20220503, 15000), } - def setup(self, data, case, backend, threads): - if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: - raise NotImplementedError("gstools_core is not available") - _num_threads(threads) - - def time_variogram_estimate(self, data, case, backend, threads): - with gstools_backend(_use_core(backend), _num_threads(threads)): - self._run_variogram(data, case) - - def peakmem_variogram_estimate(self, data, case, backend, threads): - with gstools_backend(_use_core(backend), _num_threads(threads)): - self._run_variogram(data, case) + def setup(self, data, backend, case, threads): + """Skip backend rows that cannot run in the current environment.""" + _check_backend_available(backend) - def _run_variogram(self, data, case): + def time_variogram_estimate(self, data, backend, case, threads): + """Run one variogram estimation case.""" pos, field, bins = data[case] kwargs = {} if case == "sampled_5000_to_1500": kwargs = {"sampling_size": 1500, "sampling_seed": 20220504} if case == "sampled_15000_to_4500": kwargs = {"sampling_size": 4500, "sampling_seed": 20220505} - return gs.vario_estimate( - pos, - field, - bins, - mesh_type="unstructured", - return_counts=True, - **kwargs, - ) + with gstools_backend(_use_core(backend), threads): + gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + **kwargs, + ) + + def peakmem_variogram_estimate(self, data, backend, case, threads): + """Measure peak memory for variogram estimation case.""" + pos, field, bins = data[case] + kwargs = {} + if case == "sampled_5000_to_1500": + kwargs = {"sampling_size": 1500, "sampling_seed": 20220504} + if case == "sampled_15000_to_4500": + kwargs = {"sampling_size": 4500, "sampling_seed": 20220505} + with gstools_backend(_use_core(backend), threads): + gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + **kwargs, + ) -class KrigingWorkflowBenchmarks: - """Global kriging workflow benchmarks by case and backend.""" +class KrigingBenchmarks: + """Measure global kriging cases for each backend, case, and thread count.""" - params = [KRIGE_CASES, BACKENDS, THREAD_COUNTS] - param_names = ["case", "backend", "threads"] + params = [BACKENDS, KRIGE_CASES, THREAD_COUNTS] + param_names = ["backend", "case", "threads"] def setup_cache(self): + """Create deterministic kriging inputs once per ASV environment.""" return { "small_30x500": _make_krige_data(20220506, 30, 500), "large_120x2000": _make_krige_data(20220507, 120, 2000), "extra_large_360x6000": _make_krige_data(20220508, 360, 6000), } - def setup(self, data, case, backend, threads): - if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: - raise NotImplementedError("gstools_core is not available") - _num_threads(threads) - - def time_global_krige(self, data, case, backend, threads): - with gstools_backend(_use_core(backend), _num_threads(threads)): - self._run_krige(data, case) + def setup(self, data, backend, case, threads): + """Skip backend rows that cannot run in the current environment.""" + _check_backend_available(backend) - def peakmem_global_krige(self, data, case, backend, threads): - with gstools_backend(_use_core(backend), _num_threads(threads)): - self._run_krige(data, case) - - def _run_krige(self, data, case): + def time_global_krige(self, data, backend, case, threads): + """Run one global kriging case.""" cond_pos, cond_val, target_pos = data[case] model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) krige = gs.Krige( @@ -236,21 +253,42 @@ def _run_krige(self, data, case): exact=False, cond_err=0.05, ) - return krige( - target_pos, - mesh_type="unstructured", - return_var=True, - store=False, + with gstools_backend(_use_core(backend), threads): + krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) + + def peakmem_global_krige(self, data, backend, case, threads): + """Measure peak memory for global kriging case.""" + cond_pos, cond_val, target_pos = data[case] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, ) + with gstools_backend(_use_core(backend), threads): + krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) -class RandomFieldWorkflowBenchmarks: - """SRF and CondSRF workflow benchmarks by case and backend.""" +class RandomFieldBenchmarks: + """Measure SRF and CondSRF generation cases for each backend, case, and thread count.""" - params = [FIELD_CASES, BACKENDS, THREAD_COUNTS] - param_names = ["case", "backend", "threads"] + params = [BACKENDS, FIELD_CASES, THREAD_COUNTS] + param_names = ["backend", "case", "threads"] def setup_cache(self): + """Create deterministic random-field inputs once per ASV environment.""" return { "unstructured_pos": _random_points(20220509, 2000, 100.0), "structured_pos": ( @@ -260,41 +298,52 @@ def setup_cache(self): "condsrf": _make_krige_data(20220510, 40, 1000), } - def setup(self, data, case, backend, threads): - if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: - raise NotImplementedError("gstools_core is not available") - _num_threads(threads) - - def time_field_generation(self, data, case, backend, threads): - with gstools_backend(_use_core(backend), _num_threads(threads)): - self._run_field(data, case) - - def peakmem_field_generation(self, data, case, backend, threads): - with gstools_backend(_use_core(backend), _num_threads(threads)): - self._run_field(data, case) - - def _run_field(self, data, case): - if case == "srf_unstructured_randmeth": - return self._run_srf_unstructured(data) - if case == "srf_structured_randmeth": - return self._run_srf_structured(data) - if case == "srf_structured_fourier": - return self._run_srf_fourier(data) - if case == "condsrf_unstructured": - return self._run_condsrf(data) - raise ValueError(f"Unknown field benchmark case: {case}") + def setup(self, data, backend, case, threads): + """Skip backend rows that cannot run in the current environment.""" + _check_backend_available(backend) + + def time_field_generation(self, data, backend, case, threads): + """Run one SRF or CondSRF case by label.""" + with gstools_backend(_use_core(backend), threads): + if case == "srf_unstructured_randmeth": + self._run_srf_unstructured(data) + elif case == "srf_structured_randmeth": + self._run_srf_structured(data) + elif case == "srf_structured_fourier": + self._run_srf_fourier(data) + elif case == "condsrf_unstructured": + self._run_condsrf(data) + else: + raise ValueError(f"Unknown field benchmark case: {case}") + + def peakmem_field_generation(self, data, backend, case, threads): + """Measure peak memory for one SRF or CondSRF case.""" + with gstools_backend(_use_core(backend), threads): + if case == "srf_unstructured_randmeth": + self._run_srf_unstructured(data) + elif case == "srf_structured_randmeth": + self._run_srf_structured(data) + elif case == "srf_structured_fourier": + self._run_srf_fourier(data) + elif case == "condsrf_unstructured": + self._run_condsrf(data) + else: + raise ValueError(f"Unknown field benchmark case: {case}") def _run_srf_unstructured(self, data): + """Run unstructured SRF generation with the RandMeth generator.""" model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) srf = gs.SRF(model, mean=1.0, seed=20220508, mode_no=512) return srf(data["unstructured_pos"], mesh_type="unstructured") def _run_srf_structured(self, data): + """Run structured SRF generation with the RandMeth generator.""" model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) srf = gs.SRF(model, mean=1.0, seed=20220509, mode_no=512) return srf(data["structured_pos"], mesh_type="structured") def _run_srf_fourier(self, data): + """Run structured SRF generation with the Fourier generator.""" model = gs.Gaussian(dim=2, var=2.0, len_scale=30.0) srf = gs.SRF( model, @@ -306,6 +355,7 @@ def _run_srf_fourier(self, data): return srf(data["structured_pos"], mesh_type="structured") def _run_condsrf(self, data): + """Run conditioned random-field generation on unstructured targets.""" cond_pos, cond_val, target_pos = data["condsrf"] model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) krige = gs.Krige( diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index f50e7a542..d5420cc17 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -1,12 +1,16 @@ #!/usr/bin/env python -"""Print Rust-vs-Cython speedups from local ASV result files. +"""Summarize Rust-vs-Cython speedups from local ASV result files. The summary is optional. ASV itself remains the source of truth for benchmark -storage and visualization. +storage and visualization. The helper understands both current generated +benchmark method names, which encode case and thread labels, and older +parameterized ASV result files. Usage: python benchmarks/tools/asv_speedup_summary.py python benchmarks/tools/asv_speedup_summary.py --results-dir .asv/results + python benchmarks/tools/asv_speedup_summary.py --format markdown + python benchmarks/tools/asv_speedup_summary.py --format html --output report.html python benchmarks/tools/asv_speedup_summary.py --include-legacy Speedup is calculated as: @@ -19,13 +23,24 @@ from __future__ import annotations import argparse +import html import itertools import json import math +import re +from datetime import datetime, timezone from pathlib import Path BACKENDS = ("cython_fallback", "rust_core") THREAD_PREFIX = "threads_" +BENCHMARK_PREFIXES = ( + "time_variogram_estimate", + "peakmem_variogram_estimate", + "time_global_krige", + "peakmem_global_krige", + "time_field_generation", + "peakmem_field_generation", +) LEGACY_BENCHMARKS = { "time_srf", "peakmem_srf", @@ -37,12 +52,30 @@ def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument( "--results-dir", - default=".asv/results", + default=None, + action="append", type=Path, - help="Path to the ASV results directory.", + help=( + "Path to an ASV results directory. Can be supplied more than once." + ), + ) + parser.add_argument( + "--format", + choices=("table", "markdown", "html"), + default="table", + help="Output format.", + ) + parser.add_argument( + "--output", + type=Path, + help="Write Markdown or HTML output to this file instead of stdout.", ) parser.add_argument( "--all", @@ -58,6 +91,7 @@ def parse_args(): def iter_result_files(results_dir): + """Yield ASV result JSON files below ``results_dir``.""" for path in sorted(results_dir.glob("**/*.json")): if path.name in {"benchmarks.json", "machine.json"}: continue @@ -65,6 +99,7 @@ def iter_result_files(results_dir): def load_json(path): + """Load one ASV result file, ignoring invalid JSON files.""" try: with path.open(encoding="utf8") as handle: return json.load(handle) @@ -73,6 +108,7 @@ def load_json(path): def result_entry(raw_result, result_columns): + """Normalize ASV result payloads across result schema versions.""" if isinstance(raw_result, dict): return raw_result if isinstance(raw_result, list) and result_columns: @@ -81,10 +117,12 @@ def result_entry(raw_result, result_columns): def is_number(value): + """Return whether ``value`` is a numeric, non-NaN result.""" return isinstance(value, (int, float)) and not math.isnan(value) def flatten_values(values): + """Yield scalar values from nested ASV result lists.""" if isinstance(values, list): for value in values: yield from flatten_values(value) @@ -93,6 +131,7 @@ def flatten_values(values): def backend_values(entry): + """Return backend values from a normalized ASV result entry.""" result = entry.get("result") params = entry.get("params") or [] if not isinstance(result, list) or not params: @@ -110,7 +149,30 @@ def backend_values(entry): return values -def backend_rows(entry): +def parse_benchmark_name(name): + """Split a benchmark name into benchmark, case, and thread labels.""" + short_name = short_benchmark_name(name) + threads = "-" + match = re.search(r"_(threads_\d+)$", short_name) + if match: + threads = match.group(1) + short_name = short_name[: match.start()] + + for prefix in BENCHMARK_PREFIXES: + if short_name == prefix: + return prefix, "-", threads + marker = f"{prefix}_" + if short_name.startswith(marker): + return prefix, short_name.removeprefix(marker), threads + + return short_name, "-", threads + + +def backend_rows(benchmark, entry): + """Return backend/value rows for one ASV benchmark entry.""" + parsed_benchmark, parsed_case, parsed_threads = parse_benchmark_name( + benchmark + ) result = entry.get("result") params = entry.get("params") or [] if not isinstance(result, list) or not params: @@ -134,17 +196,14 @@ def backend_rows(entry): if item not in BACKENDS and not item.startswith(THREAD_PREFIX) ] threads = next( - ( - item - for item in combo_values - if item.startswith(THREAD_PREFIX) - ), - "-", + (item for item in combo_values if item.startswith(THREAD_PREFIX)), + parsed_threads, ) rows.append( { "backend": backend, - "case": "/".join(case_values) if case_values else "-", + "benchmark": parsed_benchmark, + "case": "/".join(case_values) if case_values else parsed_case, "threads": threads, "value": float(value), } @@ -153,56 +212,118 @@ def backend_rows(entry): def short_benchmark_name(name): + """Return the method name from a fully qualified ASV benchmark name.""" return name.rsplit(".", maxsplit=1)[-1] -def collect_speedups(results_dir, include_all, include_legacy): +def collect_speedups(results_dirs, include_all, include_legacy): + """Collect Rust-vs-Cython ratios from one or more ASV result folders.""" rows = [] - for path in iter_result_files(results_dir): - data = load_json(path) - if not data: - continue - result_columns = data.get("result_columns", []) - commit = data.get("commit_hash", "unknown")[:8] - env_name = data.get("env_name", path.stem) - results = data.get("results", {}) - for benchmark, raw_result in results.items(): - benchmark_name = short_benchmark_name(benchmark) - if not include_legacy and benchmark_name in LEGACY_BENCHMARKS: - continue - if not include_all and ".time_" not in benchmark: + for results_dir in results_dirs: + for path in iter_result_files(results_dir): + data = load_json(path) + if not data: continue - by_case = {} - for row in backend_rows(result_entry(raw_result, result_columns)): - key = (row["case"], row["threads"]) - by_case.setdefault(key, {})[row["backend"]] = row["value"] - for (case, threads), values in by_case.items(): - cython = values.get("cython_fallback") - rust = values.get("rust_core") - if not is_number(cython) or not is_number(rust) or rust == 0: + result_columns = data.get("result_columns", []) + commit = data.get("commit_hash", "unknown")[:8] + date = data.get("date", 0) + env_name = data.get("env_name", path.stem) + results = data.get("results", {}) + for benchmark, raw_result in results.items(): + benchmark_name, _, _ = parse_benchmark_name(benchmark) + if not include_legacy and benchmark_name in LEGACY_BENCHMARKS: continue - rows.append( - { - "commit": commit, - "env": env_name, - "benchmark": benchmark_name, - "case": case, - "threads": threads, - "cython": cython, - "rust": rust, - "speedup": cython / rust, - } - ) + if not include_all and ".time_" not in benchmark: + continue + by_case = {} + entry = result_entry(raw_result, result_columns) + for row in backend_rows(benchmark, entry): + key = (row["benchmark"], row["case"], row["threads"]) + by_case.setdefault(key, {})[row["backend"]] = row["value"] + for (name, case, threads), values in by_case.items(): + cython = values.get("cython_fallback") + rust = values.get("rust_core") + if ( + not is_number(cython) + or not is_number(rust) + or rust == 0 + ): + continue + rows.append( + { + "source": str(results_dir), + "commit": commit, + "date": date, + "env": env_name, + "benchmark": name, + "case": case, + "threads": threads, + "cython": cython, + "rust": rust, + "speedup": cython / rust, + } + ) return rows +def sort_rows(rows): + """Sort rows for stable terminal and report output.""" + return sorted( + rows, + key=lambda row: ( + row["source"], + row["benchmark"], + row["case"], + thread_number(row["threads"]), + row["date"], + row["commit"], + ), + ) + + +def thread_number(label): + """Return the numeric part of a ``threads_N`` label.""" + if isinstance(label, str) and label.startswith(THREAD_PREFIX): + return int(label.removeprefix(THREAD_PREFIX)) + return -1 + + +def latest_rows(rows): + """Keep the newest row for each source, env, benchmark, case, and thread.""" + latest = {} + for row in rows: + key = ( + row["source"], + row["env"], + row["benchmark"], + row["case"], + row["threads"], + ) + if key not in latest or row["date"] >= latest[key]["date"]: + latest[key] = row + return sort_rows(latest.values()) + + +def format_date(timestamp): + """Format an ASV millisecond timestamp as a UTC date.""" + if not timestamp: + return "-" + return datetime.fromtimestamp( + timestamp / 1000.0, + tz=timezone.utc, + ).strftime("%Y-%m-%d") + + def print_table(rows): + """Print speedup rows as an aligned terminal table.""" if not rows: print("No matching Rust-vs-Cython ASV results found.") return headers = [ + "source", "commit", + "date", "env", "benchmark", "case", @@ -213,7 +334,9 @@ def print_table(rows): ] table = [ [ + row["source"], row["commit"], + format_date(row["date"]), row["env"], row["benchmark"], row["case"], @@ -230,6 +353,7 @@ def print_table(rows): ] def fmt(row): + """Format one aligned table row.""" return " ".join( str(item).ljust(width) for item, width in zip(row, widths) ) @@ -240,10 +364,209 @@ def fmt(row): print(fmt(row)) +def markdown_table(headers, rows): + """Render a simple Markdown table.""" + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append("| " + " | ".join(str(item) for item in row) + " |") + return "\n".join(lines) + + +def render_markdown(rows): + """Render the latest speedups and thread scaling as Markdown.""" + if not rows: + return "No matching Rust-vs-Cython ASV results found.\n" + + rows = latest_rows(rows) + lines = [ + "# GSTools ASV Backend Report", + "", + "Values greater than `1.0x` mean `rust_core` was faster than " + "`cython_fallback` for the same benchmark, case, commit, and thread " + "label.", + "", + ] + + for benchmark in sorted({row["benchmark"] for row in rows}): + benchmark_rows = [row for row in rows if row["benchmark"] == benchmark] + lines.extend([f"## {benchmark}", ""]) + table = [ + [ + row["source"], + row["commit"], + format_date(row["date"]), + row["case"], + row["threads"], + f"{row['cython']:.6g}", + f"{row['rust']:.6g}", + f"{row['speedup']:.3f}x", + ] + for row in benchmark_rows + ] + lines.extend( + [ + markdown_table( + [ + "source", + "commit", + "date", + "case", + "threads", + "cython", + "rust", + "speedup", + ], + table, + ), + "", + ] + ) + + scaling = thread_scaling_rows(rows) + if scaling: + lines.extend(["## Thread Scaling", ""]) + lines.extend( + [ + markdown_table( + [ + "source", + "commit", + "benchmark", + "case", + "backend", + "threads", + "time", + "vs min thread", + ], + scaling, + ), + "", + ] + ) + + return "\n".join(lines).rstrip() + "\n" + + +def thread_scaling_rows(rows): + """Build rows that compare each backend against its lowest thread count.""" + by_key = {} + for row in rows: + for backend, value in ( + ("cython_fallback", row["cython"]), + ("rust_core", row["rust"]), + ): + key = ( + row["source"], + row["commit"], + row["benchmark"], + row["case"], + backend, + ) + by_key.setdefault(key, {})[row["threads"]] = value + + table = [] + for key, values in sorted(by_key.items()): + if len(values) < 2: + continue + source, commit, benchmark, case, backend = key + base_thread = min(values, key=thread_number) + base_time = values[base_thread] + if not base_time: + continue + for threads in sorted(values, key=thread_number): + table.append( + [ + source, + commit, + benchmark, + case, + backend, + threads, + f"{values[threads]:.6g}", + f"{base_time / values[threads]:.3f}x", + ] + ) + return table + + +def render_html(rows): + """Render the Markdown report as a small standalone HTML page.""" + markdown = render_markdown(rows) + body_lines = [] + in_table = False + for line in markdown.splitlines(): + if line.startswith("# "): + body_lines.append(f"

{html.escape(line[2:])}

") + continue + if line.startswith("## "): + body_lines.append(f"

{html.escape(line[3:])}

") + continue + if line.startswith("| "): + cells = [cell.strip() for cell in line.strip("|").split("|")] + if all(cell == "---" for cell in cells): + continue + tag = "th" if not in_table else "td" + if tag == "th": + body_lines.append("") + in_table = True + body_lines.append( + "" + + "".join( + f"<{tag}>{html.escape(cell)}" for cell in cells + ) + + "" + ) + continue + if in_table: + body_lines.append("
") + in_table = False + if line: + body_lines.append(f"

{html.escape(line)}

") + if in_table: + body_lines.append("") + + return ( + "\n" + '\n' + "\n" + '\n' + "GSTools ASV Backend Report\n" + "\n" + "\n" + "\n" + "\n".join(body_lines) + "\n\n\n" + ) + + +def write_or_print(text, output): + """Write rendered output to a file or print it to stdout.""" + if output: + output.write_text(text, encoding="utf8") + print(f"Wrote {output}") + return + print(text, end="") + + def main(): + """Run the ASV speedup summary command.""" args = parse_args() - rows = collect_speedups(args.results_dir, args.all, args.include_legacy) - print_table(rows) + results_dirs = args.results_dir or [Path(".asv/results")] + rows = collect_speedups(results_dirs, args.all, args.include_legacy) + rows = sort_rows(rows) + if args.format == "table": + print_table(rows) + elif args.format == "markdown": + write_or_print(render_markdown(rows), args.output) + else: + write_or_print(render_html(rows), args.output) if __name__ == "__main__": diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index 2542702f4..2b929c6f7 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -"""Check that GSTools benchmark backends are ready for parallel runs. +"""Check whether GSTools benchmark backends are ready for parallel runs. This is a fast CI probe. It verifies that GSTools-Cython accepts explicit OpenMP thread counts and that the Rust backend can run a small workflow while @@ -23,7 +23,11 @@ def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument( "--threads", default=2, @@ -53,6 +57,7 @@ def parse_args(): def package_version(package_name): + """Return an installed package version or a clear missing marker.""" try: package = importlib.import_module(package_name) except ModuleNotFoundError: @@ -61,6 +66,7 @@ def package_version(package_name): def check_module(label, module_name): + """Check default and explicit OpenMP thread counts for one module.""" module = importlib.import_module(module_name) default_threads = module.set_num_threads(None) explicit = { @@ -71,6 +77,7 @@ def check_module(label, module_name): def check_cython_parallel(verbose=False, require_default_openmp=False): + """Validate GSTools-Cython OpenMP readiness across benchmark modules.""" default_values = [] for label, module_name in MODULES.items(): try: @@ -114,6 +121,7 @@ def check_cython_parallel(verbose=False, require_default_openmp=False): def check_rust_backend(threads): + """Run a small Rust-backed GSTools workflow with ``threads`` configured.""" try: import gstools as gs except ModuleNotFoundError as err: @@ -165,6 +173,7 @@ def check_rust_backend(threads): def main(): + """Run the parallel backend readiness checks.""" args = parse_args() print(f"python: {sys.executable}") print(f"gstools: {package_version('gstools')}") diff --git a/benchmarks/tools/install_openmp_cython.py b/benchmarks/tools/install_openmp_cython.py index 6c720218d..39babb088 100644 --- a/benchmarks/tools/install_openmp_cython.py +++ b/benchmarks/tools/install_openmp_cython.py @@ -2,9 +2,11 @@ """Install GSTools-Cython with OpenMP inside an ASV environment. This helper is called by ``asv.openmp.conf.json`` after ASV has created a -conda environment with the common build requirements. GitHub Actions can also -call it inside an already-active conda environment. It keeps the ASV config -portable and handles platform-specific compiler/OpenMP details here. +conda environment with the common build requirements. It is kept for local or +manual OpenMP ASV runs. GitHub Actions uses a faster readiness check with the +conda-forge ``gstools-cython`` package instead of building ``gstools-cython`` +from source. This helper keeps the ASV config portable and handles +platform-specific compiler/OpenMP details here. """ from __future__ import annotations @@ -20,11 +22,13 @@ def run(command, env=None, check=True): + """Run a subprocess command and echo it for ASV logs.""" print("+ " + " ".join(str(part) for part in command), flush=True) return subprocess.run(command, check=check, env=env) def prepend_path(build_env, *paths): + """Prepend existing paths to a build environment's ``PATH``.""" current = build_env.get("PATH", "") existing = [str(path) for path in paths if path.exists()] if existing: @@ -32,6 +36,7 @@ def prepend_path(build_env, *paths): def first_match(directory, patterns): + """Return the first file matching one of the glob patterns.""" for pattern in patterns: matches = sorted( path for path in directory.glob(pattern) if path.is_file() @@ -42,6 +47,7 @@ def first_match(directory, patterns): def require_file(path, message): + """Return true when ``path`` exists, otherwise print ``message``.""" if not path.exists(): print(message, file=sys.stderr) return False @@ -49,6 +55,7 @@ def require_file(path, message): def conda_executable(): + """Find the conda executable used to amend ASV environments.""" conda = os.environ.get("CONDA_EXE") or shutil.which("conda") if conda is None: print("No conda executable was found.", file=sys.stderr) @@ -56,6 +63,7 @@ def conda_executable(): def conda_install(env_dir, *packages): + """Install conda-forge packages into an ASV environment directory.""" conda = conda_executable() if conda is None: return False @@ -75,6 +83,7 @@ def conda_install(env_dir, *packages): def ensure_macos_llvm_openmp(env_dir): + """Ensure that macOS ASV environments contain ``llvm-openmp``.""" include_dir = env_dir / "include" lib_dir = env_dir / "lib" omp_header = include_dir / "omp.h" @@ -101,6 +110,7 @@ def ensure_macos_llvm_openmp(env_dir): def ensure_linux_libgomp(env_dir): + """Ensure that Linux ASV environments contain an OpenMP runtime.""" lib_dir = env_dir / "lib" if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: return True @@ -123,6 +133,7 @@ def ensure_linux_libgomp(env_dir): def write_macos_wrapper(path, force_cxx=False): + """Write a clang wrapper that maps ``-fopenmp`` to Apple clang flags.""" text = """#!/bin/bash set -e prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" @@ -161,6 +172,7 @@ def write_macos_wrapper(path, force_cxx=False): def macos_build_env(env_dir): + """Build environment variables for macOS OpenMP compilation.""" if not ensure_macos_llvm_openmp(env_dir): return None @@ -188,6 +200,7 @@ def macos_build_env(env_dir): def linux_build_env(env_dir): + """Build environment variables for Linux OpenMP compilation.""" if not ensure_linux_libgomp(env_dir): return None @@ -222,6 +235,7 @@ def linux_build_env(env_dir): def windows_build_env(env_dir): + """Build environment variables for native Windows OpenMP compilation.""" build_env = os.environ.copy() prepend_path( build_env, @@ -234,6 +248,7 @@ def windows_build_env(env_dir): def install_gstools_cython(build_env): + """Reinstall GSTools-Cython from source with OpenMP enabled.""" run( [ sys.executable, @@ -265,7 +280,11 @@ def install_gstools_cython(build_env): def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument( "env_dir", nargs="?", @@ -276,6 +295,7 @@ def parse_args(): def main(): + """Prepare the platform build environment and install GSTools-Cython.""" args = parse_args() env_dir = Path(args.env_dir).resolve() system = platform.system() diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py new file mode 100644 index 000000000..200f309fc --- /dev/null +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -0,0 +1,1259 @@ +#!/usr/bin/env python +"""Build a case/backend comparison report from ASV result JSON files. + +The default ASV browser remains useful for commit-history plots. This helper +creates a focused static HTML view for questions ASV does not show cleanly: +case-by-case backend comparisons and OpenMP thread comparisons. + +The script has no plotting dependency. It reads ASV result JSON and writes a +self-contained HTML file with a small inline SVG renderer. + +Usage: + python benchmarks/tools/plot_case_backend_comparison.py + python benchmarks/tools/plot_case_backend_comparison.py --results-dir .asv/results + python benchmarks/tools/plot_case_backend_comparison.py --results-dir .asv-openmp/results + python benchmarks/tools/plot_case_backend_comparison.py --metric time + python benchmarks/tools/plot_case_backend_comparison.py --metric memory + python benchmarks/tools/plot_case_backend_comparison.py --benchmark krige + python benchmarks/tools/plot_case_backend_comparison.py --output comparison.html +""" + +from __future__ import annotations + +import argparse +import base64 +import itertools +import json +import math +import os +import re +from datetime import datetime, timezone +from pathlib import Path + +BACKENDS = ("cython_fallback", "rust_core") +THREAD_PREFIX = "threads_" +BENCHMARK_PREFIXES = ( + "time_variogram_estimate", + "peakmem_variogram_estimate", + "time_global_krige", + "peakmem_global_krige", + "time_field_generation", + "peakmem_field_generation", +) +BENCHMARK_ALIASES = { + "variogram": "variogram", + "vario": "variogram", + "krige": "krige", + "kriging": "krige", + "field": "field", + "random_field": "field", + "random-field": "field", +} + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--results-dir", + action="append", + type=Path, + default=None, + help=( + "Path to an ASV results directory. Can be supplied more than " + "once. Default: .asv/results if present, otherwise " + ".asv-openmp/results." + ), + ) + parser.add_argument( + "--benchmark", + choices=sorted(BENCHMARK_ALIASES), + default=None, + help="Filter to one benchmark family. Default: include all families.", + ) + parser.add_argument( + "--metric", + choices=("all", "time", "memory"), + default="all", + help="Metric to include. Default: all, which includes time and memory.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help=( + "HTML output path. Default: case-backend-comparison.html next to " + "the first selected results directory." + ), + ) + parser.add_argument( + "--table", + action="store_true", + help="Print a compact terminal table instead of writing HTML.", + ) + return parser.parse_args() + + +def default_results_dirs(): + """Return the default ASV results directory list.""" + for candidate in (Path(".asv/results"), Path(".asv-openmp/results")): + if candidate.exists(): + return [candidate] + raise FileNotFoundError( + "Could not find .asv/results or .asv-openmp/results" + ) + + +def existing_results_dirs(paths): + """Validate and return result directories.""" + if paths is None: + return default_results_dirs() + missing = [path for path in paths if not path.exists()] + if missing: + names = ", ".join(str(path) for path in missing) + raise FileNotFoundError(f"ASV results directory not found: {names}") + return paths + + +def iter_result_files(results_dir): + """Yield ASV result JSON files below ``results_dir``.""" + for path in sorted(results_dir.glob("**/*.json")): + if path.name in {"benchmarks.json", "machine.json"}: + continue + yield path + + +def load_json(path): + """Load one ASV JSON file, ignoring invalid files.""" + try: + with path.open(encoding="utf8") as handle: + return json.load(handle) + except json.JSONDecodeError: + return None + + +def result_entry(raw_result, result_columns): + """Normalize ASV result payloads across schema versions.""" + if isinstance(raw_result, dict): + return raw_result + if isinstance(raw_result, list) and result_columns: + return dict(zip(result_columns, raw_result)) + return {"result": raw_result, "params": []} + + +def flatten_values(values): + """Yield scalar values from nested ASV result lists.""" + if isinstance(values, list): + for value in values: + yield from flatten_values(value) + return + yield values + + +def is_number(value): + """Return whether ``value`` is a numeric, non-NaN result.""" + return isinstance(value, (int, float)) and not math.isnan(value) + + +def short_benchmark_name(name): + """Return the method name from a fully qualified ASV benchmark name.""" + return name.rsplit(".", maxsplit=1)[-1] + + +def metric_from_name(name): + """Return the metric represented by one benchmark name.""" + short_name = short_benchmark_name(name) + if short_name.startswith("peakmem_"): + return "memory" + if short_name.startswith("time_"): + return "time" + return None + + +def family_from_name(name): + """Return the benchmark family label represented by one benchmark name.""" + short_name = short_benchmark_name(name) + if "variogram" in short_name: + return "variogram" + if "krige" in short_name: + return "krige" + if "field" in short_name: + return "field" + return None + + +def parse_benchmark_name(name): + """Split a benchmark name into base benchmark, case, and thread labels. + + Current OpenMP results may encode case and thread into generated method + names, while baseline results may keep them as ASV parameters. This parser + extracts the encoded part when present and leaves parameterized results for + ``result_rows`` to fill in. + """ + short_name = short_benchmark_name(name) + threads = None + match = re.search(r"_(threads_\d+)$", short_name) + if match: + threads = match.group(1) + short_name = short_name[: match.start()] + + for prefix in BENCHMARK_PREFIXES: + if short_name == prefix: + return prefix, None, threads + marker = f"{prefix}_" + if short_name.startswith(marker): + return prefix, short_name.removeprefix(marker), threads + + return short_name, None, threads + + +def normalize_thread(value): + """Normalize ASV thread values to a ``threads_N`` label.""" + if value is None: + return "threads_1" + text = str(value).strip("'\"") + if text.startswith(THREAD_PREFIX): + return text + if text.isdigit(): + return f"{THREAD_PREFIX}{text}" + return text + + +def is_thread_value(value): + """Return whether a parameter value looks like a thread count.""" + text = str(value).strip("'\"") + return text.startswith(THREAD_PREFIX) or text.isdigit() + + +def clean_param_value(value): + """Convert an ASV repr-like parameter value into display text.""" + return str(value).strip("'\"") + + +def result_rows(benchmark, entry): + """Return normalized rows for one ASV benchmark result entry.""" + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list): + return [] + + base_name, encoded_case, encoded_threads = parse_benchmark_name(benchmark) + rows = [] + values = list(flatten_values(result)) + combinations = itertools.product(*params) if params else [()] + + for combo, value in zip(combinations, values): + if not is_number(value): + continue + combo_values = [clean_param_value(item) for item in combo] + backend = next( + (candidate for candidate in BACKENDS if candidate in combo_values), + None, + ) + if backend is None: + continue + + case_values = [ + item + for item in combo_values + if item not in BACKENDS and not is_thread_value(item) + ] + thread = next( + ( + normalize_thread(item) + for item in combo_values + if is_thread_value(item) + ), + normalize_thread(encoded_threads), + ) + rows.append( + { + "benchmark": base_name, + "family": family_from_name(base_name), + "metric": metric_from_name(base_name), + "case": "/".join(case_values) if case_values else encoded_case, + "threads": thread, + "backend": backend, + "value": float(value), + } + ) + return rows + + +def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): + """Collect normalized benchmark rows from ASV result folders.""" + rows = [] + benchmark_filter = ( + BENCHMARK_ALIASES[benchmark_filter] if benchmark_filter else None + ) + for results_dir in results_dirs: + for path in iter_result_files(results_dir): + data = load_json(path) + if not data: + continue + result_columns = data.get("result_columns", []) + commit_hash = data.get("commit_hash", "unknown") + commit = commit_hash[:8] + date = data.get("date", 0) + env_name = data.get("env_name", path.stem) + python = data.get("python", "-") + for benchmark, raw_result in data.get("results", {}).items(): + entry = result_entry(raw_result, result_columns) + for row in result_rows(benchmark, entry): + if ( + not row["metric"] + or not row["family"] + or not row["case"] + ): + continue + if benchmark_filter and row["family"] != benchmark_filter: + continue + if ( + metric_filter != "all" + and row["metric"] != metric_filter + ): + continue + row.update( + { + "source": str(results_dir), + "commit": commit, + "commit_hash": commit_hash, + "date": date, + "date_label": format_date(date), + "env": env_name, + "python": python, + } + ) + rows.append(row) + return sort_rows(rows) + + +def sort_rows(rows): + """Sort rows for stable reports.""" + return sorted( + rows, + key=lambda row: ( + row["source"], + row["metric"], + row["family"], + row["benchmark"], + row["case"], + thread_number(row["threads"]), + row["date"], + row["commit"], + row["backend"], + ), + ) + + +def thread_number(label): + """Return the numeric part of a ``threads_N`` label.""" + if isinstance(label, str) and label.startswith(THREAD_PREFIX): + try: + return int(label.removeprefix(THREAD_PREFIX)) + except ValueError: + return -1 + return -1 + + +def format_date(timestamp): + """Format an ASV millisecond timestamp as a UTC date.""" + if not timestamp: + return "-" + return datetime.fromtimestamp( + timestamp / 1000.0, + tz=timezone.utc, + ).strftime("%Y-%m-%d") + + +def default_output_path(results_dirs): + """Return the default HTML report path.""" + return results_dirs[0].parent / "case-backend-comparison.html" + + +def print_table(rows): + """Print normalized rows as a compact terminal table.""" + if not rows: + print("No matching ASV backend comparison rows found.") + return + headers = [ + "source", + "commit", + "date", + "metric", + "family", + "benchmark", + "case", + "threads", + "backend", + "value", + ] + table = [ + [ + row["source"], + row["commit"], + row["date_label"], + row["metric"], + row["family"], + row["benchmark"], + row["case"], + row["threads"], + row["backend"], + f"{row['value']:.6g}", + ] + for row in rows + ] + widths = [ + max(len(str(item)) for item in column) + for column in zip(headers, *table) + ] + + def fmt(row): + return " ".join( + str(item).ljust(width) for item, width in zip(row, widths) + ) + + print(fmt(headers)) + print(fmt(["-" * width for width in widths])) + for row in table: + print(fmt(row)) + + +def gstools_logo_markup(): + """Return a compact embedded GSTools logo for the HTML report.""" + logo_path = ( + Path(__file__).resolve().parents[2] + / "docs" + / "source" + / "pics" + / "gstools_150.png" + ) + try: + encoded = base64.b64encode(logo_path.read_bytes()).decode("ascii") + except OSError: + return '' + return ( + '' + ) + + +def render_html(rows): + """Render a self-contained interactive HTML report.""" + payload = json.dumps(rows, separators=(",", ":")).replace(" + + + + +GSTools Two-Point Statistics Backend Comparison + + + +
+
+
+
ASV benchmark report
+

GSTools Two-Point Statistics Backend Comparison

+

+ Cython and Rust backend results grouped side by side for selected + two-point statistics cases, thread counts, commits, and metrics. +

+
+ __LOGO_MARKUP__ +
+
+
+
+
+ Metric +
+
+
+ Family +
+
+
+ View +
+
+
+ Case +
+
+
+ Threads +
+
+
+ Commit +
+
+
+ Backend +
+
+
+
+
+
Benchmark visualization
+
Lower values are better
+
+
+
+
+
+ + + +""" + return html.replace("__PAYLOAD__", payload).replace( + "__LOGO_MARKUP__", gstools_logo_markup() + ) + + +def main(): + """Run the report generator.""" + args = parse_args() + results_dirs = existing_results_dirs(args.results_dir) + rows = collect_rows( + results_dirs, + benchmark_filter=args.benchmark, + metric_filter=args.metric, + ) + if args.table: + try: + print_table(rows) + except BrokenPipeError: + with open(os.devnull, "w", encoding="utf8") as devnull: + os.dup2(devnull.fileno(), 1) + return + return + if not rows: + print("No matching ASV backend comparison rows found.") + return + + output = args.output or default_output_path(results_dirs) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(render_html(rows), encoding="utf8") + print(f"Wrote {len(rows)} rows to {output}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index ebd8b925d..82158474c 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -1,9 +1,9 @@ #!/usr/bin/env python -"""Profile the representative GSTools benchmark workflows with cProfile. +"""Profile representative GSTools benchmark workflows with cProfile. This is a quick measurement helper. ASV remains the source of truth for saved -benchmark results, while this script helps identify the top cumulative Python -call sites before making algorithmic changes. +benchmark results, while this script identifies the top cumulative Python call +sites for the current checkout before making algorithmic changes. Usage: cd /path/to/MPS-Tools/GSTools @@ -20,6 +20,7 @@ import argparse import cProfile +import os import pstats import sys from pathlib import Path @@ -31,54 +32,54 @@ CASES = { "variogram-full": ( - "VariogramWorkflowBenchmarks", + "VariogramBenchmarks", "time_variogram_estimate", - ("full_900",), + "full_900", ), "variogram-sampled": ( - "VariogramWorkflowBenchmarks", + "VariogramBenchmarks", "time_variogram_estimate", - ("sampled_5000_to_1500",), + "sampled_5000_to_1500", ), "variogram-extra-large": ( - "VariogramWorkflowBenchmarks", + "VariogramBenchmarks", "time_variogram_estimate", - ("sampled_15000_to_4500",), + "sampled_15000_to_4500", ), "krige-small": ( - "KrigingWorkflowBenchmarks", + "KrigingBenchmarks", "time_global_krige", - ("small_30x500",), + "small_30x500", ), "krige-large": ( - "KrigingWorkflowBenchmarks", + "KrigingBenchmarks", "time_global_krige", - ("large_120x2000",), + "large_120x2000", ), "krige-extra-large": ( - "KrigingWorkflowBenchmarks", + "KrigingBenchmarks", "time_global_krige", - ("extra_large_360x6000",), + "extra_large_360x6000", ), "srf-unstructured": ( - "RandomFieldWorkflowBenchmarks", + "RandomFieldBenchmarks", "time_field_generation", - ("srf_unstructured_randmeth",), + "srf_unstructured_randmeth", ), "srf-structured": ( - "RandomFieldWorkflowBenchmarks", + "RandomFieldBenchmarks", "time_field_generation", - ("srf_structured_randmeth",), + "srf_structured_randmeth", ), "srf-fourier": ( - "RandomFieldWorkflowBenchmarks", + "RandomFieldBenchmarks", "time_field_generation", - ("srf_structured_fourier",), + "srf_structured_fourier", ), "condsrf": ( - "RandomFieldWorkflowBenchmarks", + "RandomFieldBenchmarks", "time_field_generation", - ("condsrf_unstructured",), + "condsrf_unstructured", ), } @@ -91,7 +92,11 @@ def parse_args(): - parser = argparse.ArgumentParser(description=__doc__) + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument( "--case", default="all", @@ -137,15 +142,18 @@ def parse_args(): def iter_selected(case): + """Yield the requested workflow case definitions.""" if case == "all": yield from CASES.items() return yield case, CASES[case] -def load_suite_class(class_name): +def load_suite_class(class_name, threads): + """Import a benchmark class after selecting generated thread labels.""" + os.environ["GSTOOLS_BENCHMARK_THREADS"] = threads.removeprefix("threads_") try: - from benchmarks import benchmark_backends + from benchmarks import benchmark_two_point_statistics except ModuleNotFoundError as err: print( "Could not import GSTools benchmark dependencies. Activate the " @@ -155,29 +163,31 @@ def load_suite_class(class_name): file=sys.stderr, ) raise SystemExit(1) from err - return getattr(benchmark_backends, class_name) + return getattr(benchmark_two_point_statistics, class_name) def run_case( name, class_name, - method_name, - params, + method_base_name, + case, repeat, limit, sort, backend, threads, ): - suite_cls = load_suite_class(class_name) + """Profile one benchmark workflow case.""" + suite_cls = load_suite_class(class_name, threads) suite = suite_cls() data = suite.setup_cache() + method_name = f"{method_base_name}_{case}_{threads}" method = getattr(suite, method_name) profiler = cProfile.Profile() profiler.enable() for _ in range(repeat): - method(data, *params, backend, threads) + method(data, backend) profiler.disable() print(f"\n== {name} [{backend}, {threads}] ==") @@ -186,6 +196,7 @@ def run_case( def main(): + """Run the cProfile helper.""" args = parse_args() if args.list: for name in CASES: From c39f88e3afe378abc1c5421373f692546d0f75cc Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 1 Jun 2026 16:36:15 +0200 Subject: [PATCH 16/37] benchmark selector mismatch fix in .github/workflows/asv-benchmarks.yml --- .github/workflows/asv-benchmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index cb3e522cd..24c980181 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -151,7 +151,7 @@ jobs: # is currently Cython-only because gstools_core 1.2.0 can fail during # the Rust variogram smoke case. Existing environments cannot accept # a commit range, so the checked-out commit is recorded explicitly. - ASV_SMOKE_BENCHMARK="benchmark_two_point_statistics.VariogramBenchmarks.time_variogram_estimate_full_900_threads_1" + ASV_SMOKE_BENCHMARK="benchmark_two_point_statistics.VariogramBenchmarks.time_variogram_estimate" asv run \ --config asv.ci-existing.json \ -E existing \ From b588b50fef2b0348be8acf49421ee56c27433244 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Sun, 7 Jun 2026 22:27:55 +0200 Subject: [PATCH 17/37] rebasing from main, benchmark install dependencies & addign fix to run rust in python 3.14 --- .github/workflows/asv-benchmarks.yml | 97 ++++---------------- asv.conf.json | 2 +- asv.openmp.conf.json | 2 +- benchmarks/README.md | 9 +- benchmarks/tools/install_pyproject_extras.py | 85 +++++++++++++++++ 5 files changed, 113 insertions(+), 82 deletions(-) create mode 100644 benchmarks/tools/install_pyproject_extras.py diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 24c980181..2cdc1d057 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -27,73 +27,20 @@ permissions: jobs: benchmark_asv_existing_env: - # Use the same dependency combinations as main.yml, but run only one - # lightweight benchmark method in the already-created Python environment. - # That avoids ASV's conda environment management in CI while still proving - # that users can start the benchmark suite on each runner family. - name: ASV smoke on ${{ matrix.os }} py${{ matrix.ver.py }} + name: ASV smoke on ${{ matrix.os }} py3.14 runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: fail-fast: false matrix: - include: - - case-id: ubuntu-py311 - os: ubuntu-latest - ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - - case-id: ubuntu-py312 - os: ubuntu-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - - case-id: ubuntu-py313 - os: ubuntu-latest - ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - - case-id: ubuntu-py314 - os: ubuntu-latest - ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } - asv_backends: "cython_fallback" - - case-id: windows-py311 - os: windows-latest - ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - - case-id: windows-py312 - os: windows-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - - case-id: windows-py313 - os: windows-latest - ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - - case-id: windows-py314 - os: windows-latest - ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } - asv_backends: "cython_fallback" - - case-id: macos-latest-py311 - os: macos-latest - ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - - case-id: macos-latest-py312 - os: macos-latest - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - - case-id: macos-latest-py313 - os: macos-latest - ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - - case-id: macos-latest-py314 - os: macos-latest - ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } - asv_backends: "cython_fallback" - - case-id: macos15-intel-py311 - os: macos-15-intel - ver: { py: "3.11", np: "==1.23.2", sp: "==1.9.2" } - - case-id: macos15-intel-py312 - os: macos-15-intel - ver: { py: "3.12", np: "==1.26.2", sp: "==1.11.2" } - - case-id: macos15-intel-py313 - os: macos-15-intel - ver: { py: "3.13", np: "==2.1.0", sp: "==1.14.1" } - - case-id: macos15-intel-py314 - os: macos-15-intel - ver: { py: "3.14", np: "==2.3.2", sp: "==1.16.1" } - asv_backends: "cython_fallback" + os: + - ubuntu-latest + - windows-latest + - macos-latest + - macos-15-intel defaults: run: - # Use one shell syntax for Linux, Windows, and macOS smoke commands. shell: bash steps: @@ -103,7 +50,7 @@ jobs: - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: - python-version: ${{ matrix.ver.py }} + python-version: "3.14" - name: Install benchmark smoke dependencies run: | @@ -112,12 +59,12 @@ jobs: asv \ emcee \ "gstools-cython>=1,<2" \ - "gstools_core>=1.0.0" \ hankel \ meshio \ - "numpy${{ matrix.ver.np }}" \ + "numpy==2.3.2" \ pyevtk \ - "scipy${{ matrix.ver.sp }}" + "scipy==1.16.1" + python benchmarks/tools/install_pyproject_extras.py rust python -m pip install --no-deps --editable . - name: Configure ASV machine @@ -145,12 +92,11 @@ jobs: - name: Run ASV smoke check in existing environment env: GSTOOLS_BENCHMARK_THREADS: "1" - GSTOOLS_BENCHMARK_BACKENDS: ${{ matrix.asv_backends || '' }} + GSTOOLS_BENCHMARK_BACKENDS: "cython_fallback,rust_core" run: | - # -E existing uses the Python environment prepared above. Python 3.14 - # is currently Cython-only because gstools_core 1.2.0 can fail during - # the Rust variogram smoke case. Existing environments cannot accept - # a commit range, so the checked-out commit is recorded explicitly. + # -E existing uses the Python environment prepared above. Existing + # environments cannot accept a commit range, so the checked-out + # commit is recorded explicitly. ASV_SMOKE_BENCHMARK="benchmark_two_point_statistics.VariogramBenchmarks.time_variogram_estimate" asv run \ --config asv.ci-existing.json \ @@ -162,10 +108,7 @@ jobs: --show-stderr benchmark_parallel_readiness: - # Keep this separate from the ASV smoke job. It checks the smallest useful - # runtime proof for parallel backends without running a full ASV benchmark - # and without building gstools-cython from the PyPI source distribution. - name: Parallel readiness on ${{ matrix.os }} py3.13 + name: Parallel readiness on ${{ matrix.os }} py3.14 runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: @@ -187,7 +130,7 @@ jobs: - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 with: activate-environment: gstools-benchmark-readiness - python-version: "3.13" + python-version: "3.14" channels: conda-forge conda-remove-defaults: true @@ -198,12 +141,12 @@ jobs: gstools-cython \ hankel \ meshio \ - "numpy=2.1.0" \ + "numpy=2.3.2" \ pip \ pyevtk \ - "scipy=1.14.1" - conda run -n gstools-benchmark-readiness python -m pip install \ - "gstools_core>=1.0.0" + "scipy=1.16.1" + conda run -n gstools-benchmark-readiness python \ + benchmarks/tools/install_pyproject_extras.py rust conda run -n gstools-benchmark-readiness python -m pip install \ --no-deps --editable . diff --git a/asv.conf.json b/asv.conf.json index 87a278e13..19e6b347e 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -24,7 +24,7 @@ } }, "install_command": [ - "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_pyproject_extras.py --pyproject {conf_dir}/pyproject.toml rust", "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ], "number": 1, diff --git a/asv.openmp.conf.json b/asv.openmp.conf.json index 22f2bb6fa..5ecb75cc1 100644 --- a/asv.openmp.conf.json +++ b/asv.openmp.conf.json @@ -29,7 +29,7 @@ } }, "install_command": [ - "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_pyproject_extras.py --pyproject {conf_dir}/pyproject.toml rust", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", "in-dir={env_dir} python -m pip install --no-deps {build_dir}", "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_backend_parallel_ready.py --verbose" diff --git a/benchmarks/README.md b/benchmarks/README.md index 237e3a35f..56a6bb2b0 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -124,6 +124,9 @@ The benchmarking setup currently consists of: - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies Cython OpenMP detection and Rust backend execution with more than one GSTools thread. +- `benchmarks/tools/install_pyproject_extras.py`: installs selected optional + dependencies directly from `pyproject.toml` without installing GSTools or + its regular dependencies. - `benchmarks/tools/install_openmp_cython.py`: helper used by `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, Linux, and native Windows. @@ -154,7 +157,7 @@ The repo root `asv.conf.json` is tailored to this GSTools checkout: } }, "install_command": [ - "in-dir={env_dir} python -m pip install gstools_core>=1.0.0", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_pyproject_extras.py --pyproject {conf_dir}/pyproject.toml rust", "in-dir={env_dir} python -m pip install --no-deps {build_dir}" ], "number": 1, @@ -180,8 +183,8 @@ Important details: - `{build_dir}` is ASV's temporary checkout/build directory for the exact GSTools commit being benchmarked. - `install_command` installs the checked-out GSTools revision with `--no-deps`. - It also installs `gstools_core` with pip because `gstools-core` is not - available as a conda package in every solver/platform combination. + It also uses `install_pyproject_extras.py` to install the `rust` extra from + `pyproject.toml`, keeping the `gstools_core` requirement in one place. - ASV still needs its own `install_command` because it creates isolated environments for the commits it benchmarks. - Run the cProfile helper with the Python executable from ASV's isolated diff --git a/benchmarks/tools/install_pyproject_extras.py b/benchmarks/tools/install_pyproject_extras.py new file mode 100644 index 000000000..bc09a4b2c --- /dev/null +++ b/benchmarks/tools/install_pyproject_extras.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +"""Install selected optional dependencies declared in ``pyproject.toml``. + +This installs only the requirements listed under the requested +``[project.optional-dependencies]`` extras. It does not install the project or +its regular dependencies, so ASV can keep control of its environment matrix. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - used on Python 3.8-3.10 + import tomli as tomllib + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "extras", + nargs="+", + help="Optional dependency extras to install.", + ) + parser.add_argument( + "--pyproject", + type=Path, + default=Path("pyproject.toml"), + help="Path to pyproject.toml. Default: ./pyproject.toml", + ) + return parser.parse_args() + + +def extra_requirements(pyproject, extras): + """Return deduplicated requirements for named pyproject extras.""" + with pyproject.open("rb") as handle: + config = tomllib.load(handle) + + optional = config.get("project", {}).get("optional-dependencies", {}) + missing = sorted(set(extras) - set(optional)) + if missing: + available = ", ".join(sorted(optional)) or "none" + names = ", ".join(missing) + raise ValueError( + f"Unknown optional dependency extras: {names}. " + f"Available extras: {available}" + ) + + requirements = [] + for extra in extras: + for requirement in optional[extra]: + if requirement not in requirements: + requirements.append(requirement) + return requirements + + +def install_requirements(requirements): + """Install requirements with pip in the current Python environment.""" + if not requirements: + print("No requirements to install.", flush=True) + return + command = [sys.executable, "-m", "pip", "install", *requirements] + print("+ " + " ".join(command), flush=True) + subprocess.run(command, check=True) + + +def main(): + """Install the requested optional dependency extras.""" + args = parse_args() + try: + requirements = extra_requirements(args.pyproject, args.extras) + except (FileNotFoundError, ValueError) as err: + print(err, file=sys.stderr) + return 2 + install_requirements(requirements) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From e0861b92ed4c45303aea5628ffbdb7a9e9cf2789 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 12:29:02 +0200 Subject: [PATCH 18/37] first approach to deploy html to static github pages --- .github/workflows/asv-openmp-pr.yml | 183 ++++++++++++++++++ .github/workflows/asv-openmp-publish.yml | 118 +++++++++++ asv.conf.json | 2 +- asv.openmp.conf.json | 2 +- benchmarks/README.md | 93 ++++++++- .../tools/plot_case_backend_comparison.py | 143 +++++++++++++- tests/test_install_pyproject_extras.py | 77 ++++++++ tests/test_plot_case_backend_comparison.py | 123 ++++++++++++ 8 files changed, 731 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/asv-openmp-pr.yml create mode 100644 .github/workflows/asv-openmp-publish.yml create mode 100644 tests/test_install_pyproject_extras.py create mode 100644 tests/test_plot_case_backend_comparison.py diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml new file mode 100644 index 000000000..51be1a989 --- /dev/null +++ b/.github/workflows/asv-openmp-pr.yml @@ -0,0 +1,183 @@ +name: OpenMP PR Benchmark + +on: + pull_request: + branches: + - "main" + +permissions: + contents: read + +concurrency: + group: openmp-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + ASV_CONFIG: asv.openmp.conf.json + MACHINE_NAME: github-actions-ubuntu + +jobs: + benchmark_changes: + name: Detect benchmark-relevant changes + runs-on: ubuntu-latest + outputs: + required: ${{ steps.changes.outputs.required }} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Detect benchmark-relevant changes + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- \ + src \ + benchmarks \ + pyproject.toml \ + asv.conf.json \ + asv.openmp.conf.json \ + .github/workflows/asv-benchmarks.yml \ + .github/workflows/asv-openmp-pr.yml \ + .github/workflows/asv-openmp-publish.yml + then + echo "required=false" >> "$GITHUB_OUTPUT" + else + echo "required=true" >> "$GITHUB_OUTPUT" + fi + + benchmark: + name: Full OpenMP comparison on Linux + if: needs.benchmark_changes.outputs.required == 'true' + needs: benchmark_changes + runs-on: ubuntu-latest + timeout-minutes: 360 + + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + activate-environment: gstools-asv-driver + python-version: "3.14" + channels: conda-forge + conda-remove-defaults: true + + - name: Install ASV driver + run: python -m pip install --upgrade pip asv + + - name: Configure PR commits and ASV branches + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + printf '%s\n%s\n' "$BASE_SHA" "$HEAD_SHA" \ + > "$RUNNER_TEMP/asv-pr-commits.txt" + python - <<'PY' + import json + import os + from pathlib import Path + + config = json.loads(Path("asv.openmp.conf.json").read_text()) + config["branches"] = [os.environ["BASE_SHA"], os.environ["HEAD_SHA"]] + Path("asv.openmp.pr.conf.json").write_text( + json.dumps(config, indent=2), + encoding="utf8", + ) + PY + echo "ASV_CONFIG=asv.openmp.pr.conf.json" >> "$GITHUB_ENV" + + - name: Configure stable ASV machine name + run: | + asv --config "$ASV_CONFIG" machine \ + --machine "$MACHINE_NAME" \ + --yes + + - name: Run full OpenMP comparison + env: + GSTOOLS_BENCHMARK_THREADS: "1,2,4,8" + run: | + asv --config "$ASV_CONFIG" run \ + "HASHFILE:$RUNNER_TEMP/asv-pr-commits.txt" \ + --machine "$MACHINE_NAME" \ + --interleave-rounds \ + --no-pull \ + --bench benchmark_two_point_statistics \ + --show-stderr + + - name: Write informational ASV comparison + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + mkdir -p .asv-openmp/artifact + set +e + asv --config "$ASV_CONFIG" compare \ + "$BASE_SHA" \ + "$HEAD_SHA" \ + --machine "$MACHINE_NAME" \ + --split \ + | tee .asv-openmp/artifact/comparison.txt + compare_status="${PIPESTATUS[0]}" + set -e + echo "ASV comparison exit status: $compare_status" \ + >> .asv-openmp/artifact/comparison.txt + + - name: Build custom and native ASV reports + run: | + python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --max-commits 50 \ + --output .asv-openmp/artifact/index.html + asv --config "$ASV_CONFIG" publish \ + --no-pull \ + --html-dir .asv-openmp/artifact/asv + cp -R .asv-openmp/results .asv-openmp/artifact/results + + - name: Upload OpenMP benchmark reports + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: openmp-pr-${{ github.event.pull_request.number }} + path: .asv-openmp/artifact + if-no-files-found: warn + retention-days: 30 + + required_gate: + name: OpenMP benchmark gate + if: always() + needs: + - benchmark_changes + - benchmark + runs-on: ubuntu-latest + + steps: + - name: Check required benchmark result + env: + CHANGE_RESULT: ${{ needs.benchmark_changes.result }} + BENCHMARK_REQUIRED: ${{ needs.benchmark_changes.outputs.required }} + BENCHMARK_RESULT: ${{ needs.benchmark.result }} + run: | + if [[ "$CHANGE_RESULT" != "success" ]]; then + echo "Could not determine whether the benchmark is required." + exit 1 + fi + if [[ "$BENCHMARK_REQUIRED" == "true" \ + && "$BENCHMARK_RESULT" != "success" ]]; then + echo "The required OpenMP benchmark did not complete successfully." + exit 1 + fi + if [[ "$BENCHMARK_REQUIRED" == "true" ]]; then + echo "The required OpenMP benchmark completed successfully." + else + echo "No benchmark-relevant files changed." + fi diff --git a/.github/workflows/asv-openmp-publish.yml b/.github/workflows/asv-openmp-publish.yml new file mode 100644 index 000000000..8d31b879a --- /dev/null +++ b/.github/workflows/asv-openmp-publish.yml @@ -0,0 +1,118 @@ +name: Publish OpenMP Benchmark History + +on: + push: + branches: + - "main" + paths: + - "src/**" + - "benchmarks/**" + - "pyproject.toml" + - "asv.conf.json" + - "asv.openmp.conf.json" + - ".github/workflows/asv-benchmarks.yml" + - ".github/workflows/asv-openmp-pr.yml" + - ".github/workflows/asv-openmp-publish.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openmp-benchmark-publish + cancel-in-progress: false + +env: + ASV_CONFIG: asv.openmp.conf.json + BENCHMARKS_REPO: jeilealr/gstools-benchmarks-asv + MACHINE_NAME: github-actions-ubuntu + +jobs: + publish: + name: Benchmark accepted main commit and publish history + if: github.repository == 'jeilealr/GSTools' + runs-on: ubuntu-latest + timeout-minutes: 360 + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Check out GSTools + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Check out benchmark history repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: ${{ env.BENCHMARKS_REPO }} + token: ${{ secrets.BENCHMARKS_REPO_TOKEN }} + path: benchmark-site + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + activate-environment: gstools-asv-driver + python-version: "3.14" + channels: conda-forge + conda-remove-defaults: true + + - name: Install ASV driver + run: python -m pip install --upgrade pip asv + + - name: Expose accepted commit as local main branch + run: | + git checkout --detach "$GITHUB_SHA" + git branch --force main "$GITHUB_SHA" + + - name: Restore cumulative ASV results + run: | + mkdir -p .asv-openmp/results + if [[ -d benchmark-site/results ]]; then + cp -R benchmark-site/results/. .asv-openmp/results/ + fi + + - name: Configure stable ASV machine name + run: | + asv --config "$ASV_CONFIG" machine \ + --machine "$MACHINE_NAME" \ + --yes + + - name: Benchmark newly accepted main commit + env: + GSTOOLS_BENCHMARK_THREADS: "1,2,4,8" + run: | + asv --config "$ASV_CONFIG" run \ + "$GITHUB_SHA^!" \ + --machine "$MACHINE_NAME" \ + --skip-existing-successful \ + --no-pull \ + --bench benchmark_two_point_statistics \ + --show-stderr + + - name: Build published reports and preserve raw results + run: | + rm -rf benchmark-site/asv benchmark-site/results + python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --max-commits 50 \ + --output benchmark-site/index.html + asv --config "$ASV_CONFIG" publish \ + --no-pull \ + --html-dir benchmark-site/asv + cp -R .asv-openmp/results benchmark-site/results + touch benchmark-site/.nojekyll + + - name: Commit and push benchmark history + working-directory: benchmark-site + run: | + git config user.name "GSTools benchmark bot" + git config user.email "actions@users.noreply.github.com" + git add -A + if git diff --cached --quiet; then + echo "No benchmark history changes to publish." + exit 0 + fi + git commit -m "Update OpenMP benchmarks for ${GITHUB_SHA::8}" + git push origin main diff --git a/asv.conf.json b/asv.conf.json index 19e6b347e..97be6f086 100644 --- a/asv.conf.json +++ b/asv.conf.json @@ -11,7 +11,7 @@ "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", "environment_type": "conda", "conda_channels": ["conda-forge"], - "pythons": ["3.12"], + "pythons": ["3.14"], "matrix": { "req": { "emcee": [""], diff --git a/asv.openmp.conf.json b/asv.openmp.conf.json index 5ecb75cc1..ef4ec318b 100644 --- a/asv.openmp.conf.json +++ b/asv.openmp.conf.json @@ -11,7 +11,7 @@ "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", "environment_type": "conda", "conda_channels": ["conda-forge"], - "pythons": ["3.12"], + "pythons": ["3.14"], "matrix": { "req": { "c-compiler": [""], diff --git a/benchmarks/README.md b/benchmarks/README.md index 56a6bb2b0..a8243f8b8 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -43,6 +43,7 @@ deciding where optimization work should go: - [OpenMP ASV Configuration](#openmp-asv-configuration) - [Run on macOS and Linux](#run-on-macos-and-linux) - [Verify Parallel Backends](#verify-parallel-backends) + - [Automated OpenMP Benchmarking](#automated-openmp-benchmarking) - [Run on Windows](#run-on-windows) - [OpenMP Thread Rule](#openmp-thread-rule) - [HPC Notes](#hpc-notes) @@ -68,7 +69,7 @@ parallelism as a confounding factor. Parallel/OpenMP scaling is treated as a separate optional experiment because the correct Cython OpenMP build depends on the user's operating system, compiler, and runtime environment. -To run the benchmark and the optional cProfile helper, follow these steps (this guide will use python version 3.12. However, users can decide which version to use): +To run the benchmark and the optional cProfile helper, follow these steps (this guide uses Python 3.14): 1. Move to the GSTools repository root: @@ -79,7 +80,7 @@ cd /path/to/GSTools 2. Create and activate a conda environment for local benchmark work: ```bash -conda create -n gstools-benchmark -c conda-forge python=3.12 asv +conda create -n gstools-benchmark -c conda-forge python=3.14 asv conda activate gstools-benchmark ``` @@ -144,7 +145,7 @@ The repo root `asv.conf.json` is tailored to this GSTools checkout: "results_dir": ".asv/results", "html_dir": ".asv/html", "environment_type": "conda", - "pythons": ["3.12"], + "pythons": ["3.14"], "matrix": { "req": { "emcee": [""], @@ -170,8 +171,8 @@ Important details: - `environment_type: "conda"` means conda is required for the ASV workflow in this guide. ASV creates isolated conda environments for the commits it benchmarks. -- `pythons: ["3.12"]` means ASV creates Python 3.12 benchmark environments. - Both ASV configs use Python 3.12 by default; change `pythons` in +- `pythons: ["3.14"]` means ASV creates Python 3.14 benchmark environments. + Both ASV configs use Python 3.14 by default; change `pythons` in `asv.conf.json` and `asv.openmp.conf.json` only when intentionally validating another Python/GSTools backend stack. - `number: 1` means ASV runs the benchmark code once per iteration. @@ -637,11 +638,14 @@ Build the case/backend comparison report: ```bash python benchmarks/tools/plot_case_backend_comparison.py \ --results-dir .asv-openmp/results \ + --max-commits 50 \ --output .asv-openmp/case-backend-comparison.html ``` By default this report includes both time and peak-memory benchmarks. Use `--metric time` or `--metric memory` when you want only one metric. +`--max-commits` limits only the generated custom report; it does not delete +raw ASV results. Build and preview the OpenMP browser report: @@ -650,6 +654,85 @@ asv --config asv.openmp.conf.json publish asv --config asv.openmp.conf.json preview ``` +### Automated OpenMP Benchmarking + +The GitHub Actions benchmark workflow is split into three responsibilities: + +- `.github/workflows/asv-benchmarks.yml` keeps the fast cross-platform smoke + and parallel-readiness checks. +- `.github/workflows/asv-openmp-pr.yml` runs the full Linux OpenMP comparison + for benchmark-relevant pull requests. It benchmarks only the PR base and + head commits, uploads the custom report and native ASV site as an artifact, + and exposes the `OpenMP benchmark gate` required check. +- `.github/workflows/asv-openmp-publish.yml` runs only for trusted pushes to + `jeilealr/GSTools:main`. It benchmarks the newly accepted commit, restores + prior results, and publishes the cumulative history to + `jeilealr/gstools-benchmarks-asv`. + +Both full workflows use Python 3.14, the stable ASV machine name +`github-actions-ubuntu`, and: + +```text +GSTOOLS_BENCHMARK_THREADS=1,2,4,8 +``` + +The published benchmark repository serves: + +```text +/ custom case/backend report for the latest 50 accepted commits +/asv/ complete native ASV OpenMP history +/results/ cumulative raw ASV result JSON +``` + +GitHub-hosted `ubuntu-latest` runner hardware can change. Use the machine +metadata displayed in the custom report when interpreting long-term results. +A stable self-hosted runner is preferable if absolute performance comparisons +become a release requirement. + +Before opening a PR, compare the latest local `main` commit with the current +branch without rerunning an arbitrary history window: + +```bash +asv --config asv.openmp.conf.json machine --yes +git rev-parse main HEAD > /tmp/gstools-asv-pr-commits.txt + +GSTOOLS_BENCHMARK_THREADS=1,2,4,8 \ +asv --config asv.openmp.conf.json run \ + HASHFILE:/tmp/gstools-asv-pr-commits.txt \ + --interleave-rounds \ + --bench benchmark_two_point_statistics \ + --show-stderr +``` + +#### Benchmark Publishing Repository Setup + +Create `jeilealr/gstools-benchmarks-asv` as a public repository with an initial +`README.md`. In that repository, open **Settings > Pages**, select **Deploy +from a branch**, choose `main` and `/(root)`, then save. + +Create the restricted publishing credential from the GitHub profile settings: + +1. Open **Settings > Developer settings > Personal access tokens > + Fine-grained tokens**. +2. Generate a token named `GSTools benchmark publisher` with a 90-day + expiration. +3. Select resource owner `jeilealr` and grant access only to + `gstools-benchmarks-asv`. +4. Set repository permission **Contents** to **Read and write**. Leave every + other configurable permission at **No access**. +5. Copy the token once. Never put it in source files, chat, or workflow logs. +6. In `jeilealr/GSTools`, open **Settings > Secrets and variables > Actions** + and create the repository secret `BENCHMARKS_REPO_TOKEN`. + +The secret is used only by the trusted publishing workflow. Pull-request +workflows, including workflows from forks, never receive it. Rotate the token +before its 90-day expiration. + +After the PR workflow has run once, protect `main` in `jeilealr/GSTools` and +require the `OpenMP benchmark gate` status check. Require the gate rather than +the conditional `Full OpenMP comparison on Linux` job so pull requests without +benchmark-relevant changes can still merge. + ### Run on Windows Use native Windows, not WSL, when you want Windows benchmark results. Install diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index 200f309fc..d0acf537a 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -22,6 +22,7 @@ import argparse import base64 +import html import itertools import json import math @@ -89,6 +90,15 @@ def parse_args(): "the first selected results directory." ), ) + parser.add_argument( + "--max-commits", + type=positive_int, + default=None, + help=( + "Include only the newest N commits in the report. Raw ASV result " + "files are not changed. Default: include all commits." + ), + ) parser.add_argument( "--table", action="store_true", @@ -97,6 +107,14 @@ def parse_args(): return parser.parse_args() +def positive_int(value): + """Parse a positive integer command-line value.""" + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least 1") + return parsed + + def default_results_dirs(): """Return the default ASV results directory list.""" for candidate in (Path(".asv/results"), Path(".asv-openmp/results")): @@ -299,7 +317,19 @@ def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): commit = commit_hash[:8] date = data.get("date", 0) env_name = data.get("env_name", path.stem) - python = data.get("python", "-") + params = data.get("params", {}) + python = data.get("python") or params.get("python", "-") + machine = { + key: params.get(key, "-") + for key in ( + "machine", + "os", + "arch", + "cpu", + "num_cpu", + "ram", + ) + } for benchmark, raw_result in data.get("results", {}).items(): entry = result_entry(raw_result, result_columns) for row in result_rows(benchmark, entry): @@ -325,12 +355,27 @@ def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): "date_label": format_date(date), "env": env_name, "python": python, + **machine, } ) rows.append(row) return sort_rows(rows) +def limit_recent_commits(rows, max_commits=None): + """Return rows for only the newest requested commits.""" + if max_commits is None: + return rows + ordered_commits = sorted( + {(row["date"], row["commit_hash"]) for row in rows}, + reverse=True, + ) + included = { + commit_hash for _, commit_hash in ordered_commits[:max_commits] + } + return [row for row in rows if row["commit_hash"] in included] + + def sort_rows(rows): """Sort rows for stable reports.""" return sorted( @@ -441,6 +486,57 @@ def gstools_logo_markup(): ) +def format_ram(value): + """Format ASV RAM bytes for the machine summary.""" + try: + amount = int(value) + except (TypeError, ValueError): + return str(value) + return f"{amount / (1024**3):.1f} GiB" + + +def machine_summary_markup(rows): + """Return report cards for the distinct benchmark machines.""" + machines = {} + for row in rows: + key = tuple( + str(row.get(field, "-")) + for field in ( + "machine", + "os", + "arch", + "cpu", + "num_cpu", + "ram", + "python", + ) + ) + machines[key] = { + "machine": key[0], + "os": key[1], + "arch": key[2], + "cpu": key[3], + "num_cpu": key[4], + "ram": format_ram(key[5]), + "python": key[6], + } + + cards = [] + for machine in machines.values(): + title = html.escape(machine["machine"]) + details = html.escape( + f'{machine["os"]} · {machine["arch"]} · {machine["cpu"]} · ' + f'{machine["num_cpu"]} CPUs · {machine["ram"]} · ' + f'Python {machine["python"]}' + ) + cards.append( + '
' + f"{title}{details}" + "
" + ) + return "".join(cards) + + def render_html(rows): """Render a self-contained interactive HTML report.""" payload = json.dumps(rows, separators=(",", ":")).replace(" +
+ Complete native ASV history + + Powered by Airspeed Velocity (ASV) + +
__LOGO_MARKUP__ +
__MACHINE_MARKUP__
@@ -1223,8 +1357,10 @@ def render_html(rows): """ - return html.replace("__PAYLOAD__", payload).replace( - "__LOGO_MARKUP__", gstools_logo_markup() + return ( + html.replace("__PAYLOAD__", payload) + .replace("__LOGO_MARKUP__", gstools_logo_markup()) + .replace("__MACHINE_MARKUP__", machine_summary_markup(rows)) ) @@ -1237,6 +1373,7 @@ def main(): benchmark_filter=args.benchmark, metric_filter=args.metric, ) + rows = limit_recent_commits(rows, args.max_commits) if args.table: try: print_table(rows) diff --git a/tests/test_install_pyproject_extras.py b/tests/test_install_pyproject_extras.py new file mode 100644 index 000000000..25fb42373 --- /dev/null +++ b/tests/test_install_pyproject_extras.py @@ -0,0 +1,77 @@ +"""Tests for the benchmark pyproject extras installer.""" + +import importlib.util +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +SCRIPT = ( + Path(__file__).parents[1] + / "benchmarks" + / "tools" + / "install_pyproject_extras.py" +) +SPEC = importlib.util.spec_from_file_location( + "install_pyproject_extras", SCRIPT +) +INSTALLER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INSTALLER) + + +class TestInstallPyprojectExtras(unittest.TestCase): + """Test loading and installing pyproject optional dependencies.""" + + def setUp(self): + """Create a small pyproject fixture.""" + self.temp_dir = tempfile.TemporaryDirectory() + self.pyproject = Path(self.temp_dir.name) / "pyproject.toml" + self.pyproject.write_text( + """ +[project] +name = "example" + +[project.optional-dependencies] +first = ["one>=1", "shared>=1"] +second = ["shared>=1", "two>=2"] +""".strip(), + encoding="utf8", + ) + + def tearDown(self): + """Remove the temporary pyproject fixture.""" + self.temp_dir.cleanup() + + def test_extra_requirements_preserves_order_and_deduplicates(self): + """Requirements follow requested-extra order without duplicates.""" + requirements = INSTALLER.extra_requirements( + self.pyproject, + ["first", "second"], + ) + + self.assertEqual(requirements, ["one>=1", "shared>=1", "two>=2"]) + + def test_extra_requirements_rejects_unknown_extra(self): + """Unknown extras fail with a useful list of available extras.""" + with self.assertRaisesRegex( + ValueError, + "Available extras: first, second", + ): + INSTALLER.extra_requirements(self.pyproject, ["missing"]) + + def test_install_requirements_uses_current_python(self): + """Installation calls pip through the helper's Python executable.""" + with mock.patch.object(INSTALLER.subprocess, "run") as run: + INSTALLER.install_requirements(["one>=1", "two>=2"]) + + run.assert_called_once_with( + [ + INSTALLER.sys.executable, + "-m", + "pip", + "install", + "one>=1", + "two>=2", + ], + check=True, + ) diff --git a/tests/test_plot_case_backend_comparison.py b/tests/test_plot_case_backend_comparison.py new file mode 100644 index 000000000..8c1122553 --- /dev/null +++ b/tests/test_plot_case_backend_comparison.py @@ -0,0 +1,123 @@ +"""Tests for the custom ASV case/backend comparison report.""" + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SCRIPT = ( + Path(__file__).parents[1] + / "benchmarks" + / "tools" + / "plot_case_backend_comparison.py" +) +SPEC = importlib.util.spec_from_file_location( + "plot_case_backend_comparison", + SCRIPT, +) +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) + + +def result_row(commit_hash, date): + """Create one normalized row for report-only tests.""" + return { + "source": "results", + "commit": commit_hash[:8], + "commit_hash": commit_hash, + "date": date, + "date_label": REPORT.format_date(date), + "env": "conda-py3.14", + "python": "3.14", + "machine": "github-actions-ubuntu", + "os": "Ubuntu 24.04", + "arch": "x86_64", + "cpu": "Test CPU", + "num_cpu": "4", + "ram": str(8 * 1024**3), + "benchmark": "time_global_krige", + "family": "krige", + "metric": "time", + "case": "small_30x500", + "threads": "threads_1", + "backend": "rust_core", + "value": 0.1, + } + + +class TestPlotCaseBackendComparison(unittest.TestCase): + """Test ASV result normalization and report rendering.""" + + def test_collect_rows_includes_machine_metadata(self): + """Machine parameters are retained for the generated report.""" + with tempfile.TemporaryDirectory() as temp_dir: + results_dir = Path(temp_dir) + machine_dir = results_dir / "github-actions-ubuntu" + machine_dir.mkdir() + result = { + "commit_hash": "a" * 40, + "date": 1_700_000_000_000, + "env_name": "conda-py3.14", + "python": "3.14", + "params": { + "machine": "github-actions-ubuntu", + "os": "Ubuntu 24.04", + "arch": "x86_64", + "cpu": "Test CPU", + "num_cpu": "4", + "ram": str(8 * 1024**3), + }, + "results": { + ( + "benchmark_two_point_statistics.KrigingBenchmarks." + "time_global_krige_small_30x500_threads_1" + ): { + "result": [0.2, 0.1], + "params": [["cython_fallback", "rust_core"]], + } + }, + } + (machine_dir / "result.json").write_text( + json.dumps(result), + encoding="utf8", + ) + + rows = REPORT.collect_rows([results_dir]) + + self.assertEqual( + {row["machine"] for row in rows}, {"github-actions-ubuntu"} + ) + self.assertEqual({row["python"] for row in rows}, {"3.14"}) + self.assertEqual({row["cpu"] for row in rows}, {"Test CPU"}) + + def test_limit_recent_commits_keeps_only_newest_commits(self): + """The custom report can remain bounded without deleting results.""" + rows = [ + result_row("a" * 40, 1_700_000_000_000), + result_row("b" * 40, 1_800_000_000_000), + result_row("c" * 40, 1_900_000_000_000), + ] + + limited = REPORT.limit_recent_commits(rows, 2) + + self.assertEqual( + {row["commit_hash"] for row in limited}, + {"b" * 40, "c" * 40}, + ) + + def test_render_html_shows_machine_and_asv_links(self): + """The report identifies its machine and links to complete ASV data.""" + rendered = REPORT.render_html( + [result_row("a" * 40, 1_700_000_000_000)] + ) + + self.assertIn("github-actions-ubuntu", rendered) + self.assertIn("Ubuntu 24.04", rendered) + self.assertIn("Python 3.14", rendered) + self.assertIn('href="asv/"', rendered) + self.assertIn("Powered by Airspeed Velocity (ASV)", rendered) + + +if __name__ == "__main__": + unittest.main() From 8e7d5213db9cd21493f633ff3caa7c0007127279 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 12:53:35 +0200 Subject: [PATCH 19/37] Fix benchmark report on Python 3.8 --- benchmarks/tools/plot_case_backend_comparison.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index d0acf537a..1e570d542 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -223,7 +223,7 @@ def parse_benchmark_name(name): return prefix, None, threads marker = f"{prefix}_" if short_name.startswith(marker): - return prefix, short_name.removeprefix(marker), threads + return prefix, short_name[len(marker) :], threads return short_name, None, threads @@ -398,7 +398,7 @@ def thread_number(label): """Return the numeric part of a ``threads_N`` label.""" if isinstance(label, str) and label.startswith(THREAD_PREFIX): try: - return int(label.removeprefix(THREAD_PREFIX)) + return int(label[len(THREAD_PREFIX) :]) except ValueError: return -1 return -1 From 2bc1bcbfcdd30e73a8ee85debda3d6faae19ef28 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 12:55:51 +0200 Subject: [PATCH 20/37] Keep benchmark helpers compatible with Python 3.8 --- benchmarks/benchmark_two_point_statistics.py | 2 +- benchmarks/tools/asv_speedup_summary.py | 4 ++-- benchmarks/tools/profile_benchmark_workflows.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/benchmark_two_point_statistics.py b/benchmarks/benchmark_two_point_statistics.py index 91c98d2a7..0085af04c 100644 --- a/benchmarks/benchmark_two_point_statistics.py +++ b/benchmarks/benchmark_two_point_statistics.py @@ -64,7 +64,7 @@ def _configured_thread_counts(): if not item: continue if item.startswith("threads_"): - value = item.removeprefix("threads_") + value = item[len("threads_") :] else: value = item thread_counts.append(int(value)) diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index d5420cc17..615f1c3e0 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -163,7 +163,7 @@ def parse_benchmark_name(name): return prefix, "-", threads marker = f"{prefix}_" if short_name.startswith(marker): - return prefix, short_name.removeprefix(marker), threads + return prefix, short_name[len(marker) :], threads return short_name, "-", threads @@ -284,7 +284,7 @@ def sort_rows(rows): def thread_number(label): """Return the numeric part of a ``threads_N`` label.""" if isinstance(label, str) and label.startswith(THREAD_PREFIX): - return int(label.removeprefix(THREAD_PREFIX)) + return int(label[len(THREAD_PREFIX) :]) return -1 diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index 82158474c..1c26e9667 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -151,7 +151,7 @@ def iter_selected(case): def load_suite_class(class_name, threads): """Import a benchmark class after selecting generated thread labels.""" - os.environ["GSTOOLS_BENCHMARK_THREADS"] = threads.removeprefix("threads_") + os.environ["GSTOOLS_BENCHMARK_THREADS"] = threads[len("threads_") :] try: from benchmarks import benchmark_two_point_statistics except ModuleNotFoundError as err: From 0e9bc53102638ac71f7d67ff5acb67d8e60c7a82 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 14:17:21 +0200 Subject: [PATCH 21/37] Install pip before ASV benchmark driver --- .github/workflows/asv-openmp-pr.yml | 4 +++- .github/workflows/asv-openmp-publish.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index 51be1a989..c44e7a4bd 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -73,7 +73,9 @@ jobs: conda-remove-defaults: true - name: Install ASV driver - run: python -m pip install --upgrade pip asv + run: | + conda install -n gstools-asv-driver -y -c conda-forge pip + python -m pip install --upgrade pip asv - name: Configure PR commits and ASV branches env: diff --git a/.github/workflows/asv-openmp-publish.yml b/.github/workflows/asv-openmp-publish.yml index 8d31b879a..1dbc3b895 100644 --- a/.github/workflows/asv-openmp-publish.yml +++ b/.github/workflows/asv-openmp-publish.yml @@ -59,7 +59,9 @@ jobs: conda-remove-defaults: true - name: Install ASV driver - run: python -m pip install --upgrade pip asv + run: | + conda install -n gstools-asv-driver -y -c conda-forge pip + python -m pip install --upgrade pip asv - name: Expose accepted commit as local main branch run: | From f3a3632edbaa54b51777e83b4a07089601b58f07 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 15:24:55 +0200 Subject: [PATCH 22/37] Record OpenMP runner hardware metadata --- .github/workflows/asv-openmp-pr.yml | 4 +--- .github/workflows/asv-openmp-publish.yml | 4 +--- benchmarks/tools/configure_asv_machine.py | 23 +++++++++++++++++++ .../tools/plot_case_backend_comparison.py | 5 ++-- tests/test_plot_case_backend_comparison.py | 23 +++++++++++-------- 5 files changed, 42 insertions(+), 17 deletions(-) create mode 100644 benchmarks/tools/configure_asv_machine.py diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index c44e7a4bd..d05b649ec 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -100,9 +100,7 @@ jobs: - name: Configure stable ASV machine name run: | - asv --config "$ASV_CONFIG" machine \ - --machine "$MACHINE_NAME" \ - --yes + python benchmarks/tools/configure_asv_machine.py "$MACHINE_NAME" - name: Run full OpenMP comparison env: diff --git a/.github/workflows/asv-openmp-publish.yml b/.github/workflows/asv-openmp-publish.yml index 1dbc3b895..6195c44e7 100644 --- a/.github/workflows/asv-openmp-publish.yml +++ b/.github/workflows/asv-openmp-publish.yml @@ -77,9 +77,7 @@ jobs: - name: Configure stable ASV machine name run: | - asv --config "$ASV_CONFIG" machine \ - --machine "$MACHINE_NAME" \ - --yes + python benchmarks/tools/configure_asv_machine.py "$MACHINE_NAME" - name: Benchmark newly accepted main commit env: diff --git a/benchmarks/tools/configure_asv_machine.py b/benchmarks/tools/configure_asv_machine.py new file mode 100644 index 000000000..927d2c195 --- /dev/null +++ b/benchmarks/tools/configure_asv_machine.py @@ -0,0 +1,23 @@ +"""Store detected ASV hardware metadata under a stable machine name.""" + +import argparse + +from asv.machine import Machine, MachineCollection + + +def main(): + """Create an ASV machine profile containing the detected hardware.""" + parser = argparse.ArgumentParser() + parser.add_argument("machine_name") + args = parser.parse_args() + + machine_info = Machine.get_defaults() + machine_info["machine"] = args.machine_name + MachineCollection.save(args.machine_name, machine_info) + + for key in ("machine", "os", "arch", "cpu", "num_cpu", "ram"): + print(f"{key}: {machine_info[key]}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index 1e570d542..b59079e0a 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -149,7 +149,7 @@ def load_json(path): try: with path.open(encoding="utf8") as handle: return json.load(handle) - except json.JSONDecodeError: + except (OSError, json.JSONDecodeError): return None @@ -319,8 +319,9 @@ def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): env_name = data.get("env_name", path.stem) params = data.get("params", {}) python = data.get("python") or params.get("python", "-") + machine_data = load_json(path.parent / "machine.json") or {} machine = { - key: params.get(key, "-") + key: machine_data.get(key, params.get(key, "-")) for key in ( "machine", "os", diff --git a/tests/test_plot_case_backend_comparison.py b/tests/test_plot_case_backend_comparison.py index 8c1122553..f53d47d48 100644 --- a/tests/test_plot_case_backend_comparison.py +++ b/tests/test_plot_case_backend_comparison.py @@ -50,24 +50,25 @@ class TestPlotCaseBackendComparison(unittest.TestCase): """Test ASV result normalization and report rendering.""" def test_collect_rows_includes_machine_metadata(self): - """Machine parameters are retained for the generated report.""" + """Adjacent ASV machine metadata is retained for the report.""" with tempfile.TemporaryDirectory() as temp_dir: results_dir = Path(temp_dir) machine_dir = results_dir / "github-actions-ubuntu" machine_dir.mkdir() + machine = { + "machine": "github-actions-ubuntu", + "os": "Ubuntu 24.04", + "arch": "x86_64", + "cpu": "Test CPU", + "num_cpu": "4", + "ram": str(8 * 1024**3), + } result = { "commit_hash": "a" * 40, "date": 1_700_000_000_000, "env_name": "conda-py3.14", "python": "3.14", - "params": { - "machine": "github-actions-ubuntu", - "os": "Ubuntu 24.04", - "arch": "x86_64", - "cpu": "Test CPU", - "num_cpu": "4", - "ram": str(8 * 1024**3), - }, + "params": {"machine": "github-actions-ubuntu"}, "results": { ( "benchmark_two_point_statistics.KrigingBenchmarks." @@ -78,6 +79,10 @@ def test_collect_rows_includes_machine_metadata(self): } }, } + (machine_dir / "machine.json").write_text( + json.dumps(machine), + encoding="utf8", + ) (machine_dir / "result.json").write_text( json.dumps(result), encoding="utf8", From 30e0fe45076de3d534de09bf303e85dfb43802b6 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 23:04:00 +0200 Subject: [PATCH 23/37] cleaning code --- benchmarks/README.md | 82 +------------------ benchmarks/benchmark_two_point_statistics.py | 7 -- benchmarks/tools/asv_speedup_summary.py | 21 +---- benchmarks/tools/install_pyproject_extras.py | 2 +- .../tools/plot_case_backend_comparison.py | 6 +- .../tools/profile_benchmark_workflows.py | 6 +- 6 files changed, 9 insertions(+), 115 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index a8243f8b8..53d3ff2ed 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -43,7 +43,6 @@ deciding where optimization work should go: - [OpenMP ASV Configuration](#openmp-asv-configuration) - [Run on macOS and Linux](#run-on-macos-and-linux) - [Verify Parallel Backends](#verify-parallel-backends) - - [Automated OpenMP Benchmarking](#automated-openmp-benchmarking) - [Run on Windows](#run-on-windows) - [OpenMP Thread Rule](#openmp-thread-rule) - [HPC Notes](#hpc-notes) @@ -551,7 +550,7 @@ It also uses fewer repeats than the baseline config: "repeat": 20 ``` -That means each OpenMP benchmark case records 10 independent measurements by +That means each OpenMP benchmark case records 20 independent measurements by default, while still running the benchmark code once per measurement. During ASV installation, it runs: @@ -654,85 +653,6 @@ asv --config asv.openmp.conf.json publish asv --config asv.openmp.conf.json preview ``` -### Automated OpenMP Benchmarking - -The GitHub Actions benchmark workflow is split into three responsibilities: - -- `.github/workflows/asv-benchmarks.yml` keeps the fast cross-platform smoke - and parallel-readiness checks. -- `.github/workflows/asv-openmp-pr.yml` runs the full Linux OpenMP comparison - for benchmark-relevant pull requests. It benchmarks only the PR base and - head commits, uploads the custom report and native ASV site as an artifact, - and exposes the `OpenMP benchmark gate` required check. -- `.github/workflows/asv-openmp-publish.yml` runs only for trusted pushes to - `jeilealr/GSTools:main`. It benchmarks the newly accepted commit, restores - prior results, and publishes the cumulative history to - `jeilealr/gstools-benchmarks-asv`. - -Both full workflows use Python 3.14, the stable ASV machine name -`github-actions-ubuntu`, and: - -```text -GSTOOLS_BENCHMARK_THREADS=1,2,4,8 -``` - -The published benchmark repository serves: - -```text -/ custom case/backend report for the latest 50 accepted commits -/asv/ complete native ASV OpenMP history -/results/ cumulative raw ASV result JSON -``` - -GitHub-hosted `ubuntu-latest` runner hardware can change. Use the machine -metadata displayed in the custom report when interpreting long-term results. -A stable self-hosted runner is preferable if absolute performance comparisons -become a release requirement. - -Before opening a PR, compare the latest local `main` commit with the current -branch without rerunning an arbitrary history window: - -```bash -asv --config asv.openmp.conf.json machine --yes -git rev-parse main HEAD > /tmp/gstools-asv-pr-commits.txt - -GSTOOLS_BENCHMARK_THREADS=1,2,4,8 \ -asv --config asv.openmp.conf.json run \ - HASHFILE:/tmp/gstools-asv-pr-commits.txt \ - --interleave-rounds \ - --bench benchmark_two_point_statistics \ - --show-stderr -``` - -#### Benchmark Publishing Repository Setup - -Create `jeilealr/gstools-benchmarks-asv` as a public repository with an initial -`README.md`. In that repository, open **Settings > Pages**, select **Deploy -from a branch**, choose `main` and `/(root)`, then save. - -Create the restricted publishing credential from the GitHub profile settings: - -1. Open **Settings > Developer settings > Personal access tokens > - Fine-grained tokens**. -2. Generate a token named `GSTools benchmark publisher` with a 90-day - expiration. -3. Select resource owner `jeilealr` and grant access only to - `gstools-benchmarks-asv`. -4. Set repository permission **Contents** to **Read and write**. Leave every - other configurable permission at **No access**. -5. Copy the token once. Never put it in source files, chat, or workflow logs. -6. In `jeilealr/GSTools`, open **Settings > Secrets and variables > Actions** - and create the repository secret `BENCHMARKS_REPO_TOKEN`. - -The secret is used only by the trusted publishing workflow. Pull-request -workflows, including workflows from forks, never receive it. Rotate the token -before its 90-day expiration. - -After the PR workflow has run once, protect `main` in `jeilealr/GSTools` and -require the `OpenMP benchmark gate` status check. Require the gate rather than -the conditional `Full OpenMP comparison on Linux` job so pull requests without -benchmark-relevant changes can still merge. - ### Run on Windows Use native Windows, not WSL, when you want Windows benchmark results. Install diff --git a/benchmarks/benchmark_two_point_statistics.py b/benchmarks/benchmark_two_point_statistics.py index 0085af04c..ae19efbec 100644 --- a/benchmarks/benchmark_two_point_statistics.py +++ b/benchmarks/benchmark_two_point_statistics.py @@ -124,13 +124,6 @@ def _use_core(backend): raise ValueError(f"Unknown backend: {backend}") -def _num_threads(thread_count): - """Convert a thread count value to an integer.""" - if isinstance(thread_count, int): - return thread_count - raise ValueError(f"Unknown thread count: {thread_count}") - - def _random_points(seed, count, scale): """Create a deterministic two-dimensional point cloud.""" rng = np.random.RandomState(seed) diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index 615f1c3e0..20fe207c7 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -130,25 +130,6 @@ def flatten_values(values): yield values -def backend_values(entry): - """Return backend values from a normalized ASV result entry.""" - result = entry.get("result") - params = entry.get("params") or [] - if not isinstance(result, list) or not params: - return {} - - values = {} - combinations = itertools.product(*params) - for combo, value in zip(combinations, flatten_values(result)): - if not is_number(value): - continue - combo_values = [str(item).strip("'\"") for item in combo] - for backend in BACKENDS: - if backend in combo_values: - values[backend] = float(value) - return values - - def parse_benchmark_name(name): """Split a benchmark name into benchmark, case, and thread labels.""" short_name = short_benchmark_name(name) @@ -222,7 +203,7 @@ def collect_speedups(results_dirs, include_all, include_legacy): for results_dir in results_dirs: for path in iter_result_files(results_dir): data = load_json(path) - if not data: + if data is None: continue result_columns = data.get("result_columns", []) commit = data.get("commit_hash", "unknown")[:8] diff --git a/benchmarks/tools/install_pyproject_extras.py b/benchmarks/tools/install_pyproject_extras.py index bc09a4b2c..7c00d971d 100644 --- a/benchmarks/tools/install_pyproject_extras.py +++ b/benchmarks/tools/install_pyproject_extras.py @@ -15,7 +15,7 @@ try: import tomllib -except ModuleNotFoundError: # pragma: no cover - used on Python 3.8-3.10 +except ModuleNotFoundError: # pragma: no cover import tomli as tomllib diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index b59079e0a..3af060668 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -310,7 +310,7 @@ def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): for results_dir in results_dirs: for path in iter_result_files(results_dir): data = load_json(path) - if not data: + if data is None: continue result_columns = data.get("result_columns", []) commit_hash = data.get("commit_hash", "unknown") @@ -541,7 +541,7 @@ def machine_summary_markup(rows): def render_html(rows): """Render a self-contained interactive HTML report.""" payload = json.dumps(rows, separators=(",", ":")).replace(" + template = """ @@ -1359,7 +1359,7 @@ def render_html(rows): """ return ( - html.replace("__PAYLOAD__", payload) + template.replace("__PAYLOAD__", payload) .replace("__LOGO_MARKUP__", gstools_logo_markup()) .replace("__MACHINE_MARKUP__", machine_summary_markup(rows)) ) diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py index 1c26e9667..6995e6314 100644 --- a/benchmarks/tools/profile_benchmark_workflows.py +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -181,13 +181,13 @@ def run_case( suite_cls = load_suite_class(class_name, threads) suite = suite_cls() data = suite.setup_cache() - method_name = f"{method_base_name}_{case}_{threads}" - method = getattr(suite, method_name) + threads_int = int(threads[len("threads_"):]) + method = getattr(suite, method_base_name) profiler = cProfile.Profile() profiler.enable() for _ in range(repeat): - method(data, backend) + method(data, backend, case, threads_int) profiler.disable() print(f"\n== {name} [{backend}, {threads}] ==") From db43a099727bf12cb262623ad4ee9a0ab70d781a Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Mon, 8 Jun 2026 23:46:38 +0200 Subject: [PATCH 24/37] fix GB display in html github actions --- benchmarks/tools/plot_case_backend_comparison.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index 3af060668..a777653c9 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -488,11 +488,18 @@ def gstools_logo_markup(): def format_ram(value): - """Format ASV RAM bytes for the machine summary.""" + """Format ASV RAM for the machine summary. + + ASV 0.6+ stores RAM in bytes; older versions stored the raw /proc/meminfo + kB value. Any amount below 1 GiB expressed as bytes is implausible for a + benchmark machine, so treat it as kB and convert. + """ try: amount = int(value) except (TypeError, ValueError): return str(value) + if amount < 1024 ** 3: + amount *= 1024 return f"{amount / (1024**3):.1f} GiB" From bf6207ac7571b8ed350f53451af44ee234637ab8 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 07:56:16 +0200 Subject: [PATCH 25/37] improving scalability --- .github/workflows/asv-openmp-pr.yml | 2 +- .github/workflows/asv-openmp-publish.yml | 2 +- benchmarks/tools/plot_case_backend_comparison.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index d05b649ec..e83340fe0 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -111,7 +111,7 @@ jobs: --machine "$MACHINE_NAME" \ --interleave-rounds \ --no-pull \ - --bench benchmark_two_point_statistics \ + --bench "^benchmark_" \ --show-stderr - name: Write informational ASV comparison diff --git a/.github/workflows/asv-openmp-publish.yml b/.github/workflows/asv-openmp-publish.yml index 6195c44e7..a22659ff0 100644 --- a/.github/workflows/asv-openmp-publish.yml +++ b/.github/workflows/asv-openmp-publish.yml @@ -88,7 +88,7 @@ jobs: --machine "$MACHINE_NAME" \ --skip-existing-successful \ --no-pull \ - --bench benchmark_two_point_statistics \ + --bench "^benchmark_" \ --show-stderr - name: Build published reports and preserve raw results diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index a777653c9..2a324d841 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -196,7 +196,7 @@ def family_from_name(name): short_name = short_benchmark_name(name) if "variogram" in short_name: return "variogram" - if "krige" in short_name: + if "krig" in short_name: return "krige" if "field" in short_name: return "field" From 0caf2bcf1e23c883bfc9efb18c2ed93ac4291997 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 10:50:21 +0200 Subject: [PATCH 26/37] adding tags to the benchmarks --- .../tools/plot_case_backend_comparison.py | 258 ++++++++++++++---- 1 file changed, 207 insertions(+), 51 deletions(-) diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index 2a324d841..af671ea3a 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -28,6 +28,7 @@ import math import os import re +import subprocess from datetime import datetime, timezone from pathlib import Path @@ -144,6 +145,31 @@ def iter_result_files(results_dir): yield path +def get_git_tags(): + """Return a mapping from full commit hash to tag name, version-sorted.""" + try: + tag_out = subprocess.run( + ["git", "tag", "-l", "--sort=version:refname"], + capture_output=True, text=True, check=True, + ).stdout.strip() + if not tag_out: + return {} + result = {} + for tag in tag_out.splitlines(): + tag = tag.strip() + if not tag: + continue + rev = subprocess.run( + ["git", "rev-parse", f"{tag}^{{}}"], + capture_output=True, text=True, + ) + if rev.returncode == 0: + result[rev.stdout.strip()] = tag + return result + except (subprocess.SubprocessError, FileNotFoundError): + return {} + + def load_json(path): """Load one ASV JSON file, ignoring invalid files.""" try: @@ -301,7 +327,7 @@ def result_rows(benchmark, entry): return rows -def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): +def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all", tag_map=None): """Collect normalized benchmark rows from ASV result folders.""" rows = [] benchmark_filter = ( @@ -352,6 +378,7 @@ def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all"): "source": str(results_dir), "commit": commit, "commit_hash": commit_hash, + "tag": (tag_map or {}).get(commit_hash, ""), "date": date, "date_label": format_date(date), "env": env_name, @@ -805,6 +832,39 @@ def render_html(rows): height: 3px; width: 20px; } +.multi-select { + background: var(--panel); + border: 1px solid var(--line); + border-radius: 6px; + box-sizing: border-box; + color: var(--text); + font: 13px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + min-height: 80px; + padding: 3px; + width: 100%; +} +.multi-select option { + border-radius: 3px; + padding: 3px 6px; +} +.select-actions { + display: flex; + gap: 8px; + margin-top: 5px; +} +.select-action-btn { + background: none; + border: 1px solid var(--line); + border-radius: 4px; + color: var(--muted); + cursor: pointer; + font-size: 11px; + padding: 2px 8px; +} +.select-action-btn:hover { + border-color: var(--accent); + color: var(--accent-strong); +} @media (max-width: 700px) { header, main { padding: 18px; @@ -863,6 +923,10 @@ def render_html(rows): View
+
+ Reference +
+
Case
@@ -871,9 +935,21 @@ def render_html(rows): Threads
-
+
Commit -
+ +
+ + +
+
+
Backend @@ -905,6 +981,10 @@ def render_html(rows): bar: "Bar plot", line: "Line plot" }; +const refLabels = { + commits: "By Commits", + tags: "By Tags" +}; const metricUnits = { time: {label: "time", unit: "ms", factor: 1000}, memory: {label: "peak memory", unit: "MiB", factor: 1 / (1024 * 1024)} @@ -933,6 +1013,10 @@ def render_html(rows): const row = rows.find(item => item.commit === value); return row ? `${value} (${row.date_label})` : value; } + if (kind === "tag") { + const row = rows.find(item => item.tag === value); + return row ? `${value} — ${row.date_label}` : value; + } return value; } @@ -947,12 +1031,66 @@ def render_html(rows): }); } +function tagOrder(values) { + return Array.from(new Set(values)).sort((a, b) => { + const ra = rows.find(r => r.tag === a); + const rb = rows.find(r => r.tag === b); + const da = ra ? ra.date : 0; + const db = rb ? rb.date : 0; + if (da !== db) return da - db; + return String(a).localeCompare(String(b)); + }); +} + function checkedValues(id) { return Array.from( document.querySelectorAll(`#${id} input[type="checkbox"]:checked`) ).map(input => input.value); } +function selectedOptions(id) { + const el = document.getElementById(id); + if (!el) return []; + if (el.tagName === "SELECT") { + return Array.from(el.selectedOptions).map(opt => opt.value); + } + return checkedValues(id); +} + +function setSelectOptions(id, values, previousValues = [], textByValue = {}, defaultValues = null) { + const el = document.getElementById(id); + if (!el) return; + const retained = values.filter(v => previousValues.includes(v)); + const defaults = defaultValues === null ? values : defaultValues; + const selected = new Set(retained.length ? retained : defaults.filter(v => values.includes(v))); + if (!selected.size && values.length) selected.add(values[0]); + el.innerHTML = values.map(value => { + const sel = selected.has(value) ? " selected" : ""; + const text = textByValue[value] || optionText(id, value); + return ``; + }).join(""); +} + +function selectAllOptions(id) { + const el = document.getElementById(id); + if (!el) return; + for (const opt of el.options) opt.selected = true; + refreshOptions(); + renderAll(); +} + +function clearAllOptions(id) { + const el = document.getElementById(id); + if (!el) return; + for (const opt of el.options) opt.selected = false; + refreshOptions(); + renderAll(); +} + +function referenceModeValue() { + return checkedValues("referenceMode")[0] || "commits"; +} + function setCheckboxes( id, values, @@ -1033,15 +1171,18 @@ def render_html(rows): const benchmark = selectedBenchmark(); const cases = checkedValues("case"); const threads = checkedValues("threads"); - const commits = checkedValues("commit"); const selectedBackends = checkedValues("backend"); + const isTagMode = referenceModeValue() === "tags"; + const xFilter = isTagMode + ? (row => row.tag && selectedOptions("tag").includes(row.tag)) + : (row => selectedOptions("commit").includes(row.commit)); return rows.filter(row => (!metric || row.metric === metric) && (!family || row.family === family) && (!benchmark || row.benchmark === benchmark) && cases.includes(row.case) && threads.includes(row.threads) && - commits.includes(row.commit) && + xFilter(row) && selectedBackends.includes(row.backend) ); } @@ -1055,9 +1196,11 @@ def render_html(rows): metrics: checkedValues("metric"), families: checkedValues("family"), viewModes: checkedValues("viewMode"), + refModes: checkedValues("referenceMode"), cases: checkedValues("case"), threads: checkedValues("threads"), - commits: checkedValues("commit"), + commits: selectedOptions("commit"), + tags: selectedOptions("tag"), backends: checkedValues("backend") }; const metricOptions = unique(rows.map(row => row.metric)); @@ -1067,14 +1210,12 @@ def render_html(rows): .filter(row => row.metric === metricValue()) .map(row => row.family)); const defaultFamily = familyOptions.includes("krige") ? "krige" : familyOptions[0]; - setCheckboxes( - "family", - familyOptions, - previousMain.families, - {}, - [defaultFamily] - ); + setCheckboxes("family", familyOptions, previousMain.families, {}, [defaultFamily]); setCheckboxes("viewMode", ["line", "bar"], previousMain.viewModes, modeLabels, ["line"]); + setCheckboxes("referenceMode", ["commits", "tags"], previousMain.refModes, refLabels, ["commits"]); + const isTagMode = referenceModeValue() === "tags"; + document.getElementById("commitSection").style.display = isTagMode ? "none" : ""; + document.getElementById("tagSection").style.display = isTagMode ? "" : "none"; const benchRows = benchmarkRows(); const caseOptions = unique(benchRows.map(row => row.case)); const defaultCase = caseOptions.find(value => value.includes("extra_large")) || caseOptions[0]; @@ -1086,24 +1227,26 @@ def render_html(rows): const defaultThread = threadOptions.includes("threads_1") ? "threads_1" : threadOptions[0]; setCheckboxes("threads", threadOptions, previousMain.threads, {}, [defaultThread]); const selectedThreads = checkedValues("threads"); - const commitOptions = commitOrder(benchRows - .filter(row => selectedCases.includes(row.case) && selectedThreads.includes(row.threads)) - .map(row => row.commit)); - setCheckboxes("commit", commitOptions, previousMain.commits, {}, [lastValue(commitOptions)]); - const selectedCommits = checkedValues("commit"); - const backendOptions = backends.filter(backend => benchRows.some(row => - row.backend === backend && - selectedCases.includes(row.case) && - selectedThreads.includes(row.threads) && - selectedCommits.includes(row.commit) - )); - setCheckboxes( - "backend", - backendOptions, - previousMain.backends, - labels, - backendOptions + const baseRows = benchRows.filter(row => + selectedCases.includes(row.case) && selectedThreads.includes(row.threads) ); + if (isTagMode) { + const tagOptions = tagOrder(baseRows.filter(row => row.tag).map(row => row.tag)); + setSelectOptions("tag", tagOptions, previousMain.tags, {}, tagOptions); + const selectedTags = selectedOptions("tag"); + const backendOptions = backends.filter(backend => baseRows.some(row => + row.backend === backend && row.tag && selectedTags.includes(row.tag) + )); + setCheckboxes("backend", backendOptions, previousMain.backends, labels, backendOptions); + } else { + const commitOptions = commitOrder(baseRows.map(row => row.commit)); + setSelectOptions("commit", commitOptions, previousMain.commits, {}, [lastValue(commitOptions)]); + const selectedCommits = selectedOptions("commit"); + const backendOptions = backends.filter(backend => baseRows.some(row => + row.backend === backend && selectedCommits.includes(row.commit) + )); + setCheckboxes("backend", backendOptions, previousMain.backends, labels, backendOptions); + } const benchmark = selectedBenchmark(); const mode = viewModeValue() === "line" ? "Line plot" : "Bar plot"; document.getElementById("chart-title").textContent = @@ -1139,7 +1282,8 @@ def render_html(rows): } function barGroupKey(row) { - return `${row.case}|${row.threads}|${row.commit}`; + const xKey = referenceModeValue() === "tags" ? row.tag : row.commit; + return `${row.case}|${row.threads}|${xKey}`; } function barGroupLabel(key, showThread, showCommit) { @@ -1162,13 +1306,15 @@ def render_html(rows): function chartSubtitle(data) { const cases = checkedValues("case"); const threads = checkedValues("threads"); - const commits = checkedValues("commit"); + const isTagMode = referenceModeValue() === "tags"; + const xValues = isTagMode ? selectedOptions("tag") : selectedOptions("commit"); + const noun = isTagMode ? "tags" : "commits"; const selectedBackends = checkedValues("backend").map(value => labels[value] || value); return [ data[0].benchmark, selectionSummary(cases, "cases"), selectionSummary(threads, "thread groups"), - selectionSummary(commits, "commits"), + selectionSummary(xValues, noun), selectionSummary(selectedBackends, "backends") ].join(" · "); } @@ -1256,17 +1402,21 @@ def render_html(rows): const legend = document.getElementById("legend"); const metric = data[0].metric; const unit = metricUnits[metric]; - const commits = commitOrder(data.map(row => row.commit)); + const isTagMode = referenceModeValue() === "tags"; + const rowXKey = row => isTagMode ? row.tag : row.commit; + const xKeys = isTagMode + ? tagOrder(data.map(rowXKey)) + : commitOrder(data.map(row => row.commit)); const values = data.map(formatValue); const maxValue = axisMax(values); - const width = plotWidthFor(chart, 960, 150, commits.length); + const width = plotWidthFor(chart, 960, 150, xKeys.length); const height = lineChartHeight(width); const margin = {top: 60, right: 42, bottom: 108, left: 88}; const plotWidth = width - margin.left - margin.right; const plotHeight = height - margin.top - margin.bottom; - const x = commit => { - if (commits.length === 1) return margin.left + plotWidth / 2; - return margin.left + commits.indexOf(commit) * (plotWidth / (commits.length - 1)); + const x = key => { + if (xKeys.length === 1) return margin.left + plotWidth / 2; + return margin.left + xKeys.indexOf(key) * (plotWidth / (xKeys.length - 1)); }; const y = value => margin.top + plotHeight - (value / maxValue) * plotHeight; const groups = new Map(); @@ -1285,10 +1435,10 @@ def render_html(rows): svg += ``; svg += `${formatNumber(value)}`; } - commits.forEach(commit => { - const xx = x(commit); - const row = rows.find(item => item.commit === commit); - svg += `${escapeHtml(commit)}`; + xKeys.forEach(xKey => { + const xx = x(xKey); + const row = rows.find(item => rowXKey(item) === xKey); + svg += `${escapeHtml(xKey)}`; if (row && row.date_label !== "-") { svg += `${escapeHtml(row.date_label)}`; } @@ -1296,18 +1446,18 @@ def render_html(rows): Array.from(groups.entries()).forEach(([key, group], index) => { const color = linePalette[index % linePalette.length]; - const byCommit = new Map(group.map(row => [row.commit, row])); - const points = commits - .filter(commit => byCommit.has(commit)) - .map(commit => { - const row = byCommit.get(commit); - return {x: x(commit), y: y(formatValue(row)), row}; + const byXKey = new Map(group.map(row => [rowXKey(row), row])); + const points = xKeys + .filter(xKey => byXKey.has(xKey)) + .map(xKey => { + const row = byXKey.get(xKey); + return {x: x(xKey), y: y(formatValue(row)), row, xKey}; }); if (!points.length) return; svg += `${escapeHtml(seriesLabel(key))}`; for (const point of points) { const value = formatValue(point.row); - svg += `${escapeHtml(seriesLabel(key))} · ${point.row.commit}: ${formatNumber(value)} ${unit.unit}`; + svg += `${escapeHtml(seriesLabel(key))} · ${escapeHtml(point.xKey)}: ${formatNumber(value)} ${unit.unit}`; } }); @@ -1340,20 +1490,24 @@ def render_html(rows): renderChart(); } -for (const id of ["metric", "family", "viewMode"]) { +for (const id of ["metric", "family", "viewMode", "referenceMode"]) { document.getElementById(id).addEventListener("change", event => { enforceSingleChoice(id, event); refreshOptions(); renderAll(); }); } -for (const id of ["case", "threads", "commit", "backend"]) { +for (const id of ["case", "threads", "backend"]) { document.getElementById(id).addEventListener("change", event => { enforceAtLeastOne(id, event); refreshOptions(); renderAll(); }); } +for (const id of ["commit", "tag"]) { + const el = document.getElementById(id); + if (el) el.addEventListener("change", () => { refreshOptions(); renderAll(); }); +} let resizeTimer = null; window.addEventListener("resize", () => { clearTimeout(resizeTimer); @@ -1376,10 +1530,12 @@ def main(): """Run the report generator.""" args = parse_args() results_dirs = existing_results_dirs(args.results_dir) + tag_map = get_git_tags() rows = collect_rows( results_dirs, benchmark_filter=args.benchmark, metric_filter=args.metric, + tag_map=tag_map, ) rows = limit_recent_commits(rows, args.max_commits) if args.table: From a5611ee0363df4f4963172e3e31d9e7e63da46ef Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 13:40:08 +0200 Subject: [PATCH 27/37] implementing benchmarking html report while in PR --- .github/workflows/asv-openmp-pr.yml | 64 +++++++++++++++++++++++- .github/workflows/asv-openmp-publish.yml | 1 - 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index e83340fe0..be0edb031 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -7,6 +7,7 @@ on: permissions: contents: read + pull-requests: write concurrency: group: openmp-pr-${{ github.event.pull_request.number }} @@ -136,7 +137,6 @@ jobs: run: | python benchmarks/tools/plot_case_backend_comparison.py \ --results-dir .asv-openmp/results \ - --max-commits 50 \ --output .asv-openmp/artifact/index.html asv --config "$ASV_CONFIG" publish \ --no-pull \ @@ -144,6 +144,7 @@ jobs: cp -R .asv-openmp/results .asv-openmp/artifact/results - name: Upload OpenMP benchmark reports + id: upload if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -152,6 +153,67 @@ jobs: if-no-files-found: warn retention-days: 30 + - name: Post benchmark results as PR comment + if: always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python - <<'PY' + import os + from pathlib import Path + + base = os.environ["BASE_SHA"][:8] + head = os.environ["HEAD_SHA"][:8] + artifact_url = os.environ.get("ARTIFACT_URL", "") + report_link = ( + f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" + ) + + try: + comparison = Path(".asv-openmp/artifact/comparison.txt").read_text() + except FileNotFoundError: + comparison = "_Comparison output not available._" + + lines = comparison.splitlines() + regressed = sum( + 1 for l in lines if l.startswith("+") and l[1:2] == " " + ) + improved = sum( + 1 for l in lines if l.startswith("-") and l[1:2] == " " + ) + if regressed: + badge = f"⚠️ {regressed} benchmark(s) regressed" + elif improved: + badge = f"✅ {improved} benchmark(s) improved, none regressed" + else: + badge = "✅ No significant changes detected" + + body = ( + "\n" + "## OpenMP Benchmark Results\n\n" + f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" + f"{badge}\n\n" + "
\nFull ASV comparison\n\n" + f"```\n{comparison}\n```\n\n" + "
\n" + ) + Path("/tmp/benchmark_comment.md").write_text(body) + + summary_env = os.environ.get("GITHUB_STEP_SUMMARY", "") + if summary_env: + Path(summary_env).write_text( + body.replace("\n", "") + ) + PY + + gh pr comment "$PR_NUMBER" \ + --body-file /tmp/benchmark_comment.md \ + --edit-last + required_gate: name: OpenMP benchmark gate if: always() diff --git a/.github/workflows/asv-openmp-publish.yml b/.github/workflows/asv-openmp-publish.yml index a22659ff0..465678502 100644 --- a/.github/workflows/asv-openmp-publish.yml +++ b/.github/workflows/asv-openmp-publish.yml @@ -96,7 +96,6 @@ jobs: rm -rf benchmark-site/asv benchmark-site/results python benchmarks/tools/plot_case_backend_comparison.py \ --results-dir .asv-openmp/results \ - --max-commits 50 \ --output benchmark-site/index.html asv --config "$ASV_CONFIG" publish \ --no-pull \ From 33d5296f98d52bf7ed321d8b50729b230955eca4 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 14:18:12 +0200 Subject: [PATCH 28/37] tweak in case branch comment already exists --- .github/workflows/asv-openmp-pr.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index be0edb031..e54a19a4f 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -210,9 +210,16 @@ jobs: ) PY - gh pr comment "$PR_NUMBER" \ - --body-file /tmp/benchmark_comment.md \ - --edit-last + EXISTING_ID=$(gh api \ + "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.body | startswith("")) | .id' \ + | head -1) + if [ -n "$EXISTING_ID" ]; then + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING_ID" \ + -X PATCH -F body=@/tmp/benchmark_comment.md + else + gh pr comment "$PR_NUMBER" --body-file /tmp/benchmark_comment.md + fi required_gate: name: OpenMP benchmark gate From fc2698b6f18ed5653058c0375e302bc691e85e0d Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 14:48:50 +0200 Subject: [PATCH 29/37] add tag_regex to toml to prevent version failures, renaming github action jobs --- .github/workflows/asv-benchmarks.yml | 2 +- .github/workflows/asv-openmp-pr.yml | 2 +- AUTHORS.md | 1 + pyproject.toml | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/asv-benchmarks.yml index 2cdc1d057..290998a51 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/asv-benchmarks.yml @@ -1,4 +1,4 @@ -name: Benchmark Availability Checks +name: Benchmark Checks on: # This workflow checks whether the benchmark code can be installed and diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index e54a19a4f..9c604846d 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -1,4 +1,4 @@ -name: OpenMP PR Benchmark +name: Benchmarking on: pull_request: diff --git a/AUTHORS.md b/AUTHORS.md index 426681de3..8aafc1fd4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -8,6 +8,7 @@ and was created by following people. - Sebastian Müller, GitHub: [@MuellerSeb](https://github.com/MuellerSeb), Email: - Lennart Schüler, GitHub: [@LSchueler](https://github.com/LSchueler), Email: +- Jeisson Leal, GitHub: [@LSchueler](https://github.com/jeilealr), Email: ## Contributors (in order of contributions) diff --git a/pyproject.toml b/pyproject.toml index 828e21096..cdfcee20b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ fallback_version = "0.0.0.dev0" [tool.hatch.version.raw-options] local_scheme = "no-local-version" +tag_regex = "^v?(?P[0-9]+\\.[0-9]+.*)$" [tool.hatch.build.hooks.vcs] version-file = "src/gstools/_version.py" From befd49b327ace4f886e82fbc952f9a03e8070552 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 15:33:22 +0200 Subject: [PATCH 30/37] improve html benchmarking adding tagging --- .github/workflows/asv-openmp-pr.yml | 11 ++- .../tools/plot_case_backend_comparison.py | 77 +++++++++++-------- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index 9c604846d..86638a3ce 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -126,6 +126,7 @@ jobs: "$BASE_SHA" \ "$HEAD_SHA" \ --machine "$MACHINE_NAME" \ + --factor 1.05 \ --split \ | tee .asv-openmp/artifact/comparison.txt compare_status="${PIPESTATUS[0]}" @@ -153,11 +154,9 @@ jobs: if-no-files-found: warn retention-days: 30 - - name: Post benchmark results as PR comment + - name: Write benchmark step summary if: always() env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -210,6 +209,12 @@ jobs: ) PY + - name: Post benchmark results as PR comment + if: always() && github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | EXISTING_ID=$(gh api \ "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ --jq '.[] | select(.body | startswith("")) | .id' \ diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index af671ea3a..c786ebabd 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -865,6 +865,26 @@ def render_html(rows): border-color: var(--accent); color: var(--accent-strong); } +.toggle-btn { + background: none; + border: 1px solid var(--line); + border-radius: 4px; + color: var(--muted); + cursor: pointer; + font-size: 11px; + font-weight: 650; + letter-spacing: 0.04em; + padding: 2px 10px; + text-transform: uppercase; +} +.toggle-btn.active { + background: var(--accent-soft); + border-color: var(--accent); + color: var(--accent-strong); +} +.toggle-btn + .toggle-btn { + margin-left: 4px; +} @media (max-width: 700px) { header, main { padding: 18px; @@ -923,10 +943,6 @@ def render_html(rows): View
-
- Reference -
-
Case
@@ -935,20 +951,16 @@ def render_html(rows): Threads
-
- Commit +
+ + + + +
- - -
-
-
@@ -981,10 +993,7 @@ def render_html(rows): bar: "Bar plot", line: "Line plot" }; -const refLabels = { - commits: "By Commits", - tags: "By Tags" -}; +let xMode = "commit"; const metricUnits = { time: {label: "time", unit: "ms", factor: 1000}, memory: {label: "peak memory", unit: "MiB", factor: 1 / (1024 * 1024)} @@ -1088,7 +1097,17 @@ def render_html(rows): } function referenceModeValue() { - return checkedValues("referenceMode")[0] || "commits"; + return xMode; +} + +function setReferenceMode(mode) { + xMode = mode; + document.getElementById("modeCommit").classList.toggle("active", mode === "commit"); + document.getElementById("modeTag").classList.toggle("active", mode === "tag"); + document.getElementById("commit").style.display = mode === "tag" ? "none" : ""; + document.getElementById("tag").style.display = mode === "tag" ? "" : "none"; + refreshOptions(); + renderAll(); } function setCheckboxes( @@ -1172,7 +1191,7 @@ def render_html(rows): const cases = checkedValues("case"); const threads = checkedValues("threads"); const selectedBackends = checkedValues("backend"); - const isTagMode = referenceModeValue() === "tags"; + const isTagMode = referenceModeValue() === "tag"; const xFilter = isTagMode ? (row => row.tag && selectedOptions("tag").includes(row.tag)) : (row => selectedOptions("commit").includes(row.commit)); @@ -1196,7 +1215,6 @@ def render_html(rows): metrics: checkedValues("metric"), families: checkedValues("family"), viewModes: checkedValues("viewMode"), - refModes: checkedValues("referenceMode"), cases: checkedValues("case"), threads: checkedValues("threads"), commits: selectedOptions("commit"), @@ -1212,10 +1230,7 @@ def render_html(rows): const defaultFamily = familyOptions.includes("krige") ? "krige" : familyOptions[0]; setCheckboxes("family", familyOptions, previousMain.families, {}, [defaultFamily]); setCheckboxes("viewMode", ["line", "bar"], previousMain.viewModes, modeLabels, ["line"]); - setCheckboxes("referenceMode", ["commits", "tags"], previousMain.refModes, refLabels, ["commits"]); - const isTagMode = referenceModeValue() === "tags"; - document.getElementById("commitSection").style.display = isTagMode ? "none" : ""; - document.getElementById("tagSection").style.display = isTagMode ? "" : "none"; + const isTagMode = referenceModeValue() === "tag"; const benchRows = benchmarkRows(); const caseOptions = unique(benchRows.map(row => row.case)); const defaultCase = caseOptions.find(value => value.includes("extra_large")) || caseOptions[0]; @@ -1282,7 +1297,7 @@ def render_html(rows): } function barGroupKey(row) { - const xKey = referenceModeValue() === "tags" ? row.tag : row.commit; + const xKey = referenceModeValue() === "tag" ? row.tag : row.commit; return `${row.case}|${row.threads}|${xKey}`; } @@ -1306,7 +1321,7 @@ def render_html(rows): function chartSubtitle(data) { const cases = checkedValues("case"); const threads = checkedValues("threads"); - const isTagMode = referenceModeValue() === "tags"; + const isTagMode = referenceModeValue() === "tag"; const xValues = isTagMode ? selectedOptions("tag") : selectedOptions("commit"); const noun = isTagMode ? "tags" : "commits"; const selectedBackends = checkedValues("backend").map(value => labels[value] || value); @@ -1402,7 +1417,7 @@ def render_html(rows): const legend = document.getElementById("legend"); const metric = data[0].metric; const unit = metricUnits[metric]; - const isTagMode = referenceModeValue() === "tags"; + const isTagMode = referenceModeValue() === "tag"; const rowXKey = row => isTagMode ? row.tag : row.commit; const xKeys = isTagMode ? tagOrder(data.map(rowXKey)) @@ -1490,7 +1505,7 @@ def render_html(rows): renderChart(); } -for (const id of ["metric", "family", "viewMode", "referenceMode"]) { +for (const id of ["metric", "family", "viewMode"]) { document.getElementById(id).addEventListener("change", event => { enforceSingleChoice(id, event); refreshOptions(); From af3b3506b1024e1d49b7d706ac5da4a864de798e Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Tue, 9 Jun 2026 16:11:29 +0200 Subject: [PATCH 31/37] adding info in contributions | implementing asv-openmp-comment.yml --- .github/workflows/asv-openmp-pr.yml | 37 ++++++++++++++++++----------- CONTRIBUTING.md | 25 +++++++++++++++++++ 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr.yml index 86638a3ce..d6f9e0ad3 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr.yml @@ -7,7 +7,6 @@ on: permissions: contents: read - pull-requests: write concurrency: group: openmp-pr-${{ github.event.pull_request.number }} @@ -209,22 +208,32 @@ jobs: ) PY - - name: Post benchmark results as PR comment - if: always() && github.event.pull_request.head.repo.full_name == github.repository + - name: Save PR context for comment workflow + if: always() env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} run: | - EXISTING_ID=$(gh api \ - "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ - --jq '.[] | select(.body | startswith("")) | .id' \ - | head -1) - if [ -n "$EXISTING_ID" ]; then - gh api "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING_ID" \ - -X PATCH -F body=@/tmp/benchmark_comment.md - else - gh pr comment "$PR_NUMBER" --body-file /tmp/benchmark_comment.md - fi + mkdir -p .asv-openmp/pr-context + printf '%s' "$PR_NUMBER" > .asv-openmp/pr-context/pr_number.txt + printf '%s' "$BASE_SHA" > .asv-openmp/pr-context/base_sha.txt + printf '%s' "$HEAD_SHA" > .asv-openmp/pr-context/head_sha.txt + printf '%s' "$ARTIFACT_URL" > .asv-openmp/pr-context/artifact_url.txt + cp .asv-openmp/artifact/comparison.txt \ + .asv-openmp/pr-context/comparison.txt 2>/dev/null \ + || printf '_Comparison output not available._' \ + > .asv-openmp/pr-context/comparison.txt + + - name: Upload PR context for comment workflow + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-pr-context + path: .asv-openmp/pr-context + if-no-files-found: warn + retention-days: 1 required_gate: name: OpenMP benchmark gate diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 603590045..b4f7a1e7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,3 +36,28 @@ with your idea or suggestion and we'd love to discuss about it. - Add an example showing your new feature in one of the examples sub-folders if possible. Follow this [Sphinx-Gallary guide](https://sphinx-gallery.github.io/stable/syntax.html#embed-rst-in-your-example-python-files). - Push to your fork and submit a pull request. + + +## Do you want to contribute to benchmarking? + +GSTools tracks the runtime and memory performance +using [Airspeed Velocity (ASV)](https://asv.readthedocs.io/). +The benchmark suite lives in the `benchmarks/` directory and covers both the +Cython fallback and the optional Rust backend, as well as OpenMP thread scaling. + +Every pull request automatically runs the benchmark suite and reports any +significant regressions or improvements directly in the PR. +The cumulative benchmark history is published at +. + +If you would like to contribute to benchmarking you can: + +- **Add a new benchmark** — follow the existing patterns in + `benchmarks/` and prefix the function name with `time_` (runtime) or + `peakmem_` (peak memory) so ASV picks it up automatically. +- **Improve the benchmark infrastructure** — the helper scripts in + `benchmarks/tools/` generate the HTML comparison report and configure + ASV for CI runs. +- **Report a performance regression** — open a + [new issue](https://github.com/GeoStat-Framework/GSTools/issues) and + include the relevant ASV comparison output. From 8148b6b2f0e4fca923d812028b0bda79f5fb9489 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 10 Jun 2026 07:42:53 +0200 Subject: [PATCH 32/37] rename workflow files for clarity, add benchmark PR comment workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - asv-benchmarks.yml → benchmark-readiness.yml - asv-openmp-pr.yml → asv-openmp-pr-comparison.yml - asv-openmp-publish.yml → asv-benchmark-publish.yml - asv-openmp-comment.yml → benchmark-pr-comment.yml (new, workflow_run) Update all internal path references. benchmark-pr-comment.yml posts the ASV comparison as a PR comment via workflow_run, which runs in the base repo context and has pull-requests write access for both fork-internal and cross-fork PRs. Co-Authored-By: Claude Sonnet 4.6 --- ...-publish.yml => asv-benchmark-publish.yml} | 6 +- ...mp-pr.yml => asv-openmp-pr-comparison.yml} | 6 +- .github/workflows/benchmark-pr-comment.yml | 83 +++++++++++++++++++ ...benchmarks.yml => benchmark-readiness.yml} | 4 +- 4 files changed, 91 insertions(+), 8 deletions(-) rename .github/workflows/{asv-openmp-publish.yml => asv-benchmark-publish.yml} (95%) rename .github/workflows/{asv-openmp-pr.yml => asv-openmp-pr-comparison.yml} (98%) create mode 100644 .github/workflows/benchmark-pr-comment.yml rename .github/workflows/{asv-benchmarks.yml => benchmark-readiness.yml} (97%) diff --git a/.github/workflows/asv-openmp-publish.yml b/.github/workflows/asv-benchmark-publish.yml similarity index 95% rename from .github/workflows/asv-openmp-publish.yml rename to .github/workflows/asv-benchmark-publish.yml index 465678502..8a2156e43 100644 --- a/.github/workflows/asv-openmp-publish.yml +++ b/.github/workflows/asv-benchmark-publish.yml @@ -10,9 +10,9 @@ on: - "pyproject.toml" - "asv.conf.json" - "asv.openmp.conf.json" - - ".github/workflows/asv-benchmarks.yml" - - ".github/workflows/asv-openmp-pr.yml" - - ".github/workflows/asv-openmp-publish.yml" + - ".github/workflows/benchmark-readiness.yml" + - ".github/workflows/asv-openmp-pr-comparison.yml" + - ".github/workflows/asv-benchmark-publish.yml" workflow_dispatch: permissions: diff --git a/.github/workflows/asv-openmp-pr.yml b/.github/workflows/asv-openmp-pr-comparison.yml similarity index 98% rename from .github/workflows/asv-openmp-pr.yml rename to .github/workflows/asv-openmp-pr-comparison.yml index d6f9e0ad3..a8dc6d6f4 100644 --- a/.github/workflows/asv-openmp-pr.yml +++ b/.github/workflows/asv-openmp-pr-comparison.yml @@ -40,9 +40,9 @@ jobs: pyproject.toml \ asv.conf.json \ asv.openmp.conf.json \ - .github/workflows/asv-benchmarks.yml \ - .github/workflows/asv-openmp-pr.yml \ - .github/workflows/asv-openmp-publish.yml + .github/workflows/benchmark-readiness.yml \ + .github/workflows/asv-openmp-pr-comparison.yml \ + .github/workflows/asv-benchmark-publish.yml then echo "required=false" >> "$GITHUB_OUTPUT" else diff --git a/.github/workflows/benchmark-pr-comment.yml b/.github/workflows/benchmark-pr-comment.yml new file mode 100644 index 000000000..b8915491e --- /dev/null +++ b/.github/workflows/benchmark-pr-comment.yml @@ -0,0 +1,83 @@ +name: Post Benchmark PR Comment + +on: + workflow_run: + workflows: ["Benchmarking"] + types: [completed] + +permissions: + pull-requests: write + +jobs: + comment: + # Only runs when the triggering event was a pull_request and the run + # produced a benchmark-pr-context artifact (skipped if no relevant changes). + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + + steps: + - name: Download PR context artifact + id: download + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: benchmark-pr-context + path: pr-context + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Post or update benchmark PR comment + if: steps.download.outcome == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 - <<'PY' + from pathlib import Path + + base = Path("pr-context/base_sha.txt").read_text().strip()[:8] + head = Path("pr-context/head_sha.txt").read_text().strip()[:8] + artifact_url = Path("pr-context/artifact_url.txt").read_text().strip() + pr_number = Path("pr-context/pr_number.txt").read_text().strip() + + report_link = ( + f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" + ) + try: + comparison = Path("pr-context/comparison.txt").read_text() + except FileNotFoundError: + comparison = "_Comparison output not available._" + + lines = comparison.splitlines() + regressed = sum(1 for l in lines if l.startswith("+") and l[1:2] == " ") + improved = sum(1 for l in lines if l.startswith("-") and l[1:2] == " ") + if regressed: + badge = f"⚠️ {regressed} benchmark(s) regressed" + elif improved: + badge = f"✅ {improved} benchmark(s) improved, none regressed" + else: + badge = "✅ No significant changes detected" + + body = ( + "\n" + "## OpenMP Benchmark Results\n\n" + f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" + f"{badge}\n\n" + "
\nFull ASV comparison\n\n" + f"```\n{comparison}\n```\n\n" + "
\n" + ) + Path("/tmp/benchmark_comment.md").write_text(body) + Path("/tmp/pr_number.txt").write_text(pr_number) + PY + + PR_NUMBER=$(cat /tmp/pr_number.txt) + EXISTING_ID=$(gh api \ + "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.body | startswith("")) | .id' \ + | head -1) + if [ -n "$EXISTING_ID" ]; then + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING_ID" \ + -X PATCH -F body=@/tmp/benchmark_comment.md + else + gh pr comment "$PR_NUMBER" --body-file /tmp/benchmark_comment.md + fi diff --git a/.github/workflows/asv-benchmarks.yml b/.github/workflows/benchmark-readiness.yml similarity index 97% rename from .github/workflows/asv-benchmarks.yml rename to .github/workflows/benchmark-readiness.yml index 290998a51..d0b96f64a 100644 --- a/.github/workflows/asv-benchmarks.yml +++ b/.github/workflows/benchmark-readiness.yml @@ -8,7 +8,7 @@ on: branches: - "main" paths: - - ".github/workflows/asv-benchmarks.yml" + - ".github/workflows/benchmark-readiness.yml" - "asv*.json" - "benchmarks/**" - "pyproject.toml" @@ -16,7 +16,7 @@ on: branches: - "main" paths: - - ".github/workflows/asv-benchmarks.yml" + - ".github/workflows/benchmark-readiness.yml" - "asv*.json" - "benchmarks/**" - "pyproject.toml" From 78703ba6be72b94946d9b87677f5b20ef5b272c9 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 10 Jun 2026 10:05:50 +0200 Subject: [PATCH 33/37] improving html visualisation --- .../tools/plot_case_backend_comparison.py | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index c786ebabd..166059d07 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -1020,7 +1020,9 @@ def render_html(rows): function optionText(kind, value) { if (kind === "commit") { const row = rows.find(item => item.commit === value); - return row ? `${value} (${row.date_label})` : value; + if (!row) return value; + const tagPart = row.tag ? ` · ${row.tag}` : ""; + return `${value}${tagPart} (${row.date_label})`; } if (kind === "tag") { const row = rows.find(item => item.tag === value); @@ -1405,6 +1407,39 @@ def render_html(rows): svg += ``; svg += ``; + + // % change annotations when exactly 2 commits/tags are compared + const isTagMode = referenceModeValue() === "tag"; + const xVals = isTagMode + ? tagOrder(selectedOptions("tag")) + : commitOrder(selectedOptions("commit")); + if (xVals.length === 2) { + const baseVals = new Map(); + groups.forEach((groupKey, groupIndex) => { + const parts = groupKey.split("|"); + const xKey = parts[2]; + const groupX = plotLeft + groupIndex * groupStep + groupStep / 2; + selectedBackends.forEach((backend, backendIndex) => { + const row = data.find(item => barGroupKey(item) === groupKey && item.backend === backend); + if (!row) return; + const value = formatValue(row); + const barX = groupX - clusterWidth / 2 + backendIndex * (barWidth + barGap) + barWidth / 2; + const key = `${parts[0]}|${parts[1]}|${backend}`; + if (xKey === xVals[0]) { + baseVals.set(key, value); + } else { + const base = baseVals.get(key); + if (!base) return; + const pct = ((value - base) / base) * 100; + const sign = pct >= 0 ? "+" : ""; + const fill = Math.abs(pct) <= 5 ? "var(--muted)" : pct > 0 ? "#c0392b" : "#27ae60"; + const annotY = Math.max(margin.top + 4, y(value) - 20); + svg += `${sign}${pct.toFixed(1)}%`; + } + }); + }); + } + svg += ``; chart.innerHTML = svg; legend.innerHTML = selectedBackends.map(backend => @@ -1490,7 +1525,11 @@ def render_html(rows): const chart = document.getElementById("chart"); const legend = document.getElementById("legend"); if (!data.length) { - chart.innerHTML = "

No matching ASV rows for this selection.

"; + const emptyMsg = referenceModeValue() === "tag" + ? "No benchmarked commits carry a tag in these results. " + + "Tags appear once a tagged commit has been through the publish workflow." + : "No matching ASV rows for this selection."; + chart.innerHTML = `

${emptyMsg}

`; legend.innerHTML = ""; return; } From d684bba6b9662b0995b22d0ec914c57a77550b44 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 10 Jun 2026 10:54:39 +0200 Subject: [PATCH 34/37] restore the direct comment step in asv-openmp-pr-comparison.yml for same-repo PRs --- .github/workflows/asv-openmp-pr-comparison.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/asv-openmp-pr-comparison.yml b/.github/workflows/asv-openmp-pr-comparison.yml index a8dc6d6f4..851f9dd8c 100644 --- a/.github/workflows/asv-openmp-pr-comparison.yml +++ b/.github/workflows/asv-openmp-pr-comparison.yml @@ -7,6 +7,7 @@ on: permissions: contents: read + pull-requests: write concurrency: group: openmp-pr-${{ github.event.pull_request.number }} @@ -208,6 +209,23 @@ jobs: ) PY + - name: Post benchmark results as PR comment + if: always() && github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + EXISTING_ID=$(gh api \ + "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.body | startswith("")) | .id' \ + | head -1) + if [ -n "$EXISTING_ID" ]; then + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING_ID" \ + -X PATCH -F body=@/tmp/benchmark_comment.md + else + gh pr comment "$PR_NUMBER" --body-file /tmp/benchmark_comment.md + fi + - name: Save PR context for comment workflow if: always() env: From 302b7176daa70da88cad5f40d64ac066547d96e8 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 10 Jun 2026 12:00:35 +0200 Subject: [PATCH 35/37] add change log, improve html and github actions --- .../workflows/asv-openmp-pr-comparison.yml | 15 +- .github/workflows/benchmark-pr-comment.yml | 10 +- CHANGELOG.md | 9 + .../tools/plot_case_backend_comparison.py | 322 +++++++++--------- 4 files changed, 197 insertions(+), 159 deletions(-) diff --git a/.github/workflows/asv-openmp-pr-comparison.yml b/.github/workflows/asv-openmp-pr-comparison.yml index 851f9dd8c..17c7e81d0 100644 --- a/.github/workflows/asv-openmp-pr-comparison.yml +++ b/.github/workflows/asv-openmp-pr-comparison.yml @@ -122,6 +122,11 @@ jobs: run: | mkdir -p .asv-openmp/artifact set +e + # --factor 1.05: only flag changes whose ratio exceeds 5%. + # A benchmark is marked regressed (+) or improved (-) only when BOTH + # the ratio threshold AND ASV's Mann-Whitney U statistical test agree. + # Lines prefixed with ~ exceeded the ratio but were not statistically + # significant (measurement noise on the shared runner). asv --config "$ASV_CONFIG" compare \ "$BASE_SHA" \ "$HEAD_SHA" \ @@ -178,18 +183,22 @@ jobs: comparison = "_Comparison output not available._" lines = comparison.splitlines() + # ASV marks a line with "+" (regressed) or "-" (improved) only when + # BOTH the ratio > 1.05 AND the Mann-Whitney U test agree. + # Lines with "~" exceeded the ratio but failed the statistical test. regressed = sum( 1 for l in lines if l.startswith("+") and l[1:2] == " " ) improved = sum( 1 for l in lines if l.startswith("-") and l[1:2] == " " ) + note = "Mann-Whitney U · 5% threshold" if regressed: - badge = f"⚠️ {regressed} benchmark(s) regressed" + badge = f"⚠️ {regressed} benchmark(s) regressed · {note}" elif improved: - badge = f"✅ {improved} benchmark(s) improved, none regressed" + badge = f"✅ {improved} benchmark(s) improved, none regressed · {note}" else: - badge = "✅ No significant changes detected" + badge = f"✅ No significant changes detected · {note}" body = ( "\n" diff --git a/.github/workflows/benchmark-pr-comment.yml b/.github/workflows/benchmark-pr-comment.yml index b8915491e..897ef91bd 100644 --- a/.github/workflows/benchmark-pr-comment.yml +++ b/.github/workflows/benchmark-pr-comment.yml @@ -48,14 +48,18 @@ jobs: comparison = "_Comparison output not available._" lines = comparison.splitlines() + # ASV marks "+" (regressed) or "-" (improved) only when BOTH the + # ratio > 1.05 AND the Mann-Whitney U test agree. Lines with "~" + # exceeded the ratio threshold but were not statistically significant. regressed = sum(1 for l in lines if l.startswith("+") and l[1:2] == " ") improved = sum(1 for l in lines if l.startswith("-") and l[1:2] == " ") + note = "Mann-Whitney U · 5% threshold" if regressed: - badge = f"⚠️ {regressed} benchmark(s) regressed" + badge = f"⚠️ {regressed} benchmark(s) regressed · {note}" elif improved: - badge = f"✅ {improved} benchmark(s) improved, none regressed" + badge = f"✅ {improved} benchmark(s) improved, none regressed · {note}" else: - badge = "✅ No significant changes detected" + badge = f"✅ No significant changes detected · {note}" body = ( "\n" diff --git a/CHANGELOG.md b/CHANGELOG.md index 61aa167cb..ecbf609d6 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ All notable changes to **GSTools** will be documented in this file. - instead of spatial rules, as before, decision trees are a more flexible approach - each node of the binary tree is a decision based on the values of the SRF - replace pylint, black, and isort with ruff [#391](https://github.com/GeoStat-Framework/GSTools/pull/391) +- add comprehensive benchmark suite for two-point statistics workflows + - covers variogram estimation, kriging, and random field generation across Cython and Rust backends + - includes OpenMP thread-count scaling analysis (1/2/4/8 threads) + - interactive HTML report with per-case and per-thread-count comparisons +- add benchmarking CI workflows + - automated performance regression detection with ASV on every pull request + - benchmark results posted as PR comments with Mann-Whitney U statistical test + - cumulative benchmark history published to GitHub Pages after each merge to main +- add benchmarking guidelines to CONTRIBUTING.md ## [1.7.0] - Morphic Mint - 2025-04 diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index 166059d07..0b2a45484 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -583,49 +583,96 @@ def render_html(rows): GSTools Two-Point Statistics Backend Comparison +
@@ -929,7 +945,7 @@ def render_html(rows):
__MACHINE_MARKUP__
-
+
Metric @@ -1432,7 +1448,7 @@ def render_html(rows): if (!base) return; const pct = ((value - base) / base) * 100; const sign = pct >= 0 ? "+" : ""; - const fill = Math.abs(pct) <= 5 ? "var(--muted)" : pct > 0 ? "#c0392b" : "#27ae60"; + const fill = Math.abs(pct) <= 5 ? "var(--muted)" : pct > 0 ? "var(--danger)" : "var(--success)"; const annotY = Math.max(margin.top + 4, y(value) - 20); svg += `${sign}${pct.toFixed(1)}%`; } From d57ada2f6bca612c495757c22d2ce2288b7e2ed8 Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 10 Jun 2026 14:09:28 +0200 Subject: [PATCH 36/37] debuging and code cleaning --- .../workflows/asv-openmp-pr-comparison.yml | 60 ++-------- .github/workflows/benchmark-pr-comment.yml | 61 +++-------- benchmarks/README.md | 88 ++++++++++++--- benchmarks/tools/asv_speedup_summary.py | 9 +- .../tools/check_backend_parallel_ready.py | 16 +++ benchmarks/tools/configure_asv_machine.py | 2 +- benchmarks/tools/install_pyproject_extras.py | 6 +- .../tools/plot_case_backend_comparison.py | 14 ++- benchmarks/tools/render_benchmark_comment.py | 103 ++++++++++++++++++ 9 files changed, 240 insertions(+), 119 deletions(-) create mode 100644 benchmarks/tools/render_benchmark_comment.py diff --git a/.github/workflows/asv-openmp-pr-comparison.yml b/.github/workflows/asv-openmp-pr-comparison.yml index 17c7e81d0..bac818d0e 100644 --- a/.github/workflows/asv-openmp-pr-comparison.yml +++ b/.github/workflows/asv-openmp-pr-comparison.yml @@ -166,57 +166,15 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - python - <<'PY' - import os - from pathlib import Path - - base = os.environ["BASE_SHA"][:8] - head = os.environ["HEAD_SHA"][:8] - artifact_url = os.environ.get("ARTIFACT_URL", "") - report_link = ( - f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" - ) - - try: - comparison = Path(".asv-openmp/artifact/comparison.txt").read_text() - except FileNotFoundError: - comparison = "_Comparison output not available._" - - lines = comparison.splitlines() - # ASV marks a line with "+" (regressed) or "-" (improved) only when - # BOTH the ratio > 1.05 AND the Mann-Whitney U test agree. - # Lines with "~" exceeded the ratio but failed the statistical test. - regressed = sum( - 1 for l in lines if l.startswith("+") and l[1:2] == " " - ) - improved = sum( - 1 for l in lines if l.startswith("-") and l[1:2] == " " - ) - note = "Mann-Whitney U · 5% threshold" - if regressed: - badge = f"⚠️ {regressed} benchmark(s) regressed · {note}" - elif improved: - badge = f"✅ {improved} benchmark(s) improved, none regressed · {note}" - else: - badge = f"✅ No significant changes detected · {note}" - - body = ( - "\n" - "## OpenMP Benchmark Results\n\n" - f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" - f"{badge}\n\n" - "
\nFull ASV comparison\n\n" - f"```\n{comparison}\n```\n\n" - "
\n" - ) - Path("/tmp/benchmark_comment.md").write_text(body) - - summary_env = os.environ.get("GITHUB_STEP_SUMMARY", "") - if summary_env: - Path(summary_env).write_text( - body.replace("\n", "") - ) - PY + python benchmarks/tools/render_benchmark_comment.py \ + --base "$BASE_SHA" \ + --head "$HEAD_SHA" \ + --artifact-url "${ARTIFACT_URL:-}" \ + --comparison .asv-openmp/artifact/comparison.txt \ + --output /tmp/benchmark_comment.md + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + tail -n +2 /tmp/benchmark_comment.md > "$GITHUB_STEP_SUMMARY" + fi - name: Post benchmark results as PR comment if: always() && github.event.pull_request.head.repo.full_name == github.repository diff --git a/.github/workflows/benchmark-pr-comment.yml b/.github/workflows/benchmark-pr-comment.yml index 897ef91bd..ee2189bb5 100644 --- a/.github/workflows/benchmark-pr-comment.yml +++ b/.github/workflows/benchmark-pr-comment.yml @@ -6,6 +6,7 @@ on: types: [completed] permissions: + actions: read pull-requests: write jobs: @@ -16,10 +17,14 @@ jobs: runs-on: ubuntu-latest steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 1 + - name: Download PR context artifact id: download continue-on-error: true - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: benchmark-pr-context path: pr-context @@ -31,50 +36,18 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - python3 - <<'PY' - from pathlib import Path - - base = Path("pr-context/base_sha.txt").read_text().strip()[:8] - head = Path("pr-context/head_sha.txt").read_text().strip()[:8] - artifact_url = Path("pr-context/artifact_url.txt").read_text().strip() - pr_number = Path("pr-context/pr_number.txt").read_text().strip() - - report_link = ( - f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" - ) - try: - comparison = Path("pr-context/comparison.txt").read_text() - except FileNotFoundError: - comparison = "_Comparison output not available._" - - lines = comparison.splitlines() - # ASV marks "+" (regressed) or "-" (improved) only when BOTH the - # ratio > 1.05 AND the Mann-Whitney U test agree. Lines with "~" - # exceeded the ratio threshold but were not statistically significant. - regressed = sum(1 for l in lines if l.startswith("+") and l[1:2] == " ") - improved = sum(1 for l in lines if l.startswith("-") and l[1:2] == " ") - note = "Mann-Whitney U · 5% threshold" - if regressed: - badge = f"⚠️ {regressed} benchmark(s) regressed · {note}" - elif improved: - badge = f"✅ {improved} benchmark(s) improved, none regressed · {note}" - else: - badge = f"✅ No significant changes detected · {note}" - - body = ( - "\n" - "## OpenMP Benchmark Results\n\n" - f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" - f"{badge}\n\n" - "
\nFull ASV comparison\n\n" - f"```\n{comparison}\n```\n\n" - "
\n" - ) - Path("/tmp/benchmark_comment.md").write_text(body) - Path("/tmp/pr_number.txt").write_text(pr_number) - PY + BASE_SHA=$(cat pr-context/base_sha.txt) + HEAD_SHA=$(cat pr-context/head_sha.txt) + ARTIFACT_URL=$(cat pr-context/artifact_url.txt) + PR_NUMBER=$(cat pr-context/pr_number.txt) + + python benchmarks/tools/render_benchmark_comment.py \ + --base "$BASE_SHA" \ + --head "$HEAD_SHA" \ + --artifact-url "$ARTIFACT_URL" \ + --comparison pr-context/comparison.txt \ + --output /tmp/benchmark_comment.md - PR_NUMBER=$(cat /tmp/pr_number.txt) EXISTING_ID=$(gh api \ "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ --jq '.[] | select(.body | startswith("")) | .id' \ diff --git a/benchmarks/README.md b/benchmarks/README.md index 53d3ff2ed..7f1ea029a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -122,14 +122,21 @@ The benchmarking setup currently consists of: `cProfile`, so you can see which functions take time in the current checkout. - `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies - Cython OpenMP detection and Rust backend execution with more than one - GSTools thread. + Cython OpenMP detection and Rust backend execution (variogram, field + generation, and kriging) with more than one GSTools thread. +- `benchmarks/tools/configure_asv_machine.py`: writes a named ASV machine + profile from detected hardware; used in CI to ensure all results are stored + under a stable name regardless of the actual runner. - `benchmarks/tools/install_pyproject_extras.py`: installs selected optional dependencies directly from `pyproject.toml` without installing GSTools or its regular dependencies. - `benchmarks/tools/install_openmp_cython.py`: helper used by `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, Linux, and native Windows. +- `benchmarks/tools/render_benchmark_comment.py`: CI helper used by both the + pull-request comparison workflow and the cross-fork comment workflow to + render the GitHub PR comment body; keeps badge wording and comment format + in sync across both code paths. ### ASV Configuration @@ -187,6 +194,10 @@ Important details: `pyproject.toml`, keeping the `gstools_core` requirement in one place. - ASV still needs its own `install_command` because it creates isolated environments for the commits it benchmarks. +- In CI, `configure_asv_machine.py` replaces the interactive `asv machine + --yes` step and writes a stable machine name (`github-actions-ubuntu`) + regardless of the actual runner hardware. Locally, `asv machine --yes` is + sufficient and `configure_asv_machine.py` is not needed. - Run the cProfile helper with the Python executable from ASV's isolated environment, for example `.asv/env//bin/python`. In that mode, the ASV environment provides dependencies while the helper imports the current @@ -277,6 +288,20 @@ in generated benchmark method names. That is the default because the first benchmark target is a clean Cython-vs-Rust backend comparison without parallelism. +Both constants are driven by environment variables read at import time: + +| Variable | Default | Example override | +|---|---|---| +| `GSTOOLS_BENCHMARK_THREADS` | `"1"` | `"1,2,4,8"` — run with 1, 2, 4, and 8 threads | +| `GSTOOLS_BENCHMARK_BACKENDS` | all backends | `"rust_core"` — run Rust only | + +These must be set before ASV imports the benchmark module. When running +locally with ASV, prefix the `asv run` command: + +```bash +GSTOOLS_BENCHMARK_BACKENDS=rust_core asv run 'main^!' --bench benchmark_two_point_statistics +``` + ### Shared Helpers `gstools_backend(use_core, num_threads)` temporarily forces GSTools to use @@ -442,8 +467,18 @@ python benchmarks/tools/asv_speedup_summary.py --format html --output .asv/backe #### Visualization of Results -You can inspect the results in the ASV browser report by building and opening -the local website: +Build a self-contained interactive HTML report from the result files: + +```bash +python benchmarks/tools/plot_case_backend_comparison.py +``` + +This reads `.asv/results/` by default and writes +`case-backend-comparison.html` next to it. Open the file in any browser. Use +`--metric time` or `--metric memory` to show one metric, and `--benchmark` to +filter to one family (e.g. `variogram`, `krige`, `srf`). + +You can also inspect results in the native ASV browser report: ```bash asv publish @@ -457,10 +492,10 @@ http://127.0.0.1:8080/#/ ``` (or any other `http://127.0.0.1:/#/` URL shown by the running preview). -The browser report shows raw ASV plots and trends. ASV plot views do not draw a -line when there is only one x-axis point, so a single-commit run such as -`asv run 'main^!' --bench benchmark_two_point_statistics` may show results -without useful commit-trend graphs. +The native ASV report shows raw plots and trends. ASV does not draw a line +when there is only one x-axis point, so a single-commit run may show results +without useful trend graphs. The custom HTML report works with one or more +commits. ### Profiling With cProfile @@ -507,13 +542,29 @@ List available cases: On Windows PowerShell, replace `"$ASV_PYTHON"` with `& $asvPython`. -Possible profile selected cases: +All available cases: + +```text +variogram-full variogram on 900 points, full pairwise +variogram-sampled variogram on 5 000 pts, sampled 1 500 +variogram-extra-large variogram on 15 000 pts, sampled 4 500 +krige-small kriging 30 → 500 pts +krige-large kriging 120 → 2 000 pts +krige-extra-large kriging 360 → 6 000 pts +srf-unstructured SRF RandMeth, 2 000 unstructured pts +srf-structured SRF RandMeth, 64×64 grid +srf-fourier SRF Fourier, 64×64 grid +condsrf conditioned SRF, 40 → 1 000 pts +``` + +Profile a selected case: ```bash "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --threads threads_1 --limit 10 "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --threads threads_1 --limit 10 "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --threads threads_1 --limit 10 "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case srf-unstructured --backend rust_core --threads threads_1 --limit 10 "$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --threads threads_1 --limit 10 ``` @@ -543,15 +594,15 @@ This config keeps the OpenMP experiment separate from the default baseline: .asv-openmp/html/ ``` -It also uses fewer repeats than the baseline config: +It uses the same `number` and `repeat` settings as the baseline config: ```json "number": 1, "repeat": 20 ``` -That means each OpenMP benchmark case records 20 independent measurements by -default, while still running the benchmark code once per measurement. +Each benchmark case records 20 independent measurements, running the benchmark +code once per measurement. During ASV installation, it runs: @@ -644,7 +695,14 @@ python benchmarks/tools/plot_case_backend_comparison.py \ By default this report includes both time and peak-memory benchmarks. Use `--metric time` or `--metric memory` when you want only one metric. `--max-commits` limits only the generated custom report; it does not delete -raw ASV results. +raw ASV results. Use `--benchmark` to filter to one family: + +```bash +python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --benchmark variogram # or krige, kriging, field, srf, condsrf, vario + --output .asv-openmp/variogram-comparison.html +``` Build and preview the OpenMP browser report: @@ -706,7 +764,9 @@ python benchmarks\tools\plot_case_backend_comparison.py ` ``` By default this report includes both time and peak-memory benchmarks. Use -`--metric time` or `--metric memory` when you want only one metric. +`--metric time` or `--metric memory` when you want only one metric, and +`--benchmark` to filter to one family (e.g. `variogram`, `krige`, `field`, +`srf`, `condsrf`). Build and preview the OpenMP browser report: diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py index 20fe207c7..db56f7402 100644 --- a/benchmarks/tools/asv_speedup_summary.py +++ b/benchmarks/tools/asv_speedup_summary.py @@ -99,11 +99,11 @@ def iter_result_files(results_dir): def load_json(path): - """Load one ASV result file, ignoring invalid JSON files.""" + """Load one ASV result file, ignoring unreadable or invalid JSON files.""" try: with path.open(encoding="utf8") as handle: return json.load(handle) - except json.JSONDecodeError: + except (OSError, json.JSONDecodeError): return None @@ -265,7 +265,10 @@ def sort_rows(rows): def thread_number(label): """Return the numeric part of a ``threads_N`` label.""" if isinstance(label, str) and label.startswith(THREAD_PREFIX): - return int(label[len(THREAD_PREFIX) :]) + try: + return int(label[len(THREAD_PREFIX) :]) + except ValueError: + return -1 return -1 diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index 2b929c6f7..c03b6bec9 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -154,6 +154,8 @@ def check_rust_backend(threads): y = np.linspace(0.0, 5.0, 12) field = np.sin(x) + np.cos(y) bins = np.linspace(0.0, 8.0, 6) + + # variogram gs.vario_estimate( (x, y), field, @@ -161,6 +163,20 @@ def check_rust_backend(threads): mesh_type="unstructured", return_counts=True, ) + + # field generation + model = gs.Gaussian(dim=2, var=1.0, len_scale=3.0) + srf = gs.SRF(model, seed=1) + srf(x, y) + + # kriging + krige = gs.Krige( + model, + cond_pos=(x[:6], y[:6]), + cond_val=field[:6], + ) + krige(x[6:], y[6:]) + finally: ( gs.config._GSTOOLS_CORE_AVAIL, diff --git a/benchmarks/tools/configure_asv_machine.py b/benchmarks/tools/configure_asv_machine.py index 927d2c195..7778f9fe5 100644 --- a/benchmarks/tools/configure_asv_machine.py +++ b/benchmarks/tools/configure_asv_machine.py @@ -20,4 +20,4 @@ def main(): if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/benchmarks/tools/install_pyproject_extras.py b/benchmarks/tools/install_pyproject_extras.py index 7c00d971d..c6fae4836 100644 --- a/benchmarks/tools/install_pyproject_extras.py +++ b/benchmarks/tools/install_pyproject_extras.py @@ -77,7 +77,11 @@ def main(): except (FileNotFoundError, ValueError) as err: print(err, file=sys.stderr) return 2 - install_requirements(requirements) + try: + install_requirements(requirements) + except subprocess.CalledProcessError as err: + print(f"pip install failed with exit code {err.returncode}", file=sys.stderr) + return 1 return 0 diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py index 0b2a45484..493147f1a 100644 --- a/benchmarks/tools/plot_case_backend_comparison.py +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -50,6 +50,8 @@ "field": "field", "random_field": "field", "random-field": "field", + "srf": "field", + "condsrf": "field", } @@ -843,12 +845,10 @@ def render_html(rows): stroke-width: 1; } .grid { stroke-dasharray: 2 4; } -.tick, .bar-label, .legend, .axis-label { fill: var(--muted); font-size: 12px; } +.tick, .bar-label, .axis-label { fill: var(--muted); font-size: 12px; } .title { fill: var(--text); font-size: 16px; font-weight: 800; } .line-path { fill: none; stroke-width: 2.4; } .point { fill: var(--panel); stroke-width: 2; } -.cython { fill: var(--cython); } -.rust { fill: var(--rust); } .chart-legend { display: flex; flex-wrap: wrap; @@ -1251,7 +1251,11 @@ def render_html(rows): const isTagMode = referenceModeValue() === "tag"; const benchRows = benchmarkRows(); const caseOptions = unique(benchRows.map(row => row.case)); - const defaultCase = caseOptions.find(value => value.includes("extra_large")) || caseOptions[0]; + const defaultCase = + caseOptions.find(v => v.includes("extra_large")) || + caseOptions.find(v => v.includes("sampled_15000")) || + caseOptions.find(v => v.includes("srf_unstructured")) || + caseOptions[0]; setCheckboxes("case", caseOptions, previousMain.cases, {}, [defaultCase]); const selectedCases = checkedValues("case"); const threadOptions = unique(benchRows @@ -1371,7 +1375,7 @@ def render_html(rows): const metric = data[0].metric; const unit = metricUnits[metric]; const selectedThreads = checkedValues("threads"); - const selectedCommits = checkedValues("commit"); + const selectedCommits = selectedOptions("commit"); const selectedBackends = checkedValues("backend"); const showThread = selectedThreads.length > 1; const showCommit = selectedCommits.length > 1; diff --git a/benchmarks/tools/render_benchmark_comment.py b/benchmarks/tools/render_benchmark_comment.py new file mode 100644 index 000000000..b6e7e3c78 --- /dev/null +++ b/benchmarks/tools/render_benchmark_comment.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +"""Render a GitHub PR comment body from an ASV benchmark comparison. + +Used by both the pull-request comparison workflow (for same-repo PRs) and the +workflow_run comment workflow (for cross-fork PRs) so the badge wording and +comment format stay in sync across both code paths. + +Usage: + python benchmarks/tools/render_benchmark_comment.py \\ + --base --head \\ + [--artifact-url ] \\ + [--comparison ] \\ + --output +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--base", required=True, help="Base commit SHA.") + parser.add_argument("--head", required=True, help="Head commit SHA.") + parser.add_argument( + "--artifact-url", + default="", + help="URL to the full HTML report artifact.", + ) + parser.add_argument( + "--comparison", + type=Path, + default=None, + help="Path to the ASV comparison text file.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Write the rendered comment body to this file.", + ) + return parser.parse_args() + + +def render(base, head, artifact_url, comparison): + """Return the full PR comment body as a string.""" + base = base[:8] + head = head[:8] + report_link = ( + f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" + ) + + lines = comparison.splitlines() + # ASV marks "+" (regressed) or "-" (improved) only when BOTH the + # ratio > 1.05 AND the Mann-Whitney U test agree. Lines with "~" + # exceeded the ratio threshold but were not statistically significant. + regressed = sum(1 for l in lines if l.startswith("+") and l[1:2] == " ") + improved = sum(1 for l in lines if l.startswith("-") and l[1:2] == " ") + note = "Mann-Whitney U · 5% threshold" + if regressed: + badge = f"⚠️ {regressed} benchmark(s) regressed · {note}" + elif improved: + badge = f"✅ {improved} benchmark(s) improved, none regressed · {note}" + else: + badge = f"✅ No significant changes detected · {note}" + + return ( + "\n" + "## OpenMP Benchmark Results\n\n" + f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" + f"{badge}\n\n" + "
\nFull ASV comparison\n\n" + f"```\n{comparison}\n```\n\n" + "
\n" + ) + + +def main(): + """Render and write the PR comment body.""" + args = parse_args() + + if args.comparison is not None: + try: + comparison = args.comparison.read_text(encoding="utf8") + except OSError as err: + print(f"Cannot read comparison file: {err}", file=sys.stderr) + return 1 + else: + comparison = "_Comparison output not available._" + + body = render(args.base, args.head, args.artifact_url, comparison) + args.output.write_text(body, encoding="utf8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From cfd8b06cebd0b381e603b702dbbb0207cf4f468c Mon Sep 17 00:00:00 2001 From: Jeisson Leal Date: Wed, 10 Jun 2026 14:24:32 +0200 Subject: [PATCH 37/37] fix error in benchmark checks --- benchmarks/tools/check_backend_parallel_ready.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py index c03b6bec9..db0c6fd3b 100644 --- a/benchmarks/tools/check_backend_parallel_ready.py +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -167,7 +167,7 @@ def check_rust_backend(threads): # field generation model = gs.Gaussian(dim=2, var=1.0, len_scale=3.0) srf = gs.SRF(model, seed=1) - srf(x, y) + srf((x, y)) # kriging krige = gs.Krige( @@ -175,7 +175,7 @@ def check_rust_backend(threads): cond_pos=(x[:6], y[:6]), cond_val=field[:6], ) - krige(x[6:], y[6:]) + krige((x[6:], y[6:])) finally: (