diff --git a/.github/workflows/asv-benchmark-publish.yml b/.github/workflows/asv-benchmark-publish.yml new file mode 100644 index 00000000..8a2156e4 --- /dev/null +++ b/.github/workflows/asv-benchmark-publish.yml @@ -0,0 +1,117 @@ +name: Publish OpenMP Benchmark History + +on: + push: + branches: + - "main" + paths: + - "src/**" + - "benchmarks/**" + - "pyproject.toml" + - "asv.conf.json" + - "asv.openmp.conf.json" + - ".github/workflows/benchmark-readiness.yml" + - ".github/workflows/asv-openmp-pr-comparison.yml" + - ".github/workflows/asv-benchmark-publish.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: openmp-benchmark-publish + cancel-in-progress: false + +env: + ASV_CONFIG: asv.openmp.conf.json + BENCHMARKS_REPO: jeilealr/gstools-benchmarks-asv + MACHINE_NAME: github-actions-ubuntu + +jobs: + publish: + name: Benchmark accepted main commit and publish history + if: github.repository == 'jeilealr/GSTools' + runs-on: ubuntu-latest + timeout-minutes: 360 + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Check out GSTools + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Check out benchmark history repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + repository: ${{ env.BENCHMARKS_REPO }} + token: ${{ secrets.BENCHMARKS_REPO_TOKEN }} + path: benchmark-site + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + activate-environment: gstools-asv-driver + python-version: "3.14" + channels: conda-forge + conda-remove-defaults: true + + - name: Install ASV driver + run: | + conda install -n gstools-asv-driver -y -c conda-forge pip + python -m pip install --upgrade pip asv + + - name: Expose accepted commit as local main branch + run: | + git checkout --detach "$GITHUB_SHA" + git branch --force main "$GITHUB_SHA" + + - name: Restore cumulative ASV results + run: | + mkdir -p .asv-openmp/results + if [[ -d benchmark-site/results ]]; then + cp -R benchmark-site/results/. .asv-openmp/results/ + fi + + - name: Configure stable ASV machine name + run: | + python benchmarks/tools/configure_asv_machine.py "$MACHINE_NAME" + + - name: Benchmark newly accepted main commit + env: + GSTOOLS_BENCHMARK_THREADS: "1,2,4,8" + run: | + asv --config "$ASV_CONFIG" run \ + "$GITHUB_SHA^!" \ + --machine "$MACHINE_NAME" \ + --skip-existing-successful \ + --no-pull \ + --bench "^benchmark_" \ + --show-stderr + + - name: Build published reports and preserve raw results + run: | + rm -rf benchmark-site/asv benchmark-site/results + python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --output benchmark-site/index.html + asv --config "$ASV_CONFIG" publish \ + --no-pull \ + --html-dir benchmark-site/asv + cp -R .asv-openmp/results benchmark-site/results + touch benchmark-site/.nojekyll + + - name: Commit and push benchmark history + working-directory: benchmark-site + run: | + git config user.name "GSTools benchmark bot" + git config user.email "actions@users.noreply.github.com" + git add -A + if git diff --cached --quiet; then + echo "No benchmark history changes to publish." + exit 0 + fi + git commit -m "Update OpenMP benchmarks for ${GITHUB_SHA::8}" + git push origin main diff --git a/.github/workflows/asv-openmp-pr-comparison.yml b/.github/workflows/asv-openmp-pr-comparison.yml new file mode 100644 index 00000000..bac818d0 --- /dev/null +++ b/.github/workflows/asv-openmp-pr-comparison.yml @@ -0,0 +1,251 @@ +name: Benchmarking + +on: + pull_request: + branches: + - "main" + +permissions: + contents: read + pull-requests: write + +concurrency: + group: openmp-pr-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + ASV_CONFIG: asv.openmp.conf.json + MACHINE_NAME: github-actions-ubuntu + +jobs: + benchmark_changes: + name: Detect benchmark-relevant changes + runs-on: ubuntu-latest + outputs: + required: ${{ steps.changes.outputs.required }} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Detect benchmark-relevant changes + id: changes + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if git diff --quiet "$BASE_SHA" "$HEAD_SHA" -- \ + src \ + benchmarks \ + pyproject.toml \ + asv.conf.json \ + asv.openmp.conf.json \ + .github/workflows/benchmark-readiness.yml \ + .github/workflows/asv-openmp-pr-comparison.yml \ + .github/workflows/asv-benchmark-publish.yml + then + echo "required=false" >> "$GITHUB_OUTPUT" + else + echo "required=true" >> "$GITHUB_OUTPUT" + fi + + benchmark: + name: Full OpenMP comparison on Linux + if: needs.benchmark_changes.outputs.required == 'true' + needs: benchmark_changes + runs-on: ubuntu-latest + timeout-minutes: 360 + + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + activate-environment: gstools-asv-driver + python-version: "3.14" + channels: conda-forge + conda-remove-defaults: true + + - name: Install ASV driver + run: | + conda install -n gstools-asv-driver -y -c conda-forge pip + python -m pip install --upgrade pip asv + + - name: Configure PR commits and ASV branches + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + printf '%s\n%s\n' "$BASE_SHA" "$HEAD_SHA" \ + > "$RUNNER_TEMP/asv-pr-commits.txt" + python - <<'PY' + import json + import os + from pathlib import Path + + config = json.loads(Path("asv.openmp.conf.json").read_text()) + config["branches"] = [os.environ["BASE_SHA"], os.environ["HEAD_SHA"]] + Path("asv.openmp.pr.conf.json").write_text( + json.dumps(config, indent=2), + encoding="utf8", + ) + PY + echo "ASV_CONFIG=asv.openmp.pr.conf.json" >> "$GITHUB_ENV" + + - name: Configure stable ASV machine name + run: | + python benchmarks/tools/configure_asv_machine.py "$MACHINE_NAME" + + - name: Run full OpenMP comparison + env: + GSTOOLS_BENCHMARK_THREADS: "1,2,4,8" + run: | + asv --config "$ASV_CONFIG" run \ + "HASHFILE:$RUNNER_TEMP/asv-pr-commits.txt" \ + --machine "$MACHINE_NAME" \ + --interleave-rounds \ + --no-pull \ + --bench "^benchmark_" \ + --show-stderr + + - name: Write informational ASV comparison + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + mkdir -p .asv-openmp/artifact + set +e + # --factor 1.05: only flag changes whose ratio exceeds 5%. + # A benchmark is marked regressed (+) or improved (-) only when BOTH + # the ratio threshold AND ASV's Mann-Whitney U statistical test agree. + # Lines prefixed with ~ exceeded the ratio but were not statistically + # significant (measurement noise on the shared runner). + asv --config "$ASV_CONFIG" compare \ + "$BASE_SHA" \ + "$HEAD_SHA" \ + --machine "$MACHINE_NAME" \ + --factor 1.05 \ + --split \ + | tee .asv-openmp/artifact/comparison.txt + compare_status="${PIPESTATUS[0]}" + set -e + echo "ASV comparison exit status: $compare_status" \ + >> .asv-openmp/artifact/comparison.txt + + - name: Build custom and native ASV reports + run: | + python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --output .asv-openmp/artifact/index.html + asv --config "$ASV_CONFIG" publish \ + --no-pull \ + --html-dir .asv-openmp/artifact/asv + cp -R .asv-openmp/results .asv-openmp/artifact/results + + - name: Upload OpenMP benchmark reports + id: upload + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: openmp-pr-${{ github.event.pull_request.number }} + path: .asv-openmp/artifact + if-no-files-found: warn + retention-days: 30 + + - name: Write benchmark step summary + if: always() + env: + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python benchmarks/tools/render_benchmark_comment.py \ + --base "$BASE_SHA" \ + --head "$HEAD_SHA" \ + --artifact-url "${ARTIFACT_URL:-}" \ + --comparison .asv-openmp/artifact/comparison.txt \ + --output /tmp/benchmark_comment.md + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + tail -n +2 /tmp/benchmark_comment.md > "$GITHUB_STEP_SUMMARY" + fi + + - name: Post benchmark results as PR comment + if: always() && github.event.pull_request.head.repo.full_name == github.repository + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + EXISTING_ID=$(gh api \ + "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.body | startswith("")) | .id' \ + | head -1) + if [ -n "$EXISTING_ID" ]; then + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING_ID" \ + -X PATCH -F body=@/tmp/benchmark_comment.md + else + gh pr comment "$PR_NUMBER" --body-file /tmp/benchmark_comment.md + fi + + - name: Save PR context for comment workflow + if: always() + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + ARTIFACT_URL: ${{ steps.upload.outputs.artifact-url }} + run: | + mkdir -p .asv-openmp/pr-context + printf '%s' "$PR_NUMBER" > .asv-openmp/pr-context/pr_number.txt + printf '%s' "$BASE_SHA" > .asv-openmp/pr-context/base_sha.txt + printf '%s' "$HEAD_SHA" > .asv-openmp/pr-context/head_sha.txt + printf '%s' "$ARTIFACT_URL" > .asv-openmp/pr-context/artifact_url.txt + cp .asv-openmp/artifact/comparison.txt \ + .asv-openmp/pr-context/comparison.txt 2>/dev/null \ + || printf '_Comparison output not available._' \ + > .asv-openmp/pr-context/comparison.txt + + - name: Upload PR context for comment workflow + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-pr-context + path: .asv-openmp/pr-context + if-no-files-found: warn + retention-days: 1 + + required_gate: + name: OpenMP benchmark gate + if: always() + needs: + - benchmark_changes + - benchmark + runs-on: ubuntu-latest + + steps: + - name: Check required benchmark result + env: + CHANGE_RESULT: ${{ needs.benchmark_changes.result }} + BENCHMARK_REQUIRED: ${{ needs.benchmark_changes.outputs.required }} + BENCHMARK_RESULT: ${{ needs.benchmark.result }} + run: | + if [[ "$CHANGE_RESULT" != "success" ]]; then + echo "Could not determine whether the benchmark is required." + exit 1 + fi + if [[ "$BENCHMARK_REQUIRED" == "true" \ + && "$BENCHMARK_RESULT" != "success" ]]; then + echo "The required OpenMP benchmark did not complete successfully." + exit 1 + fi + if [[ "$BENCHMARK_REQUIRED" == "true" ]]; then + echo "The required OpenMP benchmark completed successfully." + else + echo "No benchmark-relevant files changed." + fi diff --git a/.github/workflows/benchmark-pr-comment.yml b/.github/workflows/benchmark-pr-comment.yml new file mode 100644 index 00000000..ee2189bb --- /dev/null +++ b/.github/workflows/benchmark-pr-comment.yml @@ -0,0 +1,60 @@ +name: Post Benchmark PR Comment + +on: + workflow_run: + workflows: ["Benchmarking"] + types: [completed] + +permissions: + actions: read + pull-requests: write + +jobs: + comment: + # Only runs when the triggering event was a pull_request and the run + # produced a benchmark-pr-context artifact (skipped if no relevant changes). + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 1 + + - name: Download PR context artifact + id: download + continue-on-error: true + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: benchmark-pr-context + path: pr-context + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Post or update benchmark PR comment + if: steps.download.outcome == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + BASE_SHA=$(cat pr-context/base_sha.txt) + HEAD_SHA=$(cat pr-context/head_sha.txt) + ARTIFACT_URL=$(cat pr-context/artifact_url.txt) + PR_NUMBER=$(cat pr-context/pr_number.txt) + + python benchmarks/tools/render_benchmark_comment.py \ + --base "$BASE_SHA" \ + --head "$HEAD_SHA" \ + --artifact-url "$ARTIFACT_URL" \ + --comparison pr-context/comparison.txt \ + --output /tmp/benchmark_comment.md + + EXISTING_ID=$(gh api \ + "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.body | startswith("")) | .id' \ + | head -1) + if [ -n "$EXISTING_ID" ]; then + gh api "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING_ID" \ + -X PATCH -F body=@/tmp/benchmark_comment.md + else + gh pr comment "$PR_NUMBER" --body-file /tmp/benchmark_comment.md + fi diff --git a/.github/workflows/benchmark-readiness.yml b/.github/workflows/benchmark-readiness.yml new file mode 100644 index 00000000..d0b96f64 --- /dev/null +++ b/.github/workflows/benchmark-readiness.yml @@ -0,0 +1,156 @@ +name: Benchmark Checks + +on: + # This workflow checks whether the benchmark code can be installed and + # started on the supported GitHub-hosted runners. It is a smoke check, not a + # performance benchmark suite. + pull_request: + branches: + - "main" + paths: + - ".github/workflows/benchmark-readiness.yml" + - "asv*.json" + - "benchmarks/**" + - "pyproject.toml" + push: + branches: + - "main" + paths: + - ".github/workflows/benchmark-readiness.yml" + - "asv*.json" + - "benchmarks/**" + - "pyproject.toml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + benchmark_asv_existing_env: + name: ASV smoke on ${{ matrix.os }} py3.14 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + - macos-15-intel + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: "3.14" + + - name: Install benchmark smoke dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + asv \ + emcee \ + "gstools-cython>=1,<2" \ + hankel \ + meshio \ + "numpy==2.3.2" \ + pyevtk \ + "scipy==1.16.1" + python benchmarks/tools/install_pyproject_extras.py rust + python -m pip install --no-deps --editable . + + - name: Configure ASV machine + run: | + # existing environments still need a machine profile for ASV result + # metadata. --yes keeps the command non-interactive in CI. + asv machine --yes + + - name: Write ASV smoke config for checked-out commit + run: | + # With -E existing, ASV cannot receive a range like HEAD^!. If no + # range is supplied, ASV resolves the configured branches. PR + # checkouts do not necessarily have a local "main" branch, so this + # temporary config points ASV at the exact checked-out commit. + python - <<'PY' + import json + import os + from pathlib import Path + + config = json.loads(Path("asv.conf.json").read_text()) + config["branches"] = [os.environ["GITHUB_SHA"]] + Path("asv.ci-existing.json").write_text(json.dumps(config, indent=2)) + PY + + - name: Run ASV smoke check in existing environment + env: + GSTOOLS_BENCHMARK_THREADS: "1" + GSTOOLS_BENCHMARK_BACKENDS: "cython_fallback,rust_core" + run: | + # -E existing uses the Python environment prepared above. Existing + # environments cannot accept a commit range, so the checked-out + # commit is recorded explicitly. + ASV_SMOKE_BENCHMARK="benchmark_two_point_statistics.VariogramBenchmarks.time_variogram_estimate" + asv run \ + --config asv.ci-existing.json \ + -E existing \ + --set-commit-hash "$GITHUB_SHA" \ + --no-pull \ + --quick \ + --bench "$ASV_SMOKE_BENCHMARK" \ + --show-stderr + + benchmark_parallel_readiness: + name: Parallel readiness on ${{ matrix.os }} py3.14 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + - macos-15-intel + + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3 + with: + activate-environment: gstools-benchmark-readiness + python-version: "3.14" + channels: conda-forge + conda-remove-defaults: true + + - name: Install parallel readiness dependencies + run: | + conda install -n gstools-benchmark-readiness -y -c conda-forge \ + emcee \ + gstools-cython \ + hankel \ + meshio \ + "numpy=2.3.2" \ + pip \ + pyevtk \ + "scipy=1.16.1" + conda run -n gstools-benchmark-readiness python \ + benchmarks/tools/install_pyproject_extras.py rust + conda run -n gstools-benchmark-readiness python -m pip install \ + --no-deps --editable . + + - name: Check Cython OpenMP and Rust backend readiness + run: | + conda run -n gstools-benchmark-readiness python \ + benchmarks/tools/check_backend_parallel_ready.py --verbose diff --git a/.gitignore b/.gitignore index bcdc980b..460490cd 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ htmlcov/ .coverage .coverage.* .cache +.asv/* +.asv-openmp/* +.asv-ci/* +asv.ci.*.json nosetests.xml coverage.xml *.cover diff --git a/AUTHORS.md b/AUTHORS.md index 426681de..8aafc1fd 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -8,6 +8,7 @@ and was created by following people. - Sebastian Müller, GitHub: [@MuellerSeb](https://github.com/MuellerSeb), Email: - Lennart Schüler, GitHub: [@LSchueler](https://github.com/LSchueler), Email: +- Jeisson Leal, GitHub: [@LSchueler](https://github.com/jeilealr), Email: ## Contributors (in order of contributions) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61aa167c..ecbf609d 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,15 @@ All notable changes to **GSTools** will be documented in this file. - instead of spatial rules, as before, decision trees are a more flexible approach - each node of the binary tree is a decision based on the values of the SRF - replace pylint, black, and isort with ruff [#391](https://github.com/GeoStat-Framework/GSTools/pull/391) +- add comprehensive benchmark suite for two-point statistics workflows + - covers variogram estimation, kriging, and random field generation across Cython and Rust backends + - includes OpenMP thread-count scaling analysis (1/2/4/8 threads) + - interactive HTML report with per-case and per-thread-count comparisons +- add benchmarking CI workflows + - automated performance regression detection with ASV on every pull request + - benchmark results posted as PR comments with Mann-Whitney U statistical test + - cumulative benchmark history published to GitHub Pages after each merge to main +- add benchmarking guidelines to CONTRIBUTING.md ## [1.7.0] - Morphic Mint - 2025-04 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60359004..b4f7a1e7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,3 +36,28 @@ with your idea or suggestion and we'd love to discuss about it. - Add an example showing your new feature in one of the examples sub-folders if possible. Follow this [Sphinx-Gallary guide](https://sphinx-gallery.github.io/stable/syntax.html#embed-rst-in-your-example-python-files). - Push to your fork and submit a pull request. + + +## Do you want to contribute to benchmarking? + +GSTools tracks the runtime and memory performance +using [Airspeed Velocity (ASV)](https://asv.readthedocs.io/). +The benchmark suite lives in the `benchmarks/` directory and covers both the +Cython fallback and the optional Rust backend, as well as OpenMP thread scaling. + +Every pull request automatically runs the benchmark suite and reports any +significant regressions or improvements directly in the PR. +The cumulative benchmark history is published at +. + +If you would like to contribute to benchmarking you can: + +- **Add a new benchmark** — follow the existing patterns in + `benchmarks/` and prefix the function name with `time_` (runtime) or + `peakmem_` (peak memory) so ASV picks it up automatically. +- **Improve the benchmark infrastructure** — the helper scripts in + `benchmarks/tools/` generate the HTML comparison report and configure + ASV for CI runs. +- **Report a performance regression** — open a + [new issue](https://github.com/GeoStat-Framework/GSTools/issues) and + include the relevant ASV comparison output. diff --git a/README.md b/README.md index 1947df0b..237f7587 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ Install the package by typing the following command in a command terminal: conda install gstools -In case conda forge is not set up for your system yet, see the easy to follow -instructions on [conda forge][conda_forge_link]. Using conda, the parallelized +In case conda-forge is not set up for your system yet, see the easy to follow +instructions on [conda-forge][conda_forge_link]. Using conda, the parallelized version of GSTools should be installed. diff --git a/asv.conf.json b/asv.conf.json new file mode 100644 index 00000000..97be6f08 --- /dev/null +++ b/asv.conf.json @@ -0,0 +1,32 @@ +{ + "version": 1, + "project": "GSTools", + "project_url": "https://github.com/jeilealr/GSTools", + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "pythons": ["3.14"], + "matrix": { + "req": { + "emcee": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "gstools-cython": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_pyproject_extras.py --pyproject {conf_dir}/pyproject.toml rust", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" + ], + "number": 1, + "repeat": 20 +} diff --git a/asv.openmp.conf.json b/asv.openmp.conf.json new file mode 100644 index 00000000..ef4ec318 --- /dev/null +++ b/asv.openmp.conf.json @@ -0,0 +1,39 @@ +{ + "version": 1, + "project": "GSTools", + "project_url": "https://github.com/jeilealr/GSTools", + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv-openmp/env", + "results_dir": ".asv-openmp/results", + "html_dir": ".asv-openmp/html", + "show_commit_url": "https://github.com/jeilealr/GSTools/commit/", + "environment_type": "conda", + "conda_channels": ["conda-forge"], + "pythons": ["3.14"], + "matrix": { + "req": { + "c-compiler": [""], + "cxx-compiler": [""], + "cython": [""], + "emcee": [""], + "extension-helpers": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "setuptools>=77": [""], + "wheel": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_pyproject_extras.py --pyproject {conf_dir}/pyproject.toml rust", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_openmp_cython.py {env_dir}", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}", + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/check_backend_parallel_ready.py --verbose" + ], + "number": 1, + "repeat": 20 +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..7f1ea029 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,927 @@ +# GSTools Benchmark Guide + +This directory contains the Airspeed Velocity ([ASV](https://github.com/airspeed-velocity/asv/)) benchmark suite for GSTools and a complementary profiling helper implemented with cProfile (part of the Python standard library). + +This guide benchmarks GSTools, inspects the +results, profiles where runtime is spent, and then decides what to optimize. + +Unit tests in `tests/` answer "is the code correct?". The ASV benchmarks in +`benchmarks/` answer "how fast is this workflow, how much memory does it use, +and did that change across commits?". The complementary cProfile helper +answers "inside this workflow, which Python functions are taking most of the +time right now?". + +The benchmarks compare two GSTools backends, which gives more context for +deciding where optimization work should go: + +- `cython_fallback`: the default Cython-backed fallback implementation from + [gstools-cython](https://github.com/GeoStat-Framework/GSTools-Cython). +- `rust_core`: the Rust-backed implementation from + [gstools_core](https://github.com/GeoStat-Framework/GSTools-Core). + +## Index + +- [Setup](#setup) +- [Benchmarking Scripts](#benchmarking-scripts) + - [ASV Configuration](#asv-configuration) + - [Benchmark Naming](#benchmark-naming) +- [Benchmark Coverage](#benchmark-coverage) + - [Shared Constants](#shared-constants) + - [Shared Helpers](#shared-helpers) + - [Benchmark Classes](#benchmark-classes) + - [VariogramBenchmarks](#variogrambenchmarks) + - [KrigingBenchmarks](#krigingbenchmarks) + - [RandomFieldBenchmarks](#randomfieldbenchmarks) +- [Running The Benchmarks](#running-the-benchmarks) + - [Baseline Benchmark](#baseline-benchmark) + - [Main Branch Baseline](#main-branch-baseline) + - [Several Commits Baseline](#several-commits-baseline) + - [Summary of Results](#summary-of-results) + - [Visualization of Results](#visualization-of-results) + - [Profiling With cProfile](#profiling-with-cprofile) +- [Optional Parallelization with OpenMP](#optional-parallelization-with-openmp) + - [OpenMP ASV Configuration](#openmp-asv-configuration) + - [Run on macOS and Linux](#run-on-macos-and-linux) + - [Verify Parallel Backends](#verify-parallel-backends) + - [Run on Windows](#run-on-windows) + - [OpenMP Thread Rule](#openmp-thread-rule) + - [HPC Notes](#hpc-notes) + - [Profiling With cProfile for Multiple Threads](#profiling-with-cprofile-for-multiple-threads) +- [More ASV Commands](#more-asv-commands) +- [External Reference](#external-reference) + +## Setup + +The regular installation commands in the main `README.md` install GSTools for +normal use. This benchmark guide uses conda because ASV creates isolated +benchmark environments for the commits it measures. + +The default benchmark configuration intentionally compares both backends with +one GSTools thread: + +```text +gstools.config.NUM_THREADS = 1 +``` + +That keeps the first comparison simple: Cython fallback vs Rust core without +parallelism as a confounding factor. Parallel/OpenMP scaling is treated as a +separate optional experiment because the correct Cython OpenMP build depends on +the user's operating system, compiler, and runtime environment. + +To run the benchmark and the optional cProfile helper, follow these steps (this guide uses Python 3.14): + +1. Move to the GSTools repository root: + +```bash +cd /path/to/GSTools +``` + +2. Create and activate a conda environment for local benchmark work: + +```bash +conda create -n gstools-benchmark -c conda-forge python=3.14 asv +conda activate gstools-benchmark +``` + +If you already have a suitable conda environment, activate that instead. + +3. If you use an existing environment, make sure ASV is installed: + +```bash +conda install -c conda-forge asv +``` + +4. Create a machine profile once per computer: + +```bash +asv machine --yes +``` + +The machine profile records local hardware information so ASV can label +results correctly. Do not compare absolute times across different machines. + +## Benchmarking Scripts + +The benchmarking setup currently consists of: + +- `asv.conf.json`: tells ASV how to build GSTools, where benchmarks live, where + to store results, and which Python/environment matrix to use. +- `asv.openmp.conf.json`: optional cross-platform ASV configuration that builds + `gstools-cython` from source with OpenMP inside ASV's own environment. +- `benchmarks/benchmark_two_point_statistics.py`: contains the ASV benchmark + classes. +- `benchmarks/README.md`: this practical guide. +- `benchmarks/tools/asv_speedup_summary.py`: reads ASV result JSON files and + prints Rust-vs-Cython speedup ratios or writes a curated Markdown/HTML + report. +- `benchmarks/tools/plot_case_backend_comparison.py`: reads ASV result JSON + files and writes a self-contained HTML report with grouped backend bars and + commit trends for selected cases, thread counts, and metrics. +- `benchmarks/tools/profile_benchmark_workflows.py`: runs one representative + workflow from `benchmark_two_point_statistics.py` under Python's built-in + `cProfile`, so you can see which functions take time in the current + checkout. +- `benchmarks/tools/check_backend_parallel_ready.py`: CI helper that verifies + Cython OpenMP detection and Rust backend execution (variogram, field + generation, and kriging) with more than one GSTools thread. +- `benchmarks/tools/configure_asv_machine.py`: writes a named ASV machine + profile from detected hardware; used in CI to ensure all results are stored + under a stable name regardless of the actual runner. +- `benchmarks/tools/install_pyproject_extras.py`: installs selected optional + dependencies directly from `pyproject.toml` without installing GSTools or + its regular dependencies. +- `benchmarks/tools/install_openmp_cython.py`: helper used by + `asv.openmp.conf.json` to compile `gstools-cython` with OpenMP on macOS, + Linux, and native Windows. +- `benchmarks/tools/render_benchmark_comment.py`: CI helper used by both the + pull-request comparison workflow and the cross-fork comment workflow to + render the GitHub PR comment body; keeps badge wording and comment format + in sync across both code paths. + +### ASV Configuration + +The repo root `asv.conf.json` is tailored to this GSTools checkout: + +```json +{ + "repo": ".", + "branches": ["main"], + "benchmark_dir": "benchmarks", + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "environment_type": "conda", + "pythons": ["3.14"], + "matrix": { + "req": { + "emcee": [""], + "hankel": [""], + "meshio": [""], + "numpy": [""], + "pyevtk": [""], + "scipy": [""], + "gstools-cython": [""] + } + }, + "install_command": [ + "in-dir={env_dir} python {conf_dir}/benchmarks/tools/install_pyproject_extras.py --pyproject {conf_dir}/pyproject.toml rust", + "in-dir={env_dir} python -m pip install --no-deps {build_dir}" + ], + "number": 1, + "repeat": 20 +} +``` + +Important details: + +- `environment_type: "conda"` means conda is required for the ASV workflow in + this guide. ASV creates isolated conda environments for the commits it + benchmarks. +- `pythons: ["3.14"]` means ASV creates Python 3.14 benchmark environments. + Both ASV configs use Python 3.14 by default; change `pythons` in + `asv.conf.json` and `asv.openmp.conf.json` only when intentionally + validating another Python/GSTools backend stack. +- `number: 1` means ASV runs the benchmark code once per iteration. +- `repeat: 20` means the default baseline records 20 independent measurements + per benchmark case. +- `matrix.req` asks ASV to install GSTools runtime dependencies before + installing the checked-out GSTools source. It includes `gstools-cython` + explicitly because the GSTools commit is installed with `--no-deps`. +- `{build_dir}` is ASV's temporary checkout/build directory for the exact + GSTools commit being benchmarked. +- `install_command` installs the checked-out GSTools revision with `--no-deps`. + It also uses `install_pyproject_extras.py` to install the `rust` extra from + `pyproject.toml`, keeping the `gstools_core` requirement in one place. +- ASV still needs its own `install_command` because it creates isolated + environments for the commits it benchmarks. +- In CI, `configure_asv_machine.py` replaces the interactive `asv machine + --yes` step and writes a stable machine name (`github-actions-ubuntu`) + regardless of the actual runner hardware. Locally, `asv machine --yes` is + sufficient and `configure_asv_machine.py` is not needed. +- Run the cProfile helper with the Python executable from ASV's isolated + environment, for example `.asv/env//bin/python`. In that mode, the + ASV environment provides dependencies while the helper imports the current + checkout through the repo `src/` path. + +ASV creates these generated directories: + +```text +.asv/env/ benchmark environments +.asv/results/ local benchmark result JSON files +.asv/html/ generated local benchmark website +``` + +Those directories are machine-specific generated artifacts. They should +normally stay out of git. + +If needed, users can list more than one branch, Python version, benchmark +directory, and so on. For example: + +```json +"branches": ["main", "my-feature-branch"] +``` + +Users can also benchmark any explicit branch, commit, tag, or range without +changing `asv.conf.json`: + +```bash +asv run 'my-feature-branch^!' --bench benchmark_two_point_statistics +asv run main..my-feature-branch --bench benchmark_two_point_statistics +``` + +ASV checks out package code at each git commit being benchmarked. Commit source +changes before benchmarking them with ASV. Otherwise ASV may benchmark the last +committed package code rather than your uncommitted source changes. + +### Benchmark Naming + +ASV recognizes benchmark methods by name: + +- methods starting with `time_` measure runtime +- methods starting with `peakmem_` measure peak memory +- `setup_cache()` creates reusable data once per benchmark environment +- `setup()` can skip or prepare individual parameter combinations + +## Benchmark Coverage + +This section describes what is measured by the ASV suite and how the benchmark +labels map to real GSTools workflows. The goal is to cover representative +operations that are relevant for geostatistical work, not isolated +micro-functions. + +The current suite measures runtime and peak memory for variogram estimation, +global kriging, spatial random field generation, and conditioned random field +generation. Each workflow is run with both backends so the results can show +both absolute performance and Rust-vs-Cython differences. + +### Shared Constants + +```python +BACKENDS = ("cython_fallback", "rust_core") +THREAD_COUNTS = _configured_thread_counts() +VARIOGRAM_CASES = ( + "full_900", + "sampled_5000_to_1500", + "sampled_15000_to_4500", +) +KRIGE_CASES = ("small_30x500", "large_120x2000", "extra_large_360x6000") +FIELD_CASES = ( + "srf_unstructured_randmeth", + "srf_structured_randmeth", + "srf_structured_fourier", + "condsrf_unstructured", +) +``` + +These constants define backend parameter values and the case/thread labels used +in generated benchmark method names. + +`BACKENDS` compares: + +- `cython_fallback` +- `rust_core` + +`THREAD_COUNTS` defaults to: + +- `threads_1`: force `gstools.config.NUM_THREADS = 1` + +That is the default because the first benchmark target is a clean +Cython-vs-Rust backend comparison without parallelism. + +Both constants are driven by environment variables read at import time: + +| Variable | Default | Example override | +|---|---|---| +| `GSTOOLS_BENCHMARK_THREADS` | `"1"` | `"1,2,4,8"` — run with 1, 2, 4, and 8 threads | +| `GSTOOLS_BENCHMARK_BACKENDS` | all backends | `"rust_core"` — run Rust only | + +These must be set before ASV imports the benchmark module. When running +locally with ASV, prefix the `asv run` command: + +```bash +GSTOOLS_BENCHMARK_BACKENDS=rust_core asv run 'main^!' --bench benchmark_two_point_statistics +``` + +### Shared Helpers + +`gstools_backend(use_core, num_threads)` temporarily forces GSTools to use +either the Cython fallback backend or the Rust `gstools_core` backend, and +sets `gstools.config.NUM_THREADS` for that benchmark run. + +`_random_points(seed, count, scale)` creates deterministic 2D point clouds. + +`_smooth_field(x, y)` creates deterministic synthetic values: + +```python +np.sin(x / 10.0) + np.cos(y / 15.0) +``` + +`_make_variogram_data(...)` creates positions, field values, and bins for +variogram estimation. + +`_make_krige_data(...)` creates conditioning points, conditioning values, and +target points for kriging and conditioned random fields. + +The fixed random seeds are intentional. They keep benchmark inputs stable so +changes in results are more likely to come from code changes, not new random +data. + +### Benchmark Classes + +The ASV benchmarking is organized around workflow classes. Each workflow class +compares `cython_fallback` and `rust_core`, and each class includes both +runtime and peak-memory methods. + +The suite currently measures: + +- `VariogramBenchmarks`: full pairwise work vs sampled large work +- `KrigingBenchmarks`: small vs larger global kriging systems +- `RandomFieldBenchmarks`: unstructured SRF, structured SRF, Fourier + SRF, and conditioned SRF + +This keeps the ASV suite focused on representative workflows rather than +separate duplicate backend checks. + +#### VariogramBenchmarks + +This class measures variogram estimation cases: + +```text +full_900 +sampled_5000_to_1500 +sampled_15000_to_4500 +``` + +The labels mean: + +- `full_900`: create 900 scattered points and use all 900 points for the + variogram calculation. +- `sampled_5000_to_1500`: create 5,000 scattered points, then randomly select + 1,500 of those points for the variogram calculation. +- `sampled_15000_to_4500`: create 15,000 scattered points, then randomly select + 4,500 of those points for the variogram calculation. + +The sampled cases still represent larger input datasets, but the variogram +calculation is done on the randomly selected subset so the pairwise work stays +practical. + +#### KrigingBenchmarks + +This class measures global kriging at three scales: + +```text +small_30x500 +large_120x2000 +extra_large_360x6000 +``` + +The labels mean: + +- `small_30x500`: 30 conditioning points, 500 target points +- `large_120x2000`: 120 conditioning points, 2,000 target points +- `extra_large_360x6000`: 360 conditioning points, 6,000 target points + +#### RandomFieldBenchmarks + +This class measures SRF and CondSRF generation workflows: + +```text +srf_unstructured_randmeth +srf_structured_randmeth +srf_structured_fourier +condsrf_unstructured +``` + +The cases are: + +- `srf_unstructured_randmeth`: SRF using RandMeth on 2,000 unstructured points +- `srf_structured_randmeth`: SRF using RandMeth on a 64 by 64 structured grid +- `srf_structured_fourier`: SRF using the Fourier generator on a 64 by 64 + structured grid +- `condsrf_unstructured`: conditioned SRF with 40 conditioning points and 1,000 + target points + +## Running The Benchmarks + +### Baseline Benchmark + +The baseline benchmark is the first result set to create before doing any +optimization work. It uses the default ASV configuration, so each workflow is +measured with `threads_1` for both `cython_fallback` and `rust_core`. + +#### Main Branch Baseline + +- Save a baseline for the latest local `main` commit: + +```bash +asv run 'main^!' --bench benchmark_two_point_statistics +``` + +#### Several Commits Baseline + +ASV can also compare several commits. This example runs the last three commits +on local `main`; choose a different range when that better matches your +branch. + +- Run the last three commits on the local `main` branch: + +```bash +asv run 'main~3..main' --bench benchmark_two_point_statistics +``` + +#### Summary of Results + +After running ASV, inspect the explicit Rust-vs-Cython speedup ratios: + +```bash +python benchmarks/tools/asv_speedup_summary.py +``` + +The helper reads `.asv/results/` and reports ratios per case and thread label: + +```text +speedup = cython_fallback_time / rust_core_time +``` + +Interpret the ratio as: + +- `speedup > 1.0` means Rust is faster +- `speedup = 1.0` means similar performance +- `speedup < 1.0` means Rust is slower + +The speedup helper prints the backend ratio explicitly in the terminal. By +default, the helper skips removed legacy duplicate rows from older saved +results. + +For a report that is easier to paste into a PR or issue: + +```bash +python benchmarks/tools/asv_speedup_summary.py --format markdown +``` + +For a small standalone HTML report: + +```bash +python benchmarks/tools/asv_speedup_summary.py --format html --output .asv/backend-report-overview.html +``` + +#### Visualization of Results + +Build a self-contained interactive HTML report from the result files: + +```bash +python benchmarks/tools/plot_case_backend_comparison.py +``` + +This reads `.asv/results/` by default and writes +`case-backend-comparison.html` next to it. Open the file in any browser. Use +`--metric time` or `--metric memory` to show one metric, and `--benchmark` to +filter to one family (e.g. `variogram`, `krige`, `srf`). + +You can also inspect results in the native ASV browser report: + +```bash +asv publish +asv preview +``` + +Then open the printed local URL, for example: + +```text +http://127.0.0.1:8080/#/ +``` +(or any other `http://127.0.0.1:/#/` URL shown by the running preview). + +The native ASV report shows raw plots and trends. ASV does not draw a line +when there is only one x-axis point, so a single-commit run may show results +without useful trend graphs. The custom HTML report works with one or more +commits. + +### Profiling With cProfile + +`cProfile` does not update the ASV results shown in the browser report. +Instead, it prints a table in the terminal showing which Python +functions consumed time while one workflow ran. + +The helper script is: + +```text +benchmarks/tools/profile_benchmark_workflows.py +``` + +It imports the ASV benchmark classes from `benchmark_two_point_statistics.py`, +selects one case, forces one backend, and runs that case under `cProfile`. + +Since ASV has already created an isolated Python environment, select that +environment to execute the profiling helper: + +```bash +ASV_ENV="$(ls -td .asv/env/* | head -n 1)" +ASV_PYTHON="$ASV_ENV/bin/python" +``` + +On Windows PowerShell, use: + +```powershell +$asvEnv = Get-ChildItem .asv\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvPython = Join-Path $asvEnv.FullName 'python.exe' +``` + +The helper still profiles the current checkout because +`profile_benchmark_workflows.py` adds the repository `src/` directory to +`sys.path`. The ASV environment provides the installed dependencies, including +`gstools-cython` and `gstools_core`. + +List available cases: + +```bash +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --list +``` + +On Windows PowerShell, replace `"$ASV_PYTHON"` with `& $asvPython`. + +All available cases: + +```text +variogram-full variogram on 900 points, full pairwise +variogram-sampled variogram on 5 000 pts, sampled 1 500 +variogram-extra-large variogram on 15 000 pts, sampled 4 500 +krige-small kriging 30 → 500 pts +krige-large kriging 120 → 2 000 pts +krige-extra-large kriging 360 → 6 000 pts +srf-unstructured SRF RandMeth, 2 000 unstructured pts +srf-structured SRF RandMeth, 64×64 grid +srf-fourier SRF Fourier, 64×64 grid +condsrf conditioned SRF, 40 → 1 000 pts +``` + +Profile a selected case: + +```bash +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-sampled --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case variogram-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case srf-unstructured --backend rust_core --threads threads_1 --limit 10 +"$ASV_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case condsrf --backend rust_core --threads threads_1 --limit 10 +``` + +## Optional Parallelization with OpenMP + +This section collects optional workflows for testing Cython and Rust with +several thread counts. OpenMP setup is platform-dependent, so each operating +system must be verified on the machine that produces the results. + +The default setup above remains the recommended baseline: one thread, normal +ASV environment, and no extra OpenMP build steps. Use this section only when +you explicitly want to measure backend scaling with multiple thread counts. + +### OpenMP ASV Configuration + +Use one OpenMP ASV config on all supported desktop platforms: + +```text +asv.openmp.conf.json +``` + +This config keeps the OpenMP experiment separate from the default baseline: + +```text +.asv-openmp/env/ +.asv-openmp/results/ +.asv-openmp/html/ +``` + +It uses the same `number` and `repeat` settings as the baseline config: + +```json +"number": 1, +"repeat": 20 +``` + +Each benchmark case records 20 independent measurements, running the benchmark +code once per measurement. + +During ASV installation, it runs: + +```bash +benchmarks/tools/install_openmp_cython.py +``` + +That helper compiles `gstools-cython` from source inside ASV's own +environment, not inside your active `gstools-benchmark` driver environment. +The helper sets `GSTOOLS_BUILD_PARALLEL=1` and then uses platform-specific +compiler handling: + +- macOS: uses Apple clang through wrapper scripts and conda's `llvm-openmp`. +- Linux: uses the ASV conda compiler toolchain when available. +- Windows: uses native MSVC Build Tools. + +### Run on macOS and Linux + +Use these commands from a POSIX shell on macOS or Linux. On macOS, install +Xcode command-line tools first. On Linux, make sure the conda compiler packages +from `asv.openmp.conf.json` solve for your platform. + +Create the ASV machine profile once: + +```bash +asv --config asv.openmp.conf.json machine --yes +``` + +#### Verify Parallel Backends + +Only interpret the Cython rows as OpenMP-enabled, and the Rust rows as +parallel-ready, after this check passes inside the `.asv-openmp/env/...` +environment. + +On macOS and Linux: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +"$ASV_OPENMP_ENV/bin/python" benchmarks/tools/check_backend_parallel_ready.py --verbose +``` + +On Windows PowerShell: + +```powershell +$asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' +& $asvOpenmpPython benchmarks\tools\check_backend_parallel_ready.py --verbose +``` + +Expected passing output contains: + +```text +Cython OpenMP readiness: PASS +Rust backend readiness: PASS with NUM_THREADS=2 +``` + +If the check passes, run the last five commits: + +```bash +GSTOOLS_BENCHMARK_THREADS=1,2,4,8 \ +asv --config asv.openmp.conf.json run 'main~5..main' --bench benchmark_two_point_statistics --show-stderr +``` + +Print Rust-vs-Cython ratios from the OpenMP result folder: + +```bash +python benchmarks/tools/asv_speedup_summary.py --results-dir .asv-openmp/results +``` + +Build a curated OpenMP HTML report: + +```bash +python benchmarks/tools/asv_speedup_summary.py \ + --results-dir .asv-openmp/results \ + --format html \ + --output .asv-openmp/backend-report-openmp-overview.html +``` + +Build the case/backend comparison report: + +```bash +python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --max-commits 50 \ + --output .asv-openmp/case-backend-comparison.html +``` + +By default this report includes both time and peak-memory benchmarks. Use +`--metric time` or `--metric memory` when you want only one metric. +`--max-commits` limits only the generated custom report; it does not delete +raw ASV results. Use `--benchmark` to filter to one family: + +```bash +python benchmarks/tools/plot_case_backend_comparison.py \ + --results-dir .asv-openmp/results \ + --benchmark variogram # or krige, kriging, field, srf, condsrf, vario + --output .asv-openmp/variogram-comparison.html +``` + +Build and preview the OpenMP browser report: + +```bash +asv --config asv.openmp.conf.json publish +asv --config asv.openmp.conf.json preview +``` + +### Run on Windows + +Use native Windows, not WSL, when you want Windows benchmark results. Install +Microsoft C++ Build Tools first, including the C++ build tools workload and a +Windows SDK. Run the commands from PowerShell or Anaconda Prompt after +activating the `gstools-benchmark` conda environment. + +Create the ASV machine profile once: + +```powershell +asv --config asv.openmp.conf.json machine --yes +``` + +Run a quick OpenMP smoke run for the latest local `main` commit: + +```powershell +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8' +asv --config asv.openmp.conf.json run 'main^!' --quick --bench benchmark_two_point_statistics --show-stderr +``` + +Verify the parallel backends with the PowerShell commands in +[Verify Parallel Backends](#verify-parallel-backends). If the check passes, +run the last five commits: + +```powershell +$env:GSTOOLS_BENCHMARK_THREADS = '1,2,4,8' +asv --config asv.openmp.conf.json run 'main~5..main' --bench benchmark_two_point_statistics --show-stderr +``` + +Print Rust-vs-Cython ratios from the OpenMP result folder: + +```powershell +python benchmarks\tools\asv_speedup_summary.py --results-dir .asv-openmp\results +``` + +Build a curated OpenMP HTML report: + +```powershell +python benchmarks\tools\asv_speedup_summary.py ` + --results-dir .asv-openmp\results ` + --format html ` + --output .asv-openmp\backend-report-openmp-overview.html +``` + +Build the case/backend comparison report: + +```powershell +python benchmarks\tools\plot_case_backend_comparison.py ` + --results-dir .asv-openmp\results ` + --output .asv-openmp\case-backend-comparison.html +``` + +By default this report includes both time and peak-memory benchmarks. Use +`--metric time` or `--metric memory` when you want only one metric, and +`--benchmark` to filter to one family (e.g. `variogram`, `krige`, `field`, +`srf`, `condsrf`). + +Build and preview the OpenMP browser report: + +```powershell +asv --config asv.openmp.conf.json publish +asv --config asv.openmp.conf.json preview +``` + +When finished, clear the thread-count override if the PowerShell session will +be reused: + +```powershell +Remove-Item Env:\GSTOOLS_BENCHMARK_THREADS +``` + +### OpenMP Thread Rule + +The `GSTOOLS_BENCHMARK_THREADS=1,2,4,8` setting only tells the benchmark code to +run the same workflows with different `gstools.config.NUM_THREADS` values. It +does not, by itself, make the Cython backend parallel. + +For Cython OpenMP scaling, `gstools-cython` must be compiled with OpenMP inside +the same `.asv-openmp/env/...` environment that runs the benchmark. The +`asv.openmp.conf.json` install command does that through +`benchmarks/tools/install_openmp_cython.py`; the commands in +[Verify Parallel Backends](#verify-parallel-backends) are the proof that the +environment is ready. If that check fails, the benchmark may still run, but the +Cython rows should not be interpreted as OpenMP-enabled Cython results. + +### HPC Notes + +The `asv.openmp.conf.json` workflow is intended for local macOS, Linux, and +native Windows machines. Managed HPC systems often use custom compiler modules, +MPI/OpenMP runtimes, scheduler pinning, and CPU affinity rules. Use the same +validation rule there: only interpret Cython as OpenMP-enabled if +`check_backend_parallel_ready.py --verbose` passes inside the exact ASV +environment used for the benchmark run. + +### Profiling With cProfile for Multiple Threads + +To profile how a workflow changes across configured thread counts, run the +same cProfile case several times with the OpenMP ASV environment: + +```bash +ASV_OPENMP_ENV="$(ls -td .asv-openmp/env/* | head -n 1)" +ASV_OPENMP_PYTHON="$ASV_OPENMP_ENV/bin/python" + +for threads in threads_1 threads_2 threads_4 threads_8; do + "$ASV_OPENMP_PYTHON" benchmarks/tools/profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads "$threads" --limit 10 +done +``` + +On Windows PowerShell: + +```powershell +$asvOpenmpEnv = Get-ChildItem .asv-openmp\env -Directory | + Sort-Object LastWriteTime -Descending | + Select-Object -First 1 +$asvOpenmpPython = Join-Path $asvOpenmpEnv.FullName 'python.exe' + +foreach ($threads in 'threads_1', 'threads_2', 'threads_4', 'threads_8') { + & $asvOpenmpPython benchmarks\tools\profile_benchmark_workflows.py --case krige-extra-large --backend rust_core --threads $threads --limit 10 +} +``` + +Useful options: + +- `--case`: choose one workflow, or use `all` +- `--backend`: choose `cython_fallback` or `rust_core` +- `--threads`: choose `threads_1` for a baseline profile, or `threads_2`, + `threads_4`, or `threads_8` for parallel profiles +- `--limit`: number of function rows to print from the cProfile table +- `--sort cumtime`: sort by cumulative time, usually the best first view +- `--sort tottime`: sort by time spent directly in each function +- `--repeat`: repeat a workflow inside the profiler + +For example, `--limit 10` means "print the top 10 function rows after sorting". + +## More ASV Commands + +Save results for only the latest local `main` commit: + +```bash +asv run 'main^!' --bench benchmark_two_point_statistics +``` + +Compare the latest local `main` commit with the previous local `main` commit: + +```bash +asv run 'main~1^!' --bench benchmark_two_point_statistics +asv run 'main^!' --bench benchmark_two_point_statistics +asv compare main~1 main +``` + +Compare local `main` with the latest remote `main`: + +```bash +asv run 'main^!' --bench benchmark_two_point_statistics +git fetch origin main +asv run 'origin/main^!' --bench benchmark_two_point_statistics +asv compare origin/main main +``` + +Compare the previous local `main` commit with the latest remote `main`: + +```bash +git fetch origin main +asv run 'main~1^!' --bench benchmark_two_point_statistics +asv run 'origin/main^!' --bench benchmark_two_point_statistics +asv compare main~1 origin/main +``` + +Run the last three commits on local `main`: + +```bash +asv run 'main~3..main' --bench benchmark_two_point_statistics --show-stderr +``` + +Run the last three commits on the latest remote `main`: + +```bash +git fetch origin main +asv run 'origin/main~3..origin/main' --bench benchmark_two_point_statistics --show-stderr +``` + +On a linear branch, `main~5..main` benchmarks: + +```text +main~4 +main~3 +main~2 +main~1 +main +``` + +Run a selected list of commits: + +```bash +git rev-parse main main~3 origin/main e20c88f7 > /tmp/gstools-asv-commits.txt +asv run HASHFILE:/tmp/gstools-asv-commits.txt --bench benchmark_two_point_statistics +``` + +Use full commit hashes when sharing results. Short hashes and branch names are +fine locally but can become ambiguous later. + +If running ASV from outside the repo root, pass the config explicitly: + +```bash +asv --config /path/to/MPS-Tools/GSTools/asv.conf.json run --quick --bench benchmark_two_point_statistics +``` + +## External Reference + +For complete ASV command syntax, see: + +```text +https://asv.readthedocs.io/en/stable/commands.html +``` diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 00000000..09113f74 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""ASV benchmark package for GSTools workflow performance checks.""" diff --git a/benchmarks/benchmark_two_point_statistics.py b/benchmarks/benchmark_two_point_statistics.py new file mode 100644 index 00000000..ae19efbe --- /dev/null +++ b/benchmarks/benchmark_two_point_statistics.py @@ -0,0 +1,368 @@ +"""Benchmark representative two-point statistics workflows. + +Usage: + cd /path/to/MPS-Tools/GSTools + # See benchmarks/README.md for ASV and optional cProfile setup. + asv machine --yes + asv run --quick --show-stderr --bench benchmark_two_point_statistics + asv run main^! --bench benchmark_two_point_statistics + asv run + asv publish + asv preview + asv compare main~1 main + +Backend speedup should be interpreted as: + speedup = cython_fallback_time / rust_core_time + +Values greater than 1.0 mean the Rust backend is faster on the same machine +for the same benchmark, case, commit, and thread count. + +By default the suite uses one GSTools thread. For local OpenMP scaling +experiments, set GSTOOLS_BENCHMARK_THREADS before ASV discovers the +benchmarks, for example: + GSTOOLS_BENCHMARK_THREADS=1,2,4,8 \ + asv --config asv.openmp.conf.json run 'main^!' +""" + +from __future__ import annotations + +import contextlib +import os + +import gstools as gs +import numpy as np + +AVAILABLE_BACKENDS = ("cython_fallback", "rust_core") + + +def _configured_backends(): + """Return backend labels requested through the environment.""" + raw = os.environ.get("GSTOOLS_BENCHMARK_BACKENDS", "") + if not raw.strip(): + return AVAILABLE_BACKENDS + backends = tuple(item.strip() for item in raw.split(",") if item.strip()) + unknown = sorted(set(backends) - set(AVAILABLE_BACKENDS)) + if unknown: + raise ValueError( + "GSTOOLS_BENCHMARK_BACKENDS contains unknown backends: " + + ", ".join(unknown) + ) + if not backends: + raise ValueError("GSTOOLS_BENCHMARK_BACKENDS did not define backends") + return backends + + +BACKENDS = _configured_backends() + + +def _configured_thread_counts(): + """Return thread count integers requested through the environment.""" + raw = os.environ.get("GSTOOLS_BENCHMARK_THREADS", "1") + thread_counts = [] + for item in raw.split(","): + item = item.strip() + if not item: + continue + if item.startswith("threads_"): + value = item[len("threads_") :] + else: + value = item + thread_counts.append(int(value)) + if not thread_counts: + raise ValueError("GSTOOLS_BENCHMARK_THREADS did not define threads") + return tuple(thread_counts) + + +THREAD_COUNTS = _configured_thread_counts() +VARIOGRAM_CASES = ( + "full_900", + "sampled_5000_to_1500", + "sampled_15000_to_4500", +) +KRIGE_CASES = ("small_30x500", "large_120x2000", "extra_large_360x6000") +FIELD_CASES = ( + "srf_unstructured_randmeth", + "srf_structured_randmeth", + "srf_structured_fourier", + "condsrf_unstructured", +) + + +@contextlib.contextmanager +def gstools_backend(use_core, num_threads): + """Temporarily select the GSTools backend and thread count.""" + previous = ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) + try: + if use_core: + if not previous[0]: + raise NotImplementedError("gstools_core is not available") + gs.config._GSTOOLS_CORE_AVAIL = True + gs.config.USE_GSTOOLS_CORE = True + else: + gs.config._GSTOOLS_CORE_AVAIL = False + gs.config.USE_GSTOOLS_CORE = False + gs.config.NUM_THREADS = num_threads + yield + finally: + ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) = previous + + +def _use_core(backend): + """Map a benchmark backend label to the GSTools core flag.""" + if backend == "rust_core": + return True + if backend == "cython_fallback": + return False + raise ValueError(f"Unknown backend: {backend}") + + +def _random_points(seed, count, scale): + """Create a deterministic two-dimensional point cloud.""" + rng = np.random.RandomState(seed) + return rng.rand(count) * scale, rng.rand(count) * scale + + +def _smooth_field(x, y): + """Evaluate a smooth deterministic field on matching coordinates.""" + return np.sin(x / 10.0) + np.cos(y / 15.0) + + +def _make_variogram_data(seed, count, scale=100.0): + """Create positions, values, and lag bins for variogram estimation.""" + x, y = _random_points(seed, count, scale) + field = _smooth_field(x, y) + bins = np.linspace(0.0, scale * 0.6, 16) + return (x, y), field, bins + + +def _make_krige_data(seed, cond_count, target_count, scale=50.0): + """Create conditioning and target data for kriging workflows.""" + rng = np.random.RandomState(seed) + cond_x = rng.rand(cond_count) * scale + cond_y = rng.rand(cond_count) * scale + cond_val = _smooth_field(cond_x, cond_y) + target_pos = ( + rng.rand(target_count) * scale, + rng.rand(target_count) * scale, + ) + return (cond_x, cond_y), cond_val, target_pos + + +def _check_backend_available(backend): + """Skip Rust benchmark rows when ``gstools_core`` is unavailable.""" + if backend == "rust_core" and not gs.config._GSTOOLS_CORE_AVAIL: + raise NotImplementedError("gstools_core is not available") + + +class VariogramBenchmarks: + """Measure variogram estimation cases for each backend, case, and thread count.""" + + params = [BACKENDS, VARIOGRAM_CASES, THREAD_COUNTS] + param_names = ["backend", "case", "threads"] + + def setup_cache(self): + """Create deterministic variogram inputs once per ASV environment.""" + return { + "full_900": _make_variogram_data(20220501, 900), + "sampled_5000_to_1500": _make_variogram_data(20220502, 5000), + "sampled_15000_to_4500": _make_variogram_data(20220503, 15000), + } + + def setup(self, data, backend, case, threads): + """Skip backend rows that cannot run in the current environment.""" + _check_backend_available(backend) + + def time_variogram_estimate(self, data, backend, case, threads): + """Run one variogram estimation case.""" + pos, field, bins = data[case] + kwargs = {} + if case == "sampled_5000_to_1500": + kwargs = {"sampling_size": 1500, "sampling_seed": 20220504} + if case == "sampled_15000_to_4500": + kwargs = {"sampling_size": 4500, "sampling_seed": 20220505} + with gstools_backend(_use_core(backend), threads): + gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + **kwargs, + ) + + def peakmem_variogram_estimate(self, data, backend, case, threads): + """Measure peak memory for variogram estimation case.""" + pos, field, bins = data[case] + kwargs = {} + if case == "sampled_5000_to_1500": + kwargs = {"sampling_size": 1500, "sampling_seed": 20220504} + if case == "sampled_15000_to_4500": + kwargs = {"sampling_size": 4500, "sampling_seed": 20220505} + with gstools_backend(_use_core(backend), threads): + gs.vario_estimate( + pos, + field, + bins, + mesh_type="unstructured", + return_counts=True, + **kwargs, + ) + + +class KrigingBenchmarks: + """Measure global kriging cases for each backend, case, and thread count.""" + + params = [BACKENDS, KRIGE_CASES, THREAD_COUNTS] + param_names = ["backend", "case", "threads"] + + def setup_cache(self): + """Create deterministic kriging inputs once per ASV environment.""" + return { + "small_30x500": _make_krige_data(20220506, 30, 500), + "large_120x2000": _make_krige_data(20220507, 120, 2000), + "extra_large_360x6000": _make_krige_data(20220508, 360, 6000), + } + + def setup(self, data, backend, case, threads): + """Skip backend rows that cannot run in the current environment.""" + _check_backend_available(backend) + + def time_global_krige(self, data, backend, case, threads): + """Run one global kriging case.""" + cond_pos, cond_val, target_pos = data[case] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + with gstools_backend(_use_core(backend), threads): + krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) + + def peakmem_global_krige(self, data, backend, case, threads): + """Measure peak memory for global kriging case.""" + cond_pos, cond_val, target_pos = data[case] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + with gstools_backend(_use_core(backend), threads): + krige( + target_pos, + mesh_type="unstructured", + return_var=True, + store=False, + ) + + +class RandomFieldBenchmarks: + """Measure SRF and CondSRF generation cases for each backend, case, and thread count.""" + + params = [BACKENDS, FIELD_CASES, THREAD_COUNTS] + param_names = ["backend", "case", "threads"] + + def setup_cache(self): + """Create deterministic random-field inputs once per ASV environment.""" + return { + "unstructured_pos": _random_points(20220509, 2000, 100.0), + "structured_pos": ( + np.linspace(0.0, 100.0, 64), + np.linspace(0.0, 100.0, 64), + ), + "condsrf": _make_krige_data(20220510, 40, 1000), + } + + def setup(self, data, backend, case, threads): + """Skip backend rows that cannot run in the current environment.""" + _check_backend_available(backend) + + def time_field_generation(self, data, backend, case, threads): + """Run one SRF or CondSRF case by label.""" + with gstools_backend(_use_core(backend), threads): + if case == "srf_unstructured_randmeth": + self._run_srf_unstructured(data) + elif case == "srf_structured_randmeth": + self._run_srf_structured(data) + elif case == "srf_structured_fourier": + self._run_srf_fourier(data) + elif case == "condsrf_unstructured": + self._run_condsrf(data) + else: + raise ValueError(f"Unknown field benchmark case: {case}") + + def peakmem_field_generation(self, data, backend, case, threads): + """Measure peak memory for one SRF or CondSRF case.""" + with gstools_backend(_use_core(backend), threads): + if case == "srf_unstructured_randmeth": + self._run_srf_unstructured(data) + elif case == "srf_structured_randmeth": + self._run_srf_structured(data) + elif case == "srf_structured_fourier": + self._run_srf_fourier(data) + elif case == "condsrf_unstructured": + self._run_condsrf(data) + else: + raise ValueError(f"Unknown field benchmark case: {case}") + + def _run_srf_unstructured(self, data): + """Run unstructured SRF generation with the RandMeth generator.""" + model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) + srf = gs.SRF(model, mean=1.0, seed=20220508, mode_no=512) + return srf(data["unstructured_pos"], mesh_type="unstructured") + + def _run_srf_structured(self, data): + """Run structured SRF generation with the RandMeth generator.""" + model = gs.Exponential(dim=2, var=2.0, len_scale=8.0) + srf = gs.SRF(model, mean=1.0, seed=20220509, mode_no=512) + return srf(data["structured_pos"], mesh_type="structured") + + def _run_srf_fourier(self, data): + """Run structured SRF generation with the Fourier generator.""" + model = gs.Gaussian(dim=2, var=2.0, len_scale=30.0) + srf = gs.SRF( + model, + generator="Fourier", + period=[100.0, 100.0], + mode_no=[32, 32], + seed=20220510, + ) + return srf(data["structured_pos"], mesh_type="structured") + + def _run_condsrf(self, data): + """Run conditioned random-field generation on unstructured targets.""" + cond_pos, cond_val, target_pos = data["condsrf"] + model = gs.Exponential(dim=2, var=1.5, len_scale=12.0, nugget=0.05) + krige = gs.Krige( + model, + cond_pos, + cond_val, + exact=False, + cond_err=0.05, + ) + cond_srf = gs.CondSRF(krige, seed=20220511, mode_no=512) + return cond_srf( + target_pos, + mesh_type="unstructured", + seed=20220512, + store=False, + krige_store=False, + ) diff --git a/benchmarks/tools/asv_speedup_summary.py b/benchmarks/tools/asv_speedup_summary.py new file mode 100644 index 00000000..db56f740 --- /dev/null +++ b/benchmarks/tools/asv_speedup_summary.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python +"""Summarize Rust-vs-Cython speedups from local ASV result files. + +The summary is optional. ASV itself remains the source of truth for benchmark +storage and visualization. The helper understands both current generated +benchmark method names, which encode case and thread labels, and older +parameterized ASV result files. + +Usage: + python benchmarks/tools/asv_speedup_summary.py + python benchmarks/tools/asv_speedup_summary.py --results-dir .asv/results + python benchmarks/tools/asv_speedup_summary.py --format markdown + python benchmarks/tools/asv_speedup_summary.py --format html --output report.html + python benchmarks/tools/asv_speedup_summary.py --include-legacy + +Speedup is calculated as: + cython_fallback_time / rust_core_time + +Values greater than 1.0 mean Rust was faster on the same machine, commit, +environment, benchmark, case, and thread-count combination. +""" + +from __future__ import annotations + +import argparse +import html +import itertools +import json +import math +import re +from datetime import datetime, timezone +from pathlib import Path + +BACKENDS = ("cython_fallback", "rust_core") +THREAD_PREFIX = "threads_" +BENCHMARK_PREFIXES = ( + "time_variogram_estimate", + "peakmem_variogram_estimate", + "time_global_krige", + "peakmem_global_krige", + "time_field_generation", + "peakmem_field_generation", +) +LEGACY_BENCHMARKS = { + "time_srf", + "peakmem_srf", + "time_variogram", + "peakmem_variogram", + "time_krige", + "peakmem_krige", +} + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--results-dir", + default=None, + action="append", + type=Path, + help=( + "Path to an ASV results directory. Can be supplied more than once." + ), + ) + parser.add_argument( + "--format", + choices=("table", "markdown", "html"), + default="table", + help="Output format.", + ) + parser.add_argument( + "--output", + type=Path, + help="Write Markdown or HTML output to this file instead of stdout.", + ) + parser.add_argument( + "--all", + action="store_true", + help="Include non-time benchmarks as ratios too.", + ) + parser.add_argument( + "--include-legacy", + action="store_true", + help="Include removed BackendBenchmarks rows from older saved results.", + ) + return parser.parse_args() + + +def iter_result_files(results_dir): + """Yield ASV result JSON files below ``results_dir``.""" + for path in sorted(results_dir.glob("**/*.json")): + if path.name in {"benchmarks.json", "machine.json"}: + continue + yield path + + +def load_json(path): + """Load one ASV result file, ignoring unreadable or invalid JSON files.""" + try: + with path.open(encoding="utf8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError): + return None + + +def result_entry(raw_result, result_columns): + """Normalize ASV result payloads across result schema versions.""" + if isinstance(raw_result, dict): + return raw_result + if isinstance(raw_result, list) and result_columns: + return dict(zip(result_columns, raw_result)) + return {"result": raw_result, "params": []} + + +def is_number(value): + """Return whether ``value`` is a numeric, non-NaN result.""" + return isinstance(value, (int, float)) and not math.isnan(value) + + +def flatten_values(values): + """Yield scalar values from nested ASV result lists.""" + if isinstance(values, list): + for value in values: + yield from flatten_values(value) + return + yield values + + +def parse_benchmark_name(name): + """Split a benchmark name into benchmark, case, and thread labels.""" + short_name = short_benchmark_name(name) + threads = "-" + match = re.search(r"_(threads_\d+)$", short_name) + if match: + threads = match.group(1) + short_name = short_name[: match.start()] + + for prefix in BENCHMARK_PREFIXES: + if short_name == prefix: + return prefix, "-", threads + marker = f"{prefix}_" + if short_name.startswith(marker): + return prefix, short_name[len(marker) :], threads + + return short_name, "-", threads + + +def backend_rows(benchmark, entry): + """Return backend/value rows for one ASV benchmark entry.""" + parsed_benchmark, parsed_case, parsed_threads = parse_benchmark_name( + benchmark + ) + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list) or not params: + return [] + + rows = [] + combinations = itertools.product(*params) + for combo, value in zip(combinations, flatten_values(result)): + if not is_number(value): + continue + combo_values = [str(item).strip("'\"") for item in combo] + backend = next( + (candidate for candidate in BACKENDS if candidate in combo_values), + None, + ) + if backend is None: + continue + case_values = [ + item + for item in combo_values + if item not in BACKENDS and not item.startswith(THREAD_PREFIX) + ] + threads = next( + (item for item in combo_values if item.startswith(THREAD_PREFIX)), + parsed_threads, + ) + rows.append( + { + "backend": backend, + "benchmark": parsed_benchmark, + "case": "/".join(case_values) if case_values else parsed_case, + "threads": threads, + "value": float(value), + } + ) + return rows + + +def short_benchmark_name(name): + """Return the method name from a fully qualified ASV benchmark name.""" + return name.rsplit(".", maxsplit=1)[-1] + + +def collect_speedups(results_dirs, include_all, include_legacy): + """Collect Rust-vs-Cython ratios from one or more ASV result folders.""" + rows = [] + for results_dir in results_dirs: + for path in iter_result_files(results_dir): + data = load_json(path) + if data is None: + continue + result_columns = data.get("result_columns", []) + commit = data.get("commit_hash", "unknown")[:8] + date = data.get("date", 0) + env_name = data.get("env_name", path.stem) + results = data.get("results", {}) + for benchmark, raw_result in results.items(): + benchmark_name, _, _ = parse_benchmark_name(benchmark) + if not include_legacy and benchmark_name in LEGACY_BENCHMARKS: + continue + if not include_all and ".time_" not in benchmark: + continue + by_case = {} + entry = result_entry(raw_result, result_columns) + for row in backend_rows(benchmark, entry): + key = (row["benchmark"], row["case"], row["threads"]) + by_case.setdefault(key, {})[row["backend"]] = row["value"] + for (name, case, threads), values in by_case.items(): + cython = values.get("cython_fallback") + rust = values.get("rust_core") + if ( + not is_number(cython) + or not is_number(rust) + or rust == 0 + ): + continue + rows.append( + { + "source": str(results_dir), + "commit": commit, + "date": date, + "env": env_name, + "benchmark": name, + "case": case, + "threads": threads, + "cython": cython, + "rust": rust, + "speedup": cython / rust, + } + ) + return rows + + +def sort_rows(rows): + """Sort rows for stable terminal and report output.""" + return sorted( + rows, + key=lambda row: ( + row["source"], + row["benchmark"], + row["case"], + thread_number(row["threads"]), + row["date"], + row["commit"], + ), + ) + + +def thread_number(label): + """Return the numeric part of a ``threads_N`` label.""" + if isinstance(label, str) and label.startswith(THREAD_PREFIX): + try: + return int(label[len(THREAD_PREFIX) :]) + except ValueError: + return -1 + return -1 + + +def latest_rows(rows): + """Keep the newest row for each source, env, benchmark, case, and thread.""" + latest = {} + for row in rows: + key = ( + row["source"], + row["env"], + row["benchmark"], + row["case"], + row["threads"], + ) + if key not in latest or row["date"] >= latest[key]["date"]: + latest[key] = row + return sort_rows(latest.values()) + + +def format_date(timestamp): + """Format an ASV millisecond timestamp as a UTC date.""" + if not timestamp: + return "-" + return datetime.fromtimestamp( + timestamp / 1000.0, + tz=timezone.utc, + ).strftime("%Y-%m-%d") + + +def print_table(rows): + """Print speedup rows as an aligned terminal table.""" + if not rows: + print("No matching Rust-vs-Cython ASV results found.") + return + + headers = [ + "source", + "commit", + "date", + "env", + "benchmark", + "case", + "threads", + "cython", + "rust", + "speedup", + ] + table = [ + [ + row["source"], + row["commit"], + format_date(row["date"]), + row["env"], + row["benchmark"], + row["case"], + row["threads"], + f"{row['cython']:.6g}", + f"{row['rust']:.6g}", + f"{row['speedup']:.3f}x", + ] + for row in rows + ] + widths = [ + max(len(str(item)) for item in column) + for column in zip(headers, *table) + ] + + def fmt(row): + """Format one aligned table row.""" + return " ".join( + str(item).ljust(width) for item, width in zip(row, widths) + ) + + print(fmt(headers)) + print(fmt(["-" * width for width in widths])) + for row in table: + print(fmt(row)) + + +def markdown_table(headers, rows): + """Render a simple Markdown table.""" + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append("| " + " | ".join(str(item) for item in row) + " |") + return "\n".join(lines) + + +def render_markdown(rows): + """Render the latest speedups and thread scaling as Markdown.""" + if not rows: + return "No matching Rust-vs-Cython ASV results found.\n" + + rows = latest_rows(rows) + lines = [ + "# GSTools ASV Backend Report", + "", + "Values greater than `1.0x` mean `rust_core` was faster than " + "`cython_fallback` for the same benchmark, case, commit, and thread " + "label.", + "", + ] + + for benchmark in sorted({row["benchmark"] for row in rows}): + benchmark_rows = [row for row in rows if row["benchmark"] == benchmark] + lines.extend([f"## {benchmark}", ""]) + table = [ + [ + row["source"], + row["commit"], + format_date(row["date"]), + row["case"], + row["threads"], + f"{row['cython']:.6g}", + f"{row['rust']:.6g}", + f"{row['speedup']:.3f}x", + ] + for row in benchmark_rows + ] + lines.extend( + [ + markdown_table( + [ + "source", + "commit", + "date", + "case", + "threads", + "cython", + "rust", + "speedup", + ], + table, + ), + "", + ] + ) + + scaling = thread_scaling_rows(rows) + if scaling: + lines.extend(["## Thread Scaling", ""]) + lines.extend( + [ + markdown_table( + [ + "source", + "commit", + "benchmark", + "case", + "backend", + "threads", + "time", + "vs min thread", + ], + scaling, + ), + "", + ] + ) + + return "\n".join(lines).rstrip() + "\n" + + +def thread_scaling_rows(rows): + """Build rows that compare each backend against its lowest thread count.""" + by_key = {} + for row in rows: + for backend, value in ( + ("cython_fallback", row["cython"]), + ("rust_core", row["rust"]), + ): + key = ( + row["source"], + row["commit"], + row["benchmark"], + row["case"], + backend, + ) + by_key.setdefault(key, {})[row["threads"]] = value + + table = [] + for key, values in sorted(by_key.items()): + if len(values) < 2: + continue + source, commit, benchmark, case, backend = key + base_thread = min(values, key=thread_number) + base_time = values[base_thread] + if not base_time: + continue + for threads in sorted(values, key=thread_number): + table.append( + [ + source, + commit, + benchmark, + case, + backend, + threads, + f"{values[threads]:.6g}", + f"{base_time / values[threads]:.3f}x", + ] + ) + return table + + +def render_html(rows): + """Render the Markdown report as a small standalone HTML page.""" + markdown = render_markdown(rows) + body_lines = [] + in_table = False + for line in markdown.splitlines(): + if line.startswith("# "): + body_lines.append(f"

{html.escape(line[2:])}

") + continue + if line.startswith("## "): + body_lines.append(f"

{html.escape(line[3:])}

") + continue + if line.startswith("| "): + cells = [cell.strip() for cell in line.strip("|").split("|")] + if all(cell == "---" for cell in cells): + continue + tag = "th" if not in_table else "td" + if tag == "th": + body_lines.append("") + in_table = True + body_lines.append( + "" + + "".join( + f"<{tag}>{html.escape(cell)}" for cell in cells + ) + + "" + ) + continue + if in_table: + body_lines.append("
") + in_table = False + if line: + body_lines.append(f"

{html.escape(line)}

") + if in_table: + body_lines.append("") + + return ( + "\n" + '\n' + "\n" + '\n' + "GSTools ASV Backend Report\n" + "\n" + "\n" + "\n" + "\n".join(body_lines) + "\n\n\n" + ) + + +def write_or_print(text, output): + """Write rendered output to a file or print it to stdout.""" + if output: + output.write_text(text, encoding="utf8") + print(f"Wrote {output}") + return + print(text, end="") + + +def main(): + """Run the ASV speedup summary command.""" + args = parse_args() + results_dirs = args.results_dir or [Path(".asv/results")] + rows = collect_speedups(results_dirs, args.all, args.include_legacy) + rows = sort_rows(rows) + if args.format == "table": + print_table(rows) + elif args.format == "markdown": + write_or_print(render_markdown(rows), args.output) + else: + write_or_print(render_html(rows), args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/tools/check_backend_parallel_ready.py b/benchmarks/tools/check_backend_parallel_ready.py new file mode 100644 index 00000000..db0c6fd3 --- /dev/null +++ b/benchmarks/tools/check_backend_parallel_ready.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +"""Check whether GSTools benchmark backends are ready for parallel runs. + +This is a fast CI probe. It verifies that GSTools-Cython accepts explicit +OpenMP thread counts and that the Rust backend can run a small workflow while +GSTools is configured with more than one thread. +""" + +from __future__ import annotations + +import argparse +import importlib +import sys + +import numpy as np + +MODULES = { + "variogram": "gstools_cython.variogram", + "field": "gstools_cython.field", + "krige": "gstools_cython.krige", +} +EXPLICIT_THREAD_COUNTS = (2, 4, 8) + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--threads", + default=2, + type=int, + help="Thread count to request for the Rust backend smoke workflow.", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-module Cython OpenMP thread details.", + ) + parser.add_argument( + "--cython-only", + action="store_true", + help="Only check GSTools-Cython OpenMP readiness.", + ) + parser.add_argument( + "--require-default-openmp", + action="store_true", + help=( + "Also fail when GSTools-Cython reports only one default thread. " + "CI does not use this because some runners default to one thread " + "while still accepting explicit OpenMP thread counts." + ), + ) + return parser.parse_args() + + +def package_version(package_name): + """Return an installed package version or a clear missing marker.""" + try: + package = importlib.import_module(package_name) + except ModuleNotFoundError: + return "not installed" + return getattr(package, "__version__", "unknown") + + +def check_module(label, module_name): + """Check default and explicit OpenMP thread counts for one module.""" + module = importlib.import_module(module_name) + default_threads = module.set_num_threads(None) + explicit = { + count: module.set_num_threads(count) + for count in EXPLICIT_THREAD_COUNTS + } + return label, default_threads, explicit + + +def check_cython_parallel(verbose=False, require_default_openmp=False): + """Validate GSTools-Cython OpenMP readiness across benchmark modules.""" + default_values = [] + for label, module_name in MODULES.items(): + try: + label, default_threads, explicit = check_module(label, module_name) + except ModuleNotFoundError as err: + print(f"Cython OpenMP readiness: FAIL. Missing module: {err.name}") + return False + default_values.append(default_threads) + if verbose: + explicit_text = ", ".join( + f"{request}->{actual}" for request, actual in explicit.items() + ) + print(f"{label} default None -> {default_threads}") + print(f"{label} explicit -> {explicit_text}") + mismatches = { + request: actual + for request, actual in explicit.items() + if actual != request + } + if mismatches: + mismatch_text = ", ".join( + f"{request}->{actual}" + for request, actual in mismatches.items() + ) + print( + "Cython OpenMP readiness: FAIL. " + f"{label} did not accept requested thread counts: " + f"{mismatch_text}." + ) + return False + + if require_default_openmp and min(default_values) <= 1: + print( + "Cython OpenMP readiness: FAIL. GSTools-Cython reports one " + "default thread." + ) + return False + + print("Cython OpenMP readiness: PASS") + return True + + +def check_rust_backend(threads): + """Run a small Rust-backed GSTools workflow with ``threads`` configured.""" + try: + import gstools as gs + except ModuleNotFoundError as err: + print(f"Rust backend readiness: FAIL. Missing module: {err.name}") + return False + + try: + importlib.import_module("gstools_core") + except ModuleNotFoundError as err: + print(f"Rust backend readiness: FAIL. Missing module: {err.name}") + return False + + if not gs.config._GSTOOLS_CORE_AVAIL: + print( + "Rust backend readiness: FAIL. GSTools did not detect gstools_core." + ) + return False + + previous = ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) + try: + gs.config._GSTOOLS_CORE_AVAIL = True + gs.config.USE_GSTOOLS_CORE = True + gs.config.NUM_THREADS = threads + + x = np.linspace(0.0, 10.0, 12) + y = np.linspace(0.0, 5.0, 12) + field = np.sin(x) + np.cos(y) + bins = np.linspace(0.0, 8.0, 6) + + # variogram + gs.vario_estimate( + (x, y), + field, + bins, + mesh_type="unstructured", + return_counts=True, + ) + + # field generation + model = gs.Gaussian(dim=2, var=1.0, len_scale=3.0) + srf = gs.SRF(model, seed=1) + srf((x, y)) + + # kriging + krige = gs.Krige( + model, + cond_pos=(x[:6], y[:6]), + cond_val=field[:6], + ) + krige((x[6:], y[6:])) + + finally: + ( + gs.config._GSTOOLS_CORE_AVAIL, + gs.config.USE_GSTOOLS_CORE, + gs.config.NUM_THREADS, + ) = previous + + print(f"Rust backend readiness: PASS with NUM_THREADS={threads}") + return True + + +def main(): + """Run the parallel backend readiness checks.""" + args = parse_args() + print(f"python: {sys.executable}") + print(f"gstools: {package_version('gstools')}") + print(f"gstools_cython: {package_version('gstools_cython')}") + print(f"gstools_core: {package_version('gstools_core')}") + + cython_ready = check_cython_parallel( + verbose=args.verbose, + require_default_openmp=args.require_default_openmp, + ) + rust_ready = True if args.cython_only else check_rust_backend(args.threads) + return 0 if cython_ready and rust_ready else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/configure_asv_machine.py b/benchmarks/tools/configure_asv_machine.py new file mode 100644 index 00000000..7778f9fe --- /dev/null +++ b/benchmarks/tools/configure_asv_machine.py @@ -0,0 +1,23 @@ +"""Store detected ASV hardware metadata under a stable machine name.""" + +import argparse + +from asv.machine import Machine, MachineCollection + + +def main(): + """Create an ASV machine profile containing the detected hardware.""" + parser = argparse.ArgumentParser() + parser.add_argument("machine_name") + args = parser.parse_args() + + machine_info = Machine.get_defaults() + machine_info["machine"] = args.machine_name + MachineCollection.save(args.machine_name, machine_info) + + for key in ("machine", "os", "arch", "cpu", "num_cpu", "ram"): + print(f"{key}: {machine_info[key]}") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/install_openmp_cython.py b/benchmarks/tools/install_openmp_cython.py new file mode 100644 index 00000000..39babb08 --- /dev/null +++ b/benchmarks/tools/install_openmp_cython.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python +"""Install GSTools-Cython with OpenMP inside an ASV environment. + +This helper is called by ``asv.openmp.conf.json`` after ASV has created a +conda environment with the common build requirements. It is kept for local or +manual OpenMP ASV runs. GitHub Actions uses a faster readiness check with the +conda-forge ``gstools-cython`` package instead of building ``gstools-cython`` +from source. This helper keeps the ASV config portable and handles +platform-specific compiler/OpenMP details here. +""" + +from __future__ import annotations + +import argparse +import os +import platform +import shutil +import stat +import subprocess +import sys +from pathlib import Path + + +def run(command, env=None, check=True): + """Run a subprocess command and echo it for ASV logs.""" + print("+ " + " ".join(str(part) for part in command), flush=True) + return subprocess.run(command, check=check, env=env) + + +def prepend_path(build_env, *paths): + """Prepend existing paths to a build environment's ``PATH``.""" + current = build_env.get("PATH", "") + existing = [str(path) for path in paths if path.exists()] + if existing: + build_env["PATH"] = os.pathsep.join(existing + [current]) + + +def first_match(directory, patterns): + """Return the first file matching one of the glob patterns.""" + for pattern in patterns: + matches = sorted( + path for path in directory.glob(pattern) if path.is_file() + ) + if matches: + return matches[0] + return None + + +def require_file(path, message): + """Return true when ``path`` exists, otherwise print ``message``.""" + if not path.exists(): + print(message, file=sys.stderr) + return False + return True + + +def conda_executable(): + """Find the conda executable used to amend ASV environments.""" + conda = os.environ.get("CONDA_EXE") or shutil.which("conda") + if conda is None: + print("No conda executable was found.", file=sys.stderr) + return conda + + +def conda_install(env_dir, *packages): + """Install conda-forge packages into an ASV environment directory.""" + conda = conda_executable() + if conda is None: + return False + run( + [ + conda, + "install", + "-y", + "-p", + str(env_dir), + "-c", + "conda-forge", + *packages, + ] + ) + return True + + +def ensure_macos_llvm_openmp(env_dir): + """Ensure that macOS ASV environments contain ``llvm-openmp``.""" + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + omp_header = include_dir / "omp.h" + omp_lib = lib_dir / "libomp.dylib" + + if omp_header.exists() and omp_lib.exists(): + return True + + if not conda_install(env_dir, "llvm-openmp"): + print( + "llvm-openmp is required on macOS but was not found in the ASV " + "environment, and it could not be installed with conda.", + file=sys.stderr, + ) + return False + + return require_file( + omp_header, + f"llvm-openmp install did not create expected header: {omp_header}", + ) and require_file( + omp_lib, + f"llvm-openmp install did not create expected library: {omp_lib}", + ) + + +def ensure_linux_libgomp(env_dir): + """Ensure that Linux ASV environments contain an OpenMP runtime.""" + lib_dir = env_dir / "lib" + if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: + return True + + if not conda_install(env_dir, "libgomp"): + print( + "libgomp is required on Linux but could not be installed with conda.", + file=sys.stderr, + ) + return False + + if first_match(lib_dir, ["libgomp.so*", "libomp.so*"]) is not None: + return True + + print( + f"libgomp install did not create an OpenMP runtime under {lib_dir}.", + file=sys.stderr, + ) + return False + + +def write_macos_wrapper(path, force_cxx=False): + """Write a clang wrapper that maps ``-fopenmp`` to Apple clang flags.""" + text = """#!/bin/bash +set -e +prefix="${GSTOOLS_OPENMP_PREFIX:-${CONDA_PREFIX:-}}" +name="$(basename "$0")" +if [[ "${GSTOOLS_FORCE_CXX:-0}" == "1" || "$name" == *++* ]]; then + real="${GSTOOLS_REAL_CXX:-/usr/bin/clang++}" +else + real="${GSTOOLS_REAL_CC:-/usr/bin/clang}" +fi +is_compile=0 +for arg in "$@"; do + [[ "$arg" == "-c" ]] && is_compile=1 +done +args=() +for arg in "$@"; do + if [[ "$arg" == "-fopenmp" ]]; then + if [[ "$is_compile" == "1" ]]; then + args+=("-Xpreprocessor" "-fopenmp" "-I${prefix}/include") + else + args+=("-L${prefix}/lib" "-lomp" "-Wl,-rpath,${prefix}/lib") + fi + else + args+=("$arg") + fi +done +exec "$real" "${args[@]}" +""" + if force_cxx: + text = """#!/bin/bash +GSTOOLS_FORCE_CXX=1 exec "$(dirname "$0")/gstools-asv-clang-openmp" "$@" +""" + path.write_text(text, encoding="utf8") + path.chmod( + path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) + + +def macos_build_env(env_dir): + """Build environment variables for macOS OpenMP compilation.""" + if not ensure_macos_llvm_openmp(env_dir): + return None + + bin_dir = env_dir / "bin" + include_dir = env_dir / "include" + lib_dir = env_dir / "lib" + cc_wrapper = bin_dir / "gstools-asv-clang-openmp" + cxx_wrapper = bin_dir / "gstools-asv-clang-openmp++" + write_macos_wrapper(cc_wrapper) + write_macos_wrapper(cxx_wrapper, force_cxx=True) + + build_env = os.environ.copy() + prepend_path(build_env, bin_dir) + build_env.update( + { + "GSTOOLS_BUILD_PARALLEL": "1", + "GSTOOLS_OPENMP_PREFIX": str(env_dir), + "CC": str(cc_wrapper), + "CXX": str(cxx_wrapper), + "CFLAGS": f"-I{include_dir}", + "LDFLAGS": f"-L{lib_dir}", + } + ) + return build_env + + +def linux_build_env(env_dir): + """Build environment variables for Linux OpenMP compilation.""" + if not ensure_linux_libgomp(env_dir): + return None + + bin_dir = env_dir / "bin" + cc = first_match( + bin_dir, + [ + "*-conda-linux-gnu-cc", + "*-conda-linux-gnu-gcc", + "gcc", + "cc", + ], + ) + cxx = first_match( + bin_dir, + [ + "*-conda-linux-gnu-c++", + "*-conda-linux-gnu-g++", + "g++", + "c++", + ], + ) + + build_env = os.environ.copy() + prepend_path(build_env, bin_dir) + build_env["GSTOOLS_BUILD_PARALLEL"] = "1" + if cc is not None: + build_env["CC"] = str(cc) + if cxx is not None: + build_env["CXX"] = str(cxx) + return build_env + + +def windows_build_env(env_dir): + """Build environment variables for native Windows OpenMP compilation.""" + build_env = os.environ.copy() + prepend_path( + build_env, + env_dir, + env_dir / "Scripts", + env_dir / "Library" / "bin", + ) + build_env["GSTOOLS_BUILD_PARALLEL"] = "1" + return build_env + + +def install_gstools_cython(build_env): + """Reinstall GSTools-Cython from source with OpenMP enabled.""" + run( + [ + sys.executable, + "-m", + "pip", + "uninstall", + "-y", + "gstools-cython", + "gstools_cython", + ], + env=build_env, + check=False, + ) + run( + [ + sys.executable, + "-m", + "pip", + "install", + "--no-build-isolation", + "--no-cache-dir", + "--force-reinstall", + "--no-binary=gstools-cython", + "--no-deps", + "gstools-cython", + ], + env=build_env, + ) + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "env_dir", + nargs="?", + default=sys.prefix, + help="ASV or active conda environment directory. Defaults to sys.prefix.", + ) + return parser.parse_args() + + +def main(): + """Prepare the platform build environment and install GSTools-Cython.""" + args = parse_args() + env_dir = Path(args.env_dir).resolve() + system = platform.system() + print(f"Preparing GSTools-Cython OpenMP build for {system}") + + if system == "Darwin": + build_env = macos_build_env(env_dir) + elif system == "Linux": + build_env = linux_build_env(env_dir) + elif system == "Windows": + build_env = windows_build_env(env_dir) + else: + print( + f"Unsupported platform for OpenMP ASV build: {system}", + file=sys.stderr, + ) + return 2 + + if build_env is None: + return 2 + + install_gstools_cython(build_env) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/install_pyproject_extras.py b/benchmarks/tools/install_pyproject_extras.py new file mode 100644 index 00000000..c6fae483 --- /dev/null +++ b/benchmarks/tools/install_pyproject_extras.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +"""Install selected optional dependencies declared in ``pyproject.toml``. + +This installs only the requirements listed under the requested +``[project.optional-dependencies]`` extras. It does not install the project or +its regular dependencies, so ASV can keep control of its environment matrix. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover + import tomli as tomllib + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "extras", + nargs="+", + help="Optional dependency extras to install.", + ) + parser.add_argument( + "--pyproject", + type=Path, + default=Path("pyproject.toml"), + help="Path to pyproject.toml. Default: ./pyproject.toml", + ) + return parser.parse_args() + + +def extra_requirements(pyproject, extras): + """Return deduplicated requirements for named pyproject extras.""" + with pyproject.open("rb") as handle: + config = tomllib.load(handle) + + optional = config.get("project", {}).get("optional-dependencies", {}) + missing = sorted(set(extras) - set(optional)) + if missing: + available = ", ".join(sorted(optional)) or "none" + names = ", ".join(missing) + raise ValueError( + f"Unknown optional dependency extras: {names}. " + f"Available extras: {available}" + ) + + requirements = [] + for extra in extras: + for requirement in optional[extra]: + if requirement not in requirements: + requirements.append(requirement) + return requirements + + +def install_requirements(requirements): + """Install requirements with pip in the current Python environment.""" + if not requirements: + print("No requirements to install.", flush=True) + return + command = [sys.executable, "-m", "pip", "install", *requirements] + print("+ " + " ".join(command), flush=True) + subprocess.run(command, check=True) + + +def main(): + """Install the requested optional dependency extras.""" + args = parse_args() + try: + requirements = extra_requirements(args.pyproject, args.extras) + except (FileNotFoundError, ValueError) as err: + print(err, file=sys.stderr) + return 2 + try: + install_requirements(requirements) + except subprocess.CalledProcessError as err: + print(f"pip install failed with exit code {err.returncode}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tools/plot_case_backend_comparison.py b/benchmarks/tools/plot_case_backend_comparison.py new file mode 100644 index 00000000..493147f1 --- /dev/null +++ b/benchmarks/tools/plot_case_backend_comparison.py @@ -0,0 +1,1634 @@ +#!/usr/bin/env python +"""Build a case/backend comparison report from ASV result JSON files. + +The default ASV browser remains useful for commit-history plots. This helper +creates a focused static HTML view for questions ASV does not show cleanly: +case-by-case backend comparisons and OpenMP thread comparisons. + +The script has no plotting dependency. It reads ASV result JSON and writes a +self-contained HTML file with a small inline SVG renderer. + +Usage: + python benchmarks/tools/plot_case_backend_comparison.py + python benchmarks/tools/plot_case_backend_comparison.py --results-dir .asv/results + python benchmarks/tools/plot_case_backend_comparison.py --results-dir .asv-openmp/results + python benchmarks/tools/plot_case_backend_comparison.py --metric time + python benchmarks/tools/plot_case_backend_comparison.py --metric memory + python benchmarks/tools/plot_case_backend_comparison.py --benchmark krige + python benchmarks/tools/plot_case_backend_comparison.py --output comparison.html +""" + +from __future__ import annotations + +import argparse +import base64 +import html +import itertools +import json +import math +import os +import re +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +BACKENDS = ("cython_fallback", "rust_core") +THREAD_PREFIX = "threads_" +BENCHMARK_PREFIXES = ( + "time_variogram_estimate", + "peakmem_variogram_estimate", + "time_global_krige", + "peakmem_global_krige", + "time_field_generation", + "peakmem_field_generation", +) +BENCHMARK_ALIASES = { + "variogram": "variogram", + "vario": "variogram", + "krige": "krige", + "kriging": "krige", + "field": "field", + "random_field": "field", + "random-field": "field", + "srf": "field", + "condsrf": "field", +} + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--results-dir", + action="append", + type=Path, + default=None, + help=( + "Path to an ASV results directory. Can be supplied more than " + "once. Default: .asv/results if present, otherwise " + ".asv-openmp/results." + ), + ) + parser.add_argument( + "--benchmark", + choices=sorted(BENCHMARK_ALIASES), + default=None, + help="Filter to one benchmark family. Default: include all families.", + ) + parser.add_argument( + "--metric", + choices=("all", "time", "memory"), + default="all", + help="Metric to include. Default: all, which includes time and memory.", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help=( + "HTML output path. Default: case-backend-comparison.html next to " + "the first selected results directory." + ), + ) + parser.add_argument( + "--max-commits", + type=positive_int, + default=None, + help=( + "Include only the newest N commits in the report. Raw ASV result " + "files are not changed. Default: include all commits." + ), + ) + parser.add_argument( + "--table", + action="store_true", + help="Print a compact terminal table instead of writing HTML.", + ) + return parser.parse_args() + + +def positive_int(value): + """Parse a positive integer command-line value.""" + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least 1") + return parsed + + +def default_results_dirs(): + """Return the default ASV results directory list.""" + for candidate in (Path(".asv/results"), Path(".asv-openmp/results")): + if candidate.exists(): + return [candidate] + raise FileNotFoundError( + "Could not find .asv/results or .asv-openmp/results" + ) + + +def existing_results_dirs(paths): + """Validate and return result directories.""" + if paths is None: + return default_results_dirs() + missing = [path for path in paths if not path.exists()] + if missing: + names = ", ".join(str(path) for path in missing) + raise FileNotFoundError(f"ASV results directory not found: {names}") + return paths + + +def iter_result_files(results_dir): + """Yield ASV result JSON files below ``results_dir``.""" + for path in sorted(results_dir.glob("**/*.json")): + if path.name in {"benchmarks.json", "machine.json"}: + continue + yield path + + +def get_git_tags(): + """Return a mapping from full commit hash to tag name, version-sorted.""" + try: + tag_out = subprocess.run( + ["git", "tag", "-l", "--sort=version:refname"], + capture_output=True, text=True, check=True, + ).stdout.strip() + if not tag_out: + return {} + result = {} + for tag in tag_out.splitlines(): + tag = tag.strip() + if not tag: + continue + rev = subprocess.run( + ["git", "rev-parse", f"{tag}^{{}}"], + capture_output=True, text=True, + ) + if rev.returncode == 0: + result[rev.stdout.strip()] = tag + return result + except (subprocess.SubprocessError, FileNotFoundError): + return {} + + +def load_json(path): + """Load one ASV JSON file, ignoring invalid files.""" + try: + with path.open(encoding="utf8") as handle: + return json.load(handle) + except (OSError, json.JSONDecodeError): + return None + + +def result_entry(raw_result, result_columns): + """Normalize ASV result payloads across schema versions.""" + if isinstance(raw_result, dict): + return raw_result + if isinstance(raw_result, list) and result_columns: + return dict(zip(result_columns, raw_result)) + return {"result": raw_result, "params": []} + + +def flatten_values(values): + """Yield scalar values from nested ASV result lists.""" + if isinstance(values, list): + for value in values: + yield from flatten_values(value) + return + yield values + + +def is_number(value): + """Return whether ``value`` is a numeric, non-NaN result.""" + return isinstance(value, (int, float)) and not math.isnan(value) + + +def short_benchmark_name(name): + """Return the method name from a fully qualified ASV benchmark name.""" + return name.rsplit(".", maxsplit=1)[-1] + + +def metric_from_name(name): + """Return the metric represented by one benchmark name.""" + short_name = short_benchmark_name(name) + if short_name.startswith("peakmem_"): + return "memory" + if short_name.startswith("time_"): + return "time" + return None + + +def family_from_name(name): + """Return the benchmark family label represented by one benchmark name.""" + short_name = short_benchmark_name(name) + if "variogram" in short_name: + return "variogram" + if "krig" in short_name: + return "krige" + if "field" in short_name: + return "field" + return None + + +def parse_benchmark_name(name): + """Split a benchmark name into base benchmark, case, and thread labels. + + Current OpenMP results may encode case and thread into generated method + names, while baseline results may keep them as ASV parameters. This parser + extracts the encoded part when present and leaves parameterized results for + ``result_rows`` to fill in. + """ + short_name = short_benchmark_name(name) + threads = None + match = re.search(r"_(threads_\d+)$", short_name) + if match: + threads = match.group(1) + short_name = short_name[: match.start()] + + for prefix in BENCHMARK_PREFIXES: + if short_name == prefix: + return prefix, None, threads + marker = f"{prefix}_" + if short_name.startswith(marker): + return prefix, short_name[len(marker) :], threads + + return short_name, None, threads + + +def normalize_thread(value): + """Normalize ASV thread values to a ``threads_N`` label.""" + if value is None: + return "threads_1" + text = str(value).strip("'\"") + if text.startswith(THREAD_PREFIX): + return text + if text.isdigit(): + return f"{THREAD_PREFIX}{text}" + return text + + +def is_thread_value(value): + """Return whether a parameter value looks like a thread count.""" + text = str(value).strip("'\"") + return text.startswith(THREAD_PREFIX) or text.isdigit() + + +def clean_param_value(value): + """Convert an ASV repr-like parameter value into display text.""" + return str(value).strip("'\"") + + +def result_rows(benchmark, entry): + """Return normalized rows for one ASV benchmark result entry.""" + result = entry.get("result") + params = entry.get("params") or [] + if not isinstance(result, list): + return [] + + base_name, encoded_case, encoded_threads = parse_benchmark_name(benchmark) + rows = [] + values = list(flatten_values(result)) + combinations = itertools.product(*params) if params else [()] + + for combo, value in zip(combinations, values): + if not is_number(value): + continue + combo_values = [clean_param_value(item) for item in combo] + backend = next( + (candidate for candidate in BACKENDS if candidate in combo_values), + None, + ) + if backend is None: + continue + + case_values = [ + item + for item in combo_values + if item not in BACKENDS and not is_thread_value(item) + ] + thread = next( + ( + normalize_thread(item) + for item in combo_values + if is_thread_value(item) + ), + normalize_thread(encoded_threads), + ) + rows.append( + { + "benchmark": base_name, + "family": family_from_name(base_name), + "metric": metric_from_name(base_name), + "case": "/".join(case_values) if case_values else encoded_case, + "threads": thread, + "backend": backend, + "value": float(value), + } + ) + return rows + + +def collect_rows(results_dirs, benchmark_filter=None, metric_filter="all", tag_map=None): + """Collect normalized benchmark rows from ASV result folders.""" + rows = [] + benchmark_filter = ( + BENCHMARK_ALIASES[benchmark_filter] if benchmark_filter else None + ) + for results_dir in results_dirs: + for path in iter_result_files(results_dir): + data = load_json(path) + if data is None: + continue + result_columns = data.get("result_columns", []) + commit_hash = data.get("commit_hash", "unknown") + commit = commit_hash[:8] + date = data.get("date", 0) + env_name = data.get("env_name", path.stem) + params = data.get("params", {}) + python = data.get("python") or params.get("python", "-") + machine_data = load_json(path.parent / "machine.json") or {} + machine = { + key: machine_data.get(key, params.get(key, "-")) + for key in ( + "machine", + "os", + "arch", + "cpu", + "num_cpu", + "ram", + ) + } + for benchmark, raw_result in data.get("results", {}).items(): + entry = result_entry(raw_result, result_columns) + for row in result_rows(benchmark, entry): + if ( + not row["metric"] + or not row["family"] + or not row["case"] + ): + continue + if benchmark_filter and row["family"] != benchmark_filter: + continue + if ( + metric_filter != "all" + and row["metric"] != metric_filter + ): + continue + row.update( + { + "source": str(results_dir), + "commit": commit, + "commit_hash": commit_hash, + "tag": (tag_map or {}).get(commit_hash, ""), + "date": date, + "date_label": format_date(date), + "env": env_name, + "python": python, + **machine, + } + ) + rows.append(row) + return sort_rows(rows) + + +def limit_recent_commits(rows, max_commits=None): + """Return rows for only the newest requested commits.""" + if max_commits is None: + return rows + ordered_commits = sorted( + {(row["date"], row["commit_hash"]) for row in rows}, + reverse=True, + ) + included = { + commit_hash for _, commit_hash in ordered_commits[:max_commits] + } + return [row for row in rows if row["commit_hash"] in included] + + +def sort_rows(rows): + """Sort rows for stable reports.""" + return sorted( + rows, + key=lambda row: ( + row["source"], + row["metric"], + row["family"], + row["benchmark"], + row["case"], + thread_number(row["threads"]), + row["date"], + row["commit"], + row["backend"], + ), + ) + + +def thread_number(label): + """Return the numeric part of a ``threads_N`` label.""" + if isinstance(label, str) and label.startswith(THREAD_PREFIX): + try: + return int(label[len(THREAD_PREFIX) :]) + except ValueError: + return -1 + return -1 + + +def format_date(timestamp): + """Format an ASV millisecond timestamp as a UTC date.""" + if not timestamp: + return "-" + return datetime.fromtimestamp( + timestamp / 1000.0, + tz=timezone.utc, + ).strftime("%Y-%m-%d") + + +def default_output_path(results_dirs): + """Return the default HTML report path.""" + return results_dirs[0].parent / "case-backend-comparison.html" + + +def print_table(rows): + """Print normalized rows as a compact terminal table.""" + if not rows: + print("No matching ASV backend comparison rows found.") + return + headers = [ + "source", + "commit", + "date", + "metric", + "family", + "benchmark", + "case", + "threads", + "backend", + "value", + ] + table = [ + [ + row["source"], + row["commit"], + row["date_label"], + row["metric"], + row["family"], + row["benchmark"], + row["case"], + row["threads"], + row["backend"], + f"{row['value']:.6g}", + ] + for row in rows + ] + widths = [ + max(len(str(item)) for item in column) + for column in zip(headers, *table) + ] + + def fmt(row): + return " ".join( + str(item).ljust(width) for item, width in zip(row, widths) + ) + + print(fmt(headers)) + print(fmt(["-" * width for width in widths])) + for row in table: + print(fmt(row)) + + +def gstools_logo_markup(): + """Return a compact embedded GSTools logo for the HTML report.""" + logo_path = ( + Path(__file__).resolve().parents[2] + / "docs" + / "source" + / "pics" + / "gstools_150.png" + ) + try: + encoded = base64.b64encode(logo_path.read_bytes()).decode("ascii") + except OSError: + return '' + return ( + '' + ) + + +def format_ram(value): + """Format ASV RAM for the machine summary. + + ASV 0.6+ stores RAM in bytes; older versions stored the raw /proc/meminfo + kB value. Any amount below 1 GiB expressed as bytes is implausible for a + benchmark machine, so treat it as kB and convert. + """ + try: + amount = int(value) + except (TypeError, ValueError): + return str(value) + if amount < 1024 ** 3: + amount *= 1024 + return f"{amount / (1024**3):.1f} GiB" + + +def machine_summary_markup(rows): + """Return report cards for the distinct benchmark machines.""" + machines = {} + for row in rows: + key = tuple( + str(row.get(field, "-")) + for field in ( + "machine", + "os", + "arch", + "cpu", + "num_cpu", + "ram", + "python", + ) + ) + machines[key] = { + "machine": key[0], + "os": key[1], + "arch": key[2], + "cpu": key[3], + "num_cpu": key[4], + "ram": format_ram(key[5]), + "python": key[6], + } + + cards = [] + for machine in machines.values(): + title = html.escape(machine["machine"]) + details = html.escape( + f'{machine["os"]} · {machine["arch"]} · {machine["cpu"]} · ' + f'{machine["num_cpu"]} CPUs · {machine["ram"]} · ' + f'Python {machine["python"]}' + ) + cards.append( + '
' + f"{title}{details}" + "
" + ) + return "".join(cards) + + +def render_html(rows): + """Render a self-contained interactive HTML report.""" + payload = json.dumps(rows, separators=(",", ":")).replace(" + + + + +GSTools Two-Point Statistics Backend Comparison + + + + +
+
+
+
ASV benchmark report
+

GSTools Two-Point Statistics Backend Comparison

+

+ Cython and Rust backend results grouped side by side for selected + two-point statistics cases, thread counts, commits, and metrics. +

+
+ Complete native ASV history + + Powered by Airspeed Velocity (ASV) + +
+
+ __LOGO_MARKUP__ +
+
__MACHINE_MARKUP__
+
+
+
+
+ Metric +
+
+
+ Family +
+
+
+ View +
+
+
+ Case +
+
+
+ Threads +
+
+
+ + + + + + +
+ + +
+
+
+ Backend +
+
+
+
+
+
Benchmark visualization
+
Lower values are better
+
+
+
+
+
+ + + +""" + return ( + template.replace("__PAYLOAD__", payload) + .replace("__LOGO_MARKUP__", gstools_logo_markup()) + .replace("__MACHINE_MARKUP__", machine_summary_markup(rows)) + ) + + +def main(): + """Run the report generator.""" + args = parse_args() + results_dirs = existing_results_dirs(args.results_dir) + tag_map = get_git_tags() + rows = collect_rows( + results_dirs, + benchmark_filter=args.benchmark, + metric_filter=args.metric, + tag_map=tag_map, + ) + rows = limit_recent_commits(rows, args.max_commits) + if args.table: + try: + print_table(rows) + except BrokenPipeError: + with open(os.devnull, "w", encoding="utf8") as devnull: + os.dup2(devnull.fileno(), 1) + return + return + if not rows: + print("No matching ASV backend comparison rows found.") + return + + output = args.output or default_output_path(results_dirs) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(render_html(rows), encoding="utf8") + print(f"Wrote {len(rows)} rows to {output}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/tools/profile_benchmark_workflows.py b/benchmarks/tools/profile_benchmark_workflows.py new file mode 100644 index 00000000..6995e631 --- /dev/null +++ b/benchmarks/tools/profile_benchmark_workflows.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +"""Profile representative GSTools benchmark workflows with cProfile. + +This is a quick measurement helper. ASV remains the source of truth for saved +benchmark results, while this script identifies the top cumulative Python call +sites for the current checkout before making algorithmic changes. + +Usage: + cd /path/to/MPS-Tools/GSTools + ASV_ENV="$(ls -td .asv/env/* | head -n 1)" + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py --list + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py \ + --case variogram-sampled + "$ASV_ENV/bin/python" benchmarks/tools/profile_benchmark_workflows.py \ + --case krige-large \ + --backend rust_core --threads threads_1 --limit 30 +""" + +from __future__ import annotations + +import argparse +import cProfile +import os +import pstats +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "src")) + + +CASES = { + "variogram-full": ( + "VariogramBenchmarks", + "time_variogram_estimate", + "full_900", + ), + "variogram-sampled": ( + "VariogramBenchmarks", + "time_variogram_estimate", + "sampled_5000_to_1500", + ), + "variogram-extra-large": ( + "VariogramBenchmarks", + "time_variogram_estimate", + "sampled_15000_to_4500", + ), + "krige-small": ( + "KrigingBenchmarks", + "time_global_krige", + "small_30x500", + ), + "krige-large": ( + "KrigingBenchmarks", + "time_global_krige", + "large_120x2000", + ), + "krige-extra-large": ( + "KrigingBenchmarks", + "time_global_krige", + "extra_large_360x6000", + ), + "srf-unstructured": ( + "RandomFieldBenchmarks", + "time_field_generation", + "srf_unstructured_randmeth", + ), + "srf-structured": ( + "RandomFieldBenchmarks", + "time_field_generation", + "srf_structured_randmeth", + ), + "srf-fourier": ( + "RandomFieldBenchmarks", + "time_field_generation", + "srf_structured_fourier", + ), + "condsrf": ( + "RandomFieldBenchmarks", + "time_field_generation", + "condsrf_unstructured", + ), +} + +THREAD_COUNTS = ( + "threads_1", + "threads_2", + "threads_4", + "threads_8", +) + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--case", + default="all", + choices=["all", *CASES], + help="Workflow to profile. Defaults to all workflows.", + ) + parser.add_argument( + "--repeat", + default=1, + type=int, + help="Number of times to run each selected workflow.", + ) + parser.add_argument( + "--limit", + default=25, + type=int, + help="Number of cProfile rows to print per workflow.", + ) + parser.add_argument( + "--sort", + default="cumtime", + choices=["cumtime", "tottime", "calls"], + help="pstats sort key.", + ) + parser.add_argument( + "--backend", + default="rust_core", + choices=["cython_fallback", "rust_core"], + help="Backend label to force while profiling.", + ) + parser.add_argument( + "--threads", + default="threads_1", + choices=THREAD_COUNTS, + help="GSTools thread count label.", + ) + parser.add_argument( + "--list", + action="store_true", + help="List available workflow cases and exit.", + ) + return parser.parse_args() + + +def iter_selected(case): + """Yield the requested workflow case definitions.""" + if case == "all": + yield from CASES.items() + return + yield case, CASES[case] + + +def load_suite_class(class_name, threads): + """Import a benchmark class after selecting generated thread labels.""" + os.environ["GSTOOLS_BENCHMARK_THREADS"] = threads[len("threads_") :] + try: + from benchmarks import benchmark_two_point_statistics + except ModuleNotFoundError as err: + print( + "Could not import GSTools benchmark dependencies. Activate the " + "GSTools benchmark environment, run this script with an ASV env " + "Python from .asv/env//bin/python, or install the project " + f"dependencies first. Original error: {err}", + file=sys.stderr, + ) + raise SystemExit(1) from err + return getattr(benchmark_two_point_statistics, class_name) + + +def run_case( + name, + class_name, + method_base_name, + case, + repeat, + limit, + sort, + backend, + threads, +): + """Profile one benchmark workflow case.""" + suite_cls = load_suite_class(class_name, threads) + suite = suite_cls() + data = suite.setup_cache() + threads_int = int(threads[len("threads_"):]) + method = getattr(suite, method_base_name) + + profiler = cProfile.Profile() + profiler.enable() + for _ in range(repeat): + method(data, backend, case, threads_int) + profiler.disable() + + print(f"\n== {name} [{backend}, {threads}] ==") + stats = pstats.Stats(profiler, stream=sys.stdout) + stats.strip_dirs().sort_stats(sort).print_stats(limit) + + +def main(): + """Run the cProfile helper.""" + args = parse_args() + if args.list: + for name in CASES: + print(name) + return + + for name, (suite_cls, method_name, params) in iter_selected(args.case): + run_case( + name, + suite_cls, + method_name, + params, + args.repeat, + args.limit, + args.sort, + args.backend, + args.threads, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/tools/render_benchmark_comment.py b/benchmarks/tools/render_benchmark_comment.py new file mode 100644 index 00000000..b6e7e3c7 --- /dev/null +++ b/benchmarks/tools/render_benchmark_comment.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +"""Render a GitHub PR comment body from an ASV benchmark comparison. + +Used by both the pull-request comparison workflow (for same-repo PRs) and the +workflow_run comment workflow (for cross-fork PRs) so the badge wording and +comment format stay in sync across both code paths. + +Usage: + python benchmarks/tools/render_benchmark_comment.py \\ + --base --head \\ + [--artifact-url ] \\ + [--comparison ] \\ + --output +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def parse_args(): + """Parse command-line options.""" + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--base", required=True, help="Base commit SHA.") + parser.add_argument("--head", required=True, help="Head commit SHA.") + parser.add_argument( + "--artifact-url", + default="", + help="URL to the full HTML report artifact.", + ) + parser.add_argument( + "--comparison", + type=Path, + default=None, + help="Path to the ASV comparison text file.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Write the rendered comment body to this file.", + ) + return parser.parse_args() + + +def render(base, head, artifact_url, comparison): + """Return the full PR comment body as a string.""" + base = base[:8] + head = head[:8] + report_link = ( + f" · [Full HTML Report ↗]({artifact_url})" if artifact_url else "" + ) + + lines = comparison.splitlines() + # ASV marks "+" (regressed) or "-" (improved) only when BOTH the + # ratio > 1.05 AND the Mann-Whitney U test agree. Lines with "~" + # exceeded the ratio threshold but were not statistically significant. + regressed = sum(1 for l in lines if l.startswith("+") and l[1:2] == " ") + improved = sum(1 for l in lines if l.startswith("-") and l[1:2] == " ") + note = "Mann-Whitney U · 5% threshold" + if regressed: + badge = f"⚠️ {regressed} benchmark(s) regressed · {note}" + elif improved: + badge = f"✅ {improved} benchmark(s) improved, none regressed · {note}" + else: + badge = f"✅ No significant changes detected · {note}" + + return ( + "\n" + "## OpenMP Benchmark Results\n\n" + f"**Base:** `{base}` → **Head:** `{head}`{report_link}\n\n" + f"{badge}\n\n" + "
\nFull ASV comparison\n\n" + f"```\n{comparison}\n```\n\n" + "
\n" + ) + + +def main(): + """Render and write the PR comment body.""" + args = parse_args() + + if args.comparison is not None: + try: + comparison = args.comparison.read_text(encoding="utf8") + except OSError as err: + print(f"Cannot read comparison file: {err}", file=sys.stderr) + return 1 + else: + comparison = "_Comparison output not available._" + + body = render(args.base, args.head, args.artifact_url, comparison) + args.output.write_text(body, encoding="utf8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index c51b702d..cdfcee20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ doc = [ ] plotting = ["matplotlib>=3.7", "pyvista>=0.40"] rust = ["gstools_core>=1.3.0"] +benchmark = ["asv"] test = ["pytest-cov>=3"] lint = ["ruff"] @@ -83,6 +84,7 @@ fallback_version = "0.0.0.dev0" [tool.hatch.version.raw-options] local_scheme = "no-local-version" +tag_regex = "^v?(?P[0-9]+\\.[0-9]+.*)$" [tool.hatch.build.hooks.vcs] version-file = "src/gstools/_version.py" diff --git a/tests/test_install_pyproject_extras.py b/tests/test_install_pyproject_extras.py new file mode 100644 index 00000000..25fb4237 --- /dev/null +++ b/tests/test_install_pyproject_extras.py @@ -0,0 +1,77 @@ +"""Tests for the benchmark pyproject extras installer.""" + +import importlib.util +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +SCRIPT = ( + Path(__file__).parents[1] + / "benchmarks" + / "tools" + / "install_pyproject_extras.py" +) +SPEC = importlib.util.spec_from_file_location( + "install_pyproject_extras", SCRIPT +) +INSTALLER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INSTALLER) + + +class TestInstallPyprojectExtras(unittest.TestCase): + """Test loading and installing pyproject optional dependencies.""" + + def setUp(self): + """Create a small pyproject fixture.""" + self.temp_dir = tempfile.TemporaryDirectory() + self.pyproject = Path(self.temp_dir.name) / "pyproject.toml" + self.pyproject.write_text( + """ +[project] +name = "example" + +[project.optional-dependencies] +first = ["one>=1", "shared>=1"] +second = ["shared>=1", "two>=2"] +""".strip(), + encoding="utf8", + ) + + def tearDown(self): + """Remove the temporary pyproject fixture.""" + self.temp_dir.cleanup() + + def test_extra_requirements_preserves_order_and_deduplicates(self): + """Requirements follow requested-extra order without duplicates.""" + requirements = INSTALLER.extra_requirements( + self.pyproject, + ["first", "second"], + ) + + self.assertEqual(requirements, ["one>=1", "shared>=1", "two>=2"]) + + def test_extra_requirements_rejects_unknown_extra(self): + """Unknown extras fail with a useful list of available extras.""" + with self.assertRaisesRegex( + ValueError, + "Available extras: first, second", + ): + INSTALLER.extra_requirements(self.pyproject, ["missing"]) + + def test_install_requirements_uses_current_python(self): + """Installation calls pip through the helper's Python executable.""" + with mock.patch.object(INSTALLER.subprocess, "run") as run: + INSTALLER.install_requirements(["one>=1", "two>=2"]) + + run.assert_called_once_with( + [ + INSTALLER.sys.executable, + "-m", + "pip", + "install", + "one>=1", + "two>=2", + ], + check=True, + ) diff --git a/tests/test_plot_case_backend_comparison.py b/tests/test_plot_case_backend_comparison.py new file mode 100644 index 00000000..f53d47d4 --- /dev/null +++ b/tests/test_plot_case_backend_comparison.py @@ -0,0 +1,128 @@ +"""Tests for the custom ASV case/backend comparison report.""" + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SCRIPT = ( + Path(__file__).parents[1] + / "benchmarks" + / "tools" + / "plot_case_backend_comparison.py" +) +SPEC = importlib.util.spec_from_file_location( + "plot_case_backend_comparison", + SCRIPT, +) +REPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(REPORT) + + +def result_row(commit_hash, date): + """Create one normalized row for report-only tests.""" + return { + "source": "results", + "commit": commit_hash[:8], + "commit_hash": commit_hash, + "date": date, + "date_label": REPORT.format_date(date), + "env": "conda-py3.14", + "python": "3.14", + "machine": "github-actions-ubuntu", + "os": "Ubuntu 24.04", + "arch": "x86_64", + "cpu": "Test CPU", + "num_cpu": "4", + "ram": str(8 * 1024**3), + "benchmark": "time_global_krige", + "family": "krige", + "metric": "time", + "case": "small_30x500", + "threads": "threads_1", + "backend": "rust_core", + "value": 0.1, + } + + +class TestPlotCaseBackendComparison(unittest.TestCase): + """Test ASV result normalization and report rendering.""" + + def test_collect_rows_includes_machine_metadata(self): + """Adjacent ASV machine metadata is retained for the report.""" + with tempfile.TemporaryDirectory() as temp_dir: + results_dir = Path(temp_dir) + machine_dir = results_dir / "github-actions-ubuntu" + machine_dir.mkdir() + machine = { + "machine": "github-actions-ubuntu", + "os": "Ubuntu 24.04", + "arch": "x86_64", + "cpu": "Test CPU", + "num_cpu": "4", + "ram": str(8 * 1024**3), + } + result = { + "commit_hash": "a" * 40, + "date": 1_700_000_000_000, + "env_name": "conda-py3.14", + "python": "3.14", + "params": {"machine": "github-actions-ubuntu"}, + "results": { + ( + "benchmark_two_point_statistics.KrigingBenchmarks." + "time_global_krige_small_30x500_threads_1" + ): { + "result": [0.2, 0.1], + "params": [["cython_fallback", "rust_core"]], + } + }, + } + (machine_dir / "machine.json").write_text( + json.dumps(machine), + encoding="utf8", + ) + (machine_dir / "result.json").write_text( + json.dumps(result), + encoding="utf8", + ) + + rows = REPORT.collect_rows([results_dir]) + + self.assertEqual( + {row["machine"] for row in rows}, {"github-actions-ubuntu"} + ) + self.assertEqual({row["python"] for row in rows}, {"3.14"}) + self.assertEqual({row["cpu"] for row in rows}, {"Test CPU"}) + + def test_limit_recent_commits_keeps_only_newest_commits(self): + """The custom report can remain bounded without deleting results.""" + rows = [ + result_row("a" * 40, 1_700_000_000_000), + result_row("b" * 40, 1_800_000_000_000), + result_row("c" * 40, 1_900_000_000_000), + ] + + limited = REPORT.limit_recent_commits(rows, 2) + + self.assertEqual( + {row["commit_hash"] for row in limited}, + {"b" * 40, "c" * 40}, + ) + + def test_render_html_shows_machine_and_asv_links(self): + """The report identifies its machine and links to complete ASV data.""" + rendered = REPORT.render_html( + [result_row("a" * 40, 1_700_000_000_000)] + ) + + self.assertIn("github-actions-ubuntu", rendered) + self.assertIn("Ubuntu 24.04", rendered) + self.assertIn("Python 3.14", rendered) + self.assertIn('href="asv/"', rendered) + self.assertIn("Powered by Airspeed Velocity (ASV)", rendered) + + +if __name__ == "__main__": + unittest.main()