diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7afd82f..ba320c5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -32,6 +32,11 @@ on: required: true default: false type: boolean + scaling_matrix: + description: Run the 100/500/1000/3000 adaptive scaling curve and dimension sweeps + required: true + default: true + type: boolean permissions: contents: read @@ -84,3 +89,39 @@ jobs: path: benchmark-result.json if-no-files-found: error retention-days: 30 + + scaling-matrix: + name: Scaling curve (100/500/1000/3000) + if: ${{ inputs.scaling_matrix }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + BENCHMARK_MODULES: ${{ inputs.modules }} + BENCHMARK_REPEATS: ${{ inputs.repeats }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Install uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" + python-version: "3.11" + enable-cache: false + - run: uv sync --locked --dev --no-editable + - name: Run scaling matrix + run: >- + uv run --no-editable python benchmarks/run_scaling_matrix.py + --output benchmark-scaling-matrix.json + --tests 100,500,1000,3000 + --modules "$BENCHMARK_MODULES" + --workers 1,2,4,auto + --repeats "$BENCHMARK_REPEATS" + --warmups 1 + - name: Upload scaling matrix + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: benchmark-scaling-matrix-${{ github.run_id }} + path: benchmark-scaling-matrix.json + if-no-files-found: error + retention-days: 30 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fb7991..0f2ad36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,41 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- `testenix tune` and its `testenix benchmark` alias for fresh-process, counterbalanced, + history-disabled native worker-candidate measurements, native inventory/outcome validation, + JSON reports, bounded per-run process-tree deadlines, and explicit `--write` persistence of the + measured recommendation with project-source fingerprinting and optimistic byte-drift protection + immediately before an atomic configuration replacement. +- Explicit `--shard-modules` / `shard_modules = true` support for splitting eligible modules into + finer execution units. Conservative static checks retain module affinity for module/session + fixtures, visible global mutation, and import-time lifecycle hazards, including eager calls in + assignments, decorators, function defaults, and class construction expressions. +- Versioned trusted collection manifests generated with `testenix manifest ... --output FILE` and + consumed with `testenix run --manifest FILE` or `[tool.testenix].manifest`. Exact collection + roots, selected test files, statically discoverable project-local import dependencies, and SHA-256 + digests are verified before collection imports are bypassed; stale manifests fall back to + supervised collection, and parameter values are redacted. +- Synthetic scaling-matrix tooling for 100/500/1,000/3,000 tests and balanced, dominant, and + single-module layouts, plus a redaction-safe real-project benchmark harness. + +### Changed + +- `workers = "auto"` now selects adaptively from the actual execution-unit count, available CPUs, + duration-history coverage, predicted process-start cost, and makespan instead of equalling the + logical CPU count. Explicit integer worker settings remain unchanged. +- Benchmark documentation labels the historical `3.15×` result with its Testenix 0.1.0 version, + four-worker configuration, 100,000-test/16-module synthetic workload, and `--no-history` mode; + it is not presented as a current-version or real-project claim. + +### Fixed + +- Safe-module analysis now fails closed for imported fixture providers, nested mutable containers, + mutable class state, and all import-time calls including nested `sys.path` mutations. +- Benchmark and tuning timeouts use Windows Job Objects or POSIX root-session plus identity-tracked + descendant cleanup instead of allowing observed workers to contaminate later measurements. + ## [0.2.1] - 2026-07-21 ### Added diff --git a/README.md b/README.md index 7526043..33e4786 100644 --- a/README.md +++ b/README.md @@ -201,16 +201,55 @@ workers = "auto" retries = 0 paths = ["tests"] history = ".testenix/history.sqlite3" +# shard_modules = true +# manifest = ".testenix/collection.json" # json = "reports/testenix.json" # junit = "reports/junit.xml" ``` +`workers = "auto"` is adaptive: after selection, Testenix caps concurrency by the number of +independently schedulable units and the machine capacity. With reliable duration history it models +worker startup cost and predicted makespan; on a cold run it uses a conservative cap instead of +starting one worker for every logical CPU. Use an integer when CI must have a fixed resource limit, +or measure the project directly: + +```bash +# benchmark native candidates; "benchmark" is an alias for "tune" +testenix tune tests --warmups 1 --repeats 5 +testenix benchmark tests --candidates 1,2,4,8 --json reports/tuning.json + +# write the measured recommendation to [tool.testenix].workers +testenix tune --write +``` + +Tuning disables Testenix history for its samples, gives every complete suite run a 300-second +deadline by default, and validates that every native candidate keeps the same inventory and +outcomes. On timeout it terminates the coordinator boundary and tracked workers; override the +deadline with `--run-timeout SECONDS`. Windows uses a Job Object before process resume. POSIX always +signals the new root session and identity-checks observed detached descendants; a process that calls +`setsid()` and exits between creation and the first OS snapshot cannot be given an absolute +kernel-containment guarantee, so benchmark hostile suites inside a container. The automatic sweep +is resource-aware and conservatively stays +within 1/2/4 workers; larger experiments require explicit `--candidates`. `--write` refuses to +persist a workers-only recommendation if transient `--shard-modules`, `--no-shard-modules`, or +`--manifest` settings differ from the loaded project profile, or if the configuration file changes +while measurement is running. The writer compares the original bytes again immediately before its +atomic replacement; an immutable checkout is still required to exclude an external writer racing +that final filesystem operation. Tuning fingerprints project Python/TOML sources (including linked +source directories), explicit suite files, and the trusted manifest before the probe and rechecks +content plus file identity after every sample; any observed drift discards the result. Installed +packages, non-source data, and other runtime dependencies remain external inputs. +Its main result is the project-local worker recommendation. An +optional `--pytest-source PATH` comparison is orientation for that exact source/native pair, not a +publishable speed claim by itself; use the full benchmark contract below for public comparisons. + Command-line options override this table: ```text testenix run [PATH ...] [--workers auto|N] [--retries N] [--timeout SECONDS] [--tag TAG ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] [-q | -v | -vv] + [--shard-modules] [--manifest FILE] [--color auto|always|never | --no-color] [--show-skips] [--durations N] ``` @@ -226,13 +265,53 @@ The same runner is available as a typed library API: ```python from testenix import TestenixConfig, Status, run -result = run("tests", TestenixConfig(workers="auto", history_path=None)) -failed_ids = [test.test.id for test in result.tests if test.status is not Status.PASS] + +def main() -> None: + result = run("tests", TestenixConfig(workers="auto", history_path=None)) + failed_ids = [test.test.id for test in result.tests if test.status is not Status.PASS] + print(failed_ids) + + +if __name__ == "__main__": + main() ``` Async applications can `await testenix.run_async(...)`; cancellation terminates active collection and execution process trees before returning control to the caller. +### Optional sharding and trusted collection manifests + +Module affinity remains the safe default: ordinary tests from one module stay in one worker, so +module fixtures and observable module state are not split across processes. Projects with a large +module may explicitly request finer parallelism with `--shard-modules` or +`shard_modules = true`. Testenix statically rejects splitting when it finds module/session fixtures, +module-global or mutable class state, nested mutable containers, imported fixture providers, or +import-time lifecycle hazards such as eager calls in assignments, decorators, default arguments, +or class bases. Function-scoped fixtures defined in the collected module, including autouse +fixtures, can be recreated per test. Imported providers keep module affinity because their source +is outside the manifest fingerprint boundary. Static analysis cannot prove the absence of every +dynamic side effect, so this mode is opt-in and should first be exercised in CI. + +Normal execution imports each selected module during supervised collection and again in its +execution worker. A trusted collection manifest removes the collection-side import from subsequent +unchanged runs: + +```bash +testenix manifest tests --output .testenix/collection.json +testenix run tests --manifest .testenix/collection.json +``` + +The manifest contains every selected test file plus statically discoverable project-local Python +import dependencies, SHA-256 fingerprints, collected tests, collection issues, and sharding +decisions. Parameter names are retained, but their values are +redacted because collection-time case data can contain credentials or other environment-derived +secrets. Before trusting it, Testenix compares the requested +roots, exact file inventory, and every source digest. Malformed manifest JSON is rejected; a missing, +added, deleted, unreadable, or changed source marks a valid manifest stale and makes the runner fall +back to normal isolated collection. It never executes a stale inventory. This is an explicit trust +optimization, not an automatic cache: collection affected by environment variables or external +state must be regenerated when those inputs change. + ## Exit codes | Code | Meaning | @@ -288,11 +367,29 @@ can copy its own text or the complete project reference for an LLM. ## Benchmarks -In the checked-in M4 Pro/CPython 3.11 synthetic baseline, native `testenix run` completed 100,000 -empty tests across 16 modules in a median 8.04 seconds, compared with 25.33 seconds for pytest and -21.30 seconds for pytest-xdist. That is 3.15x the throughput of pytest for this specific workload, -not a universal performance promise. The result includes one warm-up and five measured, -counterbalanced rounds. It does not describe `testenix pytest`, which executes through pytest. +The checked-in `3.15x` result is a **historical Testenix 0.1.0 synthetic baseline**, not a Testenix +0.2.1 measurement. On one M4 Pro/CPython 3.11 machine, native `testenix run` completed 100,000 +generated no-op tests across 16 modules in a median 8.04 seconds, compared with 25.33 seconds for +pytest and 21.30 seconds for pytest-xdist. The run used four workers, `--no-history`, pytest-xdist +3.8's default `load` scheduler, one warm-up, and five counterbalanced measured rounds. It does not +describe `testenix pytest`, which executes through pytest, or promise the same ratio for a real +project. + +No Testenix 0.2.1 scaling result is checked in yet. The provenance-gated matrix harness covers +100/500/1,000/3,000 tests, balanced/dominant/single-module layouts, 1/2/4/auto workers, and both +default history and `--no-history`, plus explicit safe-module sharding. Until that clean five-round +matrix is published, the project does not claim a 0.2.1 speedup. A separate manifest harness can measure a private real project +without copying its code, stdout, environment values, or absolute paths into the result. It marks a +result publishable only when a current successful migration report proves exact per-test +inventory/outcome parity, complete source/generated Python-file inventories and hashes, and binds +canonical pytest and Testenix commands to the migrated source/output roots. Without that report, +the result is explicitly diagnostic-only. Publishable source roots must be directories so support +files such as `conftest.py` are covered. Manifest commands put all options before `--` and their +exact suite roots after it, preventing an option value from being mistaken for the measured target. +Duplicate performance flags are rejected, imported module files must belong to their claimed +distributions, and every timeout performs bounded worker-tree cleanup before the next sample. The +same Windows Job Object/POSIX identity-tracking boundary and POSIX `setsid()` caveat described for +`testenix tune` apply to these harnesses. The separate safe-migration benchmark used 3,000 tests across 64 modules and four native workers. After conversion, pytest no-op tests ran in 0.521 seconds versus 1.539 seconds through sequential @@ -304,6 +401,20 @@ and is not included in those recurring-run medians. These synthetic comparisons layout, test duration, and worker count; they do not establish a universal advantage over pytest, pytest-xdist, unittest, or real project suites. +```bash +# current-version synthetic matrix; refuses dirty/version-mismatched publication input +uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json + +# redaction-safe real-project comparison driven by an argument-array manifest +cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +# edit the copied manifest for the target project +uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json +``` + See the [generated results and chart](https://polishdataengineer.github.io/testenix/benchmarks/results/), [raw JSON](https://github.com/polishdataengineer/testenix/tree/main/benchmarks), [methodology](https://polishdataengineer.github.io/testenix/benchmarking/), and @@ -312,7 +423,9 @@ See the [generated results and chart](https://polishdataengineer.github.io/teste ## Current limitations - Parallel workers are isolated processes. Normal tests from one module stay together, so a - module-scoped fixture is not duplicated merely because `--workers` is greater than one. + module-scoped fixture is not duplicated merely because `--workers` is greater than one. Optional + `--shard-modules` splitting is fail-closed for statically visible hazards but remains an explicit + trust decision for dynamic module behavior. - Reproducible 1k/10k/100k comparisons, profiler findings, memory measurements, and the Rust/PyO3 decision are documented in [the performance analysis](https://polishdataengineer.github.io/testenix/performance-analysis/). @@ -321,7 +434,9 @@ See the [generated results and chart](https://polishdataengineer.github.io/teste blocking synchronous call killable on every supported platform, but module/session fixtures used by timed tests cannot be shared with neighbouring tests. - Collection imports user modules in a supervised process. A crash becomes a collection error and - a hung import has a bounded 30-second deadline instead of taking down the coordinator. + a hung import has a bounded 30-second deadline instead of taking down the coordinator. An + explicitly generated, hash-verified `--manifest` can bypass that collection import on unchanged + sources; stale manifests safely fall back to isolated collection. - Case values are reconstructed by rediscovering the module in the worker and do not need to be pickle-serializable. They must still be reproducible during module import; reports store a JSON-safe representation when a value itself is not serializable. @@ -329,17 +444,19 @@ See the [generated results and chart](https://polishdataengineer.github.io/teste to Python's main thread, such as installing signal handlers, are not supported inside those bodies in v0.2. Migrated pytest-asyncio wrappers are synchronous from Testenix's perspective and therefore share this restriction while creating a fresh event loop for each test or case. -- On Windows, a script that calls the programmatic `run()`/`run_async()` API must use the standard - `if __name__ == "__main__":` multiprocessing guard. The `testenix` CLI handles process startup - itself. +- On every supported platform, an executable script that calls the programmatic + `run()`/`run_async()` API must use the standard `if __name__ == "__main__":` multiprocessing + guard because Testenix deliberately uses supervised spawn workers. The `testenix` CLI handles + process startup itself. - The pytest bridge does not translate delegated outcomes into Testenix `RunResult`, JSON, history, retry, timeout, or scheduling semantics. Use pytest's own flags and installed plugins in that mode. - Native migration requires a green source baseline and a new output directory. Filesystem changes inside the project are isolated by disposable copies during validation, but network, database, cloud, and other external test side effects are not sandboxed. - A normal module is one scheduler-affinity unit. Converting 3,000 tests in one module does not - create 3,000 parallel units; spread independent tests across modules and measure the generated - suite before making a project-specific speed claim. + create 3,000 parallel units unless the project opts into `--shard-modules` and its static safety + checks pass. Spread independent tests across modules or validate opt-in sharding, then run + `testenix tune` before making a project-specific speed claim. - Test impact analysis, result caching, remote workers, and deep pytest-result aggregation are not part of version 0.2. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..77b8e17 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Reproducible benchmark harnesses kept outside the Testenix runtime package.""" diff --git a/benchmarks/process_control.py b/benchmarks/process_control.py new file mode 100644 index 0000000..878fa01 --- /dev/null +++ b/benchmarks/process_control.py @@ -0,0 +1,620 @@ +"""Bounded subprocess execution for benchmark harnesses. + +The benchmark commands start their own worker processes. ``subprocess.run`` +only terminates the direct child on a timeout, which can leave Testenix or +pytest-xdist workers alive. This module uses a Windows Job Object or an +isolated POSIX process group plus identity-tracked descendant snapshots, and +keeps every cleanup wait bounded. POSIX descendants which detach before the +first snapshot cannot be given the same kernel-level guarantee as a Job Object. +""" + +from __future__ import annotations + +import contextlib +import ctypes +import functools +import os +import signal +import subprocess +import sys +import threading +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +_CLEANUP_GRACE_SECONDS = 2.0 +_PROCESS_TABLE_TIMEOUT_SECONDS = 1.0 +_TRACKER_INTERVAL_SECONDS = 0.02 + +_ProcessIdentity = tuple[int, int] | str + + +@functools.lru_cache(maxsize=1) +def _darwin_child_lister() -> Any | None: + if sys.platform != "darwin": + return None + try: + library = ctypes.CDLL("/usr/lib/libproc.dylib") + function = library.proc_listchildpids + function.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.c_int) + function.restype = ctypes.c_int + return function + except (AttributeError, OSError): + return None + + +@functools.lru_cache(maxsize=1) +def _darwin_identity_probe() -> tuple[Any, type[ctypes.Structure]] | None: + if sys.platform != "darwin": + return None + try: + + class _BsdInfo(ctypes.Structure): + _fields_ = [ + ("flags", ctypes.c_uint32), + ("status", ctypes.c_uint32), + ("xstatus", ctypes.c_uint32), + ("pid", ctypes.c_uint32), + ("ppid", ctypes.c_uint32), + ("uid", ctypes.c_uint32), + ("gid", ctypes.c_uint32), + ("ruid", ctypes.c_uint32), + ("rgid", ctypes.c_uint32), + ("svuid", ctypes.c_uint32), + ("svgid", ctypes.c_uint32), + ("reserved", ctypes.c_uint32), + ("comm", ctypes.c_char * 16), + ("name", ctypes.c_char * 32), + ("nfiles", ctypes.c_uint32), + ("pgid", ctypes.c_uint32), + ("pjobc", ctypes.c_uint32), + ("tty_device", ctypes.c_uint32), + ("tty_pgid", ctypes.c_uint32), + ("nice", ctypes.c_int32), + ("start_seconds", ctypes.c_uint64), + ("start_microseconds", ctypes.c_uint64), + ] + + library = ctypes.CDLL("/usr/lib/libproc.dylib") + function = library.proc_pidinfo + function.argtypes = ( + ctypes.c_int, + ctypes.c_int, + ctypes.c_uint64, + ctypes.c_void_p, + ctypes.c_int, + ) + function.restype = ctypes.c_int + return function, _BsdInfo + except (AttributeError, OSError): + return None + + +def _process_identity(pid: int) -> _ProcessIdentity | None: + """Return a creation token so a recycled PID is never signalled.""" + + if sys.platform.startswith("linux"): + try: + stat_line = Path(f"/proc/{pid}/stat").read_text(encoding="ascii") + fields_after_name = stat_line.rsplit(")", 1)[1].split() + return fields_after_name[19] + except (IndexError, OSError): + return None + if sys.platform == "darwin": + probe = _darwin_identity_probe() + if probe is None: + return None + function, structure = probe + information = structure() + size = ctypes.sizeof(information) + if function(pid, 3, 0, ctypes.byref(information), size) != size: + return None + return int(information.start_seconds), int(information.start_microseconds) + try: + completed = subprocess.run( + ("ps", "-o", "lstart=", "-p", str(pid)), + capture_output=True, + text=True, + timeout=_PROCESS_TABLE_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + token = completed.stdout.strip() + return token if completed.returncode == 0 and token else None + + +def _posix_direct_children(pid: int) -> tuple[int, ...]: + if sys.platform.startswith("linux"): + children: set[int] = set() + try: + task_children = Path(f"/proc/{pid}/task").glob("*/children") + for child_file in task_children: + raw = child_file.read_text(encoding="ascii") + children.update(int(value) for value in raw.split()) + except (OSError, ValueError): + pass + return tuple(sorted(children)) + if sys.platform == "darwin": + function = _darwin_child_lister() + if function is not None: + values = (ctypes.c_int * 4096)() + count = function(pid, values, ctypes.sizeof(values)) + if count > 0: + return tuple(values[: min(count, len(values))]) + return () + # This fallback is used only on less common POSIX hosts. Linux and macOS + # use cheap native snapshots so tracking does not launch processes while a + # benchmark is being timed. + return tuple(_posix_descendants(pid)) + + +class _PosixTreeTracker: + """Remember descendants before a short-lived leader can orphan them.""" + + def __init__(self, root_pid: int) -> None: + self.root_pid = root_pid + self._root_identity = _process_identity(root_pid) + self._identities: dict[int, _ProcessIdentity] = {} + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + self._capture() + self._thread.start() + + def _capture(self) -> None: + with self._lock: + known = { + pid: identity + for pid, identity in self._identities.items() + if _process_identity(pid) == identity + } + root_is_current = ( + self._root_identity is not None + and _process_identity(self.root_pid) == self._root_identity + ) + pending = [*([self.root_pid] if root_is_current else []), *known] + visited: set[int] = set() + discovered: set[int] = set() + while pending: + parent = pending.pop() + if parent in visited: + continue + visited.add(parent) + for child in _posix_direct_children(parent): + if child <= 0 or child == os.getpid(): + continue + if child not in discovered: + discovered.add(child) + pending.append(child) + live = dict(known) + for pid in discovered: + identity = _process_identity(pid) + if identity is not None: + live[pid] = identity + with self._lock: + self._identities = live + + def _run(self) -> None: + while not self._stop.wait(_TRACKER_INTERVAL_SECONDS): + self._capture() + + def stop(self) -> dict[int, _ProcessIdentity]: + self._stop.set() + self._thread.join(timeout=_CLEANUP_GRACE_SECONDS) + self._capture() + with self._lock: + return { + pid: identity + for pid, identity in self._identities.items() + if _process_identity(pid) == identity + } + + +@dataclass(slots=True) +class _WindowsJob: + kernel32: Any + handle: Any + closed: bool = False + + def terminate(self, exit_code: int = 1) -> bool: + if self.closed: + return True + try: + return bool(self.kernel32.TerminateJobObject(self.handle, exit_code)) + except Exception: + return False + + def close(self) -> bool: + if self.closed: + return True + try: + closed = bool(self.kernel32.CloseHandle(self.handle)) + except Exception: + return False + if closed: + self.closed = True + return closed + + +def _windows_kill_job(process: subprocess.Popen[str]) -> _WindowsJob | None: + """Attach *process* to a kill-on-close Job Object when Windows permits it.""" + + if os.name != "nt": + return None + job: _WindowsJob | None = None + try: # pragma: no cover - exercised by the Windows CI matrix. + from ctypes import wintypes + + class _BasicLimitInformation(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_longlong), + ("PerJobUserTimeLimit", ctypes.c_longlong), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class _IoCounters(ctypes.Structure): + _fields_ = [ + ("ReadOperationCount", ctypes.c_ulonglong), + ("WriteOperationCount", ctypes.c_ulonglong), + ("OtherOperationCount", ctypes.c_ulonglong), + ("ReadTransferCount", ctypes.c_ulonglong), + ("WriteTransferCount", ctypes.c_ulonglong), + ("OtherTransferCount", ctypes.c_ulonglong), + ] + + class _ExtendedLimitInformation(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", _BasicLimitInformation), + ("IoInfo", _IoCounters), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + kernel32.CreateJobObjectW.argtypes = (ctypes.c_void_p, wintypes.LPCWSTR) + kernel32.CreateJobObjectW.restype = wintypes.HANDLE + kernel32.SetInformationJobObject.argtypes = ( + wintypes.HANDLE, + ctypes.c_int, + ctypes.c_void_p, + wintypes.DWORD, + ) + kernel32.SetInformationJobObject.restype = wintypes.BOOL + kernel32.AssignProcessToJobObject.argtypes = (wintypes.HANDLE, wintypes.HANDLE) + kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + kernel32.TerminateJobObject.argtypes = (wintypes.HANDLE, wintypes.UINT) + kernel32.TerminateJobObject.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) + kernel32.CloseHandle.restype = wintypes.BOOL + + handle = kernel32.CreateJobObjectW(None, None) + if not handle: + return None + job = _WindowsJob(kernel32, handle) + information = _ExtendedLimitInformation() + information.BasicLimitInformation.LimitFlags = 0x00002000 # KILL_ON_JOB_CLOSE + configured = kernel32.SetInformationJobObject( + handle, + 9, # JobObjectExtendedLimitInformation + ctypes.byref(information), + ctypes.sizeof(information), + ) + raw_process_handle = vars(process).get("_handle") + if raw_process_handle is None: + job.close() + return None + process_handle = wintypes.HANDLE(int(raw_process_handle)) + if not configured or not kernel32.AssignProcessToJobObject(handle, process_handle): + job.close() + return None + return job + except (AttributeError, OSError, TypeError, ValueError): + if job is not None: + job.terminate() + job.close() + return None + + +def _resume_windows_process(process: subprocess.Popen[str]) -> None: + """Resume a CREATE_SUSPENDED process only after it belongs to the Job Object.""" + + if os.name != "nt": + raise OSError("Windows process resume requested on a non-Windows host") + try: # pragma: no cover - exercised by the Windows CI matrix. + from ctypes import wintypes + + raw_process_handle = vars(process).get("_handle") + if raw_process_handle is None: + raise OSError("subprocess has no Windows process handle") + ntdll = ctypes.WinDLL("ntdll", use_last_error=True) + ntdll.NtResumeProcess.argtypes = (wintypes.HANDLE,) + ntdll.NtResumeProcess.restype = ctypes.c_long + status = ntdll.NtResumeProcess(wintypes.HANDLE(int(raw_process_handle))) + if status != 0: + raise OSError(f"NtResumeProcess failed with status 0x{status & 0xFFFFFFFF:08x}") + except (AttributeError, TypeError, ValueError) as error: + raise OSError(f"cannot resume contained Windows benchmark process: {error}") from error + + +def _posix_descendants(root_pid: int) -> set[int]: + """Snapshot descendants, including workers which created their own session.""" + + if os.name != "posix": + return set() + try: + completed = subprocess.run( + ("ps", "-axo", "pid=,ppid="), + capture_output=True, + text=True, + timeout=_PROCESS_TABLE_TIMEOUT_SECONDS, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return set() + if completed.returncode != 0: + return set() + + children: dict[int, set[int]] = {} + for line in completed.stdout.splitlines(): + fields = line.split() + if len(fields) != 2: + continue + try: + pid, parent = (int(field) for field in fields) + except ValueError: + continue + children.setdefault(parent, set()).add(pid) + + descendants: set[int] = set() + pending = list(children.get(root_pid, ())) + while pending: + pid = pending.pop() + if pid in descendants: + continue + descendants.add(pid) + pending.extend(children.get(pid, ())) + return descendants + + +def _posix_signal_tree( + root_pid: int, + descendants: Mapping[int, _ProcessIdentity], + signum: int, + *, + root_group_owned: bool, +) -> None: + own_group = os.getpgrp() + # ``start_new_session=True`` guarantees root_pid is the initial PGID. After + # the leader is reaped, include that group only when a live, identity-checked + # descendant still proves ownership; this avoids signalling a recycled PGID. + groups: set[int] = {root_pid} if root_group_owned else set() + for pid, identity in descendants.items(): + if _process_identity(pid) != identity: + continue + with contextlib.suppress(OSError): + group = os.getpgid(pid) + # Revalidate after resolving the PGID so a process which exited in + # between cannot redirect cleanup at a recycled PID. + if group != own_group and _process_identity(pid) == identity: + groups.add(group) + # Never signal the benchmark driver's own process group, even if a stale + # PGID was recycled or a platform returned an unexpected group for a PID. + groups.discard(own_group) + for group in groups: + with contextlib.suppress(OSError): + os.killpg(group, signum) + + +def _identity_snapshot(pids: set[int]) -> dict[int, _ProcessIdentity]: + identities: dict[int, _ProcessIdentity] = {} + for pid in pids: + identity = _process_identity(pid) + if identity is not None: + identities[pid] = identity + return identities + + +def _bounded_taskkill(pid: int) -> bool: + try: # pragma: no cover - exercised by the Windows CI matrix. + completed = subprocess.run( + ("taskkill", "/PID", str(pid), "/T", "/F"), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=_CLEANUP_GRACE_SECONDS, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return completed.returncode == 0 + + +def _cleanup_windows_tree( + process: subprocess.Popen[str], + windows_job: _WindowsJob | None, +) -> bool: + if windows_job is not None: + terminated = windows_job.terminate() + closed = windows_job.close() + fallback = False if terminated or closed else _bounded_taskkill(process.pid) + cleaned = terminated or closed or fallback + else: + cleaned = _bounded_taskkill(process.pid) + with contextlib.suppress(OSError, ValueError): + process.kill() + return cleaned + + +def _bounded_drain(process: subprocess.Popen[str]) -> tuple[str, str]: + try: + return process.communicate(timeout=_CLEANUP_GRACE_SECONDS) + except subprocess.TimeoutExpired as error: + with contextlib.suppress(OSError, ValueError): + process.kill() + with contextlib.suppress(subprocess.TimeoutExpired): + process.wait(timeout=_CLEANUP_GRACE_SECONDS) + for stream in (process.stdout, process.stderr): + if stream is not None: + with contextlib.suppress(OSError): + stream.close() + return _timeout_text(error.output), _timeout_text(error.stderr) + + +def _timeout_text(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode(errors="replace") + return value + + +def _terminate_process_tree( + process: subprocess.Popen[str], + windows_job: _WindowsJob | None, + *, + tracked_pids: Mapping[int, _ProcessIdentity] | None = None, +) -> tuple[str, str]: + if os.name == "nt": # pragma: no cover - exercised by the Windows CI matrix. + cleaned = _cleanup_windows_tree(process, windows_job) + output = _bounded_drain(process) + if not cleaned: + raise RuntimeError("could not verify cleanup of the Windows benchmark process tree") + return output + + descendants = dict(tracked_pids or {}) + descendants.update(_identity_snapshot(_posix_descendants(process.pid))) + _posix_signal_tree( + process.pid, + descendants, + signal.SIGTERM, + root_group_owned=process.returncode is None, + ) + communication_complete = False + try: + stdout, stderr = process.communicate(timeout=_CLEANUP_GRACE_SECONDS) + communication_complete = True + except subprocess.TimeoutExpired: + stdout = stderr = "" + # A Testenix worker calls setsid(), so killing only the coordinator's + # process group is insufficient. Re-snapshot while the leader still + # exists and signal every captured PID/session before the final drain. + descendants.update(_identity_snapshot(_posix_descendants(process.pid))) + _posix_signal_tree( + process.pid, + descendants, + signal.SIGKILL, + root_group_owned=process.returncode is None, + ) + if communication_complete: + return stdout, stderr + return _bounded_drain(process) + + +def run_bounded_process( + command: Sequence[str], + *, + cwd: str | Path, + env: Mapping[str, str], + timeout: float, +) -> subprocess.CompletedProcess[str]: + """Run a command with bounded, cross-platform process-tree cleanup. + + The POSIX process group is always cleaned. Session-detached descendants are + captured by an immediate snapshot and a lightweight identity-aware poller; + a process which both detaches and exits its leader before that first capture + remains an inherent best-effort edge without platform-specific containment. + """ + + options: dict[str, Any] + if os.name == "nt": + creation_flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200) + creation_flags |= getattr(subprocess, "CREATE_SUSPENDED", 0x00000004) + options = {"creationflags": creation_flags} + else: + options = {"start_new_session": True} + process = subprocess.Popen( + tuple(command), + cwd=cwd, + env=dict(env), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + **options, + ) + windows_job = _windows_kill_job(process) + try: + tracker = _PosixTreeTracker(process.pid) if os.name == "posix" else None + except BaseException: + _terminate_process_tree(process, windows_job) + raise + if os.name == "nt": + if windows_job is None: + with contextlib.suppress(OSError, ValueError): + process.kill() + _bounded_drain(process) + raise RuntimeError( + "cannot place Windows benchmark process in a kill-on-close Job Object" + ) + try: + _resume_windows_process(process) + except OSError as error: + cleaned = _cleanup_windows_tree(process, windows_job) + _bounded_drain(process) + if not cleaned: + raise RuntimeError( + "could not clean up the suspended Windows benchmark process" + ) from error + raise + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as error: + tracked_pids = tracker.stop() if tracker is not None else {} + stdout, stderr = _terminate_process_tree( + process, + windows_job, + tracked_pids=tracked_pids, + ) + raise subprocess.TimeoutExpired( + tuple(command), + timeout, + output=stdout, + stderr=stderr, + ) from error + except BaseException: + tracked_pids = tracker.stop() if tracker is not None else {} + _terminate_process_tree( + process, + windows_job, + tracked_pids=tracked_pids, + ) + raise + else: + if tracker is not None: + tracked_pids = tracker.stop() + _posix_signal_tree( + process.pid, + tracked_pids, + signal.SIGKILL, + root_group_owned=False, + ) + elif windows_job is not None: + if not _cleanup_windows_tree(process, windows_job): + raise RuntimeError("could not verify cleanup of the Windows benchmark process tree") + finally: + if tracker is not None: + tracker.stop() + if windows_job is not None: + windows_job.close() + return subprocess.CompletedProcess(tuple(command), process.returncode, stdout, stderr) + + +__all__ = ["run_bounded_process"] diff --git a/benchmarks/real_project_manifest.example.json b/benchmarks/real_project_manifest.example.json new file mode 100644 index 0000000..f540c8d --- /dev/null +++ b/benchmarks/real_project_manifest.example.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "name": "private-project", + "testenix_version": "0.2.1", + "expected_tests": 118, + "expected_passed": 118, + "warmups": 1, + "repeats": 5, + "timeout_seconds": 900, + "include_project_commit": false, + "migration_report": "reports/testenix-migration.json", + "fingerprints": { + "native": "tests_testenix", + "source": "tests" + }, + "environment": { + "NO_COLOR": "1", + "PYTEST_ADDOPTS": "", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "TERM": "dumb" + }, + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": [ + "{python}", + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + "--", + "tests" + ] + }, + { + "name": "testenix-1-worker", + "kind": "testenix", + "command": [ + "{python}", + "-m", + "testenix", + "run", + "--workers", + "1", + "--no-history", + "--", + "tests_testenix" + ] + }, + { + "name": "testenix-4-workers", + "kind": "testenix", + "command": [ + "{python}", + "-m", + "testenix", + "run", + "--workers", + "4", + "--no-history", + "--", + "tests_testenix" + ] + } + ] +} diff --git a/benchmarks/run_benchmark.py b/benchmarks/run_benchmark.py index 5a8eb1e..9bb06e7 100644 --- a/benchmarks/run_benchmark.py +++ b/benchmarks/run_benchmark.py @@ -6,9 +6,11 @@ import hashlib import importlib.metadata import json +import math import os import platform import re +import shlex import statistics import subprocess import sys @@ -17,7 +19,12 @@ from dataclasses import asdict, dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import Any, Literal + +if __package__: + from benchmarks.process_control import run_bounded_process +else: # direct ``python benchmarks/run_benchmark.py`` execution + from process_control import run_bounded_process ROOT = Path(__file__).resolve().parents[1] @@ -25,7 +32,11 @@ @dataclass(frozen=True, slots=True) class Measurement: command: str + argv: tuple[str, ...] samples: tuple[float, ...] + stdout_bytes: tuple[int, ...] + stderr_bytes: tuple[int, ...] + observed_workers: tuple[int | None, ...] @property def median(self) -> float: @@ -48,8 +59,8 @@ def stdev(self) -> float: return statistics.stdev(self.samples) if len(self.samples) > 1 else 0.0 -def _display_command(command: list[str]) -> str: - parts = [] +def _display_argv(command: list[str]) -> tuple[str, ...]: + parts: list[str] = [] for value in command: if value == sys.executable: parts.append("python") @@ -57,13 +68,53 @@ def _display_command(command: list[str]) -> str: parts.append("") else: parts.append(value) - return " ".join(parts) + return tuple(parts) + + +def _display_command(command: list[str]) -> str: + return shlex.join(_display_argv(command)) + + +ModuleLayout = Literal["balanced", "dominant", "single"] +HistoryMode = Literal["disabled", "default"] +WorkerRequest = int | Literal["auto"] +XdistStrategy = Literal["load", "loadfile", "loadscope", "worksteal"] +ShardingMode = Literal["disabled", "safe"] + + +def _module_indexes( + count: int, + module_count: int, + layout: ModuleLayout, + dominant_fraction: float, +) -> tuple[int, ...]: + if layout == "single": + return (0,) * count + if layout == "balanced": + return tuple(index % module_count for index in range(count)) + if module_count == 1: + return (0,) * count + + dominant_count = min(count, max(1, math.ceil(count * dominant_fraction))) + return tuple( + 0 if index < dominant_count else 1 + ((index - dominant_count) % (module_count - 1)) + for index in range(count) + ) -def _generate_suite(directory: Path, count: int, uneven: bool, module_count: int) -> None: +def _generate_suite( + directory: Path, + count: int, + uneven: bool, + module_count: int, + *, + layout: ModuleLayout, + dominant_fraction: float, +) -> tuple[int, ...]: modules = [["import time", ""] for _ in range(module_count)] - for index in range(count): - lines = modules[index % module_count] + assignments = _module_indexes(count, module_count, layout, dominant_fraction) + for index, module_index in enumerate(assignments): + lines = modules[module_index] lines.append(f"def test_{index:05d}():") if uneven and index % 100 == 0: # Scale the amount of work with suite size and deliberately skew a @@ -79,12 +130,17 @@ def _generate_suite(directory: Path, count: int, uneven: bool, module_count: int "\n".join(lines), encoding="utf-8", ) + return tuple(assignments.count(index) for index in range(module_count)) def _benchmark_environment() -> dict[str, str]: environment = os.environ.copy() environment["PYTHONHASHSEED"] = "0" + environment["PYTHONDONTWRITEBYTECODE"] = "1" + environment["PYTEST_ADDOPTS"] = "" environment["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + environment["NO_COLOR"] = "1" + environment["TERM"] = "dumb" source_root = str(ROOT / "src") existing_pythonpath = environment.get("PYTHONPATH") environment["PYTHONPATH"] = ( @@ -123,18 +179,33 @@ def _run_once( environment: dict[str, str], test_count: int, working_directory: Path, -) -> float: + timeout: float, +) -> tuple[float, int, int, int | None]: started = time.perf_counter() - completed = subprocess.run( + completed = run_bounded_process( command, - capture_output=True, - text=True, env=environment, cwd=working_directory, + timeout=timeout, ) elapsed = time.perf_counter() - started _validate_completed_run(name, command, completed, test_count) - return elapsed + worker_match = ( + re.search(r"(?m)^Testenix\s+\|.*\|\s+(\d+)\s+workers?\s*$", completed.stdout) + if name == "testenix" + else None + ) + if name == "testenix" and worker_match is None: + raise RuntimeError( + "Testenix benchmark output did not expose the actual worker count; " + "refusing to record ambiguous performance evidence" + ) + return ( + elapsed, + len(completed.stdout.encode("utf-8")), + len(completed.stderr.encode("utf-8")), + int(worker_match.group(1)) if worker_match is not None else None, + ) def _measure_commands( @@ -144,6 +215,7 @@ def _measure_commands( warmups: int, test_count: int, working_directory: Path, + timeout: float, ) -> tuple[dict[str, Measurement], tuple[tuple[str, ...], ...]]: """Measure in deterministic rotated rounds instead of framework-sized blocks.""" @@ -157,28 +229,42 @@ def _measure_commands( environment=environment, test_count=test_count, working_directory=working_directory, + timeout=timeout, ) samples: dict[str, list[float]] = {name: [] for name in names} + stdout_bytes: dict[str, list[int]] = {name: [] for name in names} + stderr_bytes: dict[str, list[int]] = {name: [] for name in names} + observed_workers: dict[str, list[int | None]] = {name: [] for name in names} orders: list[tuple[str, ...]] = [] for round_index in range(repeats): shift = round_index % len(names) order = (*names[shift:], *names[:shift]) orders.append(order) for name in order: - samples[name].append( - _run_once( - name, - commands[name], - environment=environment, - test_count=test_count, - working_directory=working_directory, - ) + elapsed, stdout_size, stderr_size, observed_worker_count = _run_once( + name, + commands[name], + environment=environment, + test_count=test_count, + working_directory=working_directory, + timeout=timeout, ) + samples[name].append(elapsed) + stdout_bytes[name].append(stdout_size) + stderr_bytes[name].append(stderr_size) + observed_workers[name].append(observed_worker_count) return ( { - name: Measurement(_display_command(commands[name]), tuple(samples[name])) + name: Measurement( + _display_command(commands[name]), + _display_argv(commands[name]), + tuple(samples[name]), + tuple(stdout_bytes[name]), + tuple(stderr_bytes[name]), + tuple(observed_workers[name]), + ) for name in names }, tuple(orders), @@ -194,6 +280,9 @@ def _measurement_dict(measurement: Measurement, *, test_count: int) -> dict[str, mean=measurement.mean, stdev=measurement.stdev, median_tests_per_second=test_count / measurement.median, + median_stdout_bytes=statistics.median(measurement.stdout_bytes), + median_stderr_bytes=statistics.median(measurement.stderr_bytes), + observed_workers=list(measurement.observed_workers), ) return data @@ -264,17 +353,53 @@ def run_benchmark( test_count: int, repeats: int, warmups: int, - workers: int, + workers: WorkerRequest, uneven: bool, module_count: int | None = None, + module_layout: ModuleLayout = "balanced", + dominant_fraction: float = 0.5, + history_mode: HistoryMode = "disabled", + xdist_strategy: XdistStrategy = "load", + sharding_mode: ShardingMode = "disabled", + timeout: float = 900.0, ) -> dict[str, Any]: + if test_count < 1 or repeats < 1 or warmups < 0: + raise ValueError("test_count/repeats must be positive and warmups cannot be negative") + if workers != "auto" and ( + isinstance(workers, bool) or not isinstance(workers, int) or workers < 1 + ): + raise ValueError("workers must be a positive integer or 'auto'") + if module_count is not None and module_count < 1: + raise ValueError("module_count must be positive") + if not isinstance(uneven, bool): + raise TypeError("uneven must be a boolean") + if module_layout not in {"balanced", "dominant", "single"}: + raise ValueError("module_layout must be balanced, dominant, or single") + if history_mode not in {"disabled", "default"}: + raise ValueError("history_mode must be disabled or default") + if xdist_strategy not in {"load", "loadfile", "loadscope", "worksteal"}: + raise ValueError("unsupported pytest-xdist strategy") + if sharding_mode not in {"disabled", "safe"}: + raise ValueError("sharding_mode must be disabled or safe") + if not 0.0 < dominant_fraction < 1.0: + raise ValueError("dominant_fraction must be greater than zero and less than one") + if not 0 < timeout <= 3600: + raise ValueError("timeout must be greater than zero and at most 3600 seconds") with tempfile.TemporaryDirectory(prefix="testenix-benchmark-") as temporary: suite = Path(temporary) + xdist_workers = (os.cpu_count() or 1) if workers == "auto" else workers resolved_module_count = min( test_count, - module_count if module_count is not None else max(1, workers * 4), + 1 if module_layout == "single" else (module_count if module_count is not None else 16), + ) + module_sizes = _generate_suite( + suite, + test_count, + uneven, + resolved_module_count, + layout=module_layout, + dominant_fraction=dominant_fraction, ) - _generate_suite(suite, test_count, uneven, resolved_module_count) commands = { "pytest": [ sys.executable, @@ -295,7 +420,9 @@ def run_benchmark( "-p", "no:cacheprovider", "-n", - str(workers), + str(xdist_workers), + "--dist", + xdist_strategy, str(suite), ], "testenix": [ @@ -306,7 +433,8 @@ def run_benchmark( str(suite), "--workers", str(workers), - "--no-history", + *(["--no-history"] if history_mode == "disabled" else []), + *(["--shard-modules"] if sharding_mode == "safe" else []), ], } measurements, execution_orders = _measure_commands( @@ -315,10 +443,11 @@ def run_benchmark( warmups=warmups, test_count=test_count, working_directory=suite, + timeout=timeout, ) return { - "schema_version": 1, + "schema_version": 2, "recorded_at": datetime.now(UTC).isoformat(), "provenance": _provenance(), "environment": { @@ -332,9 +461,20 @@ def run_benchmark( "repeats": repeats, "test_count": test_count, "test_modules": resolved_module_count, + "module_layout": module_layout, + "module_sizes": module_sizes, + "dominant_fraction": dominant_fraction if module_layout == "dominant" else None, "uneven": uneven, "warmups": warmups, "workers": workers, + "workers_requested": workers, + "xdist_workers": xdist_workers, + "history_mode": history_mode, + "history_cli": "--no-history" if history_mode == "disabled" else "default", + "sharding_mode": sharding_mode, + "sharding_cli": "--shard-modules" if sharding_mode == "safe" else None, + "xdist_strategy": xdist_strategy, + "timeout_seconds": timeout, "measured_execution_orders": execution_orders, }, "measurements": { @@ -349,26 +489,70 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--tests", type=int, default=1_000) parser.add_argument("--repeats", type=int, default=5) parser.add_argument("--warmups", type=int, default=1) - parser.add_argument("--workers", type=int, default=min(4, os.cpu_count() or 1)) + parser.add_argument("--workers", type=_worker_request, default=min(4, os.cpu_count() or 1)) parser.add_argument( "--modules", type=int, default=None, - help="generated module count (default: workers * 4)", + help="generated module count (default: 16; single layout always uses one)", + ) + parser.add_argument( + "--sharding-mode", + choices=("disabled", "safe"), + default="disabled", + help="enable Testenix's explicit safe-module sharding with --shard-modules", ) + parser.add_argument("--timeout", type=float, default=900.0) parser.add_argument("--uneven", action="store_true") + parser.add_argument( + "--module-layout", + choices=("balanced", "dominant", "single"), + default="balanced", + help="test-count distribution across generated modules", + ) + parser.add_argument( + "--dominant-fraction", + type=float, + default=0.5, + help="fraction of tests placed in module zero for --module-layout dominant", + ) + parser.add_argument( + "--history-mode", + choices=("disabled", "default"), + default="disabled", + help="disable history (historical baseline) or exercise the default history database", + ) + parser.add_argument( + "--xdist-strategy", + choices=("load", "loadfile", "loadscope", "worksteal"), + default="load", + ) parser.add_argument("--output", type=Path) return parser +def _worker_request(value: str) -> WorkerRequest: + if value == "auto": + return "auto" + try: + workers = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("workers must be a positive integer or 'auto'") from error + if workers < 1: + raise argparse.ArgumentTypeError("workers must be positive") + return workers + + def main() -> int: arguments = build_parser().parse_args() if arguments.tests < 1 or arguments.repeats < 1 or arguments.warmups < 0: raise SystemExit("tests/repeats must be positive and warmups cannot be negative") - if arguments.workers < 1: - raise SystemExit("workers must be positive") if arguments.modules is not None and arguments.modules < 1: raise SystemExit("modules must be positive") + if not 0.0 < arguments.dominant_fraction < 1.0: + raise SystemExit("dominant-fraction must be greater than zero and less than one") + if not 0 < arguments.timeout <= 3600: + raise SystemExit("timeout must be greater than zero and at most 3600 seconds") result = run_benchmark( test_count=arguments.tests, repeats=arguments.repeats, @@ -376,6 +560,12 @@ def main() -> int: workers=arguments.workers, uneven=arguments.uneven, module_count=arguments.modules, + module_layout=arguments.module_layout, + dominant_fraction=arguments.dominant_fraction, + history_mode=arguments.history_mode, + xdist_strategy=arguments.xdist_strategy, + sharding_mode=arguments.sharding_mode, + timeout=arguments.timeout, ) rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" if arguments.output is not None: diff --git a/benchmarks/run_migration_benchmark.py b/benchmarks/run_migration_benchmark.py index 7c71ba9..d77516d 100644 --- a/benchmarks/run_migration_benchmark.py +++ b/benchmarks/run_migration_benchmark.py @@ -16,7 +16,6 @@ import os import platform import re -import signal import statistics import subprocess import sys @@ -27,6 +26,11 @@ from pathlib import Path from typing import Any, Literal, cast +if __package__: + from benchmarks.process_control import run_bounded_process +else: # direct ``python benchmarks/run_migration_benchmark.py`` execution + from process_control import run_bounded_process + ROOT = Path(__file__).resolve().parents[1] COMMAND_TIMEOUT_SECONDS = 900.0 Framework = Literal["pytest", "unittest"] @@ -193,65 +197,31 @@ def _snapshot_digest(snapshot: dict[str, str]) -> str: return digest.hexdigest() -def _terminate_process_tree(process: subprocess.Popen[str]) -> None: - if process.poll() is not None: - return - if os.name == "nt": - subprocess.run( - ("taskkill", "/PID", str(process.pid), "/T", "/F"), - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - return - try: - os.killpg(process.pid, signal.SIGTERM) - process.wait(timeout=2.0) - except (OSError, subprocess.TimeoutExpired): - try: - os.killpg(process.pid, signal.SIGKILL) - except OSError: - process.kill() - - def _run_process( command: tuple[str, ...], *, project: Path, environment: dict[str, str], ) -> ProcessOutcome: - options: dict[str, Any] = {} - if os.name == "nt": - options["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) - else: - options["start_new_session"] = True started = time.perf_counter() - process = subprocess.Popen( - command, - cwd=project, - env=environment, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - **options, - ) try: - stdout, stderr = process.communicate(timeout=COMMAND_TIMEOUT_SECONDS) + completed = run_bounded_process( + command, + cwd=project, + env=environment, + timeout=COMMAND_TIMEOUT_SECONDS, + ) except subprocess.TimeoutExpired as error: - _terminate_process_tree(process) - stdout, stderr = process.communicate() + stdout = error.output or "" + stderr = error.stderr or "" raise RuntimeError( f"benchmark command timed out after {COMMAND_TIMEOUT_SECONDS:g}s: " f"{_display_command(command, project)}\nstdout:\n{stdout}\nstderr:\n{stderr}" ) from error - except BaseException: - _terminate_process_tree(process) - process.communicate() - raise return ProcessOutcome( - returncode=process.returncode, - stdout=stdout, - stderr=stderr, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, elapsed=time.perf_counter() - started, ) diff --git a/benchmarks/run_project_benchmark.py b/benchmarks/run_project_benchmark.py new file mode 100644 index 0000000..0a05cbb --- /dev/null +++ b/benchmarks/run_project_benchmark.py @@ -0,0 +1,1092 @@ +"""Benchmark a local real project from a redaction-safe JSON manifest. + +The harness never invokes a shell and never copies project source into its result. Commands are +argument arrays executed from ``--project``. Output records timing, aggregate output sizes, +content fingerprints, and Git provenance without stdout/stderr or environment values. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import shlex +import statistics +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal + +if __package__: + from benchmarks.process_control import run_bounded_process +else: # direct ``python benchmarks/run_project_benchmark.py`` execution + from process_control import run_bounded_process + +ROOT = Path(__file__).resolve().parents[1] +RunnerKind = Literal["pytest", "testenix"] + + +@dataclass(frozen=True, slots=True) +class Runner: + name: str + kind: RunnerKind + command: tuple[str, ...] + redact_arguments: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class Sample: + elapsed: float + stdout_bytes: int + stderr_bytes: int + observed_workers: int | None + + +def _read_manifest(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"cannot read benchmark manifest {path}: {error}") from error + if not isinstance(value, dict) or value.get("schema_version") != 1: + raise RuntimeError("manifest must be a schema_version 1 JSON object") + return value + + +def _positive_integer(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RuntimeError(f"manifest {name} must be a positive integer") + return value + + +def _non_negative_integer(value: Any, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise RuntimeError(f"manifest {name} must be a non-negative integer") + return value + + +def _runners(manifest: dict[str, Any]) -> tuple[Runner, ...]: + raw_runners = manifest.get("runners") + if not isinstance(raw_runners, list) or len(raw_runners) < 2: + raise RuntimeError("manifest runners must contain at least two runner objects") + runners: list[Runner] = [] + for raw in raw_runners: + if not isinstance(raw, dict): + raise RuntimeError("every manifest runner must be an object") + name = raw.get("name") + kind = raw.get("kind") + command = raw.get("command") + if not isinstance(name, str) or not name or not re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name): + raise RuntimeError("runner names must use lowercase letters, numbers, '-' or '_'") + if kind not in {"pytest", "testenix"}: + raise RuntimeError(f"runner {name}: kind must be pytest or testenix") + if ( + not isinstance(command, list) + or not command + or any(not isinstance(argument, str) or not argument for argument in command) + ): + raise RuntimeError(f"runner {name}: command must be a non-empty string array") + if command[0] != "{python}": + raise RuntimeError( + f"runner {name}: command must start with {{python}} so the recorded " + "interpreter and package versions match the executed command" + ) + rendered = tuple( + sys.executable if argument == "{python}" else argument for argument in command + ) + if any("{" in argument or "}" in argument for argument in rendered): + raise RuntimeError(f"runner {name}: only the exact {{python}} placeholder is supported") + required_prefix = ( + (sys.executable, "-m", "pytest") + if kind == "pytest" + else (sys.executable, "-m", "testenix", "run") + ) + if rendered[: len(required_prefix)] != required_prefix: + expected = " ".join(("{python}", *required_prefix[1:])) + raise RuntimeError( + f"runner {name}: {kind} benchmarks must use the canonical {expected!r} " + "entrypoint; kind labels cannot describe arbitrary scripts" + ) + redactions = raw.get("redact_arguments", []) + if ( + not isinstance(redactions, list) + or any(isinstance(index, bool) or not isinstance(index, int) for index in redactions) + or any(index < 0 or index >= len(rendered) for index in redactions) + ): + raise RuntimeError( + f"runner {name}: redact_arguments must contain valid argument indexes" + ) + runners.append(Runner(name, kind, rendered, tuple(sorted(set(redactions))))) + if len({runner.name for runner in runners}) != len(runners): + raise RuntimeError("runner names must be unique") + kinds = {runner.kind for runner in runners} + if kinds != {"pytest", "testenix"}: + raise RuntimeError("manifest runners must include at least one pytest and one testenix run") + return tuple(runners) + + +def _environment( + manifest: dict[str, Any], +) -> tuple[dict[str, str], tuple[str, ...], str]: + environment = os.environ.copy() + environment["PYTHONHASHSEED"] = "0" + environment["PYTHONDONTWRITEBYTECODE"] = "1" + raw = manifest.get("environment", {}) + if not isinstance(raw, dict) or any( + not isinstance(key, str) or not isinstance(value, str) for key, value in raw.items() + ): + raise RuntimeError("manifest environment must map strings to strings") + environment.update(raw) + fingerprint = hashlib.sha256( + json.dumps(sorted(environment.items()), separators=(",", ":")).encode("utf-8") + ).hexdigest() + return environment, tuple(sorted(raw)), fingerprint + + +def _package_runtime_identity( + project: Path, + environment: dict[str, str], + *, + module_name: str, + distribution_name: str, +) -> dict[str, Any]: + probe = """ +import hashlib +import importlib +import importlib.metadata +import json +import sys +from pathlib import Path + +module = importlib.import_module(sys.argv[1]) +module_file = Path(module.__file__).resolve() +distribution = importlib.metadata.distribution(sys.argv[2]) +digest = hashlib.sha256() +count = 0 +owned_paths = set() +for entry in sorted(distribution.files or (), key=lambda item: str(item)): + path = Path(distribution.locate_file(entry)).resolve() + if not path.is_file(): + continue + owned_paths.add(path) + data = path.read_bytes() + digest.update(str(entry).replace("\\\\", "/").encode()) + digest.update(b"\\0") + digest.update(hashlib.sha256(data).digest()) + count += 1 +print(json.dumps({ + "version": distribution.version, + "package_sha256": digest.hexdigest(), + "package_files": count, + "source_matches_distribution": module_file in owned_paths, +}, sort_keys=True)) +""" + completed = subprocess.run( + (sys.executable, "-c", probe, module_name, distribution_name), + cwd=project, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"cannot identify the {distribution_name} runtime used by benchmark commands\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + try: + identity = json.loads(completed.stdout.splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as error: + raise RuntimeError( + f"{distribution_name} runtime identity probe returned invalid output" + ) from error + valid = ( + isinstance(identity, dict) + and isinstance(identity.get("version"), str) + and isinstance(identity.get("package_sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", identity["package_sha256"]) is not None + and isinstance(identity.get("package_files"), int) + and identity["package_files"] > 0 + and isinstance(identity.get("source_matches_distribution"), bool) + ) + if not valid: + raise RuntimeError(f"{distribution_name} runtime identity probe returned an invalid schema") + return identity + + +def _testenix_runtime_identity(project: Path, environment: dict[str, str]) -> dict[str, Any]: + return _package_runtime_identity( + project, + environment, + module_name="testenix", + distribution_name="testenix", + ) + + +def _pytest_runtime_identity(project: Path, environment: dict[str, str]) -> dict[str, Any]: + return _package_runtime_identity( + project, + environment, + module_name="pytest", + distribution_name="pytest", + ) + + +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +def _observed_testenix_workers(output: str) -> int | None: + plain = _ANSI_ESCAPE.sub("", output) + match = re.search(r"(?m)^Testenix\s+\|.*\|\s+(\d+)\s+workers?\s*$", plain) + return int(match.group(1)) if match is not None else None + + +def _validate_output( + runner: Runner, + completed: subprocess.CompletedProcess[str], + *, + expected_tests: int, + expected_passed: int, +) -> None: + if completed.returncode != 0: + raise RuntimeError( + f"runner {runner.name} failed with exit {completed.returncode}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + combined = _ANSI_ESCAPE.sub("", f"{completed.stdout}\n{completed.stderr}") + if runner.kind == "pytest": + summary_lines = [ + line for line in combined.splitlines() if re.search(r"\bin\s+\d+(?:\.\d+)?s\b", line) + ] + counts = { + label: int(count) + for count, label in re.findall( + r"(? Sample: + started = time.perf_counter() + completed = run_bounded_process( + runner.command, + cwd=project, + env=environment, + timeout=timeout, + ) + elapsed = time.perf_counter() - started + _validate_output( + runner, + completed, + expected_tests=expected_tests, + expected_passed=expected_passed, + ) + observed_workers = ( + _observed_testenix_workers(f"{completed.stdout}\n{completed.stderr}") + if runner.kind == "testenix" + else None + ) + if runner.kind == "testenix" and observed_workers is None: + raise RuntimeError( + f"runner {runner.name} did not expose the actual Testenix worker count; " + "refusing to record ambiguous benchmark evidence" + ) + return Sample( + elapsed=elapsed, + stdout_bytes=len(completed.stdout.encode("utf-8")), + stderr_bytes=len(completed.stderr.encode("utf-8")), + observed_workers=observed_workers, + ) + + +def _measure( + runners: tuple[Runner, ...], + *, + project: Path, + environment: dict[str, str], + expected_tests: int, + expected_passed: int, + warmups: int, + repeats: int, + timeout: float, +) -> tuple[dict[str, list[Sample]], list[list[str]]]: + for round_index in range(warmups): + shift = round_index % len(runners) + order = (*runners[shift:], *runners[:shift]) + for runner in order: + _run_once( + runner, + project=project, + environment=environment, + expected_tests=expected_tests, + expected_passed=expected_passed, + timeout=timeout, + ) + + samples: dict[str, list[Sample]] = {runner.name: [] for runner in runners} + orders: list[list[str]] = [] + for round_index in range(repeats): + shift = round_index % len(runners) + order = (*runners[shift:], *runners[:shift]) + orders.append([runner.name for runner in order]) + for runner in order: + samples[runner.name].append( + _run_once( + runner, + project=project, + environment=environment, + expected_tests=expected_tests, + expected_passed=expected_passed, + timeout=timeout, + ) + ) + return samples, orders + + +def _display_argv(runner: Runner, project: Path) -> tuple[str, ...]: + values: list[str] = [] + for index, argument in enumerate(runner.command): + if index in runner.redact_arguments: + values.append("") + continue + if argument == sys.executable: + values.append("python") + else: + values.append(argument.replace(str(project), "")) + return tuple(values) + + +def _display_command(runner: Runner, project: Path) -> str: + return shlex.join(_display_argv(runner, project)) + + +def _contract_arguments(command: tuple[str, ...]) -> tuple[str, ...]: + try: + delimiter = command.index("--") + except ValueError: + return command + return command[:delimiter] + + +def _option_occurrences( + command: tuple[str, ...], + *names: str, +) -> tuple[tuple[str, str | None], ...]: + arguments = _contract_arguments(command) + occurrences: list[tuple[str, str | None]] = [] + for index, argument in enumerate(arguments): + if argument in names: + value = arguments[index + 1] if index + 1 < len(arguments) else None + occurrences.append((argument, value)) + continue + for name in names: + prefix = f"{name}=" + if argument.startswith(prefix): + occurrences.append((name, argument[len(prefix) :])) + break + if len(name) == 2 and name.startswith("-") and argument.startswith(name): + occurrences.append((name, argument[len(name) :].removeprefix("="))) + break + return tuple(occurrences) + + +def _single_option_value(runner: Runner, label: str, *names: str) -> str | None: + occurrences = _option_occurrences(runner.command, *names) + if len(occurrences) > 1: + rendered = ", ".join(name for name, _value in occurrences) + raise RuntimeError( + f"runner {runner.name}: contract option {label} is configured more than once " + f"({rendered})" + ) + if not occurrences: + return None + name, value = occurrences[0] + if value is None or not value: + raise RuntimeError(f"runner {runner.name}: contract option {name} requires a value") + return value + + +def _exclusive_flags(runner: Runner, label: str, *names: str) -> str | None: + arguments = _contract_arguments(runner.command) + occurrences = tuple(argument for argument in arguments if argument in names) + if len(occurrences) > 1: + rendered = ", ".join(occurrences) + raise RuntimeError( + f"runner {runner.name}: contract option {label} is configured more than once " + f"({rendered})" + ) + return occurrences[0] if occurrences else None + + +def _runner_contract(runner: Runner) -> dict[str, Any]: + if runner.kind == "testenix": + workers = _single_option_value(runner, "workers", "--workers", "-w") + history_path = _single_option_value(runner, "history", "--history") + history_switch = _exclusive_flags(runner, "history", "--no-history") + if history_path is not None and history_switch is not None: + raise RuntimeError( + f"runner {runner.name}: contract option history is configured more than once " + "(--history, --no-history)" + ) + sharding_switch = _exclusive_flags( + runner, + "module sharding", + "--shard-modules", + "--no-shard-modules", + ) + return { + "workers_requested": workers or "configuration", + "history_mode": ( + "disabled" + if history_switch == "--no-history" + else (f"explicit:{history_path}" if history_path is not None else "configuration") + ), + "safe_module_sharding": sharding_switch == "--shard-modules", + } + return { + "workers_requested": _single_option_value( + runner, + "workers", + "-n", + "--numprocesses", + ), + "history_mode": None, + "safe_module_sharding": None, + } + + +def _measurement( + samples: list[Sample], runner: Runner, project: Path, expected_tests: int +) -> dict[str, Any]: + durations = [sample.elapsed for sample in samples] + return { + "command": _display_command(runner, project), + "argv": _display_argv(runner, project), + "contract": _runner_contract(runner), + "samples_seconds": durations, + "median_seconds": statistics.median(durations), + "minimum_seconds": min(durations), + "maximum_seconds": max(durations), + "mean_seconds": statistics.fmean(durations), + "stdev_seconds": statistics.stdev(durations) if len(durations) > 1 else 0.0, + "median_throughput_tests_per_second": expected_tests / statistics.median(durations), + "stdout_bytes": [sample.stdout_bytes for sample in samples], + "stderr_bytes": [sample.stderr_bytes for sample in samples], + "observed_workers": [sample.observed_workers for sample in samples], + } + + +def _git_value(project: Path, *arguments: str) -> str | None: + completed = subprocess.run( + ("git", *arguments), + cwd=project, + capture_output=True, + text=True, + check=False, + ) + return completed.stdout.strip() if completed.returncode == 0 else None + + +def _git_provenance(project: Path, *, allow_dirty: bool, include_commit: bool) -> dict[str, Any]: + status = _git_value(project, "status", "--porcelain") + commit = _git_value(project, "rev-parse", "HEAD") + if status is None or commit is None: + raise RuntimeError(f"benchmark project is not a readable Git checkout: {project}") + dirty = bool(status) + if dirty and not allow_dirty: + raise RuntimeError( + f"refusing to benchmark dirty checkout {project}; clean it or use the explicit " + "--allow-dirty-project smoke-only override" + ) + commit_value = commit if include_commit else hashlib.sha256(commit.encode()).hexdigest() + return { + "dirty": dirty, + "commit" if include_commit else "commit_sha256": commit_value, + "state_sha256": hashlib.sha256(f"{commit}\0{status}".encode()).hexdigest(), + } + + +def _environment_distribution_metadata( + project: Path, + environment: dict[str, str], +) -> dict[str, Any]: + """Fingerprint packages visible to the exact benchmark interpreter/environment.""" + + probe = """ +import hashlib +import importlib.metadata +import json + +distributions = set() +versions = {} +for distribution in importlib.metadata.distributions(): + name = distribution.metadata.get("Name") + if not name: + continue + canonical = name.casefold() + distributions.add(f"{canonical}=={distribution.version}") + if canonical in {"pytest", "pytest-xdist", "testenix"}: + versions[canonical] = distribution.version +serialized = "\\n".join(sorted(distributions)).encode("utf-8") +print(json.dumps({ + "count": len(distributions), + "sha256": hashlib.sha256(serialized).hexdigest(), + "versions": versions, +}, sort_keys=True)) +""" + completed = subprocess.run( + (sys.executable, "-c", probe), + cwd=project, + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + "cannot fingerprint distributions in the benchmark environment\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + try: + value = json.loads(completed.stdout.splitlines()[-1]) + except (IndexError, json.JSONDecodeError) as error: + raise RuntimeError("distribution fingerprint probe returned invalid output") from error + valid = ( + isinstance(value, dict) + and isinstance(value.get("count"), int) + and value["count"] > 0 + and isinstance(value.get("sha256"), str) + and re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is not None + and isinstance(value.get("versions"), dict) + and all( + isinstance(name, str) and isinstance(version, str) + for name, version in value["versions"].items() + ) + ) + if not valid: + raise RuntimeError("distribution fingerprint probe returned an invalid schema") + return value + + +def _cpu_model() -> str: + if sys.platform == "darwin": + completed = subprocess.run( + ("sysctl", "-n", "machdep.cpu.brand_string"), + capture_output=True, + text=True, + check=False, + ) + if completed.returncode == 0 and completed.stdout.strip(): + return completed.stdout.strip() + if sys.platform.startswith("linux"): + try: + cpuinfo = Path("/proc/cpuinfo").read_text(encoding="utf-8") + except OSError: + cpuinfo = "" + for line in cpuinfo.splitlines(): + key, separator, value = line.partition(":") + if separator and key.strip().lower() in {"hardware", "model name"}: + return value.strip() + return ( + platform.processor().strip() + or os.environ.get("PROCESSOR_IDENTIFIER", "").strip() + or "unknown" + ) + + +def _tree_fingerprint(project: Path, relative: str) -> dict[str, Any]: + root = (project / relative).resolve() + try: + root.relative_to(project.resolve()) + except ValueError as error: + raise RuntimeError(f"fingerprint path escapes the project: {relative}") from error + if not root.exists(): + raise RuntimeError(f"fingerprint path does not exist: {relative}") + files = ( + (root,) if root.is_file() else tuple(path for path in root.rglob("*.py") if path.is_file()) + ) + if not files: + raise RuntimeError(f"fingerprint path contains no Python files: {relative}") + digest = hashlib.sha256() + total_bytes = 0 + for path in sorted(files, key=lambda item: item.as_posix()): + data = path.read_bytes() + total_bytes += len(data) + digest.update(path.relative_to(project).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(hashlib.sha256(data).digest()) + return {"sha256": digest.hexdigest(), "files": len(files), "bytes": total_bytes} + + +def _fingerprints(manifest: dict[str, Any], project: Path) -> dict[str, dict[str, Any]]: + raw = manifest.get("fingerprints", {}) + if not isinstance(raw, dict) or any( + not isinstance(name, str) or not isinstance(path, str) for name, path in raw.items() + ): + raise RuntimeError("manifest fingerprints must map public labels to relative paths") + return {name: _tree_fingerprint(project, path) for name, path in sorted(raw.items())} + + +def _project_relative_path(project: Path, value: Any, *, name: str) -> Path: + if not isinstance(value, str) or not value or Path(value).is_absolute(): + raise RuntimeError(f"migration report {name} must be a non-empty relative path") + path = (project / value).resolve() + try: + path.relative_to(project.resolve()) + except ValueError as error: + raise RuntimeError(f"migration report {name} escapes the project") from error + return path + + +def _explicit_suite_targets(runner: Runner, project: Path) -> tuple[Path, ...]: + """Return unambiguous positional suite roots after the CLI ``--`` delimiter.""" + + delimiters = [index for index, argument in enumerate(runner.command) if argument == "--"] + if len(delimiters) != 1: + raise RuntimeError( + f"runner {runner.name}: publication commands must separate suite targets with " + "exactly one '--' delimiter" + ) + raw_targets = runner.command[delimiters[0] + 1 :] + if not raw_targets or any(target.startswith("-") for target in raw_targets): + raise RuntimeError( + f"runner {runner.name}: publication command has no unambiguous suite targets after '--'" + ) + targets = tuple( + (Path(target) if Path(target).is_absolute() else project / target).resolve() + for target in raw_targets + ) + if len(set(targets)) != len(targets): + raise RuntimeError(f"runner {runner.name}: publication suite targets must be unique") + return targets + + +def _summary_outcomes( + summary: Any, + *, + label: str, + expected_ids: set[str], + expected_tests: int, + expected_passed: int, +) -> dict[str, str]: + if not isinstance(summary, dict): + raise RuntimeError(f"migration report {label} summary is missing") + outcomes = summary.get("outcomes") + if not isinstance(outcomes, dict) or any( + not isinstance(test_id, str) or not isinstance(status, str) + for test_id, status in outcomes.items() + ): + raise RuntimeError(f"migration report {label} outcomes must map test IDs to statuses") + if set(outcomes) != expected_ids: + raise RuntimeError( + f"migration report {label} per-test inventory does not match converter mappings" + ) + if summary.get("tests") != expected_tests or summary.get("passed") != expected_passed: + raise RuntimeError( + f"migration report {label} summary does not match the benchmark expected counts" + ) + return outcomes + + +def _python_inventory(root: Path, *, relative_to: Path) -> set[str]: + files = (root,) if root.is_file() else tuple(root.rglob("*.py")) + inventory: set[str] = set() + for path in files: + if not path.is_file() or path.suffix != ".py": + continue + resolved = path.resolve() + try: + relative = resolved.relative_to(relative_to.resolve()) + except ValueError as error: + raise RuntimeError( + f"Python inventory path escapes its benchmark root: {path}" + ) from error + inventory.add(relative.as_posix()) + return inventory + + +def _migration_gate( + manifest: dict[str, Any], + project: Path, + expected_tests: int, + expected_passed: int, + runners: tuple[Runner, ...], +) -> dict[str, Any] | None: + relative = manifest.get("migration_report") + if relative is None: + return None + path = _project_relative_path(project, relative, name="migration_report") + try: + report = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RuntimeError(f"cannot read migration report: {error}") from error + if not isinstance(report, dict): + raise RuntimeError("migration report must be a JSON object") + valid_header = ( + report.get("format") == "testenix.migration-report" + and report.get("schema_version") == 1 + and report.get("framework") == "pytest" + and report.get("status") == "published" + and report.get("published") is True + and report.get("converted_tests") == expected_tests + and report.get("originals_modified") is False + ) + if not valid_header: + raise RuntimeError("migration report did not pass publication/integrity gates") + + mappings = report.get("mappings") + if not isinstance(mappings, list) or len(mappings) != expected_tests: + raise RuntimeError("migration report mappings do not match expected_tests") + try: + source_ids = [mapping["source_id"] for mapping in mappings] + target_files = [mapping["target_file"] for mapping in mappings] + except (KeyError, TypeError) as error: + raise RuntimeError("migration report mappings have an invalid schema") from error + if ( + any(not isinstance(test_id, str) or not test_id for test_id in source_ids) + or len(set(source_ids)) != expected_tests + or any(not isinstance(target, str) or not target for target in target_files) + ): + raise RuntimeError("migration report mappings must contain unique source test IDs") + expected_ids = set(source_ids) + baseline_outcomes = _summary_outcomes( + report.get("baseline"), + label="baseline", + expected_ids=expected_ids, + expected_tests=expected_tests, + expected_passed=expected_passed, + ) + serial_outcomes = _summary_outcomes( + report.get("native_serial"), + label="native_serial", + expected_ids=expected_ids, + expected_tests=expected_tests, + expected_passed=expected_passed, + ) + parallel_outcomes = _summary_outcomes( + report.get("native_parallel"), + label="native_parallel", + expected_ids=expected_ids, + expected_tests=expected_tests, + expected_passed=expected_passed, + ) + if baseline_outcomes != serial_outcomes or baseline_outcomes != parallel_outcomes: + raise RuntimeError("migration report per-test outcomes are not equivalent") + + raw_sources = report.get("sources") + output = report.get("output") + if ( + not isinstance(raw_sources, list) + or not raw_sources + or any(not isinstance(source, str) or not source for source in raw_sources) + or not isinstance(output, str) + or not output + ): + raise RuntimeError("migration report sources/output have an invalid schema") + source_roots: list[Path] = [] + for source in raw_sources: + source_root = _project_relative_path(project, source, name="sources[]") + if not source_root.is_dir(): + raise RuntimeError( + "publishable migration benchmarks require directory source roots so pytest " + "support files such as conftest.py are part of the verified inventory" + ) + source_roots.append(source_root) + output_path = _project_relative_path(project, output, name="output") + source_targets = {(project / source).resolve() for source in raw_sources} + native_targets = {output_path} + for runner in runners: + expected_targets = source_targets if runner.kind == "pytest" else native_targets + if set(_explicit_suite_targets(runner, project)) != expected_targets: + raise RuntimeError( + "benchmark runner paths do not match the migration report source/output roots" + ) + + source_hashes = report.get("source_hashes") + if not isinstance(source_hashes, dict) or not source_hashes: + raise RuntimeError("migration report source_hashes are missing") + declared_source_inventory: set[str] = set() + for source, expected_sha256 in source_hashes.items(): + source_path = _project_relative_path(project, source, name="source_hashes key") + declared_source_inventory.add(source_path.relative_to(project.resolve()).as_posix()) + if ( + not isinstance(expected_sha256, str) + or re.fullmatch(r"[0-9a-f]{64}", expected_sha256) is None + or not source_path.is_file() + or hashlib.sha256(source_path.read_bytes()).hexdigest() != expected_sha256 + ): + raise RuntimeError(f"migration report source hash is stale for {source!r}") + actual_source_inventory = set().union( + *(_python_inventory(root, relative_to=project) for root in source_roots) + ) + if actual_source_inventory != declared_source_inventory: + raise RuntimeError( + "migration report source Python inventory is stale or incomplete; regenerate migration" + ) + + generated_files = report.get("generated_files") + if ( + not isinstance(generated_files, list) + or not generated_files + or any(not isinstance(generated, str) or not generated for generated in generated_files) + ): + raise RuntimeError("migration report generated_files are missing") + generated_digest = hashlib.sha256() + generated_paths: dict[str, Path] = {} + for generated in sorted(generated_files): + generated_path = _project_relative_path( + output_path, + generated, + name="generated_files[]", + ) + if not generated_path.is_file(): + raise RuntimeError(f"migration report generated file is missing: {generated!r}") + generated_paths[generated] = generated_path + generated_digest.update(generated.encode("utf-8")) + generated_digest.update(b"\0") + generated_digest.update(hashlib.sha256(generated_path.read_bytes()).digest()) + if any(target not in generated_paths for target in target_files): + raise RuntimeError("migration mappings reference files outside generated_files") + actual_generated_inventory = _python_inventory(output_path, relative_to=output_path) + if actual_generated_inventory != set(generated_files): + raise RuntimeError( + "migration report generated Python inventory is stale or incomplete; " + "regenerate migration" + ) + + serialized_outcomes = json.dumps( + sorted(baseline_outcomes.items()), separators=(",", ":") + ).encode("utf-8") + return { + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "status": report["status"], + "converted_tests": report["converted_tests"], + "originals_modified": report["originals_modified"], + "inventory_sha256": hashlib.sha256( + "\n".join(sorted(source_ids)).encode("utf-8") + ).hexdigest(), + "outcomes_sha256": hashlib.sha256(serialized_outcomes).hexdigest(), + "source_files_verified": len(source_hashes), + "generated_files_verified": len(generated_files), + "generated_files_sha256": generated_digest.hexdigest(), + "runner_paths_verified": True, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--allow-dirty-project", action="store_true") + return parser + + +def main() -> int: + arguments = build_parser().parse_args() + project = arguments.project.resolve() + if not project.is_dir(): + raise RuntimeError(f"benchmark project is not a directory: {project}") + manifest_path = arguments.manifest.resolve() + manifest = _read_manifest(manifest_path) + expected_tests = _positive_integer(manifest.get("expected_tests"), "expected_tests") + expected_passed = _positive_integer( + manifest.get("expected_passed", expected_tests), "expected_passed" + ) + if expected_passed > expected_tests: + raise RuntimeError("expected_passed cannot exceed expected_tests") + warmups = _non_negative_integer(manifest.get("warmups", 1), "warmups") + repeats = _positive_integer(manifest.get("repeats", 5), "repeats") + raw_timeout = manifest.get("timeout_seconds", 900.0) + if isinstance(raw_timeout, bool) or not isinstance(raw_timeout, (int, float)): + raise RuntimeError("timeout_seconds must be a number") + timeout = float(raw_timeout) + if not 0 < timeout <= 3600: + raise RuntimeError("timeout_seconds must be greater than zero and at most 3600") + + expected_version = manifest.get("testenix_version") + if not isinstance(expected_version, str) or not expected_version: + raise RuntimeError("manifest testenix_version must be a non-empty string") + runners = _runners(manifest) + runner_contracts = {runner.name: _runner_contract(runner) for runner in runners} + explicit_testenix_contracts = all( + contract["workers_requested"] != "configuration" + and contract["history_mode"] != "configuration" + for runner in runners + if runner.kind == "testenix" + for contract in (runner_contracts[runner.name],) + ) + environment, environment_override_keys, environment_sha256 = _environment(manifest) + testenix_runtime = _testenix_runtime_identity(project, environment) + pytest_runtime = _pytest_runtime_identity(project, environment) + distribution_metadata = _environment_distribution_metadata(project, environment) + installed_version = testenix_runtime["version"] + if expected_version != installed_version: + raise RuntimeError( + f"manifest requires Testenix {expected_version!r}, executed version is " + f"{installed_version!r}" + ) + project_provenance = _git_provenance( + project, + allow_dirty=arguments.allow_dirty_project, + include_commit=manifest.get("include_project_commit") is True, + ) + fingerprints = _fingerprints(manifest, project) + migration_gate = _migration_gate( + manifest, + project, + expected_tests, + expected_passed, + runners, + ) + samples, orders = _measure( + runners, + project=project, + environment=environment, + expected_tests=expected_tests, + expected_passed=expected_passed, + warmups=warmups, + repeats=repeats, + timeout=timeout, + ) + final_project_provenance = _git_provenance( + project, + allow_dirty=arguments.allow_dirty_project, + include_commit=manifest.get("include_project_commit") is True, + ) + final_fingerprints = _fingerprints(manifest, project) + final_migration_gate = _migration_gate( + manifest, + project, + expected_tests, + expected_passed, + runners, + ) + final_testenix_runtime = _testenix_runtime_identity(project, environment) + final_pytest_runtime = _pytest_runtime_identity(project, environment) + final_distribution_metadata = _environment_distribution_metadata(project, environment) + if ( + final_project_provenance != project_provenance + or final_fingerprints != fingerprints + or final_migration_gate != migration_gate + or final_testenix_runtime != testenix_runtime + or final_pytest_runtime != pytest_runtime + or final_distribution_metadata != distribution_metadata + ): + raise RuntimeError( + "benchmark project changed during execution; discard this run and restore a stable " + "checkout before measuring again" + ) + + project_name = manifest.get("name", "private-project") + if not isinstance(project_name, str) or not project_name.strip(): + raise RuntimeError("manifest name must be a non-empty string") + publication_eligible = ( + not project_provenance["dirty"] + and repeats >= 5 + and warmups >= 1 + and explicit_testenix_contracts + and testenix_runtime["source_matches_distribution"] + and pytest_runtime["source_matches_distribution"] + and migration_gate is not None + ) + + result = { + "schema_version": 1, + "kind": "testenix.real-project-benchmark", + "recorded_at": datetime.now(UTC).isoformat(), + "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), + "project": { + "name": project_name, + "provenance": project_provenance, + "fingerprints": fingerprints, + "migration_gate": migration_gate, + }, + "environment": { + "cpu_count": os.cpu_count(), + "cpu_model": _cpu_model(), + "machine": platform.machine(), + "platform": platform.platform(), + "python": platform.python_version(), + "testenix": installed_version, + "testenix_runtime": testenix_runtime, + "pytest_runtime": pytest_runtime, + "versions": { + "pytest": pytest_runtime["version"], + "pytest_xdist": distribution_metadata["versions"].get( + "pytest-xdist", "not-installed" + ), + "testenix": installed_version, + }, + "distributions": { + "count": distribution_metadata["count"], + "sha256": distribution_metadata["sha256"], + }, + "override_keys": environment_override_keys, + "environment_sha256": environment_sha256, + }, + "scenario": { + "expected_tests": expected_tests, + "expected_passed": expected_passed, + "warmups": warmups, + "repeats": repeats, + "measured_execution_orders": orders, + "timeout_seconds": timeout, + "publication_eligible": publication_eligible, + "publication_contract": ( + "clean project, >=5 measured rounds, >=1 warm-up, explicit Testenix workers " + "and history mode, canonical module entrypoints, installed-distribution runtimes, " + "and a current published migration report with exact per-test inventory/outcome " + "parity and runner-path binding" + ), + "runner_contracts": runner_contracts, + }, + "measurements": { + runner.name: _measurement( + samples[runner.name], + runner, + project, + expected_tests, + ) + for runner in runners + }, + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(arguments.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/run_scaling_matrix.py b/benchmarks/run_scaling_matrix.py new file mode 100644 index 0000000..f10c423 --- /dev/null +++ b/benchmarks/run_scaling_matrix.py @@ -0,0 +1,456 @@ +"""Run a provenance-gated Testenix scaling matrix without publishing unmeasured claims. + +The default design uses dimension sweeps rather than an expensive Cartesian product: + +* 100, 500, 1,000, and 3,000 tests at the reference configuration; +* 1, 2, 4, and ``auto`` workers at the largest test count; +* balanced, one-dominant-module, and single-module layouts at the largest count; +* default history and ``--no-history`` at the largest count. +* explicit safe-module sharding for every layout at the largest count. + +Pass ``--full-cross-product`` when every combination is required. +""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import os +import tomllib +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal, TypeAlias + +if __package__: + from benchmarks.run_benchmark import run_benchmark +else: # direct ``python benchmarks/run_scaling_matrix.py`` execution + from run_benchmark import run_benchmark + +ModuleLayout: TypeAlias = Literal["balanced", "dominant", "single"] +HistoryMode: TypeAlias = Literal["disabled", "default"] +WorkerRequest: TypeAlias = int | Literal["auto"] +ShardingMode: TypeAlias = Literal["disabled", "safe"] + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_COUNTS = (100, 500, 1_000, 3_000) +DEFAULT_WORKERS: tuple[WorkerRequest, ...] = (1, 2, 4, "auto") +DEFAULT_LAYOUTS: tuple[ModuleLayout, ...] = ("balanced", "dominant", "single") +DEFAULT_HISTORIES: tuple[HistoryMode, ...] = ("disabled", "default") +DEFAULT_SHARDING_MODES: tuple[ShardingMode, ...] = ("disabled", "safe") + + +@dataclass(frozen=True, slots=True) +class MatrixScenario: + id: str + test_count: int + module_count: int + workers: WorkerRequest + module_layout: ModuleLayout + history_mode: HistoryMode + sharding_mode: ShardingMode = "disabled" + uneven: bool = False + dominant_fraction: float = 0.5 + + +def _project_version() -> str: + with (ROOT / "pyproject.toml").open("rb") as source: + return str(tomllib.load(source)["project"]["version"]) + + +def _git_value(*arguments: str) -> str | None: + import subprocess + + completed = subprocess.run( + ("git", *arguments), + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + return completed.stdout.strip() if completed.returncode == 0 else None + + +def _provenance(*, allow_dirty: bool) -> dict[str, Any]: + status = _git_value("status", "--porcelain") + if status is None: + raise RuntimeError("cannot read the Testenix Git worktree state") + dirty = bool(status) + if dirty and not allow_dirty: + raise RuntimeError( + "refusing to record a publishable matrix from a dirty worktree; " + "commit/stash changes or use --allow-dirty for an unpublished smoke run" + ) + + expected_version = _project_version() + try: + installed_version = importlib.metadata.version("testenix") + except importlib.metadata.PackageNotFoundError as error: + raise RuntimeError("Testenix must be installed in the benchmark environment") from error + if installed_version != expected_version: + raise RuntimeError( + f"installed Testenix {installed_version} does not match pyproject {expected_version}" + ) + return { + "commit": _git_value("rev-parse", "HEAD"), + "dirty": dirty, + "testenix_version": installed_version, + "pyproject_version": expected_version, + } + + +def _deduplicate(scenarios: list[MatrixScenario]) -> tuple[MatrixScenario, ...]: + seen: set[tuple[object, ...]] = set() + unique: list[MatrixScenario] = [] + for scenario in scenarios: + key = ( + scenario.test_count, + scenario.module_count, + scenario.workers, + scenario.module_layout, + scenario.history_mode, + scenario.sharding_mode, + scenario.uneven, + scenario.dominant_fraction, + ) + if key not in seen: + seen.add(key) + unique.append(scenario) + return tuple(unique) + + +def _reference_worker(workers: tuple[WorkerRequest, ...]) -> WorkerRequest: + return "auto" if "auto" in workers else (4 if 4 in workers else workers[0]) + + +def build_scenarios( + *, + counts: tuple[int, ...], + module_count: int, + workers: tuple[WorkerRequest, ...], + layouts: tuple[ModuleLayout, ...], + histories: tuple[HistoryMode, ...], + sharding_modes: tuple[ShardingMode, ...], + dominant_fraction: float, + full_cross_product: bool, + include_duration_skew: bool, +) -> tuple[MatrixScenario, ...]: + largest = max(counts) + if full_cross_product: + scenarios = [ + MatrixScenario( + id=( + f"tests-{count}-layout-{layout}-workers-{worker}-history-{history}" + f"-sharding-{sharding}" + ), + test_count=count, + module_count=1 if layout == "single" else min(module_count, count), + workers=worker, + module_layout=layout, + history_mode=history, + sharding_mode=sharding, + dominant_fraction=dominant_fraction, + ) + for count in counts + for layout in layouts + for worker in workers + for history in histories + for sharding in sharding_modes + ] + return _deduplicate(scenarios) + + reference_workers = _reference_worker(workers) + scenarios = [ + MatrixScenario( + id=f"scale-{count}", + test_count=count, + module_count=min(module_count, count), + workers=reference_workers, + module_layout="balanced", + history_mode="disabled", + sharding_mode="disabled", + ) + for count in counts + ] + scenarios.extend( + MatrixScenario( + id=f"workers-{worker}", + test_count=largest, + module_count=min(module_count, largest), + workers=worker, + module_layout="balanced", + history_mode="disabled", + sharding_mode="disabled", + ) + for worker in workers + ) + scenarios.extend( + MatrixScenario( + id=f"layout-{layout}", + test_count=largest, + module_count=1 if layout == "single" else min(module_count, largest), + workers=reference_workers, + module_layout=layout, + history_mode="disabled", + sharding_mode="disabled", + dominant_fraction=dominant_fraction, + ) + for layout in layouts + ) + scenarios.extend( + MatrixScenario( + id=f"history-{history}", + test_count=largest, + module_count=min(module_count, largest), + workers=reference_workers, + module_layout="balanced", + history_mode=history, + sharding_mode="disabled", + ) + for history in histories + ) + if "safe" in sharding_modes: + scenarios.extend( + MatrixScenario( + id=f"sharding-safe-layout-{layout}", + test_count=largest, + module_count=1 if layout == "single" else min(module_count, largest), + workers=reference_workers, + module_layout=layout, + history_mode="disabled", + sharding_mode="safe", + dominant_fraction=dominant_fraction, + ) + for layout in layouts + ) + if include_duration_skew: + scenarios.append( + MatrixScenario( + id="duration-skew", + test_count=largest, + module_count=min(module_count, largest), + workers=reference_workers, + module_layout="balanced", + history_mode="disabled", + uneven=True, + ) + ) + return _deduplicate(scenarios) + + +def _reference_curve( + results: list[dict[str, Any]], + *, + reference_workers: WorkerRequest, +) -> list[dict[str, Any]]: + """Project one canonical balanced/no-history point for every test count.""" + + points = ( + { + "test_count": entry["scenario"].test_count, + "workers_requested": entry["scenario"].workers, + "history_mode": entry["scenario"].history_mode, + "sharding_mode": entry["scenario"].sharding_mode, + "measurements": { + name: { + "median_seconds": measurement["median"], + "median_tests_per_second": measurement["median_tests_per_second"], + "observed_workers": measurement["observed_workers"], + } + for name, measurement in entry["result"]["measurements"].items() + }, + } + for entry in results + if entry["scenario"].workers == reference_workers + and entry["scenario"].module_layout == "balanced" + and entry["scenario"].history_mode == "disabled" + and entry["scenario"].sharding_mode == "disabled" + and not entry["scenario"].uneven + ) + curve = sorted(points, key=lambda point: point["test_count"]) + counts = [point["test_count"] for point in curve] + if len(counts) != len(set(counts)): + raise RuntimeError("canonical reference curve contains duplicate test counts") + return curve + + +def _parse_positive_csv(value: str) -> tuple[int, ...]: + try: + parsed = tuple(int(item.strip()) for item in value.split(",") if item.strip()) + except ValueError as error: + raise argparse.ArgumentTypeError("expected comma-separated positive integers") from error + if not parsed or any(item < 1 for item in parsed): + raise argparse.ArgumentTypeError("expected comma-separated positive integers") + return parsed + + +def _parse_workers(value: str) -> tuple[WorkerRequest, ...]: + parsed: list[WorkerRequest] = [] + for raw in value.split(","): + item = raw.strip() + if not item: + continue + if item == "auto": + parsed.append("auto") + continue + try: + number = int(item) + except ValueError as error: + raise argparse.ArgumentTypeError("workers must be integers or 'auto'") from error + if number < 1: + raise argparse.ArgumentTypeError("workers must be positive") + parsed.append(number) + if not parsed: + raise argparse.ArgumentTypeError("at least one worker value is required") + return tuple(parsed) + + +def _validate_coverage( + scenarios: tuple[MatrixScenario, ...], + *, + counts: tuple[int, ...], + workers: tuple[WorkerRequest, ...], + layouts: tuple[ModuleLayout, ...], + histories: tuple[HistoryMode, ...], + sharding_modes: tuple[ShardingMode, ...], +) -> None: + if not set(counts).issubset({scenario.test_count for scenario in scenarios}): + raise RuntimeError("matrix does not cover every requested test count") + if not set(workers).issubset({scenario.workers for scenario in scenarios}): + raise RuntimeError("matrix does not cover every requested worker value") + if not set(layouts).issubset({scenario.module_layout for scenario in scenarios}): + raise RuntimeError("matrix does not cover every requested module layout") + if not set(histories).issubset({scenario.history_mode for scenario in scenarios}): + raise RuntimeError("matrix does not cover both history modes") + if not set(sharding_modes).issubset({scenario.sharding_mode for scenario in scenarios}): + raise RuntimeError("matrix does not cover every requested sharding mode") + if "safe" in sharding_modes: + missing_safe_layouts = { + layout + for layout in layouts + if not any( + scenario.module_layout == layout and scenario.sharding_mode == "safe" + for scenario in scenarios + ) + } + if missing_safe_layouts: + raise RuntimeError( + "safe sharding is missing layouts: " + ", ".join(sorted(missing_safe_layouts)) + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--tests", type=_parse_positive_csv, default=DEFAULT_COUNTS) + parser.add_argument("--modules", type=int, default=16) + parser.add_argument("--workers", type=_parse_workers, default=DEFAULT_WORKERS) + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--dominant-fraction", type=float, default=0.5) + parser.add_argument("--full-cross-product", action="store_true") + parser.add_argument("--include-duration-skew", action="store_true") + parser.add_argument("--allow-dirty", action="store_true") + parser.add_argument( + "--quick", + action="store_true", + help="unpublished smoke mode: one measured round and no warm-up", + ) + return parser + + +def main() -> int: + arguments = build_parser().parse_args() + if arguments.modules < 1: + raise SystemExit("modules must be positive") + if arguments.repeats < 1 or arguments.warmups < 0: + raise SystemExit("repeats must be positive and warmups cannot be negative") + if not 0.0 < arguments.dominant_fraction < 1.0: + raise SystemExit("dominant-fraction must be greater than zero and less than one") + + provenance = _provenance(allow_dirty=arguments.allow_dirty) + repeats = 1 if arguments.quick else arguments.repeats + warmups = 0 if arguments.quick else arguments.warmups + scenarios = build_scenarios( + counts=arguments.tests, + module_count=arguments.modules, + workers=arguments.workers, + layouts=DEFAULT_LAYOUTS, + histories=DEFAULT_HISTORIES, + sharding_modes=DEFAULT_SHARDING_MODES, + dominant_fraction=arguments.dominant_fraction, + full_cross_product=arguments.full_cross_product, + include_duration_skew=arguments.include_duration_skew, + ) + _validate_coverage( + scenarios, + counts=arguments.tests, + workers=arguments.workers, + layouts=DEFAULT_LAYOUTS, + histories=DEFAULT_HISTORIES, + sharding_modes=DEFAULT_SHARDING_MODES, + ) + + results: list[dict[str, Any]] = [] + for index, scenario in enumerate(scenarios, start=1): + print(f"[{index}/{len(scenarios)}] {scenario.id}", flush=True) + result = run_benchmark( + test_count=scenario.test_count, + repeats=repeats, + warmups=warmups, + workers=scenario.workers, + uneven=scenario.uneven, + module_count=scenario.module_count, + module_layout=scenario.module_layout, + dominant_fraction=scenario.dominant_fraction, + history_mode=scenario.history_mode, + xdist_strategy="load", + sharding_mode=scenario.sharding_mode, + ) + results.append({"id": scenario.id, "scenario": scenario, "result": result}) + + reference_curve = _reference_curve( + results, + reference_workers=_reference_worker(arguments.workers), + ) + serialized_results = [{"id": entry["id"], "result": entry["result"]} for entry in results] + publication_eligible = ( + not provenance["dirty"] and not arguments.quick and repeats >= 5 and warmups >= 1 + ) + + output = { + "schema_version": 1, + "kind": "testenix.scaling-matrix", + "recorded_at": datetime.now(UTC).isoformat(), + "provenance": provenance, + "design": { + "mode": "full-cross-product" if arguments.full_cross_product else "dimension-sweeps", + "tests": arguments.tests, + "workers": arguments.workers, + "module_layouts": DEFAULT_LAYOUTS, + "history_modes": DEFAULT_HISTORIES, + "sharding_modes": DEFAULT_SHARDING_MODES, + "modules": arguments.modules, + "dominant_fraction": arguments.dominant_fraction, + "repeats": repeats, + "warmups": warmups, + "xdist_strategy": "load", + "testenix_auto_semantics": "adaptive", + "xdist_auto_workers": os.cpu_count() or 1, + "publication_eligible": publication_eligible, + "publication_contract": "clean commit, >=5 measured rounds, >=1 warm-up", + }, + "reference_curve": reference_curve, + "scenarios": serialized_results, + } + arguments.output.parent.mkdir(parents=True, exist_ok=True) + arguments.output.write_text( + json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(arguments.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/_static/benchmark-speedup.svg b/docs/_static/benchmark-speedup.svg index 51d4970..cd5518e 100644 --- a/docs/_static/benchmark-speedup.svg +++ b/docs/_static/benchmark-speedup.svg @@ -1,10 +1,10 @@ -Preliminary Testenix benchmark throughput ratios -Horizontal bars compare Testenix throughput with pytest and pytest-xdist for three checked-in synthetic benchmark scenarios. +Historical Testenix 0.1.0 benchmark throughput ratios +Horizontal bars compare historical Testenix 0.1.0 throughput with pytest and pytest-xdist for three checked-in synthetic benchmark scenarios using four workers and disabled Testenix history. -Synthetic benchmark throughput ratio -Higher is better · 1× means equal throughput +Historical Testenix 0.1.0 synthetic ratios +4 workers · --no-history · higher is better 0× @@ -36,5 +36,5 @@ vs pytest-xdist 2.65× -Development baseline · Apple M4 Pro · CPython 3.11.14 · raw samples linked below +Historical baseline · Apple M4 Pro · CPython 3.11.14 · raw samples linked below diff --git a/docs/architecture.md b/docs/architecture.md index 6b0eb04..f9d811d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,11 +55,11 @@ test outcomes. A retry never overwrites an earlier attempt, infrastructure failu from test failures, and setup/call/teardown are preserved as separate phases. ```text -Authoring API -> supervised collection -> inert manifest -> affinity scheduler -> process workers - | -> streamed attempts - +----------> append-only events - -> reducer - -> reports/history +Authoring API -> supervised collection -------> inert manifest -> scheduler -> process workers + ^ ^ | -> streamed attempts + | | +------------> append-only events +trusted manifest +-- roots/inventory/SHA-256 verify -> reducer + exact match bypasses collection imports -> reports/history ``` ## Dependency rules @@ -68,6 +68,9 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - `api`, `discovery`, `fixtures`, and `executor` form the native engine. - `events`, `aggregate`, and `scheduler` remain engine-independent. - `runner` is the application service connecting the native engine with execution policy. +- `tuning` models adaptive worker selection and runs explicit project-local candidate measurements. +- `sharding` contains fail-closed static module decisions and the versioned trusted-manifest + serialization/verification boundary. - reporters and storage consume completed domain results or versioned events. - optional compatibility adapters stay at the CLI boundary; the native core never imports pytest. - migration analyzers depend on serializable migration contracts, while shadow execution and @@ -81,6 +84,10 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - fixture scopes: test, module, session, with broader scopes currently bounded by a worker shard; - sequential and local process execution; - deterministic scheduling based on historical durations; +- adaptive `workers = "auto"` capped by real execution units, history-informed predicted makespan, + process-start cost, and CPU capacity; +- explicit project-local worker tuning, conservative opt-in intra-module sharding, and an + explicitly generated source-verified collection manifest; - append-only JSONL events and a pure reducer; - console, JSON, and JUnit output plus local SQLite duration history; - retries represented as immutable attempts and finalized as `FLAKY` when appropriate; @@ -96,10 +103,16 @@ plugin SDK are deliberately outside version 0.2. ## Fixture scopes and process isolation -The scheduler treats every normal test module as one affinity unit and never splits that unit -between parallel shared workers. Multiple modules assigned to one shard execute in one persistent -process and fixture runtime. A test with an explicit timeout (including a global timeout applied at -selection) is instead a single-test isolation unit with a hard process deadline. +By default, the scheduler treats every normal test module as one affinity unit and never splits +that unit between parallel shared workers. Multiple modules assigned to one shard execute in one +persistent process and fixture runtime. A test with an explicit timeout (including a global timeout +applied at selection) is instead a single-test isolation unit with a hard process deadline. + +An explicit intra-module sharding policy can turn tests in an eligible module into finer units. +The static analysis fails closed for module/session fixtures, writes or obvious mutations of module +globals, and import-time lifecycle behavior. Function-scoped fixtures can be recreated per worker. +Because arbitrary dynamic calls and external effects cannot be proven safe, passing this policy is +a caller trust decision; ineligible modules keep normal affinity. Scope therefore has the following concrete meaning in version 0.2: @@ -133,6 +146,18 @@ top-level import crash or deadline becomes a `CollectionIssue`, so user import c indefinitely block the coordinator. Timed execution units send a ready handshake after rediscovery; the test deadline therefore does not include interpreter startup or module import. +A caller may instead supply a `TrustedCollectionManifest` created by an earlier explicit +collection. Before using it, Testenix enumerates the current roots and verifies the complete file +set and every SHA-256 digest. An exact match bypasses collection imports; a stale manifest falls +back to the supervised collector. Malformed serialized input is rejected at the adapter boundary. +Manifest parameter values are redacted at creation and serialization; only their names remain. +The manifest also carries prior sharding decisions so collection and scheduling agree. A module +using a fixture provider from outside its fingerprinted source fails closed to module affinity. +This removes +one module import per unchanged run, not the execution-worker import needed to reconstruct Python +objects. Inputs to dynamic collection beyond fingerprinted source bytes remain the producer's trust +responsibility. + Workers normally create a separate POSIX process session (or use recursive tree termination on Windows). During migration validation they remain in the validator's process group so an outer validation deadline can terminate native workers too. Timeout and cancellation terminate ordinary diff --git a/docs/benchmarking.md b/docs/benchmarking.md index ca68faf..434be3e 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -17,19 +17,84 @@ For every scenario record wall-clock duration, collection time, execution time, worker utilization, number of process starts, result completeness, and output size. Performance claims require at least five runs per configuration and must publish the environment fingerprint. -The current v0.1 harness automates wall-clock samples and the environment fingerprint across 16 -generated modules for a four-worker run. It also validates the completed test count, rotates runner -order between measured rounds, supports an explicit module count, and records throughput, mean, and -standard deviation. Pytest plugin autoloading and its cache provider are disabled, pytest-xdist is -loaded explicitly, and every tool runs from the generated suite directory so repository-level -pytest configuration does not affect the comparison. New output also records the commit, dirty -state, lockfile hash, timestamp, and installed framework versions. The remaining telemetry above is -the acceptance contract for the next harness iteration, not data claimed by the checked-in baseline -files. +The checked-in headline files were recorded with Testenix 0.1.0 across 16 generated modules, four +workers, and `--no-history`. They automate subprocess wall-clock samples and the environment +fingerprint, validate the completed test count, rotate runner order between measured rounds, and +record throughput, mean, standard deviation, provenance, and raw samples. Pytest plugin autoloading +and its cache provider are disabled, pytest-xdist 3.8 is loaded explicitly with its default `load` +distribution, and every tool runs from the generated suite directory so repository-level pytest +configuration does not affect the comparison. Those historical records do not measure Testenix +0.2.1. + +Schema-version 2 harness output additionally records the requested and resolved worker counts, +balanced/dominant/single-module test distributions, default-history versus `--no-history`, the +pytest-xdist distribution strategy, and captured stdout/stderr byte counts. Collection/execution +splits, peak memory, utilization, process counts, and the remaining telemetry above are still the +acceptance contract for a later harness iteration, not data claimed by the historical baseline. Correctness wins over speed: a run with a missing, duplicated, or incorrectly finalized result is invalid and excluded from performance comparisons. +## Evidence levels + +1. **Historical synthetic baseline.** The committed `3.15×` figure is Testenix 0.1.0 on 100,000 + generated no-op tests, 16 modules, four workers, disabled history, and one M4 Pro. It remains + useful provenance but must not be labelled as current-version performance. +2. **Current-version scaling matrix.** `run_scaling_matrix.py` requires the installed Testenix + version to match `pyproject.toml` and refuses a dirty worktree by default. The dimension-sweep + design covers 100, 500, 1,000, and 3,000 tests; 1, 2, 4, and `auto` workers; + balanced/dominant/single-module layouts; and default history versus `--no-history`. The reference + configuration changes one axis at a time. `--full-cross-product` is available when every + combination is worth the substantially larger runtime; its artifact still projects one + canonical balanced/no-history reference curve. `auto` remains an adaptive Testenix + request; the harness records the worker count observed in each Testenix sample. For the xdist + side of that row, `auto` resolves separately to Python's logical CPU count and is labelled as + such. +3. **Real-project evidence.** `run_project_benchmark.py` executes argument arrays from a local JSON + manifest without a shell. Its result excludes source, stdout/stderr, environment values, + absolute project paths, and Git remotes. Publication requires a successful migration report: + the harness verifies its exact per-test inventory and outcomes, complete current Python-file + inventories and hashes under directory source roots, the complete generated Python inventory, + and that canonical `python -m pytest` / `python -m testenix run` commands point at the report's + source and output roots. It records only aggregate timings/output sizes, digests, redacted Git + state, and optional content fingerprints. Repeated or conflicting worker/history/sharding flags + fail closed, and an imported runtime must belong to the exact files owned by its installed + distribution rather than merely sharing the same `site-packages` directory. + +Every synthetic and real-project command has a bounded deadline and bounded cleanup. Windows starts +the command suspended, attaches a kill-on-close Job Object, then resumes it. POSIX always signals +the new root session and identity-checks every observed detached descendant. Polling cannot provide +an absolute kernel guarantee for a child that calls `setsid()` and exits before the first snapshot; +run hostile benchmark inputs inside a container or other kernel containment boundary. +On POSIX the harness also discovers workers that created their own sessions; on Windows it uses a +kill-on-close Job Object with a bounded recursive fallback. A timed-out runner therefore cannot +continue consuming CPU or contaminate later counterbalanced samples. + +The 118-test project mentioned in the v0.2.0 release was a differential semantic-validation gate. +Its three timings were single observations, not a committed multi-round benchmark, and therefore do +not establish a real-project speedup. + +## Project-local tuning is not a published benchmark + +`testenix tune` and its `testenix benchmark` alias answer a narrow operational question: which +native worker count is fastest for this selected project suite on this machine now? They use fresh +CLI processes, disable history, validate a green one-worker inventory, alternate candidate order, +require native candidate inventory/outcome parity, and recommend the smallest worker count within +a narrow tolerance of the best measured median. This is appropriate for writing a local +`[tool.testenix].workers` value. + +The optional `--pytest-source` measurement is orientation for a corresponding source suite. A +tuning report alone is not sufficient for marketing because a migrated native path and its pytest +source may differ in collection, wrappers, output, or configuration. Publishing a ratio still +requires all of this contract: equivalent inventories and outcomes, full-process wall time, +counterbalanced order, at least one warm-up and five measured rounds, command/version/environment +provenance, output sizes, and an explanation of history, sharding, manifest, and plugin settings. + +For every published Testenix result, record both the requested and observed worker count. `auto` is +adaptive and must never be relabelled as the logical CPU count. Also record whether +`--shard-modules` was enabled and whether a trusted collection manifest was accepted or fell back +to supervised collection; either choice changes the amount and shape of schedulable work. + ## Pytest compatibility bridge Measurements of `testenix pytest` must be reported separately from native `testenix run` @@ -43,25 +108,62 @@ environment, pytest configuration, plugins, and arguments for both `python -m py Testenix execution speedup. Native comparisons continue to use `testenix run` and must validate that both runners execute the same tests and produce equivalent outcomes. -Run the reproducible local harness with: +Run one reproducible synthetic scenario with: + +```bash +uv run --no-editable python benchmarks/run_benchmark.py \ + --tests 1000 --modules 16 --workers 4 --repeats 5 --warmups 1 \ + --module-layout balanced --history-mode disabled --xdist-strategy load +``` + +Generate the current-version dimension sweeps from a clean checkout with: + +```bash +uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +For an unpublished smoke test only, add `--quick --allow-dirty`. A publishable matrix must retain +five rounds, one warm-up, clean provenance, and the complete requested coverage. + +Measure a real project without committing its code or private paths: ```bash -uv run python benchmarks/run_benchmark.py --tests 1000 --workers 4 --repeats 5 -uv run python benchmarks/run_benchmark.py --tests 1000 --workers 4 --repeats 5 --uneven -uv run python benchmarks/run_benchmark.py --tests 10000 --modules 1000 --workers 4 --repeats 5 +cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +# Edit expected counts, migration-report path, fingerprints, and runner commands. +uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /absolute/path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json ``` +Keep private manifests and results outside the repository until their labels, commit policy, and +fingerprints have been reviewed for publication. Environment override **values** are never written +to the result; only their key names are retained. Commands are recorded for reproducibility, so +secrets should stay in the environment. If an unavoidable command argument is sensitive, list its +zero-based index in that runner's `redact_arguments` field. + +Without `migration_report`, the harness can still produce a private diagnostic, but it always sets +`publication_eligible` to `false`. A publishable run also requires the canonical module +entrypoints, a clean project, at least one warm-up and five measured rounds, explicit Testenix +workers and history mode, and installed-distribution identities for both pytest and Testenix. The +pytest version is probed inside the same project environment used by the timed commands. In each +runner command, place options before `--` and the exact benchmark suite roots after it. The harness +uses that delimiter to distinguish positional targets from values of options such as `-k` or +`--tag` and requires those targets to match the migration report exactly. + Maintainers can run the same comparison from GitHub's **Benchmarks** workflow and download its raw JSON artifact. Shared GitHub runners are appropriate for reproducibility checks, not for silently replacing the approved marketing baseline: their timing variance is outside this project's control. -The checked-in baseline files are development evidence, not universal performance claims. See -`docs/performance-analysis.md` for the current large-suite results, optimization profile, memory -notes, and native-code decision. Real project suites and cross-platform repetitions remain required -before publishing broad comparative claims. +The checked-in baseline files are historical development evidence, not universal or current-version +performance claims. See `docs/performance-analysis.md` for the large-suite results, optimization +profile, memory notes, and native-code decision. A clean Testenix 0.2.1 scaling matrix, publishable +real-project suites, alternative pytest-xdist strategies, and cross-platform repetitions remain +required before publishing broad comparative claims. An approved public baseline must be committed through a reviewed pull request. Do not remove slow but valid samples as outliers; invalid commands remain evidence and must be explained. The current checked-in 10,000- and 100,000-test files each contain five measured rounds, one warm-up, and clean -commit provenance. They remain single-machine synthetic evidence, so broader claims still require -the real-project and cross-platform scenarios above. +commit provenance. They remain Testenix 0.1.0 single-machine synthetic evidence, so broader claims +still require the current-version, real-project, and cross-platform scenarios above. diff --git a/docs/benchmarks/results.md b/docs/benchmarks/results.md index d4acd73..bb0eeed 100644 --- a/docs/benchmarks/results.md +++ b/docs/benchmarks/results.md @@ -5,9 +5,68 @@ evidence for specific synthetic workloads, not a universal claim that Testenix i than pytest. `Testenix` in these results means the native `testenix run` engine. The `testenix pytest` compatibility bridge delegates to pytest and is not represented here. -![Preliminary Testenix throughput ratios](../_static/benchmark-speedup.svg) +## Testenix 0.2.1 scaling matrix -## Median wall-clock time +No current-version matrix is checked in yet. The historical results below must therefore not be +described as Testenix 0.2.1 performance. The new provenance-gated harness covers +100/500/1,000/3,000 tests, balanced/dominant/single-module layouts, 1/2/4/auto workers, and both +default history and `--no-history`, plus explicit safe-module sharding. Its default design uses +dimension sweeps; use +`--full-cross-product` only when the much larger run is intentional. + +`auto` is passed literally to Testenix and remains adaptive; observed Testenix worker counts are +stored per sample. pytest-xdist resolves its side of an `auto` row separately to the machine's +logical CPU count. + +```console +$ uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +The command refuses a dirty worktree or an installed Testenix version that differs from +`pyproject.toml`. `--allow-dirty` is available only for unpublished smoke runs. A matrix becomes +publishable here only after five measured rounds, one warm-up, clean commit provenance, and full +axis coverage pass the documentation generator's validation. + + +## Real-project harness + +The 118-test project used during v0.2 migration validation was a semantic parity gate, not a +publishable benchmark: its release-note timings were single observations without a committed +multi-round record. Use the redaction-safe manifest harness for a real repository: + +```console +$ cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +$ uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /absolute/path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json +``` + +The manifest stores argument arrays, never shell fragments. The result omits stdout, stderr, +environment values, absolute project paths, and private source. It records only timings, aggregate +output sizes, optional tree fingerprints, and redacted Git provenance. A migrated-suite comparison +must point the manifest at a successful migration report to become publication-eligible. The +harness verifies the report's exact per-test inventory and outcomes, complete source and generated +Python-file inventories, current hashes, and binds canonical `python -m pytest` / +`python -m testenix run` commands to the report's source/output roots. Publishable source roots are +directories so support files such as `conftest.py` are covered. Without the report the result is +diagnostic-only. Commands are retained for +reproducibility. Publishable commands put options before `--` and exact suite targets after it, so +an option value cannot impersonate a migration root. Keep secrets in the environment or list +sensitive argument indexes in `redact_arguments`. + + +## Historical Testenix 0.1.0 synthetic baseline + +The checked-in `3.15×` figure is a Testenix 0.1.0 result for 100,000 generated no-op +tests across 16 modules, four workers, disabled history (`--no-history`), and pytest-xdist's default +`load` strategy. It is retained as transparent historical evidence; it is not a measurement of +Testenix 0.2.1. + +![Historical Testenix 0.1.0 throughput ratios](../_static/benchmark-speedup.svg) + +### Median wall-clock time Lower time is better. A speedup of `2.85×` means pytest's median wall time was 2.85 times the Testenix median for that exact @@ -24,19 +83,24 @@ The 100,000-test result meets the project's local five-run, one-warmup minimum. It remains a synthetic result from one machine, not a universal performance promise. -## Environment +### Environment and controls - CPU: Apple M4 Pro (14 logical CPUs) - Machine: `arm64` - Platform: `macOS-26.5.1-arm64-arm-64bit` - Python: `3.11.14` +- Testenix: `0.1.0` +- Workers: four for Testenix and pytest-xdist +- Testenix history: disabled with `--no-history` +- pytest-xdist: version `3.8.0`, + default `load` distribution - Measurement: complete subprocess wall-clock time, including discovery, execution, aggregation, and console rendering - Correctness gate: every command had to exit successfully and report the expected test count -## Raw samples and variance +### Raw samples and variance ### 10,000 no-op tests / 16 modules @@ -51,6 +115,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 2.170, 2.267, 2.077, 2.075, 2.106 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:12:43.635798+00:00` - Commit: `8f24f8a7bd72fa876988a8ce96364be97e35c2b6` - Clean working tree at capture: yes @@ -69,6 +135,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 2.146, 2.129, 2.109, 2.138, 2.176 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:13:53.369799+00:00` - Commit: `18d9bba6cb5c8e39c2d5b211ee4384ae8f824524` - Clean working tree at capture: yes @@ -87,6 +155,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 21.239, 21.120, 22.216, 21.300, 21.949 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:19:39.942492+00:00` - Commit: `24b877c2f98420e91dcd2c8bcbc9417c7cf1ac96` - Clean working tree at capture: yes @@ -97,7 +167,9 @@ It remains a synthetic result from one machine, not a universal performance prom These separate measurements start with generated pytest or unittest sources, complete one safe copy-and-validate migration, and then compare recurring source-suite runs with recurring native Testenix runs. The migration transaction is a one-time cost shown separately; it is not included -in either execution median. +in either execution median. These records came from the pre-v0.2 source commit linked below; its +distribution metadata still reported `0.1.0`. They are historical evidence, not measurements of +the current release. | Source runner | Workload | Tests / modules | Source median | Native median | Native vs source | Migration transaction | | --- | --- | ---: | ---: | ---: | ---: | ---: | @@ -191,9 +263,9 @@ therefore material, and none of these synthetic rows predicts a specific real pr ## Interpretation -The checked-in results show that Testenix has low per-test overhead for large generated suites and -that its built-in process model is competitive with both sequential pytest and pytest-xdist in -those scenarios. +The historical checked-in results show that Testenix 0.1.0 had low per-test overhead for the large +generated suites above and was competitive with sequential pytest and pytest-xdist's default +`load` strategy in those scenarios. They are not evidence for the current release. They do **not** yet answer how Testenix performs for import-heavy applications, complex fixture graphs, assertion failures, real repositories, or different operating systems. Pytest also has a diff --git a/docs/for-llms.md b/docs/for-llms.md index b7f16f2..da5e883 100644 --- a/docs/for-llms.md +++ b/docs/for-llms.md @@ -32,7 +32,8 @@ workload-specific and preserve all documented limitations. - installation and first-run instructions; - the pytest compatibility bridge, capability matrix, and migration boundary; - native tests, cases, tags, skips, expected failures, and fixtures; -- parallelism, timeouts, retries, crash recovery, reports, and history; +- adaptive parallelism, tuning, optional safety-checked module sharding, trusted collection + manifests, timeouts, retries, crash recovery, reports, and history; - CLI, configuration, and generated Python API reference; - architecture and roadmap; - benchmark results, raw-data links, methodology, and caveats. diff --git a/docs/getting-started.md b/docs/getting-started.md index 6e2727c..fe88037 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -126,6 +126,8 @@ paths = ["tests"] workers = "auto" retries = 0 history = ".testenix/history.sqlite3" +# shard_modules = true +# manifest = ".testenix/collection.json" # timeout = 10 # json = "reports/testenix.json" # junit = "reports/junit.xml" @@ -140,6 +142,33 @@ $ testenix run --workers 4 --retries 1 --json reports/result.json Use `--no-history` for a side-effect-free run. Duration history normally helps later runs schedule long tests earlier. +`workers = "auto"` adapts to the selected suite instead of copying the logical CPU count. It is +capped by the actual schedulable units and uses duration history when enough is available. Measure +an explicit project setting with: + +```console +$ testenix tune tests --warmups 1 --repeats 5 +$ testenix tune --write +``` + +`testenix benchmark` is an alias for `testenix tune`. + +Module affinity is the safe default. For a large module whose tests are known to be independent, +`--shard-modules` opts eligible tests into finer scheduling after conservative static checks. Read +[parallel execution](guides/parallelism.md) before enabling it. + +To avoid repeating collection imports on later unchanged runs, create and trust a source-hashed +manifest explicitly: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +Testenix verifies the complete file inventory and every SHA-256 before reuse. A stale manifest +falls back to supervised collection rather than running a stale test selection. Parameter names +remain visible in the artifact, but their values are redacted. + ## Use it from Python The runner also exposes a typed API: @@ -147,12 +176,20 @@ The runner also exposes a typed API: ```python from testenix import Status, TestenixConfig, run -result = run("tests", TestenixConfig(workers=4, history_path=None)) -failed = [item.test.id for item in result.tests if item.status is not Status.PASS] + +def main() -> None: + result = run("tests", TestenixConfig(workers=4, history_path=None)) + failed = [item.test.id for item in result.tests if item.status is not Status.PASS] + print(failed) + + +if __name__ == "__main__": + main() ``` Async applications can call `await testenix.run_async(...)`. Cancellation terminates active -collection and execution process trees before control returns to the caller. +collection and execution process trees before control returns to the caller. Executable scripts +must place the top-level call behind the same `if __name__ == "__main__":` guard on every platform. ## Next steps diff --git a/docs/guides/migration.md b/docs/guides/migration.md index 47bf83c..91c61a9 100644 --- a/docs/guides/migration.md +++ b/docs/guides/migration.md @@ -197,10 +197,13 @@ those tests were published. ## Performance with thousands of migrated tests Migration unlocks the native scheduler, process supervision, async engine, retries, history, and -Testenix reports. It does not guarantee that every converted suite is faster. Testenix keeps a -normal module as one affinity unit so module-scoped fixtures are not duplicated. Consequently, -3,000 tests in one source module still form one schedulable unit; 3,000 tests spread across enough -independent modules can use multiple workers. +Testenix reports. It does not guarantee that every converted suite is faster. By default Testenix +keeps a normal module as one affinity unit so module-scoped fixtures are not duplicated. +Consequently, 3,000 tests in one source module still form one schedulable unit; 3,000 tests spread +across enough independent modules can use multiple workers. A project may later opt eligible native +modules into `--shard-modules`, but only after validating the generated suite and the documented +static-analysis trust boundary. Run `testenix tune` on the published native copy before fixing its +CI worker count. The checked-in migration baseline used an Apple M4 Pro, CPython 3.11, 3,000 generated tests in 64 modules, four native workers, one warm-up, and five measured rounds: diff --git a/docs/guides/parallelism.md b/docs/guides/parallelism.md index 35c4647..591532b 100644 --- a/docs/guides/parallelism.md +++ b/docs/guides/parallelism.md @@ -10,8 +10,53 @@ $ testenix run --workers 4 $ testenix run --workers 1 ``` -`auto` resolves to the logical CPU count reported by Python. For CI and benchmark runs, an explicit -count makes resource use and results easier to reproduce. +`auto` is adaptive; it is not an alias for the logical CPU count. Testenix first counts the +execution units it can actually schedule (module-affinity groups, isolated timeout tests, and any +eligible opt-in module shards). It caps the choice by that count and the available CPUs. With +reliable duration history, a startup-cost/makespan model chooses the smallest worker count within a +narrow tolerance of the predicted best. Without enough history, cold-start auto uses a conservative +cap so a short suite does not launch one process per CPU. + +An explicit number remains a hard project setting. Prefer one in tightly budgeted CI; for a +project-specific measured value, use the tuner. + +## Tune a project + +```console +$ testenix tune tests --warmups 1 --repeats 5 +$ testenix benchmark tests --candidates 1,2,4,8 +$ testenix tune --json reports/tuning.json --write +``` + +`benchmark` is an alias for `tune`. The command runs fresh CLI processes, uses a green one-worker +inventory probe, disables history for all timing samples, measures native candidates in alternating +order, and rejects a candidate whose test inventory or outcomes differ. The recommendation is the +smallest worker count within a narrow tolerance of the best median, avoiding a larger setting for +measurement noise. `--write` stores that integer in +`[tool.testenix].workers`; it is an explicit file change, never an effect of `workers = "auto"`. +The automatic sweep respects the process-visible CPU capacity and tests at most 1/2/4 workers; +larger counts must be requested explicitly with `--candidates`. Each complete suite run has a +300-second deadline by default (`--run-timeout SECONDS`). Windows starts the command suspended and +attaches a kill-on-close Job Object before resume. POSIX always signals the new root session and +identity-checks observed detached descendants. A child that calls `setsid()` and exits between +creation and the first process snapshot is outside an absolute kernel-containment guarantee; use a +container for hostile suites. + +Pass `--shard-modules` to tune that explicit mode and `--manifest FILE` to include the verified +single-import collection path in every native sample. When `--write` is present, Testenix refuses a +transient sharding or manifest override that differs from `[tool.testenix]`, because writing only +`workers` would persist a recommendation for a different execution profile. Configure the profile +first or tune it without `--write`. It also fingerprints project Python/TOML sources, explicit suite +files (including linked source directories), and the trusted manifest, then rejects any observed +content or file-identity drift after a sample. The final configuration update rechecks the original +bytes immediately before an atomic replacement. Use an immutable checkout for publishable tuning: +it excludes a writer racing that last filesystem operation, while installed packages, non-source +data, and other runtime dependencies remain external inputs. + +Use `--pytest-source PATH` to add an optional pytest timing for a corresponding source suite. The +tuner's primary contract is worker selection for the native suite. Its optional pytest ratio is +orientation for that exact invocation, not sufficient evidence for a public speed claim; public +comparisons must also satisfy the [benchmarking contract](../benchmarking.md). ## Scheduling @@ -21,6 +66,62 @@ preserves module-fixture reuse and avoids splitting hidden module state between When duration history exists, Testenix schedules longer units first. This longest-processing-time strategy is deterministic and reduces the chance that one slow shard becomes the tail of the run. +## Opt-in intra-module sharding + +One large module normally exposes one unit no matter how many tests it contains. If its tests are +known to be independent, explicitly request finer units: + +```console +$ testenix run tests --shard-modules +``` + +or configure: + +```toml +[tool.testenix] +shard_modules = true +``` + +This is deliberately off by default. Before splitting a module, Testenix statically fails closed +when it detects module- or session-scoped fixtures, imported fixture providers, direct writes to +module globals, nested mutable containers, mutable class state, or executable import-time lifecycle +behavior. Function-scoped fixtures defined in the collected module, including autouse fixtures, +may be recreated in separate workers and do not block sharding. Eager calls in module assignments, +annotations, decorators, function defaults, and class bases or keywords are treated as import-time +lifecycle behavior and keep the module intact. + +Static analysis cannot prove that arbitrary calls, imported libraries, environment state, or +external services are free of shared effects. Enabling the option is therefore a project trust +decision. Validate the suite both with and without sharding before adopting it in CI. Modules that +fail the safety check retain normal module affinity while eligible modules can be split. + +## Avoid the collection-side import + +Without a manifest, safe supervised collection imports selected modules once, then execution +workers import their assigned modules again to materialize functions and case values. Projects for +which imports are a meaningful part of wall time can generate an explicit trusted manifest: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +The manifest records collection roots, every selected test file plus statically discoverable +project-local Python import dependencies and their SHA-256 hashes, test metadata, collection issues, +and sharding decisions. Parameter names are retained but values +are redacted so collection-time environment data is not persisted. Each run verifies the requested roots, +the exact file set, and every source digest before reusing it. Malformed manifest JSON is rejected +as invalid input. If a file was added or removed, a source changed, or current-source verification +cannot establish an exact match, the manifest is stale and Testenix falls back to the normal +isolated collection process. It does not execute a stale selection. + +This optimization removes the collection-side import on an unchanged run; execution workers still +import the code they execute. It is not an implicit cache. If collection depends on environment +variables, generated files outside the selected roots, network state, or other inputs not captured +by source hashes, the producer must regenerate the manifest when those inputs change. Set +`manifest = ".testenix/collection.json"` in `[tool.testenix]` only when that trust boundary is +appropriate. + ## Process isolation Workers are spawned processes. A worker streams each completed attempt to the coordinator before @@ -67,7 +168,8 @@ async def main() -> None: print(result.exit_code) -asyncio.run(main()) +if __name__ == "__main__": + asyncio.run(main()) ``` Cancelling `run_async` terminates active collection and execution process trees before the @@ -75,8 +177,9 @@ coroutine returns control. ## Platform note -On Windows, scripts that call `run()` or `run_async()` directly must use the standard -multiprocessing guard: +On every supported platform, executable scripts that call `run()` or `run_async()` directly must +use the standard multiprocessing guard because the supervised worker protocol uses the `spawn` +start method: ```python if __name__ == "__main__": diff --git a/docs/index.md b/docs/index.md index f54a427..3fd239e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -71,15 +71,15 @@ for-llms
0dependencies in the native runtime
12Python and OS combinations in CI
3console, JSON, and JUnit reports
-
3.15×native testenix run on the 100k synthetic workload vs pytest
+
3.15×historical v0.1.0 result: 100k synthetic no-op tests, 4 workers, --no-history
Testenix is deliberately built around a few strong guarantees: - **Async is native.** Coroutine tests and async-generator fixtures use the same model as synchronous code and do not require a plugin. -- **Parallelism is part of the runner.** Module affinity, process isolation, and - duration-aware scheduling are designed together. +- **Parallelism is part of the runner.** Adaptive worker selection, module affinity, optional + safety-checked sharding, process isolation, and duration-aware scheduling are designed together. - **Retries preserve evidence.** A failed attempt followed by a pass is `FLAKY`, never silently rewritten as a clean pass. - **Crashes cannot erase completed work.** Workers stream results as tests finish, and unfinished @@ -139,20 +139,28 @@ $ python -m pip install testenix $ testenix run tests ``` +`workers = "auto"` adapts to the schedulable work instead of launching one process per CPU. Use +`testenix tune` (also available as `testenix benchmark`) for a measured project recommendation. +Import-heavy native suites can explicitly generate a source-hashed collection manifest, and large +independent modules can opt into conservative `--shard-modules` scheduling. See +[parallel execution](guides/parallelism/) for both trust boundaries. + To evaluate unreleased source changes, install the current `main` branch with `python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main"`. ## Performance evidence, with context -The checked-in development baseline measured native `testenix run` on 100,000 empty tests across -16 generated modules on an Apple M4 Pro and CPython 3.11. Native Testenix completed that specific -workload in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds for -pytest-xdist. These measurements do not apply to the delegated `testenix pytest` command. +The checked-in development baseline measured **Testenix 0.1.0**, not the current release. Native +`testenix run` completed 100,000 generated no-op tests across 16 modules on one Apple M4 Pro and +CPython 3.11 machine in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 +seconds for pytest-xdist. It used four workers, `--no-history`, and pytest-xdist's default `load` +scheduler. These measurements do not apply to Testenix 0.2.1, a real project, the default-history +mode, or the delegated `testenix pytest` command.
-This is a preliminary synthetic result from one machine, not a promise that every project will be -3.15× faster. The benchmark page publishes the raw samples, environment, variance, methodology, -and limitations so that the claim can be evaluated rather than taken on trust. +This is historical synthetic evidence from one machine, not a promise that every project will be +3.15× faster. No clean Testenix 0.2.1 scaling matrix is checked in yet. The benchmark page publishes +the raw samples, environment, variance, methodology, current matrix status, and limitations.
[Inspect the benchmark data](benchmarks/results/) or diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 2eb96ff..4513014 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -85,15 +85,15 @@ for-llms
0dependencies in the native runtime
12Python and OS combinations in CI
3console, JSON, and JUnit reports
-
3.15×native testenix run on the 100k synthetic workload vs pytest
+
3.15×historical v0.1.0 result: 100k synthetic no-op tests, 4 workers, --no-history
Testenix is deliberately built around a few strong guarantees: - **Async is native.** Coroutine tests and async-generator fixtures use the same model as synchronous code and do not require a plugin. -- **Parallelism is part of the runner.** Module affinity, process isolation, and - duration-aware scheduling are designed together. +- **Parallelism is part of the runner.** Adaptive worker selection, module affinity, optional + safety-checked sharding, process isolation, and duration-aware scheduling are designed together. - **Retries preserve evidence.** A failed attempt followed by a pass is `FLAKY`, never silently rewritten as a clean pass. - **Crashes cannot erase completed work.** Workers stream results as tests finish, and unfinished @@ -153,20 +153,28 @@ $ python -m pip install testenix $ testenix run tests ``` +`workers = "auto"` adapts to the schedulable work instead of launching one process per CPU. Use +`testenix tune` (also available as `testenix benchmark`) for a measured project recommendation. +Import-heavy native suites can explicitly generate a source-hashed collection manifest, and large +independent modules can opt into conservative `--shard-modules` scheduling. See +[parallel execution](https://polishdataengineer.github.io/testenix/guides/parallelism/) for both trust boundaries. + To evaluate unreleased source changes, install the current `main` branch with `python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main"`. ## Performance evidence, with context -The checked-in development baseline measured native `testenix run` on 100,000 empty tests across -16 generated modules on an Apple M4 Pro and CPython 3.11. Native Testenix completed that specific -workload in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds for -pytest-xdist. These measurements do not apply to the delegated `testenix pytest` command. +The checked-in development baseline measured **Testenix 0.1.0**, not the current release. Native +`testenix run` completed 100,000 generated no-op tests across 16 modules on one Apple M4 Pro and +CPython 3.11 machine in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 +seconds for pytest-xdist. It used four workers, `--no-history`, and pytest-xdist's default `load` +scheduler. These measurements do not apply to Testenix 0.2.1, a real project, the default-history +mode, or the delegated `testenix pytest` command.
-This is a preliminary synthetic result from one machine, not a promise that every project will be -3.15× faster. The benchmark page publishes the raw samples, environment, variance, methodology, -and limitations so that the claim can be evaluated rather than taken on trust. +This is historical synthetic evidence from one machine, not a promise that every project will be +3.15× faster. No clean Testenix 0.2.1 scaling matrix is checked in yet. The benchmark page publishes +the raw samples, environment, variance, methodology, current matrix status, and limitations.
[Inspect the benchmark data](https://polishdataengineer.github.io/testenix/benchmarks/results/) or @@ -316,6 +324,8 @@ paths = ["tests"] workers = "auto" retries = 0 history = ".testenix/history.sqlite3" +# shard_modules = true +# manifest = ".testenix/collection.json" # timeout = 10 # json = "reports/testenix.json" # junit = "reports/junit.xml" @@ -330,6 +340,33 @@ $ testenix run --workers 4 --retries 1 --json reports/result.json Use `--no-history` for a side-effect-free run. Duration history normally helps later runs schedule long tests earlier. +`workers = "auto"` adapts to the selected suite instead of copying the logical CPU count. It is +capped by the actual schedulable units and uses duration history when enough is available. Measure +an explicit project setting with: + +```console +$ testenix tune tests --warmups 1 --repeats 5 +$ testenix tune --write +``` + +`testenix benchmark` is an alias for `testenix tune`. + +Module affinity is the safe default. For a large module whose tests are known to be independent, +`--shard-modules` opts eligible tests into finer scheduling after conservative static checks. Read +[parallel execution](https://polishdataengineer.github.io/testenix/guides/parallelism/) before enabling it. + +To avoid repeating collection imports on later unchanged runs, create and trust a source-hashed +manifest explicitly: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +Testenix verifies the complete file inventory and every SHA-256 before reuse. A stale manifest +falls back to supervised collection rather than running a stale test selection. Parameter names +remain visible in the artifact, but their values are redacted. + ## Use it from Python The runner also exposes a typed API: @@ -337,12 +374,20 @@ The runner also exposes a typed API: ```python from testenix import Status, TestenixConfig, run -result = run("tests", TestenixConfig(workers=4, history_path=None)) -failed = [item.test.id for item in result.tests if item.status is not Status.PASS] + +def main() -> None: + result = run("tests", TestenixConfig(workers=4, history_path=None)) + failed = [item.test.id for item in result.tests if item.status is not Status.PASS] + print(failed) + + +if __name__ == "__main__": + main() ``` Async applications can call `await testenix.run_async(...)`. Cancellation terminates active -collection and execution process trees before control returns to the caller. +collection and execution process trees before control returns to the caller. Executable scripts +must place the top-level call behind the same `if __name__ == "__main__":` guard on every platform. ## Next steps @@ -712,10 +757,13 @@ those tests were published. ## Performance with thousands of migrated tests Migration unlocks the native scheduler, process supervision, async engine, retries, history, and -Testenix reports. It does not guarantee that every converted suite is faster. Testenix keeps a -normal module as one affinity unit so module-scoped fixtures are not duplicated. Consequently, -3,000 tests in one source module still form one schedulable unit; 3,000 tests spread across enough -independent modules can use multiple workers. +Testenix reports. It does not guarantee that every converted suite is faster. By default Testenix +keeps a normal module as one affinity unit so module-scoped fixtures are not duplicated. +Consequently, 3,000 tests in one source module still form one schedulable unit; 3,000 tests spread +across enough independent modules can use multiple workers. A project may later opt eligible native +modules into `--shard-modules`, but only after validating the generated suite and the documented +static-analysis trust boundary. Run `testenix tune` on the published native copy before fixing its +CI worker count. The checked-in migration baseline used an Apple M4 Pro, CPython 3.11, 3,000 generated tests in 64 modules, four native workers, one warm-up, and five measured rounds: @@ -1091,8 +1139,53 @@ $ testenix run --workers 4 $ testenix run --workers 1 ``` -`auto` resolves to the logical CPU count reported by Python. For CI and benchmark runs, an explicit -count makes resource use and results easier to reproduce. +`auto` is adaptive; it is not an alias for the logical CPU count. Testenix first counts the +execution units it can actually schedule (module-affinity groups, isolated timeout tests, and any +eligible opt-in module shards). It caps the choice by that count and the available CPUs. With +reliable duration history, a startup-cost/makespan model chooses the smallest worker count within a +narrow tolerance of the predicted best. Without enough history, cold-start auto uses a conservative +cap so a short suite does not launch one process per CPU. + +An explicit number remains a hard project setting. Prefer one in tightly budgeted CI; for a +project-specific measured value, use the tuner. + +## Tune a project + +```console +$ testenix tune tests --warmups 1 --repeats 5 +$ testenix benchmark tests --candidates 1,2,4,8 +$ testenix tune --json reports/tuning.json --write +``` + +`benchmark` is an alias for `tune`. The command runs fresh CLI processes, uses a green one-worker +inventory probe, disables history for all timing samples, measures native candidates in alternating +order, and rejects a candidate whose test inventory or outcomes differ. The recommendation is the +smallest worker count within a narrow tolerance of the best median, avoiding a larger setting for +measurement noise. `--write` stores that integer in +`[tool.testenix].workers`; it is an explicit file change, never an effect of `workers = "auto"`. +The automatic sweep respects the process-visible CPU capacity and tests at most 1/2/4 workers; +larger counts must be requested explicitly with `--candidates`. Each complete suite run has a +300-second deadline by default (`--run-timeout SECONDS`). Windows starts the command suspended and +attaches a kill-on-close Job Object before resume. POSIX always signals the new root session and +identity-checks observed detached descendants. A child that calls `setsid()` and exits between +creation and the first process snapshot is outside an absolute kernel-containment guarantee; use a +container for hostile suites. + +Pass `--shard-modules` to tune that explicit mode and `--manifest FILE` to include the verified +single-import collection path in every native sample. When `--write` is present, Testenix refuses a +transient sharding or manifest override that differs from `[tool.testenix]`, because writing only +`workers` would persist a recommendation for a different execution profile. Configure the profile +first or tune it without `--write`. It also fingerprints project Python/TOML sources, explicit suite +files (including linked source directories), and the trusted manifest, then rejects any observed +content or file-identity drift after a sample. The final configuration update rechecks the original +bytes immediately before an atomic replacement. Use an immutable checkout for publishable tuning: +it excludes a writer racing that last filesystem operation, while installed packages, non-source +data, and other runtime dependencies remain external inputs. + +Use `--pytest-source PATH` to add an optional pytest timing for a corresponding source suite. The +tuner's primary contract is worker selection for the native suite. Its optional pytest ratio is +orientation for that exact invocation, not sufficient evidence for a public speed claim; public +comparisons must also satisfy the [benchmarking contract](https://polishdataengineer.github.io/testenix/benchmarking/). ## Scheduling @@ -1102,6 +1195,62 @@ preserves module-fixture reuse and avoids splitting hidden module state between When duration history exists, Testenix schedules longer units first. This longest-processing-time strategy is deterministic and reduces the chance that one slow shard becomes the tail of the run. +## Opt-in intra-module sharding + +One large module normally exposes one unit no matter how many tests it contains. If its tests are +known to be independent, explicitly request finer units: + +```console +$ testenix run tests --shard-modules +``` + +or configure: + +```toml +[tool.testenix] +shard_modules = true +``` + +This is deliberately off by default. Before splitting a module, Testenix statically fails closed +when it detects module- or session-scoped fixtures, imported fixture providers, direct writes to +module globals, nested mutable containers, mutable class state, or executable import-time lifecycle +behavior. Function-scoped fixtures defined in the collected module, including autouse fixtures, +may be recreated in separate workers and do not block sharding. Eager calls in module assignments, +annotations, decorators, function defaults, and class bases or keywords are treated as import-time +lifecycle behavior and keep the module intact. + +Static analysis cannot prove that arbitrary calls, imported libraries, environment state, or +external services are free of shared effects. Enabling the option is therefore a project trust +decision. Validate the suite both with and without sharding before adopting it in CI. Modules that +fail the safety check retain normal module affinity while eligible modules can be split. + +## Avoid the collection-side import + +Without a manifest, safe supervised collection imports selected modules once, then execution +workers import their assigned modules again to materialize functions and case values. Projects for +which imports are a meaningful part of wall time can generate an explicit trusted manifest: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +The manifest records collection roots, every selected test file plus statically discoverable +project-local Python import dependencies and their SHA-256 hashes, test metadata, collection issues, +and sharding decisions. Parameter names are retained but values +are redacted so collection-time environment data is not persisted. Each run verifies the requested roots, +the exact file set, and every source digest before reusing it. Malformed manifest JSON is rejected +as invalid input. If a file was added or removed, a source changed, or current-source verification +cannot establish an exact match, the manifest is stale and Testenix falls back to the normal +isolated collection process. It does not execute a stale selection. + +This optimization removes the collection-side import on an unchanged run; execution workers still +import the code they execute. It is not an implicit cache. If collection depends on environment +variables, generated files outside the selected roots, network state, or other inputs not captured +by source hashes, the producer must regenerate the manifest when those inputs change. Set +`manifest = ".testenix/collection.json"` in `[tool.testenix]` only when that trust boundary is +appropriate. + ## Process isolation Workers are spawned processes. A worker streams each completed attempt to the coordinator before @@ -1148,7 +1297,8 @@ async def main() -> None: print(result.exit_code) -asyncio.run(main()) +if __name__ == "__main__": + asyncio.run(main()) ``` Cancelling `run_async` terminates active collection and execution process trees before the @@ -1156,8 +1306,9 @@ coroutine returns control. ## Platform note -On Windows, scripts that call `run()` or `run_async()` directly must use the standard -multiprocessing guard: +On every supported platform, executable scripts that call `run()` or `run_async()` directly must +use the standard multiprocessing guard because the supervised worker protocol uses the `spawn` +start method: ```python if __name__ == "__main__": @@ -1322,13 +1473,16 @@ testenix [-h] [--version] testenix [--config PYPROJECT] run [RUN_ARGS ...] testenix pytest [PYTEST_ARGS ...] testenix migrate FRAMEWORK PATH [PATH ...] [MIGRATION_ARGS ...] +testenix [--config PYPROJECT] tune [PATH ...] [TUNING_ARGS ...] +testenix [--config PYPROJECT] benchmark [PATH ...] [TUNING_ARGS ...] +testenix manifest PATH [PATH ...] --output FILE ``` | Option | Description | | --- | --- | | `-h`, `--help` | Show command help. | | `--version` | Print the installed Testenix version. | -| `--config PATH` | Load `[tool.testenix]` for native `testenix run`. | +| `--config PATH` | Load `[tool.testenix]` for native `run` and `tune`/`benchmark` commands. | ## `testenix run` @@ -1342,6 +1496,8 @@ testenix run [PATH ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] + [--shard-modules] + [--manifest FILE] [-q | -v | -vv] [--color {auto,always,never} | --no-color] [--show-skips] @@ -1351,7 +1507,7 @@ testenix run [PATH ...] | Argument | Default | Description | | --- | --- | --- | | `PATH ...` | configured `paths`, otherwise `tests` | Files or directories to discover. | -| `-w`, `--workers` | `auto` | Worker process count or logical CPU count. | +| `-w`, `--workers` | `auto` | Positive worker count, or adaptive selection from schedulable units, history, startup cost, and CPU capacity. | | `--retries` | `0` | Additional attempts after a gating outcome. | | `--timeout` | none | Global hard deadline for every selected test. | | `-t`, `--tag` | none | Required tag; repeat for AND selection. | @@ -1359,6 +1515,8 @@ testenix run [PATH ...] | `--junit` | none | Write a JUnit XML report. | | `--history` | `.testenix/history.sqlite3` | Override the duration-history database. | | `--no-history` | off | Disable reading and writing history. | +| `--shard-modules` | off | Opt eligible modules into per-test execution units after conservative static safety checks. | +| `--manifest FILE` | none | Reuse an explicitly generated, source-verified trusted collection manifest. | | `-q`, `--quiet` | off | Hide the run header and compact per-file table. Collection errors, failure details, and the final summary remain visible. | | `-v`, `--verbose` | off | Print one result row per test in the stable detailed format. Repeat for `-vv`. | | `-vv` | off | Add worker, attempt, and phase metadata, including captured output. | @@ -1375,6 +1533,20 @@ order after execution completes; these modes do not promise live progress update flags change only terminal rendering, not selection, scheduling, result statuses, JSON, JUnit, or exit codes. +`workers = auto` never means “start one process per logical CPU.” Testenix caps it by the number of +units that can run independently. Reliable duration history feeds a makespan/startup-cost model; +cold runs use a conservative cap. The final console and JSON results report the worker count +actually used. An explicit integer remains useful for fixed CI resource limits and reproducible +published benchmarks. + +`--shard-modules` is an explicit safety/performance trade-off. Modules with module/session +fixtures, statically visible mutable global state, or import-time lifecycle hazards keep module +affinity; eligible modules may be split. Static analysis cannot prove every dynamic side effect. + +`--manifest` accepts the versioned JSON produced by `testenix manifest`. Malformed input is a usage +error. An otherwise valid manifest whose roots, complete Python-file inventory, or SHA-256 digests +no longer match is treated as stale, and the run safely performs ordinary supervised collection. + In `auto` color mode, Testenix requires a terminal, respects `NO_COLOR`, allows `FORCE_COLOR`, and disables styling for a truthy `CI` value or `TERM=dumb`. Explicit `always` or `never` takes precedence. @@ -1451,6 +1623,80 @@ grouped by severity and code, with the first source location shown. Use `--repor worker setting. It is emitted only for `--check` or publication after static analysis succeeds, because dry-run and unsupported transactions never execute the parallel gate. +## `testenix tune` / `testenix benchmark` + +```text +testenix tune [PATH ...] + [--config PYPROJECT] + [--candidates N[,N...]] + [--warmups N] + [--repeats N] + [--pytest-source PATH] ... + [--json FILE|-] + [--shard-modules] + [--manifest FILE] + [--write] +``` + +`testenix benchmark` is an exact alias for `testenix tune`. + +| Argument | Default | Description | +| --- | --- | --- | +| `PATH ...` | configured `paths`, otherwise `tests` | Native Testenix suite to tune. | +| `--candidates N[,N...]` | resource-aware 1/2/4 sweep | Positive worker counts to measure. Counts above the number of execution units are deduplicated at that limit; values above four require this explicit option. | +| `--warmups N` | `1` | Unrecorded warmups for each native candidate and optional pytest source. | +| `--repeats N` | `5` | Recorded samples for each candidate. Candidate order alternates between rounds. | +| `--run-timeout SECONDS` | `300` | Deadline for each complete native or pytest suite run; cleanup uses a Windows Job Object or POSIX root-session plus identity-tracked descendants. | +| `--pytest-source PATH` | none | Also time a corresponding pytest source path; repeat for multiple paths. | +| `--json FILE\|-` | none | Write the complete tuning report to a new file, or standard output with `-`. | +| `--shard-modules` / `--no-shard-modules` | configured value | Tune with explicit safe intra-module sharding or module affinity. | +| `--manifest FILE` | configured value | Use one source-verified collection manifest for every native sample. | +| `--write` | off | Persist the measured recommendation as `[tool.testenix].workers`. | + +The command measures fresh CLI processes with Testenix history disabled. A one-worker probe +establishes the inventory and outcomes, and every native candidate must match them. To avoid +persisting noise, it recommends the smallest worker count within a narrow tolerance of the best +median. `--write` is the only configuration-mutating mode; normal tuning and adaptive auto do not +edit configuration. A workers-only write is rejected when a transient sharding or manifest override +differs from the loaded project configuration, or when that configuration file changes during the +measurement, because the recommendation would not describe the persisted execution profile. The +writer compares the pre-tuning bytes again immediately before an atomic replacement. Project +Python/TOML sources, linked source directories, explicit suite files, and a trusted manifest are +fingerprinted by content and file identity after every sample; observed drift discards the complete +result. Run publishable tuning from an immutable checkout to exclude a writer racing the final +filesystem operation; installed packages, non-source data, and other runtime inputs are also outside +that fingerprint. + +The optional pytest row is useful for local orientation only when it represents the corresponding +source suite. A public pytest/Testenix claim still needs equivalent inventories and outcomes, +counterbalanced commands, environment/version provenance, and the complete +[benchmarking contract](https://polishdataengineer.github.io/testenix/benchmarking/). + +## `testenix manifest` + +```text +testenix manifest PATH [PATH ...] --output FILE +``` + +The command performs supervised native collection and creates a new deterministic trusted +manifest. It records collection roots, the complete selected Python-source inventory and SHA-256 +fingerprints, collected tests and issues, and conservative module-sharding decisions. Test case +parameter names are retained while their values are redacted, so environment-derived secrets are +not copied into the trust artifact. `FILE` must +not already exist; Testenix does not silently replace a previous trust artifact. + +Use it explicitly on later runs: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +This can avoid importing every selected module once for collection and once again for execution. +Execution workers still import the modules they run. Source verification cannot cover dynamic +collection inputs such as environment variables or external services; regenerate the manifest +when those inputs change. + ## Examples ```console @@ -1461,6 +1707,12 @@ $ testenix run --retries 1 --timeout 10 $ testenix run -v --show-skips --durations 10 $ testenix run --color never $ testenix run --json reports/run.json --junit reports/junit.xml +$ testenix run tests --shard-modules +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +$ testenix tune tests --candidates 1,2,4,8 --warmups 1 --repeats 5 +$ testenix benchmark tests --json reports/tuning.json +$ testenix tune --write $ testenix --config config/pyproject.toml run $ testenix pytest -q --tb=short tests $ testenix migrate pytest tests --dry-run @@ -1518,6 +1770,8 @@ workers = "auto" retries = 0 # timeout = 10.0 tags = [] +# shard_modules = true +# manifest = ".testenix/collection.json" # json = "reports/testenix.json" # junit = "reports/junit.xml" history = ".testenix/history.sqlite3" @@ -1539,8 +1793,15 @@ Files and directories used when no positional path is passed to `testenix run`. - Type: positive integer or `"auto"` - Default: `"auto"` -`auto` uses Python's logical CPU count. An explicit number is recommended for reproducible CI and -benchmark runs. +`auto` is adaptive rather than equal to Python's logical CPU count. After collection and selection, +Testenix caps concurrency by the available CPUs and the number of independently schedulable units. +Reliable duration history feeds a worker-startup/makespan estimate; without enough history, a +conservative cold-start cap avoids oversubscribing short suites. The smallest worker count within a +narrow tolerance of the predicted best is selected. + +An explicit number is recommended for strict CI resource limits and publishable benchmark runs. +Use `testenix tune` (or its `testenix benchmark` alias) to measure native candidates for this +project, and `testenix tune --write` to persist its recommendation explicitly. ### `retries` @@ -1593,6 +1854,40 @@ history = false The programmatic field is `history_path` and accepts a `pathlib.Path` or `None`. +### `shard_modules` + +- Type: boolean +- Default: `false` + +Allow eligible modules to be divided into finer per-test execution units. The static analyzer +keeps module affinity when it sees module/session fixtures, mutation of obvious module-global +state, or import-time lifecycle hazards. Function-scoped fixtures, including autouse fixtures, do +not block sharding. + +This option is explicitly opt-in because static analysis cannot prove the absence of all dynamic +side effects. Validate the project with and without sharding before enabling it in CI. + +### `manifest` + +- Type: filesystem path or `null` +- Default: none + +Path to a trusted collection manifest generated explicitly with: + +```console +$ testenix manifest tests --output .testenix/collection.json +``` + +On each run Testenix verifies the requested collection roots, selected test files, statically +discoverable project-local Python import dependencies, and SHA-256 source digests. An exact match +bypasses the collection-side imports; a stale +but well-formed manifest falls back to normal supervised collection. Malformed JSON is rejected. +Execution workers still import the modules they run. Test parameter names remain available for +diagnostics, but their values are stored only as `` to avoid persisting secrets obtained +during collection. Regenerate the manifest when source files or dynamic collection inputs change. + +The programmatic field is `manifest_path` and accepts a `pathlib.Path` or `None`. + ## Programmatic configuration ```python @@ -1606,6 +1901,8 @@ config = TestenixConfig( retries=1, timeout=5.0, tags=("unit",), + shard_modules=True, + manifest_path=Path(".testenix/collection.json"), json_path=Path("reports/run.json"), history_path=None, ) @@ -1755,9 +2052,68 @@ evidence for specific synthetic workloads, not a universal claim that Testenix i than pytest. `Testenix` in these results means the native `testenix run` engine. The `testenix pytest` compatibility bridge delegates to pytest and is not represented here. -![Preliminary Testenix throughput ratios](https://polishdataengineer.github.io/testenix/_static/benchmark-speedup.svg) +## Testenix 0.2.1 scaling matrix + +No current-version matrix is checked in yet. The historical results below must therefore not be +described as Testenix 0.2.1 performance. The new provenance-gated harness covers +100/500/1,000/3,000 tests, balanced/dominant/single-module layouts, 1/2/4/auto workers, and both +default history and `--no-history`, plus explicit safe-module sharding. Its default design uses +dimension sweeps; use +`--full-cross-product` only when the much larger run is intentional. -## Median wall-clock time +`auto` is passed literally to Testenix and remains adaptive; observed Testenix worker counts are +stored per sample. pytest-xdist resolves its side of an `auto` row separately to the machine's +logical CPU count. + +```console +$ uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +The command refuses a dirty worktree or an installed Testenix version that differs from +`pyproject.toml`. `--allow-dirty` is available only for unpublished smoke runs. A matrix becomes +publishable here only after five measured rounds, one warm-up, clean commit provenance, and full +axis coverage pass the documentation generator's validation. + + +## Real-project harness + +The 118-test project used during v0.2 migration validation was a semantic parity gate, not a +publishable benchmark: its release-note timings were single observations without a committed +multi-round record. Use the redaction-safe manifest harness for a real repository: + +```console +$ cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +$ uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /absolute/path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json +``` + +The manifest stores argument arrays, never shell fragments. The result omits stdout, stderr, +environment values, absolute project paths, and private source. It records only timings, aggregate +output sizes, optional tree fingerprints, and redacted Git provenance. A migrated-suite comparison +must point the manifest at a successful migration report to become publication-eligible. The +harness verifies the report's exact per-test inventory and outcomes, complete source and generated +Python-file inventories, current hashes, and binds canonical `python -m pytest` / +`python -m testenix run` commands to the report's source/output roots. Publishable source roots are +directories so support files such as `conftest.py` are covered. Without the report the result is +diagnostic-only. Commands are retained for +reproducibility. Publishable commands put options before `--` and exact suite targets after it, so +an option value cannot impersonate a migration root. Keep secrets in the environment or list +sensitive argument indexes in `redact_arguments`. + + +## Historical Testenix 0.1.0 synthetic baseline + +The checked-in `3.15×` figure is a Testenix 0.1.0 result for 100,000 generated no-op +tests across 16 modules, four workers, disabled history (`--no-history`), and pytest-xdist's default +`load` strategy. It is retained as transparent historical evidence; it is not a measurement of +Testenix 0.2.1. + +![Historical Testenix 0.1.0 throughput ratios](https://polishdataengineer.github.io/testenix/_static/benchmark-speedup.svg) + +### Median wall-clock time Lower time is better. A speedup of `2.85×` means pytest's median wall time was 2.85 times the Testenix median for that exact @@ -1774,19 +2130,24 @@ The 100,000-test result meets the project's local five-run, one-warmup minimum. It remains a synthetic result from one machine, not a universal performance promise. -## Environment +### Environment and controls - CPU: Apple M4 Pro (14 logical CPUs) - Machine: `arm64` - Platform: `macOS-26.5.1-arm64-arm-64bit` - Python: `3.11.14` +- Testenix: `0.1.0` +- Workers: four for Testenix and pytest-xdist +- Testenix history: disabled with `--no-history` +- pytest-xdist: version `3.8.0`, + default `load` distribution - Measurement: complete subprocess wall-clock time, including discovery, execution, aggregation, and console rendering - Correctness gate: every command had to exit successfully and report the expected test count -## Raw samples and variance +### Raw samples and variance ### 10,000 no-op tests / 16 modules @@ -1801,6 +2162,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 2.170, 2.267, 2.077, 2.075, 2.106 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:12:43.635798+00:00` - Commit: `8f24f8a7bd72fa876988a8ce96364be97e35c2b6` - Clean working tree at capture: yes @@ -1819,6 +2182,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 2.146, 2.129, 2.109, 2.138, 2.176 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:13:53.369799+00:00` - Commit: `18d9bba6cb5c8e39c2d5b211ee4384ae8f824524` - Clean working tree at capture: yes @@ -1837,6 +2202,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 21.239, 21.120, 22.216, 21.300, 21.949 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:19:39.942492+00:00` - Commit: `24b877c2f98420e91dcd2c8bcbc9417c7cf1ac96` - Clean working tree at capture: yes @@ -1847,7 +2214,9 @@ It remains a synthetic result from one machine, not a universal performance prom These separate measurements start with generated pytest or unittest sources, complete one safe copy-and-validate migration, and then compare recurring source-suite runs with recurring native Testenix runs. The migration transaction is a one-time cost shown separately; it is not included -in either execution median. +in either execution median. These records came from the pre-v0.2 source commit linked below; its +distribution metadata still reported `0.1.0`. They are historical evidence, not measurements of +the current release. | Source runner | Workload | Tests / modules | Source median | Native median | Native vs source | Migration transaction | | --- | --- | ---: | ---: | ---: | ---: | ---: | @@ -1941,9 +2310,9 @@ therefore material, and none of these synthetic rows predicts a specific real pr ## Interpretation -The checked-in results show that Testenix has low per-test overhead for large generated suites and -that its built-in process model is competitive with both sequential pytest and pytest-xdist in -those scenarios. +The historical checked-in results show that Testenix 0.1.0 had low per-test overhead for the large +generated suites above and was competitive with sequential pytest and pytest-xdist's default +`load` strategy in those scenarios. They are not evidence for the current release. They do **not** yet answer how Testenix performs for import-heavy applications, complex fixture graphs, assertion failures, real repositories, or different operating systems. Pytest also has a @@ -1999,19 +2368,84 @@ For every scenario record wall-clock duration, collection time, execution time, worker utilization, number of process starts, result completeness, and output size. Performance claims require at least five runs per configuration and must publish the environment fingerprint. -The current v0.1 harness automates wall-clock samples and the environment fingerprint across 16 -generated modules for a four-worker run. It also validates the completed test count, rotates runner -order between measured rounds, supports an explicit module count, and records throughput, mean, and -standard deviation. Pytest plugin autoloading and its cache provider are disabled, pytest-xdist is -loaded explicitly, and every tool runs from the generated suite directory so repository-level -pytest configuration does not affect the comparison. New output also records the commit, dirty -state, lockfile hash, timestamp, and installed framework versions. The remaining telemetry above is -the acceptance contract for the next harness iteration, not data claimed by the checked-in baseline -files. +The checked-in headline files were recorded with Testenix 0.1.0 across 16 generated modules, four +workers, and `--no-history`. They automate subprocess wall-clock samples and the environment +fingerprint, validate the completed test count, rotate runner order between measured rounds, and +record throughput, mean, standard deviation, provenance, and raw samples. Pytest plugin autoloading +and its cache provider are disabled, pytest-xdist 3.8 is loaded explicitly with its default `load` +distribution, and every tool runs from the generated suite directory so repository-level pytest +configuration does not affect the comparison. Those historical records do not measure Testenix +0.2.1. + +Schema-version 2 harness output additionally records the requested and resolved worker counts, +balanced/dominant/single-module test distributions, default-history versus `--no-history`, the +pytest-xdist distribution strategy, and captured stdout/stderr byte counts. Collection/execution +splits, peak memory, utilization, process counts, and the remaining telemetry above are still the +acceptance contract for a later harness iteration, not data claimed by the historical baseline. Correctness wins over speed: a run with a missing, duplicated, or incorrectly finalized result is invalid and excluded from performance comparisons. +## Evidence levels + +1. **Historical synthetic baseline.** The committed `3.15×` figure is Testenix 0.1.0 on 100,000 + generated no-op tests, 16 modules, four workers, disabled history, and one M4 Pro. It remains + useful provenance but must not be labelled as current-version performance. +2. **Current-version scaling matrix.** `run_scaling_matrix.py` requires the installed Testenix + version to match `pyproject.toml` and refuses a dirty worktree by default. The dimension-sweep + design covers 100, 500, 1,000, and 3,000 tests; 1, 2, 4, and `auto` workers; + balanced/dominant/single-module layouts; and default history versus `--no-history`. The reference + configuration changes one axis at a time. `--full-cross-product` is available when every + combination is worth the substantially larger runtime; its artifact still projects one + canonical balanced/no-history reference curve. `auto` remains an adaptive Testenix + request; the harness records the worker count observed in each Testenix sample. For the xdist + side of that row, `auto` resolves separately to Python's logical CPU count and is labelled as + such. +3. **Real-project evidence.** `run_project_benchmark.py` executes argument arrays from a local JSON + manifest without a shell. Its result excludes source, stdout/stderr, environment values, + absolute project paths, and Git remotes. Publication requires a successful migration report: + the harness verifies its exact per-test inventory and outcomes, complete current Python-file + inventories and hashes under directory source roots, the complete generated Python inventory, + and that canonical `python -m pytest` / `python -m testenix run` commands point at the report's + source and output roots. It records only aggregate timings/output sizes, digests, redacted Git + state, and optional content fingerprints. Repeated or conflicting worker/history/sharding flags + fail closed, and an imported runtime must belong to the exact files owned by its installed + distribution rather than merely sharing the same `site-packages` directory. + +Every synthetic and real-project command has a bounded deadline and bounded cleanup. Windows starts +the command suspended, attaches a kill-on-close Job Object, then resumes it. POSIX always signals +the new root session and identity-checks every observed detached descendant. Polling cannot provide +an absolute kernel guarantee for a child that calls `setsid()` and exits before the first snapshot; +run hostile benchmark inputs inside a container or other kernel containment boundary. +On POSIX the harness also discovers workers that created their own sessions; on Windows it uses a +kill-on-close Job Object with a bounded recursive fallback. A timed-out runner therefore cannot +continue consuming CPU or contaminate later counterbalanced samples. + +The 118-test project mentioned in the v0.2.0 release was a differential semantic-validation gate. +Its three timings were single observations, not a committed multi-round benchmark, and therefore do +not establish a real-project speedup. + +## Project-local tuning is not a published benchmark + +`testenix tune` and its `testenix benchmark` alias answer a narrow operational question: which +native worker count is fastest for this selected project suite on this machine now? They use fresh +CLI processes, disable history, validate a green one-worker inventory, alternate candidate order, +require native candidate inventory/outcome parity, and recommend the smallest worker count within +a narrow tolerance of the best measured median. This is appropriate for writing a local +`[tool.testenix].workers` value. + +The optional `--pytest-source` measurement is orientation for a corresponding source suite. A +tuning report alone is not sufficient for marketing because a migrated native path and its pytest +source may differ in collection, wrappers, output, or configuration. Publishing a ratio still +requires all of this contract: equivalent inventories and outcomes, full-process wall time, +counterbalanced order, at least one warm-up and five measured rounds, command/version/environment +provenance, output sizes, and an explanation of history, sharding, manifest, and plugin settings. + +For every published Testenix result, record both the requested and observed worker count. `auto` is +adaptive and must never be relabelled as the logical CPU count. Also record whether +`--shard-modules` was enabled and whether a trusted collection manifest was accepted or fell back +to supervised collection; either choice changes the amount and shape of schedulable work. + ## Pytest compatibility bridge Measurements of `testenix pytest` must be reported separately from native `testenix run` @@ -2025,28 +2459,65 @@ environment, pytest configuration, plugins, and arguments for both `python -m py Testenix execution speedup. Native comparisons continue to use `testenix run` and must validate that both runners execute the same tests and produce equivalent outcomes. -Run the reproducible local harness with: +Run one reproducible synthetic scenario with: ```bash -uv run python benchmarks/run_benchmark.py --tests 1000 --workers 4 --repeats 5 -uv run python benchmarks/run_benchmark.py --tests 1000 --workers 4 --repeats 5 --uneven -uv run python benchmarks/run_benchmark.py --tests 10000 --modules 1000 --workers 4 --repeats 5 +uv run --no-editable python benchmarks/run_benchmark.py \ + --tests 1000 --modules 16 --workers 4 --repeats 5 --warmups 1 \ + --module-layout balanced --history-mode disabled --xdist-strategy load ``` +Generate the current-version dimension sweeps from a clean checkout with: + +```bash +uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +For an unpublished smoke test only, add `--quick --allow-dirty`. A publishable matrix must retain +five rounds, one warm-up, clean provenance, and the complete requested coverage. + +Measure a real project without committing its code or private paths: + +```bash +cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +# Edit expected counts, migration-report path, fingerprints, and runner commands. +uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /absolute/path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json +``` + +Keep private manifests and results outside the repository until their labels, commit policy, and +fingerprints have been reviewed for publication. Environment override **values** are never written +to the result; only their key names are retained. Commands are recorded for reproducibility, so +secrets should stay in the environment. If an unavoidable command argument is sensitive, list its +zero-based index in that runner's `redact_arguments` field. + +Without `migration_report`, the harness can still produce a private diagnostic, but it always sets +`publication_eligible` to `false`. A publishable run also requires the canonical module +entrypoints, a clean project, at least one warm-up and five measured rounds, explicit Testenix +workers and history mode, and installed-distribution identities for both pytest and Testenix. The +pytest version is probed inside the same project environment used by the timed commands. In each +runner command, place options before `--` and the exact benchmark suite roots after it. The harness +uses that delimiter to distinguish positional targets from values of options such as `-k` or +`--tag` and requires those targets to match the migration report exactly. + Maintainers can run the same comparison from GitHub's **Benchmarks** workflow and download its raw JSON artifact. Shared GitHub runners are appropriate for reproducibility checks, not for silently replacing the approved marketing baseline: their timing variance is outside this project's control. -The checked-in baseline files are development evidence, not universal performance claims. See -`docs/performance-analysis.md` for the current large-suite results, optimization profile, memory -notes, and native-code decision. Real project suites and cross-platform repetitions remain required -before publishing broad comparative claims. +The checked-in baseline files are historical development evidence, not universal or current-version +performance claims. See `docs/performance-analysis.md` for the large-suite results, optimization +profile, memory notes, and native-code decision. A clean Testenix 0.2.1 scaling matrix, publishable +real-project suites, alternative pytest-xdist strategies, and cross-platform repetitions remain +required before publishing broad comparative claims. An approved public baseline must be committed through a reviewed pull request. Do not remove slow but valid samples as outliers; invalid commands remain evidence and must be explained. The current checked-in 10,000- and 100,000-test files each contain five measured rounds, one warm-up, and clean -commit provenance. They remain single-machine synthetic evidence, so broader claims still require -the real-project and cross-platform scenarios above. +commit provenance. They remain Testenix 0.1.0 single-machine synthetic evidence, so broader claims +still require the current-version, real-project, and cross-platform scenarios above. --- @@ -2059,22 +2530,25 @@ Source: docs/performance-analysis.md ## Executive summary -The optimized native v0.1 `testenix run` engine is faster than pytest and pytest-xdist in the checked-in synthetic -large-suite scenarios. The largest recorded comparison is 100,000 passing tests across 16 modules: -Testenix completed the suite in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist -in 21.300 seconds. Every measured command had to report the expected test count or the harness -rejected the sample. The baseline contains one warm-up and five counterbalanced measured rounds, -and Testenix's samples ranged from 7.912 to 8.096 seconds. +The checked-in headline numbers are a **historical Testenix 0.1.0 synthetic baseline**, not current +Testenix 0.2.1 results. In the largest recorded comparison, 100,000 generated no-op tests were spread +evenly across 16 modules and run with four workers and `--no-history`. Testenix completed the suite +in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist 3.8's default `load` scheduler +in 21.300 seconds. Every command had to report the expected test count or the harness rejected the +sample. The baseline contains one warm-up and five counterbalanced measured rounds, and Testenix's +samples ranged from 7.912 to 8.096 seconds. This is evidence for the tested workload and machine, not a universal claim about every Python -project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, different operating -systems, and real repositories still need independent measurements. +project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, default history, +alternative pytest-xdist schedulers, different operating systems, and real repositories still need +independent measurements. No clean Testenix 0.2.1 scaling matrix is checked in yet, so `3.15×` must +not be presented as a 0.2.1 speedup. These results do not apply to `testenix pytest`. The compatibility command delegates to pytest and has pytest execution performance plus launcher and adapter overhead, which has not yet been measured separately. -The new safe-migration baseline is deliberately mixed rather than uniformly positive. For 3,000 +The pre-v0.2 safe-migration baseline is deliberately mixed rather than uniformly positive. For 3,000 no-op pytest tests across 64 modules, the generated native suite was 2.96x faster than sequential pytest. Empty unittest wrappers were 7.40x slower than the sequential stdlib-based outcome probe, while the same layout with 1 ms of work in each method was 1.58x faster under four native workers. These are @@ -2092,6 +2566,7 @@ parallel unittest runners, or a real repository. - deterministic rotated execution order instead of always running Testenix last; - test-count validation from each runner's final output; - disabled pytest plugin autoloading and cache, with pytest-xdist loaded explicitly; +- pytest-xdist's default `load` distribution rather than `loadfile`, `loadscope`, or `worksteal`; - generated-suite working directory, isolated from repository-level pytest configuration; - `--no-history` for the primary runner-overhead comparison. @@ -2100,7 +2575,7 @@ median throughput. All checked-in comparison files use one warm-up and five meas They also record the clean source commit, lockfile hash, timestamp, CPU model, and installed Testenix, pytest, and pytest-xdist versions. -## Results +## Historical Testenix 0.1.0 results | Scenario | pytest | pytest-xdist | Native Testenix | Native Testenix advantage | |---|---:|---:|---:|---:| @@ -2108,7 +2583,7 @@ Testenix, pytest, and pytest-xdist versions. | 10,000 uneven tests, 16 modules | 3.076 s | 2.138 s | 1.345 s | 2.29x vs pytest | | 100,000 no-op tests, 16 modules | 25.333 s | 21.300 s | 8.038 s | 3.15x vs pytest | -The 100,000-test median throughputs were 12,440 tests/s for Testenix, 3,947 tests/s for pytest, and +The 100,000-test median throughputs were 12,440 tests/s for Testenix 0.1.0, 3,947 tests/s for pytest, and 4,695 tests/s for pytest-xdist. The raw five-sample ranges and standard deviations are published in the generated benchmark page and the checked-in JSON files. @@ -2116,7 +2591,9 @@ In a separate exploratory 100,000-test profile, the coordinator's measured maxim approximately 513 MiB after the final manifest/event optimization. Sequential pytest measured approximately 520 MiB on the same generated suite. These older macOS `time` figures are not part of the current baseline JSON and are process maxima, not aggregate memory across every xdist/Testenix -child process. +child process. The console renderer changed substantially after these captures, and the historical +harness did not record output byte counts. Current schema-version 2 runs do record stdout/stderr +sizes, but a clean 0.2.1 matrix is still pending. ### Migrated-suite measurements @@ -2142,16 +2619,36 @@ The no-op unittest row exposes the adapter's fixed cost: each native test is a w the unchanged source and translates the result of `TestCase.run()`. The sequential source probe uses the stdlib loader and result semantics, then serializes per-test outcomes; it wins when the test body does essentially nothing. With 1 ms per method, four-worker execution across 64 -modules amortizes that cost and overtakes the sequential source runner. A suite concentrated in -one module would expose only one affinity unit, while too many workers can add process and import -overhead. Test duration, module distribution, imports, fixtures, I/O, failures, operating system, -and competing parallel runners must all be measured on the target project. +modules amortizes that cost and overtakes the sequential source runner. By default, a suite +concentrated in one module exposes only one affinity unit; opt-in safety-checked sharding may change +that for an independent module. Too many workers can still add process and import overhead. Test +duration, module distribution, imports, fixtures, I/O, failures, operating system, and competing +parallel runners must all be measured on the target project. The generated [benchmark results](https://polishdataengineer.github.io/testenix/benchmarks/results/) publish every sample, range, standard deviation, command, environment field, and raw JSON link. These three synthetic records are useful for finding overhead boundaries; they are not evidence that converted suites are universally faster. +### The 118-test validation was not a benchmark + +The [v0.2.0 release notes](https://github.com/polishdataengineer/testenix/releases/tag/v0.2.0) +reported one final validation observation from a real 118-test pytest project: 3.120 seconds for +pytest, 2.870 seconds for native serial execution, and 2.423 seconds for native parallel execution. +Those values correspond to roughly 1.09× and 1.29× for that observation, not 3.15×. They had no +committed raw rounds, warm-up series, counterbalanced order, or publishable environment manifest, +so their role was outcome parity (118/118 in all modes), not performance marketing. + +A small real suite can differ sharply from the 100,000-test no-op baseline. Interpreter spawn and +application import costs are a much larger fraction of its wall time; fixtures, mocks, files, +databases, and actual test bodies dominate framework overhead; and module affinity prevents one +large module from being split between workers. `workers=auto` is adaptive in current Testenix and +must be recorded as the worker count actually observed, not assumed to equal logical CPU count. +For a demonstrably independent large module, opt-in `--shard-modules` can create finer units after +static safety checks; it is not a safe default for arbitrary module state. Use `testenix tune` for +a local worker recommendation, then use the real-project manifest harness and publish five +counterbalanced rounds before drawing a project-specific conclusion. + ### Worker-count sensitivity An earlier exploratory run of 10,000 no-op tests across 16 modules produced these Testenix medians: @@ -2199,6 +2696,9 @@ The optimized profile performed approximately 3.46 million calls. The main chang 7. Unchanged selected specifications reuse discovered objects; selection events are omitted when every test is selected and no effective contract changed. 8. JSONL keeps one append descriptor for the run rather than performing open/close per event. +9. An explicit trusted collection manifest can remove the collection-side import from later + unchanged runs. Roots, the complete file inventory, and every source SHA-256 are verified first; + stale manifests fall back to supervised collection. Execution still imports assigned modules. Correctness was retained throughout: the framework's full resilience suite passes after every optimization, including worker crashes, timeout process-tree cleanup, collection crashes/hangs, @@ -2243,6 +2743,9 @@ Relevant upstream constraints are documented in the ## Next measurement gates +- publish the clean Testenix 0.2.1 dimension-sweep matrix for 100/500/1,000/3,000 tests, + balanced/dominant/single-module layouts, 1/2/4/adaptive-auto workers, and both history modes; +- compare pytest-xdist `load`, `loadfile`, `loadscope`, and `worksteal` where each strategy is valid; - collection, execution, IPC-byte, process-start, CPU, and aggregate-memory telemetry; - 1,000/10,000/100,000 tests across 1, 16, 1,000, and 10,000 modules; - synchronous and asynchronous fixtures, failures, captured output, timeouts, and retries; @@ -2251,12 +2754,14 @@ Relevant upstream constraints are documented in the - real migrated pytest and unittest suites, including sequential and established parallel source runners, multiple module layouts, and a break-even curve by median test duration; - checkpoint-batched IPC followed by another profile; -- persistent workers that collect and execute without importing every module twice. +- measure trusted-manifest hit and stale-fallback paths on import-heavy real projects, and consider + a persistent collect-and-execute worker only if the remaining execution import is still material. -No universal “always faster than pytest” statement should be published until the real-project and -cross-platform gates pass. The supported claim today is narrower: Testenix is materially faster in the -measured native large passing-suite scenarios while retaining supervised isolation and complete -results. +No universal “always faster than pytest” statement should be published until the current-version, +real-project, and cross-platform gates pass. The supported claim today is historical and narrower: +Testenix 0.1.0 was materially faster in the recorded native large passing-suite scenarios while +retaining supervised isolation and complete results. Testenix 0.2.1 has no checked-in speedup claim +yet. --- @@ -2322,11 +2827,11 @@ test outcomes. A retry never overwrites an earlier attempt, infrastructure failu from test failures, and setup/call/teardown are preserved as separate phases. ```text -Authoring API -> supervised collection -> inert manifest -> affinity scheduler -> process workers - | -> streamed attempts - +----------> append-only events - -> reducer - -> reports/history +Authoring API -> supervised collection -------> inert manifest -> scheduler -> process workers + ^ ^ | -> streamed attempts + | | +------------> append-only events +trusted manifest +-- roots/inventory/SHA-256 verify -> reducer + exact match bypasses collection imports -> reports/history ``` ## Dependency rules @@ -2335,6 +2840,9 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - `api`, `discovery`, `fixtures`, and `executor` form the native engine. - `events`, `aggregate`, and `scheduler` remain engine-independent. - `runner` is the application service connecting the native engine with execution policy. +- `tuning` models adaptive worker selection and runs explicit project-local candidate measurements. +- `sharding` contains fail-closed static module decisions and the versioned trusted-manifest + serialization/verification boundary. - reporters and storage consume completed domain results or versioned events. - optional compatibility adapters stay at the CLI boundary; the native core never imports pytest. - migration analyzers depend on serializable migration contracts, while shadow execution and @@ -2348,6 +2856,10 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - fixture scopes: test, module, session, with broader scopes currently bounded by a worker shard; - sequential and local process execution; - deterministic scheduling based on historical durations; +- adaptive `workers = "auto"` capped by real execution units, history-informed predicted makespan, + process-start cost, and CPU capacity; +- explicit project-local worker tuning, conservative opt-in intra-module sharding, and an + explicitly generated source-verified collection manifest; - append-only JSONL events and a pure reducer; - console, JSON, and JUnit output plus local SQLite duration history; - retries represented as immutable attempts and finalized as `FLAKY` when appropriate; @@ -2363,10 +2875,16 @@ plugin SDK are deliberately outside version 0.2. ## Fixture scopes and process isolation -The scheduler treats every normal test module as one affinity unit and never splits that unit -between parallel shared workers. Multiple modules assigned to one shard execute in one persistent -process and fixture runtime. A test with an explicit timeout (including a global timeout applied at -selection) is instead a single-test isolation unit with a hard process deadline. +By default, the scheduler treats every normal test module as one affinity unit and never splits +that unit between parallel shared workers. Multiple modules assigned to one shard execute in one +persistent process and fixture runtime. A test with an explicit timeout (including a global timeout +applied at selection) is instead a single-test isolation unit with a hard process deadline. + +An explicit intra-module sharding policy can turn tests in an eligible module into finer units. +The static analysis fails closed for module/session fixtures, writes or obvious mutations of module +globals, and import-time lifecycle behavior. Function-scoped fixtures can be recreated per worker. +Because arbitrary dynamic calls and external effects cannot be proven safe, passing this policy is +a caller trust decision; ineligible modules keep normal affinity. Scope therefore has the following concrete meaning in version 0.2: @@ -2400,6 +2918,18 @@ top-level import crash or deadline becomes a `CollectionIssue`, so user import c indefinitely block the coordinator. Timed execution units send a ready handshake after rediscovery; the test deadline therefore does not include interpreter startup or module import. +A caller may instead supply a `TrustedCollectionManifest` created by an earlier explicit +collection. Before using it, Testenix enumerates the current roots and verifies the complete file +set and every SHA-256 digest. An exact match bypasses collection imports; a stale manifest falls +back to the supervised collector. Malformed serialized input is rejected at the adapter boundary. +Manifest parameter values are redacted at creation and serialization; only their names remain. +The manifest also carries prior sharding decisions so collection and scheduling agree. A module +using a fixture provider from outside its fingerprinted source fails closed to module affinity. +This removes +one module import per unchanged run, not the execution-worker import needed to reconstruct Python +objects. Inputs to dynamic collection beyond fingerprinted source bytes remain the producer's trust +responsibility. + Workers normally create a separate POSIX process session (or use recursive tree termination on Windows). During migration validation they remain in the validator's process group so an outer validation deadline can terminate native workers too. Timeout and cancellation terminate ordinary @@ -2443,6 +2973,10 @@ Source: docs/roadmap.md ## 0.3 — fast feedback +- Adaptive worker selection based on real execution units, duration history, and process cost, + plus an explicit project-local `tune`/`benchmark` command. +- Conservative opt-in intra-module sharding and a source-verified trusted collection manifest that + can bypass duplicate collection imports without trusting stale source metadata. - Dynamic micro-shards and work stealing. - `--last-failed`, watch mode, and failure fingerprints. - Test-impact analysis in shadow mode with an explanation for every selection decision. @@ -2454,7 +2988,7 @@ Source: docs/roadmap.md - Expand migration beyond the v0.2 static subset only when new transformations have differential semantics tests on real projects. - Versioned reporter and selector plugin interfaces. -- IDE protocol and machine-readable collection manifest. +- IDE protocol built on the versioned machine-readable collection manifest. ## Later @@ -2480,6 +3014,41 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- `testenix tune` and its `testenix benchmark` alias for fresh-process, counterbalanced, + history-disabled native worker-candidate measurements, native inventory/outcome validation, + JSON reports, bounded per-run process-tree deadlines, and explicit `--write` persistence of the + measured recommendation with project-source fingerprinting and optimistic byte-drift protection + immediately before an atomic configuration replacement. +- Explicit `--shard-modules` / `shard_modules = true` support for splitting eligible modules into + finer execution units. Conservative static checks retain module affinity for module/session + fixtures, visible global mutation, and import-time lifecycle hazards, including eager calls in + assignments, decorators, function defaults, and class construction expressions. +- Versioned trusted collection manifests generated with `testenix manifest ... --output FILE` and + consumed with `testenix run --manifest FILE` or `[tool.testenix].manifest`. Exact collection + roots, selected test files, statically discoverable project-local import dependencies, and SHA-256 + digests are verified before collection imports are bypassed; stale manifests fall back to + supervised collection, and parameter values are redacted. +- Synthetic scaling-matrix tooling for 100/500/1,000/3,000 tests and balanced, dominant, and + single-module layouts, plus a redaction-safe real-project benchmark harness. + +### Changed + +- `workers = "auto"` now selects adaptively from the actual execution-unit count, available CPUs, + duration-history coverage, predicted process-start cost, and makespan instead of equalling the + logical CPU count. Explicit integer worker settings remain unchanged. +- Benchmark documentation labels the historical `3.15×` result with its Testenix 0.1.0 version, + four-worker configuration, 100,000-test/16-module synthetic workload, and `--no-history` mode; + it is not presented as a current-version or real-project claim. + +### Fixed + +- Safe-module analysis now fails closed for imported fixture providers, nested mutable containers, + mutable class state, and all import-time calls including nested `sys.path` mutations. +- Benchmark and tuning timeouts use Windows Job Objects or POSIX root-session plus identity-tracked + descendant cleanup instead of allowing observed workers to contaminate later measurements. + ## [0.2.1] - 2026-07-21 ### Added @@ -2632,6 +3201,10 @@ Fields: Complete collection output, including non-fatal authoring issues. +## `testenix.CollectionManifestError` + +A trusted collection manifest is malformed or unsafe to resolve. + ## `testenix.Event` ```text @@ -2722,7 +3295,7 @@ Terminal state of one migration transaction. ## `testenix.TestenixConfig` ```text -TestenixConfig(paths: 'tuple[str, ...]' = ('tests',), workers: "int | Literal['auto']" = 'auto', retries: 'int' = 0, timeout: 'float | None' = None, tags: 'tuple[str, ...]' = (), json_path: 'Path | None' = None, junit_path: 'Path | None' = None, history_path: 'Path | None' = PosixPath('.testenix/history.sqlite3')) -> None +TestenixConfig(paths: 'tuple[str, ...]' = ('tests',), workers: "int | Literal['auto']" = 'auto', retries: 'int' = 0, timeout: 'float | None' = None, tags: 'tuple[str, ...]' = (), json_path: 'Path | None' = None, junit_path: 'Path | None' = None, history_path: 'Path | None' = PosixPath('.testenix/history.sqlite3'), shard_modules: 'bool' = False, manifest_path: 'Path | None' = None) -> None ``` Fields: @@ -2735,6 +3308,8 @@ Fields: - `json_path`: `Path | None` - `junit_path`: `Path | None` - `history_path`: `Path | None` +- `shard_modules`: `bool` +- `manifest_path`: `Path | None` Validated execution and reporting settings. @@ -2745,7 +3320,7 @@ a worker process without retaining hidden project state. ## `testenix.RunResult` ```text -RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float') -> None +RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float', workers_used: 'int | None' = None, shardable_paths: 'tuple[str, ...]' = ()) -> None ``` Fields: @@ -2755,8 +3330,26 @@ Fields: - `collection_issues`: `tuple[CollectionIssue, ...]` - `started_at`: `float` - `finished_at`: `float` +- `workers_used`: `int | None` +- `shardable_paths`: `tuple[str, ...]` + +RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float', workers_used: 'int | None' = None, shardable_paths: 'tuple[str, ...]' = ()) + +## `testenix.ShardingPolicy` + +```text +ShardingPolicy(intra_module: 'bool' = False) -> None +``` + +Fields: -RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float') +- `intra_module`: `bool` + +Core scheduling policy independent of CLI/configuration concerns. + +Intra-module sharding is deliberately opt-in. Callers which do not pass a +policy, or pass the default instance, retain the original module-affinity +behaviour exactly. ## `testenix.Scope` @@ -2816,6 +3409,32 @@ Fields: Serializable description of one concrete test case. +## `testenix.TrustedCollectionManifest` + +```text +TrustedCollectionManifest(collection_roots: 'tuple[str, ...]', files: 'tuple[SourceFingerprint, ...]', tests: 'tuple[TestSpec, ...]', issues: 'tuple[CollectionIssue, ...]' = (), sharding: 'tuple[ModuleShardingDecision, ...]' = ()) -> None +``` + +Fields: + +- `collection_roots`: `tuple[str, ...]` +- `files`: `tuple[SourceFingerprint, ...]` +- `tests`: `tuple[TestSpec, ...]` +- `issues`: `tuple[CollectionIssue, ...]` +- `sharding`: `tuple[ModuleShardingDecision, ...]` + +Explicit portable collection result that may bypass collection imports. + +This is deliberately not an implicit cache. A caller creates or loads the +manifest and opts into trusting it for a run. Parameter names are retained +for diagnostics, but every parameter value is replaced with the explicit +```` sentinel. Testenix still compares the complete test-file +inventory plus local import dependencies and every SHA-256 digest before +using it. +Dynamic collection influenced by anything other than source bytes (for +example environment variables) remains the manifest producer's trust +decision. + ## `testenix.ValidationSummary` ```text @@ -2866,6 +3485,22 @@ Attach several cases, either explicitly or as a Cartesian product. ``@cases(role=["admin", "editor"], active=[True, False])`` creates the Cartesian product of the supplied dimensions. +## `testenix.collect_trusted_manifest` + +```text +collect_trusted_manifest(paths: 'Sequence[str] | str', *, project_root: 'str | Path | None' = None) -> 'TrustedCollectionManifest' +``` + +Create an explicit collection manifest in a supervised worker process. + +## `testenix.deserialize_trusted_collection_manifest` + +```text +deserialize_trusted_collection_manifest(data: 'str | bytes | bytearray | Mapping[str, Any]') -> 'TrustedCollectionManifest' +``` + +Decode and validate trusted collection manifest JSON or mapping data. + ## `testenix.discover` ```text @@ -2902,7 +3537,7 @@ checked for concurrent changes. ## `testenix.run` ```text -run(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None) -> 'RunResult' +run(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None, sharding_policy: 'ShardingPolicy | None' = None, trusted_manifest: 'TrustedCollectionManifest | None' = None) -> 'RunResult' ``` Discover and execute a native Testenix suite. @@ -2914,11 +3549,19 @@ reduced after execution. ## `testenix.run_async` ```text -run_async(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None) -> 'RunResult' +run_async(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None, sharding_policy: 'ShardingPolicy | None' = None, trusted_manifest: 'TrustedCollectionManifest | None' = None) -> 'RunResult' ``` Cancellable embedding facade around the process-oriented coordinator. +## `testenix.serialize_trusted_collection_manifest` + +```text +serialize_trusted_collection_manifest(manifest: 'TrustedCollectionManifest') -> 'str' +``` + +Serialize a trusted collection manifest as deterministic JSON. + ## `testenix.skip` ```text diff --git a/docs/performance-analysis.md b/docs/performance-analysis.md index 3bd0e64..8207dea 100644 --- a/docs/performance-analysis.md +++ b/docs/performance-analysis.md @@ -2,22 +2,25 @@ ## Executive summary -The optimized native v0.1 `testenix run` engine is faster than pytest and pytest-xdist in the checked-in synthetic -large-suite scenarios. The largest recorded comparison is 100,000 passing tests across 16 modules: -Testenix completed the suite in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist -in 21.300 seconds. Every measured command had to report the expected test count or the harness -rejected the sample. The baseline contains one warm-up and five counterbalanced measured rounds, -and Testenix's samples ranged from 7.912 to 8.096 seconds. +The checked-in headline numbers are a **historical Testenix 0.1.0 synthetic baseline**, not current +Testenix 0.2.1 results. In the largest recorded comparison, 100,000 generated no-op tests were spread +evenly across 16 modules and run with four workers and `--no-history`. Testenix completed the suite +in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist 3.8's default `load` scheduler +in 21.300 seconds. Every command had to report the expected test count or the harness rejected the +sample. The baseline contains one warm-up and five counterbalanced measured rounds, and Testenix's +samples ranged from 7.912 to 8.096 seconds. This is evidence for the tested workload and machine, not a universal claim about every Python -project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, different operating -systems, and real repositories still need independent measurements. +project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, default history, +alternative pytest-xdist schedulers, different operating systems, and real repositories still need +independent measurements. No clean Testenix 0.2.1 scaling matrix is checked in yet, so `3.15×` must +not be presented as a 0.2.1 speedup. These results do not apply to `testenix pytest`. The compatibility command delegates to pytest and has pytest execution performance plus launcher and adapter overhead, which has not yet been measured separately. -The new safe-migration baseline is deliberately mixed rather than uniformly positive. For 3,000 +The pre-v0.2 safe-migration baseline is deliberately mixed rather than uniformly positive. For 3,000 no-op pytest tests across 64 modules, the generated native suite was 2.96x faster than sequential pytest. Empty unittest wrappers were 7.40x slower than the sequential stdlib-based outcome probe, while the same layout with 1 ms of work in each method was 1.58x faster under four native workers. These are @@ -35,6 +38,7 @@ parallel unittest runners, or a real repository. - deterministic rotated execution order instead of always running Testenix last; - test-count validation from each runner's final output; - disabled pytest plugin autoloading and cache, with pytest-xdist loaded explicitly; +- pytest-xdist's default `load` distribution rather than `loadfile`, `loadscope`, or `worksteal`; - generated-suite working directory, isolated from repository-level pytest configuration; - `--no-history` for the primary runner-overhead comparison. @@ -43,7 +47,7 @@ median throughput. All checked-in comparison files use one warm-up and five meas They also record the clean source commit, lockfile hash, timestamp, CPU model, and installed Testenix, pytest, and pytest-xdist versions. -## Results +## Historical Testenix 0.1.0 results | Scenario | pytest | pytest-xdist | Native Testenix | Native Testenix advantage | |---|---:|---:|---:|---:| @@ -51,7 +55,7 @@ Testenix, pytest, and pytest-xdist versions. | 10,000 uneven tests, 16 modules | 3.076 s | 2.138 s | 1.345 s | 2.29x vs pytest | | 100,000 no-op tests, 16 modules | 25.333 s | 21.300 s | 8.038 s | 3.15x vs pytest | -The 100,000-test median throughputs were 12,440 tests/s for Testenix, 3,947 tests/s for pytest, and +The 100,000-test median throughputs were 12,440 tests/s for Testenix 0.1.0, 3,947 tests/s for pytest, and 4,695 tests/s for pytest-xdist. The raw five-sample ranges and standard deviations are published in the generated benchmark page and the checked-in JSON files. @@ -59,7 +63,9 @@ In a separate exploratory 100,000-test profile, the coordinator's measured maxim approximately 513 MiB after the final manifest/event optimization. Sequential pytest measured approximately 520 MiB on the same generated suite. These older macOS `time` figures are not part of the current baseline JSON and are process maxima, not aggregate memory across every xdist/Testenix -child process. +child process. The console renderer changed substantially after these captures, and the historical +harness did not record output byte counts. Current schema-version 2 runs do record stdout/stderr +sizes, but a clean 0.2.1 matrix is still pending. ### Migrated-suite measurements @@ -85,16 +91,36 @@ The no-op unittest row exposes the adapter's fixed cost: each native test is a w the unchanged source and translates the result of `TestCase.run()`. The sequential source probe uses the stdlib loader and result semantics, then serializes per-test outcomes; it wins when the test body does essentially nothing. With 1 ms per method, four-worker execution across 64 -modules amortizes that cost and overtakes the sequential source runner. A suite concentrated in -one module would expose only one affinity unit, while too many workers can add process and import -overhead. Test duration, module distribution, imports, fixtures, I/O, failures, operating system, -and competing parallel runners must all be measured on the target project. +modules amortizes that cost and overtakes the sequential source runner. By default, a suite +concentrated in one module exposes only one affinity unit; opt-in safety-checked sharding may change +that for an independent module. Too many workers can still add process and import overhead. Test +duration, module distribution, imports, fixtures, I/O, failures, operating system, and competing +parallel runners must all be measured on the target project. The generated [benchmark results](benchmarks/results.md) publish every sample, range, standard deviation, command, environment field, and raw JSON link. These three synthetic records are useful for finding overhead boundaries; they are not evidence that converted suites are universally faster. +### The 118-test validation was not a benchmark + +The [v0.2.0 release notes](https://github.com/polishdataengineer/testenix/releases/tag/v0.2.0) +reported one final validation observation from a real 118-test pytest project: 3.120 seconds for +pytest, 2.870 seconds for native serial execution, and 2.423 seconds for native parallel execution. +Those values correspond to roughly 1.09× and 1.29× for that observation, not 3.15×. They had no +committed raw rounds, warm-up series, counterbalanced order, or publishable environment manifest, +so their role was outcome parity (118/118 in all modes), not performance marketing. + +A small real suite can differ sharply from the 100,000-test no-op baseline. Interpreter spawn and +application import costs are a much larger fraction of its wall time; fixtures, mocks, files, +databases, and actual test bodies dominate framework overhead; and module affinity prevents one +large module from being split between workers. `workers=auto` is adaptive in current Testenix and +must be recorded as the worker count actually observed, not assumed to equal logical CPU count. +For a demonstrably independent large module, opt-in `--shard-modules` can create finer units after +static safety checks; it is not a safe default for arbitrary module state. Use `testenix tune` for +a local worker recommendation, then use the real-project manifest harness and publish five +counterbalanced rounds before drawing a project-specific conclusion. + ### Worker-count sensitivity An earlier exploratory run of 10,000 no-op tests across 16 modules produced these Testenix medians: @@ -142,6 +168,9 @@ The optimized profile performed approximately 3.46 million calls. The main chang 7. Unchanged selected specifications reuse discovered objects; selection events are omitted when every test is selected and no effective contract changed. 8. JSONL keeps one append descriptor for the run rather than performing open/close per event. +9. An explicit trusted collection manifest can remove the collection-side import from later + unchanged runs. Roots, the complete file inventory, and every source SHA-256 are verified first; + stale manifests fall back to supervised collection. Execution still imports assigned modules. Correctness was retained throughout: the framework's full resilience suite passes after every optimization, including worker crashes, timeout process-tree cleanup, collection crashes/hangs, @@ -186,6 +215,9 @@ Relevant upstream constraints are documented in the ## Next measurement gates +- publish the clean Testenix 0.2.1 dimension-sweep matrix for 100/500/1,000/3,000 tests, + balanced/dominant/single-module layouts, 1/2/4/adaptive-auto workers, and both history modes; +- compare pytest-xdist `load`, `loadfile`, `loadscope`, and `worksteal` where each strategy is valid; - collection, execution, IPC-byte, process-start, CPU, and aggregate-memory telemetry; - 1,000/10,000/100,000 tests across 1, 16, 1,000, and 10,000 modules; - synchronous and asynchronous fixtures, failures, captured output, timeouts, and retries; @@ -194,9 +226,11 @@ Relevant upstream constraints are documented in the - real migrated pytest and unittest suites, including sequential and established parallel source runners, multiple module layouts, and a break-even curve by median test duration; - checkpoint-batched IPC followed by another profile; -- persistent workers that collect and execute without importing every module twice. - -No universal “always faster than pytest” statement should be published until the real-project and -cross-platform gates pass. The supported claim today is narrower: Testenix is materially faster in the -measured native large passing-suite scenarios while retaining supervised isolation and complete -results. +- measure trusted-manifest hit and stale-fallback paths on import-heavy real projects, and consider + a persistent collect-and-execute worker only if the remaining execution import is still material. + +No universal “always faster than pytest” statement should be published until the current-version, +real-project, and cross-platform gates pass. The supported claim today is historical and narrower: +Testenix 0.1.0 was materially faster in the recorded native large passing-suite scenarios while +retaining supervised isolation and complete results. Testenix 0.2.1 has no checked-in speedup claim +yet. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 0751f72..efd3bae 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -7,13 +7,16 @@ testenix [-h] [--version] testenix [--config PYPROJECT] run [RUN_ARGS ...] testenix pytest [PYTEST_ARGS ...] testenix migrate FRAMEWORK PATH [PATH ...] [MIGRATION_ARGS ...] +testenix [--config PYPROJECT] tune [PATH ...] [TUNING_ARGS ...] +testenix [--config PYPROJECT] benchmark [PATH ...] [TUNING_ARGS ...] +testenix manifest PATH [PATH ...] --output FILE ``` | Option | Description | | --- | --- | | `-h`, `--help` | Show command help. | | `--version` | Print the installed Testenix version. | -| `--config PATH` | Load `[tool.testenix]` for native `testenix run`. | +| `--config PATH` | Load `[tool.testenix]` for native `run` and `tune`/`benchmark` commands. | ## `testenix run` @@ -27,6 +30,8 @@ testenix run [PATH ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] + [--shard-modules] + [--manifest FILE] [-q | -v | -vv] [--color {auto,always,never} | --no-color] [--show-skips] @@ -36,7 +41,7 @@ testenix run [PATH ...] | Argument | Default | Description | | --- | --- | --- | | `PATH ...` | configured `paths`, otherwise `tests` | Files or directories to discover. | -| `-w`, `--workers` | `auto` | Worker process count or logical CPU count. | +| `-w`, `--workers` | `auto` | Positive worker count, or adaptive selection from schedulable units, history, startup cost, and CPU capacity. | | `--retries` | `0` | Additional attempts after a gating outcome. | | `--timeout` | none | Global hard deadline for every selected test. | | `-t`, `--tag` | none | Required tag; repeat for AND selection. | @@ -44,6 +49,8 @@ testenix run [PATH ...] | `--junit` | none | Write a JUnit XML report. | | `--history` | `.testenix/history.sqlite3` | Override the duration-history database. | | `--no-history` | off | Disable reading and writing history. | +| `--shard-modules` | off | Opt eligible modules into per-test execution units after conservative static safety checks. | +| `--manifest FILE` | none | Reuse an explicitly generated, source-verified trusted collection manifest. | | `-q`, `--quiet` | off | Hide the run header and compact per-file table. Collection errors, failure details, and the final summary remain visible. | | `-v`, `--verbose` | off | Print one result row per test in the stable detailed format. Repeat for `-vv`. | | `-vv` | off | Add worker, attempt, and phase metadata, including captured output. | @@ -60,6 +67,20 @@ order after execution completes; these modes do not promise live progress update flags change only terminal rendering, not selection, scheduling, result statuses, JSON, JUnit, or exit codes. +`workers = auto` never means “start one process per logical CPU.” Testenix caps it by the number of +units that can run independently. Reliable duration history feeds a makespan/startup-cost model; +cold runs use a conservative cap. The final console and JSON results report the worker count +actually used. An explicit integer remains useful for fixed CI resource limits and reproducible +published benchmarks. + +`--shard-modules` is an explicit safety/performance trade-off. Modules with module/session +fixtures, statically visible mutable global state, or import-time lifecycle hazards keep module +affinity; eligible modules may be split. Static analysis cannot prove every dynamic side effect. + +`--manifest` accepts the versioned JSON produced by `testenix manifest`. Malformed input is a usage +error. An otherwise valid manifest whose roots, complete Python-file inventory, or SHA-256 digests +no longer match is treated as stale, and the run safely performs ordinary supervised collection. + In `auto` color mode, Testenix requires a terminal, respects `NO_COLOR`, allows `FORCE_COLOR`, and disables styling for a truthy `CI` value or `TERM=dumb`. Explicit `always` or `never` takes precedence. @@ -136,6 +157,80 @@ grouped by severity and code, with the first source location shown. Use `--repor worker setting. It is emitted only for `--check` or publication after static analysis succeeds, because dry-run and unsupported transactions never execute the parallel gate. +## `testenix tune` / `testenix benchmark` + +```text +testenix tune [PATH ...] + [--config PYPROJECT] + [--candidates N[,N...]] + [--warmups N] + [--repeats N] + [--pytest-source PATH] ... + [--json FILE|-] + [--shard-modules] + [--manifest FILE] + [--write] +``` + +`testenix benchmark` is an exact alias for `testenix tune`. + +| Argument | Default | Description | +| --- | --- | --- | +| `PATH ...` | configured `paths`, otherwise `tests` | Native Testenix suite to tune. | +| `--candidates N[,N...]` | resource-aware 1/2/4 sweep | Positive worker counts to measure. Counts above the number of execution units are deduplicated at that limit; values above four require this explicit option. | +| `--warmups N` | `1` | Unrecorded warmups for each native candidate and optional pytest source. | +| `--repeats N` | `5` | Recorded samples for each candidate. Candidate order alternates between rounds. | +| `--run-timeout SECONDS` | `300` | Deadline for each complete native or pytest suite run; cleanup uses a Windows Job Object or POSIX root-session plus identity-tracked descendants. | +| `--pytest-source PATH` | none | Also time a corresponding pytest source path; repeat for multiple paths. | +| `--json FILE\|-` | none | Write the complete tuning report to a new file, or standard output with `-`. | +| `--shard-modules` / `--no-shard-modules` | configured value | Tune with explicit safe intra-module sharding or module affinity. | +| `--manifest FILE` | configured value | Use one source-verified collection manifest for every native sample. | +| `--write` | off | Persist the measured recommendation as `[tool.testenix].workers`. | + +The command measures fresh CLI processes with Testenix history disabled. A one-worker probe +establishes the inventory and outcomes, and every native candidate must match them. To avoid +persisting noise, it recommends the smallest worker count within a narrow tolerance of the best +median. `--write` is the only configuration-mutating mode; normal tuning and adaptive auto do not +edit configuration. A workers-only write is rejected when a transient sharding or manifest override +differs from the loaded project configuration, or when that configuration file changes during the +measurement, because the recommendation would not describe the persisted execution profile. The +writer compares the pre-tuning bytes again immediately before an atomic replacement. Project +Python/TOML sources, linked source directories, explicit suite files, and a trusted manifest are +fingerprinted by content and file identity after every sample; observed drift discards the complete +result. Run publishable tuning from an immutable checkout to exclude a writer racing the final +filesystem operation; installed packages, non-source data, and other runtime inputs are also outside +that fingerprint. + +The optional pytest row is useful for local orientation only when it represents the corresponding +source suite. A public pytest/Testenix claim still needs equivalent inventories and outcomes, +counterbalanced commands, environment/version provenance, and the complete +[benchmarking contract](../benchmarking.md). + +## `testenix manifest` + +```text +testenix manifest PATH [PATH ...] --output FILE +``` + +The command performs supervised native collection and creates a new deterministic trusted +manifest. It records collection roots, the complete selected Python-source inventory and SHA-256 +fingerprints, collected tests and issues, and conservative module-sharding decisions. Test case +parameter names are retained while their values are redacted, so environment-derived secrets are +not copied into the trust artifact. `FILE` must +not already exist; Testenix does not silently replace a previous trust artifact. + +Use it explicitly on later runs: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +This can avoid importing every selected module once for collection and once again for execution. +Execution workers still import the modules they run. Source verification cannot cover dynamic +collection inputs such as environment variables or external services; regenerate the manifest +when those inputs change. + ## Examples ```console @@ -146,6 +241,12 @@ $ testenix run --retries 1 --timeout 10 $ testenix run -v --show-skips --durations 10 $ testenix run --color never $ testenix run --json reports/run.json --junit reports/junit.xml +$ testenix run tests --shard-modules +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +$ testenix tune tests --candidates 1,2,4,8 --warmups 1 --repeats 5 +$ testenix benchmark tests --json reports/tuning.json +$ testenix tune --write $ testenix --config config/pyproject.toml run $ testenix pytest -q --tb=short tests $ testenix migrate pytest tests --dry-run diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 634553f..19926ae 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -13,6 +13,8 @@ workers = "auto" retries = 0 # timeout = 10.0 tags = [] +# shard_modules = true +# manifest = ".testenix/collection.json" # json = "reports/testenix.json" # junit = "reports/junit.xml" history = ".testenix/history.sqlite3" @@ -34,8 +36,15 @@ Files and directories used when no positional path is passed to `testenix run`. - Type: positive integer or `"auto"` - Default: `"auto"` -`auto` uses Python's logical CPU count. An explicit number is recommended for reproducible CI and -benchmark runs. +`auto` is adaptive rather than equal to Python's logical CPU count. After collection and selection, +Testenix caps concurrency by the available CPUs and the number of independently schedulable units. +Reliable duration history feeds a worker-startup/makespan estimate; without enough history, a +conservative cold-start cap avoids oversubscribing short suites. The smallest worker count within a +narrow tolerance of the predicted best is selected. + +An explicit number is recommended for strict CI resource limits and publishable benchmark runs. +Use `testenix tune` (or its `testenix benchmark` alias) to measure native candidates for this +project, and `testenix tune --write` to persist its recommendation explicitly. ### `retries` @@ -88,6 +97,40 @@ history = false The programmatic field is `history_path` and accepts a `pathlib.Path` or `None`. +### `shard_modules` + +- Type: boolean +- Default: `false` + +Allow eligible modules to be divided into finer per-test execution units. The static analyzer +keeps module affinity when it sees module/session fixtures, mutation of obvious module-global +state, or import-time lifecycle hazards. Function-scoped fixtures, including autouse fixtures, do +not block sharding. + +This option is explicitly opt-in because static analysis cannot prove the absence of all dynamic +side effects. Validate the project with and without sharding before enabling it in CI. + +### `manifest` + +- Type: filesystem path or `null` +- Default: none + +Path to a trusted collection manifest generated explicitly with: + +```console +$ testenix manifest tests --output .testenix/collection.json +``` + +On each run Testenix verifies the requested collection roots, selected test files, statically +discoverable project-local Python import dependencies, and SHA-256 source digests. An exact match +bypasses the collection-side imports; a stale +but well-formed manifest falls back to normal supervised collection. Malformed JSON is rejected. +Execution workers still import the modules they run. Test parameter names remain available for +diagnostics, but their values are stored only as `` to avoid persisting secrets obtained +during collection. Regenerate the manifest when source files or dynamic collection inputs change. + +The programmatic field is `manifest_path` and accepts a `pathlib.Path` or `None`. + ## Programmatic configuration ```python @@ -101,6 +144,8 @@ config = TestenixConfig( retries=1, timeout=5.0, tags=("unit",), + shard_modules=True, + manifest_path=Path(".testenix/collection.json"), json_path=Path("reports/run.json"), history_path=None, ) diff --git a/docs/roadmap.md b/docs/roadmap.md index c480d6c..2e7f9a1 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -28,6 +28,10 @@ ## 0.3 — fast feedback +- Adaptive worker selection based on real execution units, duration history, and process cost, + plus an explicit project-local `tune`/`benchmark` command. +- Conservative opt-in intra-module sharding and a source-verified trusted collection manifest that + can bypass duplicate collection imports without trusting stale source metadata. - Dynamic micro-shards and work stealing. - `--last-failed`, watch mode, and failure fingerprints. - Test-impact analysis in shadow mode with an explanation for every selection decision. @@ -39,7 +43,7 @@ - Expand migration beyond the v0.2 static subset only when new transformations have differential semantics tests on real projects. - Versioned reporter and selector plugin interfaces. -- IDE protocol and machine-readable collection manifest. +- IDE protocol built on the versioned machine-readable collection manifest. ## Later diff --git a/llms-full.txt b/llms-full.txt index 2eb96ff..4513014 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -85,15 +85,15 @@ for-llms
0dependencies in the native runtime
12Python and OS combinations in CI
3console, JSON, and JUnit reports
-
3.15×native testenix run on the 100k synthetic workload vs pytest
+
3.15×historical v0.1.0 result: 100k synthetic no-op tests, 4 workers, --no-history
Testenix is deliberately built around a few strong guarantees: - **Async is native.** Coroutine tests and async-generator fixtures use the same model as synchronous code and do not require a plugin. -- **Parallelism is part of the runner.** Module affinity, process isolation, and - duration-aware scheduling are designed together. +- **Parallelism is part of the runner.** Adaptive worker selection, module affinity, optional + safety-checked sharding, process isolation, and duration-aware scheduling are designed together. - **Retries preserve evidence.** A failed attempt followed by a pass is `FLAKY`, never silently rewritten as a clean pass. - **Crashes cannot erase completed work.** Workers stream results as tests finish, and unfinished @@ -153,20 +153,28 @@ $ python -m pip install testenix $ testenix run tests ``` +`workers = "auto"` adapts to the schedulable work instead of launching one process per CPU. Use +`testenix tune` (also available as `testenix benchmark`) for a measured project recommendation. +Import-heavy native suites can explicitly generate a source-hashed collection manifest, and large +independent modules can opt into conservative `--shard-modules` scheduling. See +[parallel execution](https://polishdataengineer.github.io/testenix/guides/parallelism/) for both trust boundaries. + To evaluate unreleased source changes, install the current `main` branch with `python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main"`. ## Performance evidence, with context -The checked-in development baseline measured native `testenix run` on 100,000 empty tests across -16 generated modules on an Apple M4 Pro and CPython 3.11. Native Testenix completed that specific -workload in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 seconds for -pytest-xdist. These measurements do not apply to the delegated `testenix pytest` command. +The checked-in development baseline measured **Testenix 0.1.0**, not the current release. Native +`testenix run` completed 100,000 generated no-op tests across 16 modules on one Apple M4 Pro and +CPython 3.11 machine in a median 8.04 seconds, compared with 25.33 seconds for pytest and 21.30 +seconds for pytest-xdist. It used four workers, `--no-history`, and pytest-xdist's default `load` +scheduler. These measurements do not apply to Testenix 0.2.1, a real project, the default-history +mode, or the delegated `testenix pytest` command.
-This is a preliminary synthetic result from one machine, not a promise that every project will be -3.15× faster. The benchmark page publishes the raw samples, environment, variance, methodology, -and limitations so that the claim can be evaluated rather than taken on trust. +This is historical synthetic evidence from one machine, not a promise that every project will be +3.15× faster. No clean Testenix 0.2.1 scaling matrix is checked in yet. The benchmark page publishes +the raw samples, environment, variance, methodology, current matrix status, and limitations.
[Inspect the benchmark data](https://polishdataengineer.github.io/testenix/benchmarks/results/) or @@ -316,6 +324,8 @@ paths = ["tests"] workers = "auto" retries = 0 history = ".testenix/history.sqlite3" +# shard_modules = true +# manifest = ".testenix/collection.json" # timeout = 10 # json = "reports/testenix.json" # junit = "reports/junit.xml" @@ -330,6 +340,33 @@ $ testenix run --workers 4 --retries 1 --json reports/result.json Use `--no-history` for a side-effect-free run. Duration history normally helps later runs schedule long tests earlier. +`workers = "auto"` adapts to the selected suite instead of copying the logical CPU count. It is +capped by the actual schedulable units and uses duration history when enough is available. Measure +an explicit project setting with: + +```console +$ testenix tune tests --warmups 1 --repeats 5 +$ testenix tune --write +``` + +`testenix benchmark` is an alias for `testenix tune`. + +Module affinity is the safe default. For a large module whose tests are known to be independent, +`--shard-modules` opts eligible tests into finer scheduling after conservative static checks. Read +[parallel execution](https://polishdataengineer.github.io/testenix/guides/parallelism/) before enabling it. + +To avoid repeating collection imports on later unchanged runs, create and trust a source-hashed +manifest explicitly: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +Testenix verifies the complete file inventory and every SHA-256 before reuse. A stale manifest +falls back to supervised collection rather than running a stale test selection. Parameter names +remain visible in the artifact, but their values are redacted. + ## Use it from Python The runner also exposes a typed API: @@ -337,12 +374,20 @@ The runner also exposes a typed API: ```python from testenix import Status, TestenixConfig, run -result = run("tests", TestenixConfig(workers=4, history_path=None)) -failed = [item.test.id for item in result.tests if item.status is not Status.PASS] + +def main() -> None: + result = run("tests", TestenixConfig(workers=4, history_path=None)) + failed = [item.test.id for item in result.tests if item.status is not Status.PASS] + print(failed) + + +if __name__ == "__main__": + main() ``` Async applications can call `await testenix.run_async(...)`. Cancellation terminates active -collection and execution process trees before control returns to the caller. +collection and execution process trees before control returns to the caller. Executable scripts +must place the top-level call behind the same `if __name__ == "__main__":` guard on every platform. ## Next steps @@ -712,10 +757,13 @@ those tests were published. ## Performance with thousands of migrated tests Migration unlocks the native scheduler, process supervision, async engine, retries, history, and -Testenix reports. It does not guarantee that every converted suite is faster. Testenix keeps a -normal module as one affinity unit so module-scoped fixtures are not duplicated. Consequently, -3,000 tests in one source module still form one schedulable unit; 3,000 tests spread across enough -independent modules can use multiple workers. +Testenix reports. It does not guarantee that every converted suite is faster. By default Testenix +keeps a normal module as one affinity unit so module-scoped fixtures are not duplicated. +Consequently, 3,000 tests in one source module still form one schedulable unit; 3,000 tests spread +across enough independent modules can use multiple workers. A project may later opt eligible native +modules into `--shard-modules`, but only after validating the generated suite and the documented +static-analysis trust boundary. Run `testenix tune` on the published native copy before fixing its +CI worker count. The checked-in migration baseline used an Apple M4 Pro, CPython 3.11, 3,000 generated tests in 64 modules, four native workers, one warm-up, and five measured rounds: @@ -1091,8 +1139,53 @@ $ testenix run --workers 4 $ testenix run --workers 1 ``` -`auto` resolves to the logical CPU count reported by Python. For CI and benchmark runs, an explicit -count makes resource use and results easier to reproduce. +`auto` is adaptive; it is not an alias for the logical CPU count. Testenix first counts the +execution units it can actually schedule (module-affinity groups, isolated timeout tests, and any +eligible opt-in module shards). It caps the choice by that count and the available CPUs. With +reliable duration history, a startup-cost/makespan model chooses the smallest worker count within a +narrow tolerance of the predicted best. Without enough history, cold-start auto uses a conservative +cap so a short suite does not launch one process per CPU. + +An explicit number remains a hard project setting. Prefer one in tightly budgeted CI; for a +project-specific measured value, use the tuner. + +## Tune a project + +```console +$ testenix tune tests --warmups 1 --repeats 5 +$ testenix benchmark tests --candidates 1,2,4,8 +$ testenix tune --json reports/tuning.json --write +``` + +`benchmark` is an alias for `tune`. The command runs fresh CLI processes, uses a green one-worker +inventory probe, disables history for all timing samples, measures native candidates in alternating +order, and rejects a candidate whose test inventory or outcomes differ. The recommendation is the +smallest worker count within a narrow tolerance of the best median, avoiding a larger setting for +measurement noise. `--write` stores that integer in +`[tool.testenix].workers`; it is an explicit file change, never an effect of `workers = "auto"`. +The automatic sweep respects the process-visible CPU capacity and tests at most 1/2/4 workers; +larger counts must be requested explicitly with `--candidates`. Each complete suite run has a +300-second deadline by default (`--run-timeout SECONDS`). Windows starts the command suspended and +attaches a kill-on-close Job Object before resume. POSIX always signals the new root session and +identity-checks observed detached descendants. A child that calls `setsid()` and exits between +creation and the first process snapshot is outside an absolute kernel-containment guarantee; use a +container for hostile suites. + +Pass `--shard-modules` to tune that explicit mode and `--manifest FILE` to include the verified +single-import collection path in every native sample. When `--write` is present, Testenix refuses a +transient sharding or manifest override that differs from `[tool.testenix]`, because writing only +`workers` would persist a recommendation for a different execution profile. Configure the profile +first or tune it without `--write`. It also fingerprints project Python/TOML sources, explicit suite +files (including linked source directories), and the trusted manifest, then rejects any observed +content or file-identity drift after a sample. The final configuration update rechecks the original +bytes immediately before an atomic replacement. Use an immutable checkout for publishable tuning: +it excludes a writer racing that last filesystem operation, while installed packages, non-source +data, and other runtime dependencies remain external inputs. + +Use `--pytest-source PATH` to add an optional pytest timing for a corresponding source suite. The +tuner's primary contract is worker selection for the native suite. Its optional pytest ratio is +orientation for that exact invocation, not sufficient evidence for a public speed claim; public +comparisons must also satisfy the [benchmarking contract](https://polishdataengineer.github.io/testenix/benchmarking/). ## Scheduling @@ -1102,6 +1195,62 @@ preserves module-fixture reuse and avoids splitting hidden module state between When duration history exists, Testenix schedules longer units first. This longest-processing-time strategy is deterministic and reduces the chance that one slow shard becomes the tail of the run. +## Opt-in intra-module sharding + +One large module normally exposes one unit no matter how many tests it contains. If its tests are +known to be independent, explicitly request finer units: + +```console +$ testenix run tests --shard-modules +``` + +or configure: + +```toml +[tool.testenix] +shard_modules = true +``` + +This is deliberately off by default. Before splitting a module, Testenix statically fails closed +when it detects module- or session-scoped fixtures, imported fixture providers, direct writes to +module globals, nested mutable containers, mutable class state, or executable import-time lifecycle +behavior. Function-scoped fixtures defined in the collected module, including autouse fixtures, +may be recreated in separate workers and do not block sharding. Eager calls in module assignments, +annotations, decorators, function defaults, and class bases or keywords are treated as import-time +lifecycle behavior and keep the module intact. + +Static analysis cannot prove that arbitrary calls, imported libraries, environment state, or +external services are free of shared effects. Enabling the option is therefore a project trust +decision. Validate the suite both with and without sharding before adopting it in CI. Modules that +fail the safety check retain normal module affinity while eligible modules can be split. + +## Avoid the collection-side import + +Without a manifest, safe supervised collection imports selected modules once, then execution +workers import their assigned modules again to materialize functions and case values. Projects for +which imports are a meaningful part of wall time can generate an explicit trusted manifest: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +The manifest records collection roots, every selected test file plus statically discoverable +project-local Python import dependencies and their SHA-256 hashes, test metadata, collection issues, +and sharding decisions. Parameter names are retained but values +are redacted so collection-time environment data is not persisted. Each run verifies the requested roots, +the exact file set, and every source digest before reusing it. Malformed manifest JSON is rejected +as invalid input. If a file was added or removed, a source changed, or current-source verification +cannot establish an exact match, the manifest is stale and Testenix falls back to the normal +isolated collection process. It does not execute a stale selection. + +This optimization removes the collection-side import on an unchanged run; execution workers still +import the code they execute. It is not an implicit cache. If collection depends on environment +variables, generated files outside the selected roots, network state, or other inputs not captured +by source hashes, the producer must regenerate the manifest when those inputs change. Set +`manifest = ".testenix/collection.json"` in `[tool.testenix]` only when that trust boundary is +appropriate. + ## Process isolation Workers are spawned processes. A worker streams each completed attempt to the coordinator before @@ -1148,7 +1297,8 @@ async def main() -> None: print(result.exit_code) -asyncio.run(main()) +if __name__ == "__main__": + asyncio.run(main()) ``` Cancelling `run_async` terminates active collection and execution process trees before the @@ -1156,8 +1306,9 @@ coroutine returns control. ## Platform note -On Windows, scripts that call `run()` or `run_async()` directly must use the standard -multiprocessing guard: +On every supported platform, executable scripts that call `run()` or `run_async()` directly must +use the standard multiprocessing guard because the supervised worker protocol uses the `spawn` +start method: ```python if __name__ == "__main__": @@ -1322,13 +1473,16 @@ testenix [-h] [--version] testenix [--config PYPROJECT] run [RUN_ARGS ...] testenix pytest [PYTEST_ARGS ...] testenix migrate FRAMEWORK PATH [PATH ...] [MIGRATION_ARGS ...] +testenix [--config PYPROJECT] tune [PATH ...] [TUNING_ARGS ...] +testenix [--config PYPROJECT] benchmark [PATH ...] [TUNING_ARGS ...] +testenix manifest PATH [PATH ...] --output FILE ``` | Option | Description | | --- | --- | | `-h`, `--help` | Show command help. | | `--version` | Print the installed Testenix version. | -| `--config PATH` | Load `[tool.testenix]` for native `testenix run`. | +| `--config PATH` | Load `[tool.testenix]` for native `run` and `tune`/`benchmark` commands. | ## `testenix run` @@ -1342,6 +1496,8 @@ testenix run [PATH ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] + [--shard-modules] + [--manifest FILE] [-q | -v | -vv] [--color {auto,always,never} | --no-color] [--show-skips] @@ -1351,7 +1507,7 @@ testenix run [PATH ...] | Argument | Default | Description | | --- | --- | --- | | `PATH ...` | configured `paths`, otherwise `tests` | Files or directories to discover. | -| `-w`, `--workers` | `auto` | Worker process count or logical CPU count. | +| `-w`, `--workers` | `auto` | Positive worker count, or adaptive selection from schedulable units, history, startup cost, and CPU capacity. | | `--retries` | `0` | Additional attempts after a gating outcome. | | `--timeout` | none | Global hard deadline for every selected test. | | `-t`, `--tag` | none | Required tag; repeat for AND selection. | @@ -1359,6 +1515,8 @@ testenix run [PATH ...] | `--junit` | none | Write a JUnit XML report. | | `--history` | `.testenix/history.sqlite3` | Override the duration-history database. | | `--no-history` | off | Disable reading and writing history. | +| `--shard-modules` | off | Opt eligible modules into per-test execution units after conservative static safety checks. | +| `--manifest FILE` | none | Reuse an explicitly generated, source-verified trusted collection manifest. | | `-q`, `--quiet` | off | Hide the run header and compact per-file table. Collection errors, failure details, and the final summary remain visible. | | `-v`, `--verbose` | off | Print one result row per test in the stable detailed format. Repeat for `-vv`. | | `-vv` | off | Add worker, attempt, and phase metadata, including captured output. | @@ -1375,6 +1533,20 @@ order after execution completes; these modes do not promise live progress update flags change only terminal rendering, not selection, scheduling, result statuses, JSON, JUnit, or exit codes. +`workers = auto` never means “start one process per logical CPU.” Testenix caps it by the number of +units that can run independently. Reliable duration history feeds a makespan/startup-cost model; +cold runs use a conservative cap. The final console and JSON results report the worker count +actually used. An explicit integer remains useful for fixed CI resource limits and reproducible +published benchmarks. + +`--shard-modules` is an explicit safety/performance trade-off. Modules with module/session +fixtures, statically visible mutable global state, or import-time lifecycle hazards keep module +affinity; eligible modules may be split. Static analysis cannot prove every dynamic side effect. + +`--manifest` accepts the versioned JSON produced by `testenix manifest`. Malformed input is a usage +error. An otherwise valid manifest whose roots, complete Python-file inventory, or SHA-256 digests +no longer match is treated as stale, and the run safely performs ordinary supervised collection. + In `auto` color mode, Testenix requires a terminal, respects `NO_COLOR`, allows `FORCE_COLOR`, and disables styling for a truthy `CI` value or `TERM=dumb`. Explicit `always` or `never` takes precedence. @@ -1451,6 +1623,80 @@ grouped by severity and code, with the first source location shown. Use `--repor worker setting. It is emitted only for `--check` or publication after static analysis succeeds, because dry-run and unsupported transactions never execute the parallel gate. +## `testenix tune` / `testenix benchmark` + +```text +testenix tune [PATH ...] + [--config PYPROJECT] + [--candidates N[,N...]] + [--warmups N] + [--repeats N] + [--pytest-source PATH] ... + [--json FILE|-] + [--shard-modules] + [--manifest FILE] + [--write] +``` + +`testenix benchmark` is an exact alias for `testenix tune`. + +| Argument | Default | Description | +| --- | --- | --- | +| `PATH ...` | configured `paths`, otherwise `tests` | Native Testenix suite to tune. | +| `--candidates N[,N...]` | resource-aware 1/2/4 sweep | Positive worker counts to measure. Counts above the number of execution units are deduplicated at that limit; values above four require this explicit option. | +| `--warmups N` | `1` | Unrecorded warmups for each native candidate and optional pytest source. | +| `--repeats N` | `5` | Recorded samples for each candidate. Candidate order alternates between rounds. | +| `--run-timeout SECONDS` | `300` | Deadline for each complete native or pytest suite run; cleanup uses a Windows Job Object or POSIX root-session plus identity-tracked descendants. | +| `--pytest-source PATH` | none | Also time a corresponding pytest source path; repeat for multiple paths. | +| `--json FILE\|-` | none | Write the complete tuning report to a new file, or standard output with `-`. | +| `--shard-modules` / `--no-shard-modules` | configured value | Tune with explicit safe intra-module sharding or module affinity. | +| `--manifest FILE` | configured value | Use one source-verified collection manifest for every native sample. | +| `--write` | off | Persist the measured recommendation as `[tool.testenix].workers`. | + +The command measures fresh CLI processes with Testenix history disabled. A one-worker probe +establishes the inventory and outcomes, and every native candidate must match them. To avoid +persisting noise, it recommends the smallest worker count within a narrow tolerance of the best +median. `--write` is the only configuration-mutating mode; normal tuning and adaptive auto do not +edit configuration. A workers-only write is rejected when a transient sharding or manifest override +differs from the loaded project configuration, or when that configuration file changes during the +measurement, because the recommendation would not describe the persisted execution profile. The +writer compares the pre-tuning bytes again immediately before an atomic replacement. Project +Python/TOML sources, linked source directories, explicit suite files, and a trusted manifest are +fingerprinted by content and file identity after every sample; observed drift discards the complete +result. Run publishable tuning from an immutable checkout to exclude a writer racing the final +filesystem operation; installed packages, non-source data, and other runtime inputs are also outside +that fingerprint. + +The optional pytest row is useful for local orientation only when it represents the corresponding +source suite. A public pytest/Testenix claim still needs equivalent inventories and outcomes, +counterbalanced commands, environment/version provenance, and the complete +[benchmarking contract](https://polishdataengineer.github.io/testenix/benchmarking/). + +## `testenix manifest` + +```text +testenix manifest PATH [PATH ...] --output FILE +``` + +The command performs supervised native collection and creates a new deterministic trusted +manifest. It records collection roots, the complete selected Python-source inventory and SHA-256 +fingerprints, collected tests and issues, and conservative module-sharding decisions. Test case +parameter names are retained while their values are redacted, so environment-derived secrets are +not copied into the trust artifact. `FILE` must +not already exist; Testenix does not silently replace a previous trust artifact. + +Use it explicitly on later runs: + +```console +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +``` + +This can avoid importing every selected module once for collection and once again for execution. +Execution workers still import the modules they run. Source verification cannot cover dynamic +collection inputs such as environment variables or external services; regenerate the manifest +when those inputs change. + ## Examples ```console @@ -1461,6 +1707,12 @@ $ testenix run --retries 1 --timeout 10 $ testenix run -v --show-skips --durations 10 $ testenix run --color never $ testenix run --json reports/run.json --junit reports/junit.xml +$ testenix run tests --shard-modules +$ testenix manifest tests --output .testenix/collection.json +$ testenix run tests --manifest .testenix/collection.json +$ testenix tune tests --candidates 1,2,4,8 --warmups 1 --repeats 5 +$ testenix benchmark tests --json reports/tuning.json +$ testenix tune --write $ testenix --config config/pyproject.toml run $ testenix pytest -q --tb=short tests $ testenix migrate pytest tests --dry-run @@ -1518,6 +1770,8 @@ workers = "auto" retries = 0 # timeout = 10.0 tags = [] +# shard_modules = true +# manifest = ".testenix/collection.json" # json = "reports/testenix.json" # junit = "reports/junit.xml" history = ".testenix/history.sqlite3" @@ -1539,8 +1793,15 @@ Files and directories used when no positional path is passed to `testenix run`. - Type: positive integer or `"auto"` - Default: `"auto"` -`auto` uses Python's logical CPU count. An explicit number is recommended for reproducible CI and -benchmark runs. +`auto` is adaptive rather than equal to Python's logical CPU count. After collection and selection, +Testenix caps concurrency by the available CPUs and the number of independently schedulable units. +Reliable duration history feeds a worker-startup/makespan estimate; without enough history, a +conservative cold-start cap avoids oversubscribing short suites. The smallest worker count within a +narrow tolerance of the predicted best is selected. + +An explicit number is recommended for strict CI resource limits and publishable benchmark runs. +Use `testenix tune` (or its `testenix benchmark` alias) to measure native candidates for this +project, and `testenix tune --write` to persist its recommendation explicitly. ### `retries` @@ -1593,6 +1854,40 @@ history = false The programmatic field is `history_path` and accepts a `pathlib.Path` or `None`. +### `shard_modules` + +- Type: boolean +- Default: `false` + +Allow eligible modules to be divided into finer per-test execution units. The static analyzer +keeps module affinity when it sees module/session fixtures, mutation of obvious module-global +state, or import-time lifecycle hazards. Function-scoped fixtures, including autouse fixtures, do +not block sharding. + +This option is explicitly opt-in because static analysis cannot prove the absence of all dynamic +side effects. Validate the project with and without sharding before enabling it in CI. + +### `manifest` + +- Type: filesystem path or `null` +- Default: none + +Path to a trusted collection manifest generated explicitly with: + +```console +$ testenix manifest tests --output .testenix/collection.json +``` + +On each run Testenix verifies the requested collection roots, selected test files, statically +discoverable project-local Python import dependencies, and SHA-256 source digests. An exact match +bypasses the collection-side imports; a stale +but well-formed manifest falls back to normal supervised collection. Malformed JSON is rejected. +Execution workers still import the modules they run. Test parameter names remain available for +diagnostics, but their values are stored only as `` to avoid persisting secrets obtained +during collection. Regenerate the manifest when source files or dynamic collection inputs change. + +The programmatic field is `manifest_path` and accepts a `pathlib.Path` or `None`. + ## Programmatic configuration ```python @@ -1606,6 +1901,8 @@ config = TestenixConfig( retries=1, timeout=5.0, tags=("unit",), + shard_modules=True, + manifest_path=Path(".testenix/collection.json"), json_path=Path("reports/run.json"), history_path=None, ) @@ -1755,9 +2052,68 @@ evidence for specific synthetic workloads, not a universal claim that Testenix i than pytest. `Testenix` in these results means the native `testenix run` engine. The `testenix pytest` compatibility bridge delegates to pytest and is not represented here. -![Preliminary Testenix throughput ratios](https://polishdataengineer.github.io/testenix/_static/benchmark-speedup.svg) +## Testenix 0.2.1 scaling matrix + +No current-version matrix is checked in yet. The historical results below must therefore not be +described as Testenix 0.2.1 performance. The new provenance-gated harness covers +100/500/1,000/3,000 tests, balanced/dominant/single-module layouts, 1/2/4/auto workers, and both +default history and `--no-history`, plus explicit safe-module sharding. Its default design uses +dimension sweeps; use +`--full-cross-product` only when the much larger run is intentional. -## Median wall-clock time +`auto` is passed literally to Testenix and remains adaptive; observed Testenix worker counts are +stored per sample. pytest-xdist resolves its side of an `auto` row separately to the machine's +logical CPU count. + +```console +$ uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +The command refuses a dirty worktree or an installed Testenix version that differs from +`pyproject.toml`. `--allow-dirty` is available only for unpublished smoke runs. A matrix becomes +publishable here only after five measured rounds, one warm-up, clean commit provenance, and full +axis coverage pass the documentation generator's validation. + + +## Real-project harness + +The 118-test project used during v0.2 migration validation was a semantic parity gate, not a +publishable benchmark: its release-note timings were single observations without a committed +multi-round record. Use the redaction-safe manifest harness for a real repository: + +```console +$ cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +$ uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /absolute/path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json +``` + +The manifest stores argument arrays, never shell fragments. The result omits stdout, stderr, +environment values, absolute project paths, and private source. It records only timings, aggregate +output sizes, optional tree fingerprints, and redacted Git provenance. A migrated-suite comparison +must point the manifest at a successful migration report to become publication-eligible. The +harness verifies the report's exact per-test inventory and outcomes, complete source and generated +Python-file inventories, current hashes, and binds canonical `python -m pytest` / +`python -m testenix run` commands to the report's source/output roots. Publishable source roots are +directories so support files such as `conftest.py` are covered. Without the report the result is +diagnostic-only. Commands are retained for +reproducibility. Publishable commands put options before `--` and exact suite targets after it, so +an option value cannot impersonate a migration root. Keep secrets in the environment or list +sensitive argument indexes in `redact_arguments`. + + +## Historical Testenix 0.1.0 synthetic baseline + +The checked-in `3.15×` figure is a Testenix 0.1.0 result for 100,000 generated no-op +tests across 16 modules, four workers, disabled history (`--no-history`), and pytest-xdist's default +`load` strategy. It is retained as transparent historical evidence; it is not a measurement of +Testenix 0.2.1. + +![Historical Testenix 0.1.0 throughput ratios](https://polishdataengineer.github.io/testenix/_static/benchmark-speedup.svg) + +### Median wall-clock time Lower time is better. A speedup of `2.85×` means pytest's median wall time was 2.85 times the Testenix median for that exact @@ -1774,19 +2130,24 @@ The 100,000-test result meets the project's local five-run, one-warmup minimum. It remains a synthetic result from one machine, not a universal performance promise. -## Environment +### Environment and controls - CPU: Apple M4 Pro (14 logical CPUs) - Machine: `arm64` - Platform: `macOS-26.5.1-arm64-arm-64bit` - Python: `3.11.14` +- Testenix: `0.1.0` +- Workers: four for Testenix and pytest-xdist +- Testenix history: disabled with `--no-history` +- pytest-xdist: version `3.8.0`, + default `load` distribution - Measurement: complete subprocess wall-clock time, including discovery, execution, aggregation, and console rendering - Correctness gate: every command had to exit successfully and report the expected test count -## Raw samples and variance +### Raw samples and variance ### 10,000 no-op tests / 16 modules @@ -1801,6 +2162,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 2.170, 2.267, 2.077, 2.075, 2.106 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:12:43.635798+00:00` - Commit: `8f24f8a7bd72fa876988a8ce96364be97e35c2b6` - Clean working tree at capture: yes @@ -1819,6 +2182,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 2.146, 2.129, 2.109, 2.138, 2.176 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:13:53.369799+00:00` - Commit: `18d9bba6cb5c8e39c2d5b211ee4384ae8f824524` - Clean working tree at capture: yes @@ -1837,6 +2202,8 @@ It remains a synthetic result from one machine, not a universal performance prom - pytest-xdist raw samples: 21.239, 21.120, 22.216, 21.300, 21.949 seconds - Measured rounds: 5; warmups: 1 - Workers: 4 +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` - Recorded at: `2026-07-20T12:19:39.942492+00:00` - Commit: `24b877c2f98420e91dcd2c8bcbc9417c7cf1ac96` - Clean working tree at capture: yes @@ -1847,7 +2214,9 @@ It remains a synthetic result from one machine, not a universal performance prom These separate measurements start with generated pytest or unittest sources, complete one safe copy-and-validate migration, and then compare recurring source-suite runs with recurring native Testenix runs. The migration transaction is a one-time cost shown separately; it is not included -in either execution median. +in either execution median. These records came from the pre-v0.2 source commit linked below; its +distribution metadata still reported `0.1.0`. They are historical evidence, not measurements of +the current release. | Source runner | Workload | Tests / modules | Source median | Native median | Native vs source | Migration transaction | | --- | --- | ---: | ---: | ---: | ---: | ---: | @@ -1941,9 +2310,9 @@ therefore material, and none of these synthetic rows predicts a specific real pr ## Interpretation -The checked-in results show that Testenix has low per-test overhead for large generated suites and -that its built-in process model is competitive with both sequential pytest and pytest-xdist in -those scenarios. +The historical checked-in results show that Testenix 0.1.0 had low per-test overhead for the large +generated suites above and was competitive with sequential pytest and pytest-xdist's default +`load` strategy in those scenarios. They are not evidence for the current release. They do **not** yet answer how Testenix performs for import-heavy applications, complex fixture graphs, assertion failures, real repositories, or different operating systems. Pytest also has a @@ -1999,19 +2368,84 @@ For every scenario record wall-clock duration, collection time, execution time, worker utilization, number of process starts, result completeness, and output size. Performance claims require at least five runs per configuration and must publish the environment fingerprint. -The current v0.1 harness automates wall-clock samples and the environment fingerprint across 16 -generated modules for a four-worker run. It also validates the completed test count, rotates runner -order between measured rounds, supports an explicit module count, and records throughput, mean, and -standard deviation. Pytest plugin autoloading and its cache provider are disabled, pytest-xdist is -loaded explicitly, and every tool runs from the generated suite directory so repository-level -pytest configuration does not affect the comparison. New output also records the commit, dirty -state, lockfile hash, timestamp, and installed framework versions. The remaining telemetry above is -the acceptance contract for the next harness iteration, not data claimed by the checked-in baseline -files. +The checked-in headline files were recorded with Testenix 0.1.0 across 16 generated modules, four +workers, and `--no-history`. They automate subprocess wall-clock samples and the environment +fingerprint, validate the completed test count, rotate runner order between measured rounds, and +record throughput, mean, standard deviation, provenance, and raw samples. Pytest plugin autoloading +and its cache provider are disabled, pytest-xdist 3.8 is loaded explicitly with its default `load` +distribution, and every tool runs from the generated suite directory so repository-level pytest +configuration does not affect the comparison. Those historical records do not measure Testenix +0.2.1. + +Schema-version 2 harness output additionally records the requested and resolved worker counts, +balanced/dominant/single-module test distributions, default-history versus `--no-history`, the +pytest-xdist distribution strategy, and captured stdout/stderr byte counts. Collection/execution +splits, peak memory, utilization, process counts, and the remaining telemetry above are still the +acceptance contract for a later harness iteration, not data claimed by the historical baseline. Correctness wins over speed: a run with a missing, duplicated, or incorrectly finalized result is invalid and excluded from performance comparisons. +## Evidence levels + +1. **Historical synthetic baseline.** The committed `3.15×` figure is Testenix 0.1.0 on 100,000 + generated no-op tests, 16 modules, four workers, disabled history, and one M4 Pro. It remains + useful provenance but must not be labelled as current-version performance. +2. **Current-version scaling matrix.** `run_scaling_matrix.py` requires the installed Testenix + version to match `pyproject.toml` and refuses a dirty worktree by default. The dimension-sweep + design covers 100, 500, 1,000, and 3,000 tests; 1, 2, 4, and `auto` workers; + balanced/dominant/single-module layouts; and default history versus `--no-history`. The reference + configuration changes one axis at a time. `--full-cross-product` is available when every + combination is worth the substantially larger runtime; its artifact still projects one + canonical balanced/no-history reference curve. `auto` remains an adaptive Testenix + request; the harness records the worker count observed in each Testenix sample. For the xdist + side of that row, `auto` resolves separately to Python's logical CPU count and is labelled as + such. +3. **Real-project evidence.** `run_project_benchmark.py` executes argument arrays from a local JSON + manifest without a shell. Its result excludes source, stdout/stderr, environment values, + absolute project paths, and Git remotes. Publication requires a successful migration report: + the harness verifies its exact per-test inventory and outcomes, complete current Python-file + inventories and hashes under directory source roots, the complete generated Python inventory, + and that canonical `python -m pytest` / `python -m testenix run` commands point at the report's + source and output roots. It records only aggregate timings/output sizes, digests, redacted Git + state, and optional content fingerprints. Repeated or conflicting worker/history/sharding flags + fail closed, and an imported runtime must belong to the exact files owned by its installed + distribution rather than merely sharing the same `site-packages` directory. + +Every synthetic and real-project command has a bounded deadline and bounded cleanup. Windows starts +the command suspended, attaches a kill-on-close Job Object, then resumes it. POSIX always signals +the new root session and identity-checks every observed detached descendant. Polling cannot provide +an absolute kernel guarantee for a child that calls `setsid()` and exits before the first snapshot; +run hostile benchmark inputs inside a container or other kernel containment boundary. +On POSIX the harness also discovers workers that created their own sessions; on Windows it uses a +kill-on-close Job Object with a bounded recursive fallback. A timed-out runner therefore cannot +continue consuming CPU or contaminate later counterbalanced samples. + +The 118-test project mentioned in the v0.2.0 release was a differential semantic-validation gate. +Its three timings were single observations, not a committed multi-round benchmark, and therefore do +not establish a real-project speedup. + +## Project-local tuning is not a published benchmark + +`testenix tune` and its `testenix benchmark` alias answer a narrow operational question: which +native worker count is fastest for this selected project suite on this machine now? They use fresh +CLI processes, disable history, validate a green one-worker inventory, alternate candidate order, +require native candidate inventory/outcome parity, and recommend the smallest worker count within +a narrow tolerance of the best measured median. This is appropriate for writing a local +`[tool.testenix].workers` value. + +The optional `--pytest-source` measurement is orientation for a corresponding source suite. A +tuning report alone is not sufficient for marketing because a migrated native path and its pytest +source may differ in collection, wrappers, output, or configuration. Publishing a ratio still +requires all of this contract: equivalent inventories and outcomes, full-process wall time, +counterbalanced order, at least one warm-up and five measured rounds, command/version/environment +provenance, output sizes, and an explanation of history, sharding, manifest, and plugin settings. + +For every published Testenix result, record both the requested and observed worker count. `auto` is +adaptive and must never be relabelled as the logical CPU count. Also record whether +`--shard-modules` was enabled and whether a trusted collection manifest was accepted or fell back +to supervised collection; either choice changes the amount and shape of schedulable work. + ## Pytest compatibility bridge Measurements of `testenix pytest` must be reported separately from native `testenix run` @@ -2025,28 +2459,65 @@ environment, pytest configuration, plugins, and arguments for both `python -m py Testenix execution speedup. Native comparisons continue to use `testenix run` and must validate that both runners execute the same tests and produce equivalent outcomes. -Run the reproducible local harness with: +Run one reproducible synthetic scenario with: ```bash -uv run python benchmarks/run_benchmark.py --tests 1000 --workers 4 --repeats 5 -uv run python benchmarks/run_benchmark.py --tests 1000 --workers 4 --repeats 5 --uneven -uv run python benchmarks/run_benchmark.py --tests 10000 --modules 1000 --workers 4 --repeats 5 +uv run --no-editable python benchmarks/run_benchmark.py \ + --tests 1000 --modules 16 --workers 4 --repeats 5 --warmups 1 \ + --module-layout balanced --history-mode disabled --xdist-strategy load ``` +Generate the current-version dimension sweeps from a clean checkout with: + +```bash +uv run --no-editable python benchmarks/run_scaling_matrix.py \ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +For an unpublished smoke test only, add `--quick --allow-dirty`. A publishable matrix must retain +five rounds, one warm-up, clean provenance, and the complete requested coverage. + +Measure a real project without committing its code or private paths: + +```bash +cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +# Edit expected counts, migration-report path, fingerprints, and runner commands. +uv run --no-editable python benchmarks/run_project_benchmark.py \ + --project /absolute/path/to/project \ + --manifest /tmp/testenix-project-benchmark.json \ + --output /tmp/testenix-project-result.json +``` + +Keep private manifests and results outside the repository until their labels, commit policy, and +fingerprints have been reviewed for publication. Environment override **values** are never written +to the result; only their key names are retained. Commands are recorded for reproducibility, so +secrets should stay in the environment. If an unavoidable command argument is sensitive, list its +zero-based index in that runner's `redact_arguments` field. + +Without `migration_report`, the harness can still produce a private diagnostic, but it always sets +`publication_eligible` to `false`. A publishable run also requires the canonical module +entrypoints, a clean project, at least one warm-up and five measured rounds, explicit Testenix +workers and history mode, and installed-distribution identities for both pytest and Testenix. The +pytest version is probed inside the same project environment used by the timed commands. In each +runner command, place options before `--` and the exact benchmark suite roots after it. The harness +uses that delimiter to distinguish positional targets from values of options such as `-k` or +`--tag` and requires those targets to match the migration report exactly. + Maintainers can run the same comparison from GitHub's **Benchmarks** workflow and download its raw JSON artifact. Shared GitHub runners are appropriate for reproducibility checks, not for silently replacing the approved marketing baseline: their timing variance is outside this project's control. -The checked-in baseline files are development evidence, not universal performance claims. See -`docs/performance-analysis.md` for the current large-suite results, optimization profile, memory -notes, and native-code decision. Real project suites and cross-platform repetitions remain required -before publishing broad comparative claims. +The checked-in baseline files are historical development evidence, not universal or current-version +performance claims. See `docs/performance-analysis.md` for the large-suite results, optimization +profile, memory notes, and native-code decision. A clean Testenix 0.2.1 scaling matrix, publishable +real-project suites, alternative pytest-xdist strategies, and cross-platform repetitions remain +required before publishing broad comparative claims. An approved public baseline must be committed through a reviewed pull request. Do not remove slow but valid samples as outliers; invalid commands remain evidence and must be explained. The current checked-in 10,000- and 100,000-test files each contain five measured rounds, one warm-up, and clean -commit provenance. They remain single-machine synthetic evidence, so broader claims still require -the real-project and cross-platform scenarios above. +commit provenance. They remain Testenix 0.1.0 single-machine synthetic evidence, so broader claims +still require the current-version, real-project, and cross-platform scenarios above. --- @@ -2059,22 +2530,25 @@ Source: docs/performance-analysis.md ## Executive summary -The optimized native v0.1 `testenix run` engine is faster than pytest and pytest-xdist in the checked-in synthetic -large-suite scenarios. The largest recorded comparison is 100,000 passing tests across 16 modules: -Testenix completed the suite in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist -in 21.300 seconds. Every measured command had to report the expected test count or the harness -rejected the sample. The baseline contains one warm-up and five counterbalanced measured rounds, -and Testenix's samples ranged from 7.912 to 8.096 seconds. +The checked-in headline numbers are a **historical Testenix 0.1.0 synthetic baseline**, not current +Testenix 0.2.1 results. In the largest recorded comparison, 100,000 generated no-op tests were spread +evenly across 16 modules and run with four workers and `--no-history`. Testenix completed the suite +in a median 8.038 seconds, pytest in 25.333 seconds, and pytest-xdist 3.8's default `load` scheduler +in 21.300 seconds. Every command had to report the expected test count or the harness rejected the +sample. The baseline contains one warm-up and five counterbalanced measured rounds, and Testenix's +samples ranged from 7.912 to 8.096 seconds. This is evidence for the tested workload and machine, not a universal claim about every Python -project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, different operating -systems, and real repositories still need independent measurements. +project. Import-heavy suites, fixture-heavy suites, slow tests, failure output, default history, +alternative pytest-xdist schedulers, different operating systems, and real repositories still need +independent measurements. No clean Testenix 0.2.1 scaling matrix is checked in yet, so `3.15×` must +not be presented as a 0.2.1 speedup. These results do not apply to `testenix pytest`. The compatibility command delegates to pytest and has pytest execution performance plus launcher and adapter overhead, which has not yet been measured separately. -The new safe-migration baseline is deliberately mixed rather than uniformly positive. For 3,000 +The pre-v0.2 safe-migration baseline is deliberately mixed rather than uniformly positive. For 3,000 no-op pytest tests across 64 modules, the generated native suite was 2.96x faster than sequential pytest. Empty unittest wrappers were 7.40x slower than the sequential stdlib-based outcome probe, while the same layout with 1 ms of work in each method was 1.58x faster under four native workers. These are @@ -2092,6 +2566,7 @@ parallel unittest runners, or a real repository. - deterministic rotated execution order instead of always running Testenix last; - test-count validation from each runner's final output; - disabled pytest plugin autoloading and cache, with pytest-xdist loaded explicitly; +- pytest-xdist's default `load` distribution rather than `loadfile`, `loadscope`, or `worksteal`; - generated-suite working directory, isolated from repository-level pytest configuration; - `--no-history` for the primary runner-overhead comparison. @@ -2100,7 +2575,7 @@ median throughput. All checked-in comparison files use one warm-up and five meas They also record the clean source commit, lockfile hash, timestamp, CPU model, and installed Testenix, pytest, and pytest-xdist versions. -## Results +## Historical Testenix 0.1.0 results | Scenario | pytest | pytest-xdist | Native Testenix | Native Testenix advantage | |---|---:|---:|---:|---:| @@ -2108,7 +2583,7 @@ Testenix, pytest, and pytest-xdist versions. | 10,000 uneven tests, 16 modules | 3.076 s | 2.138 s | 1.345 s | 2.29x vs pytest | | 100,000 no-op tests, 16 modules | 25.333 s | 21.300 s | 8.038 s | 3.15x vs pytest | -The 100,000-test median throughputs were 12,440 tests/s for Testenix, 3,947 tests/s for pytest, and +The 100,000-test median throughputs were 12,440 tests/s for Testenix 0.1.0, 3,947 tests/s for pytest, and 4,695 tests/s for pytest-xdist. The raw five-sample ranges and standard deviations are published in the generated benchmark page and the checked-in JSON files. @@ -2116,7 +2591,9 @@ In a separate exploratory 100,000-test profile, the coordinator's measured maxim approximately 513 MiB after the final manifest/event optimization. Sequential pytest measured approximately 520 MiB on the same generated suite. These older macOS `time` figures are not part of the current baseline JSON and are process maxima, not aggregate memory across every xdist/Testenix -child process. +child process. The console renderer changed substantially after these captures, and the historical +harness did not record output byte counts. Current schema-version 2 runs do record stdout/stderr +sizes, but a clean 0.2.1 matrix is still pending. ### Migrated-suite measurements @@ -2142,16 +2619,36 @@ The no-op unittest row exposes the adapter's fixed cost: each native test is a w the unchanged source and translates the result of `TestCase.run()`. The sequential source probe uses the stdlib loader and result semantics, then serializes per-test outcomes; it wins when the test body does essentially nothing. With 1 ms per method, four-worker execution across 64 -modules amortizes that cost and overtakes the sequential source runner. A suite concentrated in -one module would expose only one affinity unit, while too many workers can add process and import -overhead. Test duration, module distribution, imports, fixtures, I/O, failures, operating system, -and competing parallel runners must all be measured on the target project. +modules amortizes that cost and overtakes the sequential source runner. By default, a suite +concentrated in one module exposes only one affinity unit; opt-in safety-checked sharding may change +that for an independent module. Too many workers can still add process and import overhead. Test +duration, module distribution, imports, fixtures, I/O, failures, operating system, and competing +parallel runners must all be measured on the target project. The generated [benchmark results](https://polishdataengineer.github.io/testenix/benchmarks/results/) publish every sample, range, standard deviation, command, environment field, and raw JSON link. These three synthetic records are useful for finding overhead boundaries; they are not evidence that converted suites are universally faster. +### The 118-test validation was not a benchmark + +The [v0.2.0 release notes](https://github.com/polishdataengineer/testenix/releases/tag/v0.2.0) +reported one final validation observation from a real 118-test pytest project: 3.120 seconds for +pytest, 2.870 seconds for native serial execution, and 2.423 seconds for native parallel execution. +Those values correspond to roughly 1.09× and 1.29× for that observation, not 3.15×. They had no +committed raw rounds, warm-up series, counterbalanced order, or publishable environment manifest, +so their role was outcome parity (118/118 in all modes), not performance marketing. + +A small real suite can differ sharply from the 100,000-test no-op baseline. Interpreter spawn and +application import costs are a much larger fraction of its wall time; fixtures, mocks, files, +databases, and actual test bodies dominate framework overhead; and module affinity prevents one +large module from being split between workers. `workers=auto` is adaptive in current Testenix and +must be recorded as the worker count actually observed, not assumed to equal logical CPU count. +For a demonstrably independent large module, opt-in `--shard-modules` can create finer units after +static safety checks; it is not a safe default for arbitrary module state. Use `testenix tune` for +a local worker recommendation, then use the real-project manifest harness and publish five +counterbalanced rounds before drawing a project-specific conclusion. + ### Worker-count sensitivity An earlier exploratory run of 10,000 no-op tests across 16 modules produced these Testenix medians: @@ -2199,6 +2696,9 @@ The optimized profile performed approximately 3.46 million calls. The main chang 7. Unchanged selected specifications reuse discovered objects; selection events are omitted when every test is selected and no effective contract changed. 8. JSONL keeps one append descriptor for the run rather than performing open/close per event. +9. An explicit trusted collection manifest can remove the collection-side import from later + unchanged runs. Roots, the complete file inventory, and every source SHA-256 are verified first; + stale manifests fall back to supervised collection. Execution still imports assigned modules. Correctness was retained throughout: the framework's full resilience suite passes after every optimization, including worker crashes, timeout process-tree cleanup, collection crashes/hangs, @@ -2243,6 +2743,9 @@ Relevant upstream constraints are documented in the ## Next measurement gates +- publish the clean Testenix 0.2.1 dimension-sweep matrix for 100/500/1,000/3,000 tests, + balanced/dominant/single-module layouts, 1/2/4/adaptive-auto workers, and both history modes; +- compare pytest-xdist `load`, `loadfile`, `loadscope`, and `worksteal` where each strategy is valid; - collection, execution, IPC-byte, process-start, CPU, and aggregate-memory telemetry; - 1,000/10,000/100,000 tests across 1, 16, 1,000, and 10,000 modules; - synchronous and asynchronous fixtures, failures, captured output, timeouts, and retries; @@ -2251,12 +2754,14 @@ Relevant upstream constraints are documented in the - real migrated pytest and unittest suites, including sequential and established parallel source runners, multiple module layouts, and a break-even curve by median test duration; - checkpoint-batched IPC followed by another profile; -- persistent workers that collect and execute without importing every module twice. +- measure trusted-manifest hit and stale-fallback paths on import-heavy real projects, and consider + a persistent collect-and-execute worker only if the remaining execution import is still material. -No universal “always faster than pytest” statement should be published until the real-project and -cross-platform gates pass. The supported claim today is narrower: Testenix is materially faster in the -measured native large passing-suite scenarios while retaining supervised isolation and complete -results. +No universal “always faster than pytest” statement should be published until the current-version, +real-project, and cross-platform gates pass. The supported claim today is historical and narrower: +Testenix 0.1.0 was materially faster in the recorded native large passing-suite scenarios while +retaining supervised isolation and complete results. Testenix 0.2.1 has no checked-in speedup claim +yet. --- @@ -2322,11 +2827,11 @@ test outcomes. A retry never overwrites an earlier attempt, infrastructure failu from test failures, and setup/call/teardown are preserved as separate phases. ```text -Authoring API -> supervised collection -> inert manifest -> affinity scheduler -> process workers - | -> streamed attempts - +----------> append-only events - -> reducer - -> reports/history +Authoring API -> supervised collection -------> inert manifest -> scheduler -> process workers + ^ ^ | -> streamed attempts + | | +------------> append-only events +trusted manifest +-- roots/inventory/SHA-256 verify -> reducer + exact match bypasses collection imports -> reports/history ``` ## Dependency rules @@ -2335,6 +2840,9 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - `api`, `discovery`, `fixtures`, and `executor` form the native engine. - `events`, `aggregate`, and `scheduler` remain engine-independent. - `runner` is the application service connecting the native engine with execution policy. +- `tuning` models adaptive worker selection and runs explicit project-local candidate measurements. +- `sharding` contains fail-closed static module decisions and the versioned trusted-manifest + serialization/verification boundary. - reporters and storage consume completed domain results or versioned events. - optional compatibility adapters stay at the CLI boundary; the native core never imports pytest. - migration analyzers depend on serializable migration contracts, while shadow execution and @@ -2348,6 +2856,10 @@ Authoring API -> supervised collection -> inert manifest -> affinity scheduler - - fixture scopes: test, module, session, with broader scopes currently bounded by a worker shard; - sequential and local process execution; - deterministic scheduling based on historical durations; +- adaptive `workers = "auto"` capped by real execution units, history-informed predicted makespan, + process-start cost, and CPU capacity; +- explicit project-local worker tuning, conservative opt-in intra-module sharding, and an + explicitly generated source-verified collection manifest; - append-only JSONL events and a pure reducer; - console, JSON, and JUnit output plus local SQLite duration history; - retries represented as immutable attempts and finalized as `FLAKY` when appropriate; @@ -2363,10 +2875,16 @@ plugin SDK are deliberately outside version 0.2. ## Fixture scopes and process isolation -The scheduler treats every normal test module as one affinity unit and never splits that unit -between parallel shared workers. Multiple modules assigned to one shard execute in one persistent -process and fixture runtime. A test with an explicit timeout (including a global timeout applied at -selection) is instead a single-test isolation unit with a hard process deadline. +By default, the scheduler treats every normal test module as one affinity unit and never splits +that unit between parallel shared workers. Multiple modules assigned to one shard execute in one +persistent process and fixture runtime. A test with an explicit timeout (including a global timeout +applied at selection) is instead a single-test isolation unit with a hard process deadline. + +An explicit intra-module sharding policy can turn tests in an eligible module into finer units. +The static analysis fails closed for module/session fixtures, writes or obvious mutations of module +globals, and import-time lifecycle behavior. Function-scoped fixtures can be recreated per worker. +Because arbitrary dynamic calls and external effects cannot be proven safe, passing this policy is +a caller trust decision; ineligible modules keep normal affinity. Scope therefore has the following concrete meaning in version 0.2: @@ -2400,6 +2918,18 @@ top-level import crash or deadline becomes a `CollectionIssue`, so user import c indefinitely block the coordinator. Timed execution units send a ready handshake after rediscovery; the test deadline therefore does not include interpreter startup or module import. +A caller may instead supply a `TrustedCollectionManifest` created by an earlier explicit +collection. Before using it, Testenix enumerates the current roots and verifies the complete file +set and every SHA-256 digest. An exact match bypasses collection imports; a stale manifest falls +back to the supervised collector. Malformed serialized input is rejected at the adapter boundary. +Manifest parameter values are redacted at creation and serialization; only their names remain. +The manifest also carries prior sharding decisions so collection and scheduling agree. A module +using a fixture provider from outside its fingerprinted source fails closed to module affinity. +This removes +one module import per unchanged run, not the execution-worker import needed to reconstruct Python +objects. Inputs to dynamic collection beyond fingerprinted source bytes remain the producer's trust +responsibility. + Workers normally create a separate POSIX process session (or use recursive tree termination on Windows). During migration validation they remain in the validator's process group so an outer validation deadline can terminate native workers too. Timeout and cancellation terminate ordinary @@ -2443,6 +2973,10 @@ Source: docs/roadmap.md ## 0.3 — fast feedback +- Adaptive worker selection based on real execution units, duration history, and process cost, + plus an explicit project-local `tune`/`benchmark` command. +- Conservative opt-in intra-module sharding and a source-verified trusted collection manifest that + can bypass duplicate collection imports without trusting stale source metadata. - Dynamic micro-shards and work stealing. - `--last-failed`, watch mode, and failure fingerprints. - Test-impact analysis in shadow mode with an explanation for every selection decision. @@ -2454,7 +2988,7 @@ Source: docs/roadmap.md - Expand migration beyond the v0.2 static subset only when new transformations have differential semantics tests on real projects. - Versioned reporter and selector plugin interfaces. -- IDE protocol and machine-readable collection manifest. +- IDE protocol built on the versioned machine-readable collection manifest. ## Later @@ -2480,6 +3014,41 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- `testenix tune` and its `testenix benchmark` alias for fresh-process, counterbalanced, + history-disabled native worker-candidate measurements, native inventory/outcome validation, + JSON reports, bounded per-run process-tree deadlines, and explicit `--write` persistence of the + measured recommendation with project-source fingerprinting and optimistic byte-drift protection + immediately before an atomic configuration replacement. +- Explicit `--shard-modules` / `shard_modules = true` support for splitting eligible modules into + finer execution units. Conservative static checks retain module affinity for module/session + fixtures, visible global mutation, and import-time lifecycle hazards, including eager calls in + assignments, decorators, function defaults, and class construction expressions. +- Versioned trusted collection manifests generated with `testenix manifest ... --output FILE` and + consumed with `testenix run --manifest FILE` or `[tool.testenix].manifest`. Exact collection + roots, selected test files, statically discoverable project-local import dependencies, and SHA-256 + digests are verified before collection imports are bypassed; stale manifests fall back to + supervised collection, and parameter values are redacted. +- Synthetic scaling-matrix tooling for 100/500/1,000/3,000 tests and balanced, dominant, and + single-module layouts, plus a redaction-safe real-project benchmark harness. + +### Changed + +- `workers = "auto"` now selects adaptively from the actual execution-unit count, available CPUs, + duration-history coverage, predicted process-start cost, and makespan instead of equalling the + logical CPU count. Explicit integer worker settings remain unchanged. +- Benchmark documentation labels the historical `3.15×` result with its Testenix 0.1.0 version, + four-worker configuration, 100,000-test/16-module synthetic workload, and `--no-history` mode; + it is not presented as a current-version or real-project claim. + +### Fixed + +- Safe-module analysis now fails closed for imported fixture providers, nested mutable containers, + mutable class state, and all import-time calls including nested `sys.path` mutations. +- Benchmark and tuning timeouts use Windows Job Objects or POSIX root-session plus identity-tracked + descendant cleanup instead of allowing observed workers to contaminate later measurements. + ## [0.2.1] - 2026-07-21 ### Added @@ -2632,6 +3201,10 @@ Fields: Complete collection output, including non-fatal authoring issues. +## `testenix.CollectionManifestError` + +A trusted collection manifest is malformed or unsafe to resolve. + ## `testenix.Event` ```text @@ -2722,7 +3295,7 @@ Terminal state of one migration transaction. ## `testenix.TestenixConfig` ```text -TestenixConfig(paths: 'tuple[str, ...]' = ('tests',), workers: "int | Literal['auto']" = 'auto', retries: 'int' = 0, timeout: 'float | None' = None, tags: 'tuple[str, ...]' = (), json_path: 'Path | None' = None, junit_path: 'Path | None' = None, history_path: 'Path | None' = PosixPath('.testenix/history.sqlite3')) -> None +TestenixConfig(paths: 'tuple[str, ...]' = ('tests',), workers: "int | Literal['auto']" = 'auto', retries: 'int' = 0, timeout: 'float | None' = None, tags: 'tuple[str, ...]' = (), json_path: 'Path | None' = None, junit_path: 'Path | None' = None, history_path: 'Path | None' = PosixPath('.testenix/history.sqlite3'), shard_modules: 'bool' = False, manifest_path: 'Path | None' = None) -> None ``` Fields: @@ -2735,6 +3308,8 @@ Fields: - `json_path`: `Path | None` - `junit_path`: `Path | None` - `history_path`: `Path | None` +- `shard_modules`: `bool` +- `manifest_path`: `Path | None` Validated execution and reporting settings. @@ -2745,7 +3320,7 @@ a worker process without retaining hidden project state. ## `testenix.RunResult` ```text -RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float') -> None +RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float', workers_used: 'int | None' = None, shardable_paths: 'tuple[str, ...]' = ()) -> None ``` Fields: @@ -2755,8 +3330,26 @@ Fields: - `collection_issues`: `tuple[CollectionIssue, ...]` - `started_at`: `float` - `finished_at`: `float` +- `workers_used`: `int | None` +- `shardable_paths`: `tuple[str, ...]` + +RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float', workers_used: 'int | None' = None, shardable_paths: 'tuple[str, ...]' = ()) + +## `testenix.ShardingPolicy` + +```text +ShardingPolicy(intra_module: 'bool' = False) -> None +``` + +Fields: -RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tuple[CollectionIssue, ...]', started_at: 'float', finished_at: 'float') +- `intra_module`: `bool` + +Core scheduling policy independent of CLI/configuration concerns. + +Intra-module sharding is deliberately opt-in. Callers which do not pass a +policy, or pass the default instance, retain the original module-affinity +behaviour exactly. ## `testenix.Scope` @@ -2816,6 +3409,32 @@ Fields: Serializable description of one concrete test case. +## `testenix.TrustedCollectionManifest` + +```text +TrustedCollectionManifest(collection_roots: 'tuple[str, ...]', files: 'tuple[SourceFingerprint, ...]', tests: 'tuple[TestSpec, ...]', issues: 'tuple[CollectionIssue, ...]' = (), sharding: 'tuple[ModuleShardingDecision, ...]' = ()) -> None +``` + +Fields: + +- `collection_roots`: `tuple[str, ...]` +- `files`: `tuple[SourceFingerprint, ...]` +- `tests`: `tuple[TestSpec, ...]` +- `issues`: `tuple[CollectionIssue, ...]` +- `sharding`: `tuple[ModuleShardingDecision, ...]` + +Explicit portable collection result that may bypass collection imports. + +This is deliberately not an implicit cache. A caller creates or loads the +manifest and opts into trusting it for a run. Parameter names are retained +for diagnostics, but every parameter value is replaced with the explicit +```` sentinel. Testenix still compares the complete test-file +inventory plus local import dependencies and every SHA-256 digest before +using it. +Dynamic collection influenced by anything other than source bytes (for +example environment variables) remains the manifest producer's trust +decision. + ## `testenix.ValidationSummary` ```text @@ -2866,6 +3485,22 @@ Attach several cases, either explicitly or as a Cartesian product. ``@cases(role=["admin", "editor"], active=[True, False])`` creates the Cartesian product of the supplied dimensions. +## `testenix.collect_trusted_manifest` + +```text +collect_trusted_manifest(paths: 'Sequence[str] | str', *, project_root: 'str | Path | None' = None) -> 'TrustedCollectionManifest' +``` + +Create an explicit collection manifest in a supervised worker process. + +## `testenix.deserialize_trusted_collection_manifest` + +```text +deserialize_trusted_collection_manifest(data: 'str | bytes | bytearray | Mapping[str, Any]') -> 'TrustedCollectionManifest' +``` + +Decode and validate trusted collection manifest JSON or mapping data. + ## `testenix.discover` ```text @@ -2902,7 +3537,7 @@ checked for concurrent changes. ## `testenix.run` ```text -run(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None) -> 'RunResult' +run(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None, sharding_policy: 'ShardingPolicy | None' = None, trusted_manifest: 'TrustedCollectionManifest | None' = None) -> 'RunResult' ``` Discover and execute a native Testenix suite. @@ -2914,11 +3549,19 @@ reduced after execution. ## `testenix.run_async` ```text -run_async(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None) -> 'RunResult' +run_async(paths: 'Sequence[str] | str | None' = None, config: 'TestenixConfig | None' = None, *, event_sink: 'EventSink | None' = None, sharding_policy: 'ShardingPolicy | None' = None, trusted_manifest: 'TrustedCollectionManifest | None' = None) -> 'RunResult' ``` Cancellable embedding facade around the process-oriented coordinator. +## `testenix.serialize_trusted_collection_manifest` + +```text +serialize_trusted_collection_manifest(manifest: 'TrustedCollectionManifest') -> 'str' +``` + +Serialize a trusted collection manifest as deterministic JSON. + ## `testenix.skip` ```text diff --git a/scripts/generate_docs_assets.py b/scripts/generate_docs_assets.py index aa507c8..644eee6 100644 --- a/scripts/generate_docs_assets.py +++ b/scripts/generate_docs_assets.py @@ -10,6 +10,7 @@ import posixpath import re import sys +import tomllib from dataclasses import dataclass, fields, is_dataclass from enum import Enum from html import escape @@ -33,6 +34,8 @@ ROOT / "benchmarks" / "migration_baseline_unittest_3000_delay_1ms.json", ) +SCALING_MATRIX = ROOT / "benchmarks" / "scaling_matrix_0_2_1.json" + LLM_DOCUMENTS = ( ("Overview", Path("docs/index.md"), ""), ("Getting started", Path("docs/getting-started.md"), "getting-started/"), @@ -131,6 +134,111 @@ def workload(self) -> str: return "no-op" if self.delay_ms == 0 else f"{self.delay_ms:g} ms body" +def _project_version() -> str: + with (ROOT / "pyproject.toml").open("rb") as source: + return str(tomllib.load(source)["project"]["version"]) + + +def _historical_version(benchmarks: tuple[Benchmark, ...]) -> str: + versions = { + str(benchmark.provenance.get("versions", {}).get("testenix", "unknown")) + for benchmark in benchmarks + } + if len(versions) != 1: + raise ValueError("published historical baselines do not share one Testenix version") + return versions.pop() + + +def _load_scaling_matrix(path: Path, *, expected_version: str) -> dict[str, Any] | None: + if not path.exists(): + return None + data = json.loads(path.read_text(encoding="utf-8")) + provenance = data.get("provenance", {}) + design = data.get("design", {}) + scenarios = data.get("scenarios") + if ( + data.get("schema_version") != 1 + or data.get("kind") != "testenix.scaling-matrix" + or provenance.get("dirty") is not False + or provenance.get("testenix_version") != expected_version + or provenance.get("pyproject_version") != expected_version + or not provenance.get("commit") + or not isinstance(scenarios, list) + or not scenarios + or int(design.get("repeats", 0)) < 5 + or int(design.get("warmups", 0)) < 1 + ): + raise ValueError(f"{path}: current-version scaling publication gates did not pass") + + counts: set[int] = set() + workers: set[str] = set() + layouts: set[str] = set() + histories: set[str] = set() + sharding_modes: set[str] = set() + for entry in scenarios: + if not isinstance(entry, dict) or not isinstance(entry.get("id"), str): + raise ValueError(f"{path}: invalid scaling scenario entry") + result = entry.get("result") + if not isinstance(result, dict) or result.get("schema_version") != 2: + raise ValueError(f"{path}: invalid scaling scenario result") + scenario = result.get("scenario", {}) + measurements = result.get("measurements", {}) + result_provenance = result.get("provenance", {}) + if ( + result_provenance.get("dirty") is not False + or result_provenance.get("commit") != provenance["commit"] + or result_provenance.get("versions", {}).get("testenix") != expected_version + or int(scenario.get("repeats", 0)) != int(design["repeats"]) + or int(scenario.get("warmups", -1)) != int(design["warmups"]) + or scenario.get("xdist_strategy") != design.get("xdist_strategy") + ): + raise ValueError(f"{path}: scenario provenance/design mismatch in {entry['id']}") + counts.add(int(scenario["test_count"])) + workers.add(str(scenario["workers_requested"])) + layouts.add(str(scenario["module_layout"])) + histories.add(str(scenario["history_mode"])) + sharding_modes.add(str(scenario.get("sharding_mode", "disabled"))) + for runner in ("pytest", "pytest_xdist", "testenix"): + measurement = measurements.get(runner, {}) + samples = measurement.get("samples") + stdout_bytes = measurement.get("stdout_bytes") + stderr_bytes = measurement.get("stderr_bytes") + if ( + not isinstance(samples, list) + or len(samples) != int(scenario["repeats"]) + or any(float(sample) <= 0 for sample in samples) + or not isinstance(stdout_bytes, list) + or len(stdout_bytes) != len(samples) + or not isinstance(stderr_bytes, list) + or len(stderr_bytes) != len(samples) + ): + raise ValueError(f"{path}: invalid {runner} samples in {entry['id']}") + if scenario["workers_requested"] == "auto": + observed_workers = measurements["testenix"].get("observed_workers") + if ( + not isinstance(observed_workers, list) + or len(observed_workers) != int(scenario["repeats"]) + or any( + isinstance(worker, bool) or not isinstance(worker, int) or worker < 1 + for worker in observed_workers + ) + ): + raise ValueError( + f"{path}: auto scenario {entry['id']} has no valid observed worker counts" + ) + if not {100, 500, 1_000, 3_000}.issubset(counts): + raise ValueError(f"{path}: scaling counts are incomplete") + if not {"1", "2", "4", "auto"}.issubset(workers): + raise ValueError(f"{path}: worker coverage is incomplete") + if not {"balanced", "dominant", "single"}.issubset(layouts): + raise ValueError(f"{path}: module-layout coverage is incomplete") + if not {"disabled", "default"}.issubset(histories): + raise ValueError(f"{path}: history coverage is incomplete") + if not {"disabled", "safe"}.issubset(sharding_modes): + raise ValueError(f"{path}: safe-sharding coverage is incomplete") + return data + + def _load_benchmark(path: Path) -> Benchmark: data = json.loads(path.read_text(encoding="utf-8")) scenario = data["scenario"] @@ -315,7 +423,9 @@ def _render_migration_results(benchmarks: tuple[MigrationBenchmark, ...]) -> str These separate measurements start with generated pytest or unittest sources, complete one safe copy-and-validate migration, and then compare recurring source-suite runs with recurring native Testenix runs. The migration transaction is a one-time cost shown separately; it is not included -in either execution median. +in either execution median. These records came from the pre-v0.2 source commit linked below; its +distribution metadata still reported `0.1.0`. They are historical evidence, not measurements of +the current release. {table_header} | --- | --- | ---: | ---: | ---: | ---: | ---: | @@ -336,10 +446,129 @@ def _render_migration_results(benchmarks: tuple[MigrationBenchmark, ...]) -> str """ +def _render_current_matrix(matrix: dict[str, Any] | None, *, current_version: str) -> str: + if matrix is None: + matrix_section = f"""## Testenix {current_version} scaling matrix + +No current-version matrix is checked in yet. The historical results below must therefore not be +described as Testenix {current_version} performance. The new provenance-gated harness covers +100/500/1,000/3,000 tests, balanced/dominant/single-module layouts, 1/2/4/auto workers, and both +default history and `--no-history`, plus explicit safe-module sharding. Its default design uses +dimension sweeps; use +`--full-cross-product` only when the much larger run is intentional. + +`auto` is passed literally to Testenix and remains adaptive; observed Testenix worker counts are +stored per sample. pytest-xdist resolves its side of an `auto` row separately to the machine's +logical CPU count. + +```console +$ uv run --no-editable python benchmarks/run_scaling_matrix.py \\ + --output benchmarks/scaling_matrix_0_2_1.json +``` + +The command refuses a dirty worktree or an installed Testenix version that differs from +`pyproject.toml`. `--allow-dirty` is available only for unpublished smoke runs. A matrix becomes +publishable here only after five measured rounds, one warm-up, clean commit provenance, and full +axis coverage pass the documentation generator's validation. +""" + else: + rows: list[str] = [] + for entry in matrix["scenarios"]: + result = entry["result"] + scenario = result["scenario"] + measurements = result["measurements"] + native = float(measurements["testenix"]["median"]) + pytest = float(measurements["pytest"]["median"]) + xdist = float(measurements["pytest_xdist"]["median"]) + history = "default" if scenario["history_mode"] == "default" else "disabled" + sharding = str(scenario.get("sharding_mode", "disabled")) + workers = str(scenario["workers_requested"]) + if workers == "auto": + observed = sorted(set(measurements["testenix"]["observed_workers"])) + workers = f"auto ({'/'.join(str(worker) for worker in observed)} observed)" + rows.append( + "| " + + " | ".join( + ( + str(entry["id"]), + f"{int(scenario['test_count']):,}", + f"{int(scenario['test_modules']):,}", + str(scenario["module_layout"]), + workers, + history, + sharding, + _seconds(native), + _seconds(pytest), + _seconds(xdist), + f"{pytest / native:.2f}×", + ) + ) + + " |" + ) + commit = str(matrix["provenance"]["commit"]) + matrix_section = f"""## Testenix {current_version} scaling matrix + +This current-version matrix passed the clean-worktree, version, sample-count, and axis-coverage +publication gates. Ratios still apply only to the recorded environment and exact row. + +| Scenario | Tests | Mods | Layout | Workers | History | Shard | Native | pytest | xdist | ratio | +| --- | ---: | ---: | --- | ---: | --- | --- | ---: | ---: | ---: | ---: | +{chr(10).join(rows)} + +- Measured rounds: {matrix["design"]["repeats"]}; warmups: {matrix["design"]["warmups"]} +- pytest-xdist strategy: `{matrix["design"]["xdist_strategy"]}` +- Clean source commit: [`{commit}`]({REPOSITORY_URL}/commit/{commit}) +- [Raw JSON]({_raw_link_path(SCALING_MATRIX)}) +""" + + return ( + matrix_section + + """ + +## Real-project harness + +The 118-test project used during v0.2 migration validation was a semantic parity gate, not a +publishable benchmark: its release-note timings were single observations without a committed +multi-round record. Use the redaction-safe manifest harness for a real repository: + +```console +$ cp benchmarks/real_project_manifest.example.json /tmp/testenix-project-benchmark.json +$ uv run --no-editable python benchmarks/run_project_benchmark.py \\ + --project /absolute/path/to/project \\ + --manifest /tmp/testenix-project-benchmark.json \\ + --output /tmp/testenix-project-result.json +``` + +The manifest stores argument arrays, never shell fragments. The result omits stdout, stderr, +environment values, absolute project paths, and private source. It records only timings, aggregate +output sizes, optional tree fingerprints, and redacted Git provenance. A migrated-suite comparison +must point the manifest at a successful migration report to become publication-eligible. The +harness verifies the report's exact per-test inventory and outcomes, complete source and generated +Python-file inventories, current hashes, and binds canonical `python -m pytest` / +`python -m testenix run` commands to the report's source/output roots. Publishable source roots are +directories so support files such as `conftest.py` are covered. Without the report the result is +diagnostic-only. Commands are retained for +reproducibility. Publishable commands put options before `--` and exact suite targets after it, so +an option value cannot impersonate a migration root. Keep secrets in the environment or list +sensitive argument indexes in `redact_arguments`. +""" + ) + + +def _raw_link_path(path: Path) -> str: + relative = path.relative_to(ROOT).as_posix() + return f"{REPOSITORY_URL}/blob/main/{relative}" + + def _render_benchmark_results( benchmarks: tuple[Benchmark, ...], migration_benchmarks: tuple[MigrationBenchmark, ...], + scaling_matrix: dict[str, Any] | None, + *, + current_version: str, ) -> str: + historical_version = _historical_version(benchmarks) + xdist_version = benchmarks[0].provenance.get("versions", {}).get("pytest_xdist", "unknown") rows = [] detail_sections = [] for benchmark in benchmarks: @@ -389,6 +618,8 @@ def _render_benchmark_results( {chr(10).join(runner_details)} - Measured rounds: {benchmark.repeats}; warmups: {benchmark.warmups} - Workers: {benchmark.workers} +- Testenix history: disabled with `--no-history` +- pytest-xdist strategy: default `load` {chr(10).join(provenance_details)} - [Raw JSON]({_raw_link(benchmark)}) """ @@ -428,9 +659,18 @@ def _render_benchmark_results( than pytest. `Testenix` in these results means the native `testenix run` engine. The `testenix pytest` compatibility bridge delegates to pytest and is not represented here. -![Preliminary Testenix throughput ratios](../_static/benchmark-speedup.svg) +{_render_current_matrix(scaling_matrix, current_version=current_version)} + +## Historical Testenix {historical_version} synthetic baseline + +The checked-in `3.15×` figure is a Testenix {historical_version} result for 100,000 generated no-op +tests across 16 modules, four workers, disabled history (`--no-history`), and pytest-xdist's default +`load` strategy. It is retained as transparent historical evidence; it is not a measurement of +Testenix {current_version}. + +![Historical Testenix {historical_version} throughput ratios](../_static/benchmark-speedup.svg) -## Median wall-clock time +### Median wall-clock time Lower time is better. A speedup of `{benchmarks[0].speedup_vs_pytest:.2f}×` means pytest's median wall time was {benchmarks[0].speedup_vs_pytest:.2f} times the Testenix median for that exact @@ -446,19 +686,24 @@ def _render_benchmark_results( {publication_note} -## Environment +### Environment and controls - CPU: {cpu_model} ({environment["cpu_count"]} logical CPUs) - Machine: `{environment["machine"]}` - Platform: `{environment["platform"]}` - Python: `{environment["python"]}` +- Testenix: `{historical_version}` +- Workers: four for Testenix and pytest-xdist +- Testenix history: disabled with `--no-history` +- pytest-xdist: version `{xdist_version}`, + default `load` distribution - Measurement: complete subprocess wall-clock time, including discovery, execution, aggregation, and console rendering - Correctness gate: every command had to exit successfully and report the expected test count {legacy_note} -## Raw samples and variance +### Raw samples and variance """ + "\n".join(detail_sections) @@ -467,9 +712,9 @@ def _render_benchmark_results( + """ ## Interpretation -The checked-in results show that Testenix has low per-test overhead for large generated suites and -that its built-in process model is competitive with both sequential pytest and pytest-xdist in -those scenarios. +The historical checked-in results show that Testenix 0.1.0 had low per-test overhead for the large +generated suites above and was competitive with sequential pytest and pytest-xdist's default +`load` strategy in those scenarios. They are not evidence for the current release. They do **not** yet answer how Testenix performs for import-heavy applications, complex fixture graphs, assertion failures, real repositories, or different operating systems. Pytest also has a @@ -503,6 +748,7 @@ def _render_benchmark_results( def _render_benchmark_svg(benchmarks: tuple[Benchmark, ...]) -> str: + historical_version = _historical_version(benchmarks) width = 980 height = 130 + len(benchmarks) * 112 plot_x = 285 @@ -514,9 +760,11 @@ def _render_benchmark_svg(benchmarks: tuple[Benchmark, ...]) -> str: f'', - 'Preliminary Testenix benchmark throughput ratios', - 'Horizontal bars compare Testenix throughput with pytest and ' - "pytest-xdist for three checked-in synthetic benchmark scenarios.", + f'Historical Testenix {historical_version} ' + "benchmark throughput ratios", + 'Horizontal bars compare historical Testenix ' + f"{historical_version} throughput with pytest and pytest-xdist for three checked-in " + "synthetic benchmark scenarios using four workers and disabled Testenix history.", "", '', - 'Synthetic benchmark throughput ratio', - 'Higher is better · 1× means equal throughput', + f'Historical Testenix {historical_version} ' + "synthetic ratios", + '4 workers · --no-history · higher is better', ] for tick in range(axis_max + 1): x = plot_x + tick * scale @@ -566,7 +815,7 @@ def _render_benchmark_svg(benchmarks: tuple[Benchmark, ...]) -> str: python_version = escape(str(environment["python"])) elements.append( f'' - f"Development baseline · {cpu_model} · CPython {python_version} · raw samples linked below" + f"Historical baseline · {cpu_model} · CPython {python_version} · raw samples linked below" "" ) elements.append("") @@ -753,11 +1002,18 @@ def _render_llms_full(generated: dict[Path, str]) -> str: def _outputs() -> dict[Path, str]: + current_version = _project_version() benchmarks = tuple(_load_benchmark(path) for path in BASELINES) migration_benchmarks = tuple(_load_migration_benchmark(path) for path in MIGRATION_BASELINES) + scaling_matrix = _load_scaling_matrix(SCALING_MATRIX, expected_version=current_version) results_path = Path("docs/benchmarks/results.md") generated = { - results_path: _render_benchmark_results(benchmarks, migration_benchmarks), + results_path: _render_benchmark_results( + benchmarks, + migration_benchmarks, + scaling_matrix, + current_version=current_version, + ), Path("docs/_static/benchmark-speedup.svg"): _render_benchmark_svg(benchmarks), } llms_index = _render_llms_index() diff --git a/src/testenix/__init__.py b/src/testenix/__init__.py index 33ff9d0..d6af89f 100644 --- a/src/testenix/__init__.py +++ b/src/testenix/__init__.py @@ -14,11 +14,19 @@ ValidationSummary, migrate, ) -from testenix.runner import run, run_async +from testenix.runner import collect_trusted_manifest, run, run_async +from testenix.sharding import ( + CollectionManifestError, + ShardingPolicy, + TrustedCollectionManifest, + deserialize_trusted_collection_manifest, + serialize_trusted_collection_manifest, +) __all__ = [ "CaseDefinition", "CollectionResult", + "CollectionManifestError", "Event", "EventSink", "MigrationOptions", @@ -26,18 +34,23 @@ "MigrationStatus", "TestenixConfig", "RunResult", + "ShardingPolicy", "Scope", "Status", "TestResult", "TestSpec", + "TrustedCollectionManifest", "ValidationSummary", "case", "cases", + "collect_trusted_manifest", + "deserialize_trusted_collection_manifest", "discover", "fixture", "migrate", "run", "run_async", + "serialize_trusted_collection_manifest", "skip", "test", "xfail", diff --git a/src/testenix/aggregate.py b/src/testenix/aggregate.py index 2cac44f..baea9b6 100644 --- a/src/testenix/aggregate.py +++ b/src/testenix/aggregate.py @@ -410,6 +410,8 @@ def reduce_events( selection_observed = False started_at: float | None = None finished_at: float | None = None + workers_used: int | None = None + shardable_paths: tuple[str, ...] = () def remember_spec( spec: TestSpec | None, event: Event, *, replace_existing: bool = False @@ -434,6 +436,16 @@ def builder_for(test_id: str, attempt: int, event: Event) -> _AttemptBuilder: if event.event_type is EventType.RUN_FINISHED: candidate = _finite_number(payload.get("finished_at"), event.timestamp) finished_at = candidate if finished_at is None else max(finished_at, candidate) + raw_workers = payload.get("workers_used") + if ( + isinstance(raw_workers, int) + and not isinstance(raw_workers, bool) + and raw_workers >= 0 + ): + workers_used = raw_workers + raw_shardable = payload.get("shardable_paths", ()) + if isinstance(raw_shardable, Iterable) and not isinstance(raw_shardable, (str, bytes)): + shardable_paths = tuple(sorted(str(path) for path in raw_shardable)) continue if event.event_type is EventType.COLLECTION_ERROR: collection_issues.append(_collection_issue(payload)) @@ -588,6 +600,8 @@ def test_order(test_id: str) -> tuple[int, int, str, int, str]: collection_issues=tuple(collection_issues), started_at=started_at, finished_at=finished_at, + workers_used=workers_used, + shardable_paths=shardable_paths, ) diff --git a/src/testenix/cli.py b/src/testenix/cli.py index 5e72b1e..20e160f 100644 --- a/src/testenix/cli.py +++ b/src/testenix/cli.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from testenix.migration_service import MigrationOptions, MigrationReport + from testenix.sharding import TrustedCollectionManifest EXIT_OK = 0 EXIT_TEST_FAILURE = 1 @@ -40,7 +41,7 @@ def build_parser() -> argparse.ArgumentParser: "--config", dest="global_config", type=Path, - help="pyproject.toml containing [tool.testenix] for the native run command", + help="pyproject.toml containing [tool.testenix] for native run/tune commands", ) subparsers = parser.add_subparsers(dest="command", required=True) @@ -112,6 +113,27 @@ def build_parser() -> argparse.ArgumentParser: history_group = run_parser.add_mutually_exclusive_group() history_group.add_argument("--history", dest="history_path", type=Path, default=None) history_group.add_argument("--no-history", action="store_true") + sharding_group = run_parser.add_mutually_exclusive_group() + sharding_group.add_argument( + "--shard-modules", + dest="shard_modules", + action="store_true", + default=None, + help="opt in to splitting statically safe modules across workers", + ) + sharding_group.add_argument( + "--no-shard-modules", + dest="shard_modules", + action="store_false", + help="disable intra-module sharding configured in pyproject.toml", + ) + run_parser.add_argument( + "--manifest", + dest="manifest_path", + type=Path, + default=None, + help="trusted collection manifest; stale manifests fall back to isolated collection", + ) run_parser.set_defaults(handler=_run_command) # Pytest owns the complete argument grammar for this compatibility command. @@ -179,6 +201,104 @@ def build_parser() -> argparse.ArgumentParser: ), ) migrate_parser.set_defaults(handler=_migrate_command) + + tune_parser = subparsers.add_parser( + "tune", + aliases=("benchmark",), + help="benchmark native worker counts and recommend a project-local setting", + ) + tune_parser.add_argument( + "paths", + nargs="*", + help="native test files or directories (default: [tool.testenix].paths)", + ) + tune_parser.add_argument( + "--config", + dest="tune_config", + type=Path, + help="pyproject.toml containing [tool.testenix]", + ) + tune_parser.add_argument( + "--candidates", + type=_worker_candidates, + metavar="N[,N...]", + help="explicit worker counts to measure (default: adaptive powers-of-two sweep)", + ) + tune_parser.add_argument( + "--warmups", + type=_non_negative_int, + default=1, + metavar="N", + help="warmup runs per native candidate and pytest (default: 1)", + ) + tune_parser.add_argument( + "--repeats", + type=_positive_int, + default=5, + metavar="N", + help="measured runs per candidate (default: 5)", + ) + tune_parser.add_argument( + "--run-timeout", + type=_positive_float, + default=300.0, + metavar="SECONDS", + help="deadline for each complete suite run (default: 300)", + ) + tune_parser.add_argument( + "--pytest-source", + dest="pytest_paths", + action="append", + default=[], + metavar="PATH", + help="also time pytest on this source path; repeat for multiple paths", + ) + tune_parser.add_argument( + "--json", + dest="tuning_json", + metavar="FILE|-", + help="write the tuning report as JSON, or '-' for stdout", + ) + tune_parser.add_argument( + "--write", + action="store_true", + help="persist the measured recommendation to [tool.testenix].workers", + ) + tune_sharding_group = tune_parser.add_mutually_exclusive_group() + tune_sharding_group.add_argument( + "--shard-modules", + dest="tune_shard_modules", + action="store_true", + default=None, + help="benchmark opt-in safe intra-module sharding", + ) + tune_sharding_group.add_argument( + "--no-shard-modules", + dest="tune_shard_modules", + action="store_false", + help="benchmark with module affinity even if sharding is configured", + ) + tune_parser.add_argument( + "--manifest", + dest="tune_manifest_path", + type=Path, + default=None, + help="trusted collection manifest used by every native sample", + ) + tune_parser.set_defaults(handler=_tune_command) + + manifest_parser = subparsers.add_parser( + "manifest", + help="create an explicit trusted collection manifest in an isolated worker", + ) + manifest_parser.add_argument("paths", nargs="+", help="native test files or directories") + manifest_parser.add_argument( + "--output", + required=True, + metavar="FILE|-", + help="write a new manifest file, or '-' for stdout; never overwrite", + ) + manifest_parser.set_defaults(handler=_manifest_command) return parser @@ -203,7 +323,15 @@ def _run_command(arguments: argparse.Namespace) -> int: config_path = arguments.run_config or arguments.global_config config = load_config(config_path) overrides: dict[str, Any] = {} - for name in ("workers", "retries", "timeout", "json_path", "junit_path"): + for name in ( + "workers", + "retries", + "timeout", + "json_path", + "junit_path", + "manifest_path", + "shard_modules", + ): value = getattr(arguments, name) if value is not None: overrides[name] = value @@ -216,8 +344,22 @@ def _run_command(arguments: argparse.Namespace) -> int: config = config.with_overrides(**overrides) paths = tuple(arguments.paths) if arguments.paths else config.paths + trusted_manifest = _load_trusted_manifest(config.manifest_path) + if trusted_manifest is not None: + from testenix.sharding import verify_trusted_collection_manifest + + if not verify_trusted_collection_manifest(trusted_manifest, paths): + print( + "testenix: collection manifest is stale or does not match these paths; " + "falling back to isolated collection", + file=sys.stderr, + ) try: - result = _call_runner(paths, config) + result = ( + _call_runner(paths, config) + if trusted_manifest is None + else _call_runner(paths, config, trusted_manifest=trusted_manifest) + ) except KeyboardInterrupt: raise except Exception as error: # the CLI is the final application boundary @@ -248,17 +390,45 @@ def _run_command(arguments: argparse.Namespace) -> int: return result.exit_code -def _call_runner(paths: Sequence[str], config: TestenixConfig) -> RunResult: +def _call_runner( + paths: Sequence[str], + config: TestenixConfig, + *, + trusted_manifest: TrustedCollectionManifest | None = None, +) -> RunResult: # Importing here keeps `testenix --help` usable even if an optional execution # backend cannot be imported in the current environment. from testenix.runner import run + from testenix.sharding import ShardingPolicy + + return run( + paths, + config, + sharding_policy=ShardingPolicy(intra_module=config.shard_modules), + trusted_manifest=trusted_manifest, + ) - return run(paths, config) + +def _load_trusted_manifest(path: Path | None) -> TrustedCollectionManifest | None: + if path is None: + return None + from testenix.sharding import ( + CollectionManifestError, + deserialize_trusted_collection_manifest, + ) + + try: + data = path.read_bytes() + return deserialize_trusted_collection_manifest(data) + except (OSError, CollectionManifestError) as error: + raise ConfigError(f"cannot read collection manifest {path}: {error}") from error def _reporter_worker_count(result: RunResult, config: TestenixConfig) -> int: """Mirror the native runner's initial module/timeout execution units.""" + if result.workers_used is not None: + return result.workers_used shared_modules = {test.test.path for test in result.tests if test.test.timeout is None} isolated_tests = sum(test.test.timeout is not None for test in result.tests) return min(config.resolved_workers, len(shared_modules) + isolated_tests) @@ -382,6 +552,215 @@ def _call_migrator(options: MigrationOptions) -> MigrationReport: return migrate(options) +def _manifest_command(arguments: argparse.Namespace) -> int: + from testenix.runner import collect_trusted_manifest + from testenix.sharding import CollectionManifestError, serialize_trusted_collection_manifest + + output = None if arguments.output == "-" else Path(arguments.output).expanduser() + if output is not None: + try: + _validate_new_output(output, label="collection manifest") + except ConfigError as error: + print(f"testenix: cannot write collection manifest: {error}", file=sys.stderr) + return EXIT_USAGE + try: + manifest = collect_trusted_manifest(tuple(arguments.paths)) + encoded = serialize_trusted_collection_manifest(manifest) + "\n" + if output is None: + print(encoded, end="") + else: + _write_new_text(output, encoded) + except KeyboardInterrupt: + raise + except (CollectionManifestError, OSError, ValueError) as error: + print(f"testenix: manifest error: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + except Exception as error: # the CLI is the final application boundary + print(f"testenix: manifest error: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + if output is not None: + print(f"Wrote trusted collection manifest to {output}") + return EXIT_OK + + +def _tune_command(arguments: argparse.Namespace) -> int: + from testenix.config import write_worker_recommendation + from testenix.tuning import TuningError, render_tuning_report, run_tuning + + config_path = arguments.tune_config or arguments.global_config + loaded_config = load_config(config_path) + tune_overrides: dict[str, Any] = {} + if arguments.tune_shard_modules is not None: + tune_overrides["shard_modules"] = arguments.tune_shard_modules + if arguments.tune_manifest_path is not None: + tune_overrides["manifest_path"] = arguments.tune_manifest_path + transient_profile = _different_tune_profile_overrides(loaded_config, tune_overrides) + if arguments.write and transient_profile: + rendered = ", ".join(transient_profile) + print( + "testenix: tuning error: --write refuses a workers-only recommendation " + f"measured with transient execution-profile override(s): {rendered}; " + "add the profile to [tool.testenix] or rerun without --write", + file=sys.stderr, + ) + return EXIT_USAGE + config = loaded_config.with_overrides(**tune_overrides) + paths = tuple(arguments.paths) if arguments.paths else config.paths + destination = config_path or Path("pyproject.toml") + configuration_snapshot: bytes | None = None + if arguments.write: + try: + configuration_snapshot = _read_optional_file(destination) + except ConfigError as error: + print(f"testenix: tuning error: {error}", file=sys.stderr) + return EXIT_USAGE + if arguments.write and arguments.paths and not _same_paths(paths, config.paths): + print( + "testenix: tuning error: --write refuses to persist a recommendation for " + "paths different from [tool.testenix].paths", + file=sys.stderr, + ) + return EXIT_USAGE + + report_output = ( + None + if not arguments.tuning_json or arguments.tuning_json == "-" + else Path(arguments.tuning_json).expanduser() + ) + if report_output is not None: + try: + _validate_new_output(report_output, label="tuning report") + if arguments.write and _same_file_target(report_output, destination): + raise ConfigError("tuning report must not target the active configuration file") + except ConfigError as error: + print(f"testenix: cannot write tuning report: {error}", file=sys.stderr) + return EXIT_USAGE + + trusted_manifest = _load_trusted_manifest(config.manifest_path) + if trusted_manifest is not None: + from testenix.sharding import verify_trusted_collection_manifest + + if not verify_trusted_collection_manifest(trusted_manifest, paths): + print( + "testenix: tuning error: manifest is stale or does not match the tuned paths; " + "regenerate it before measuring", + file=sys.stderr, + ) + return EXIT_USAGE + + # The automatic sweep is bounded by the four-worker cold-start ceiling. + candidate_limit = len(arguments.candidates) if arguments.candidates else 4 + native_runs = 1 + candidate_limit * (arguments.warmups + arguments.repeats) + pytest_runs = arguments.warmups + arguments.repeats if arguments.pytest_paths else 0 + print( + f"Testenix tune: up to {native_runs + pytest_runs} complete suite runs " + f"({candidate_limit} native candidates).", + file=sys.stderr, + ) + try: + report = run_tuning( + paths, + config, + candidates=arguments.candidates, + warmups=arguments.warmups, + repeats=arguments.repeats, + run_timeout=arguments.run_timeout, + pytest_paths=tuple(arguments.pytest_paths), + ) + except KeyboardInterrupt: + raise + except (TuningError, OSError, ValueError) as error: + print(f"testenix: tuning error: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + except Exception as error: # the CLI is the final application boundary + print(f"testenix: tuning error: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + + summary_stream = sys.stderr if arguments.tuning_json == "-" else sys.stdout + print(render_tuning_report(report), end="", file=summary_stream) + if arguments.tuning_json == "-": + print(report.to_json(), end="") + elif report_output is not None: + try: + _write_new_text(report_output, report.to_json()) + except OSError as error: + print(f"testenix: cannot write tuning report: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + + if arguments.write: + try: + if _read_optional_file(destination) != configuration_snapshot: + raise ConfigError( + "configuration changed while tuning; recommendation was not written" + ) + changed = write_worker_recommendation( + destination, + report.recommended_workers, + expected_source=configuration_snapshot, + ) + except ConfigError as error: + print(f"testenix: cannot write tuning recommendation: {error}", file=sys.stderr) + return EXIT_INTERNAL_ERROR + action = "Wrote" if changed else "Kept" + print( + f"{action} workers = {report.recommended_workers} in {destination}", + file=summary_stream, + ) + return EXIT_OK + + +def _same_file_target(first: Path, second: Path) -> bool: + return first.resolve(strict=False) == second.resolve(strict=False) + + +def _read_optional_file(path: Path) -> bytes | None: + if not os.path.lexists(path): + return None + try: + return path.read_bytes() + except OSError as error: + raise ConfigError(f"cannot snapshot configuration {path}: {error}") from error + + +def _different_tune_profile_overrides( + config: TestenixConfig, + overrides: dict[str, Any], +) -> tuple[str, ...]: + """Name transient measurement settings that differ from project config.""" + + labels = { + "manifest_path": "--manifest", + "shard_modules": "--shard-modules/--no-shard-modules", + } + different: list[str] = [] + for name, requested in overrides.items(): + configured = getattr(config, name) + if name.endswith("_path") and configured is not None and requested is not None: + matches = _same_file_target(Path(configured), Path(requested)) + else: + matches = configured == requested + if not matches: + different.append(labels.get(name, f"--{name.replace('_', '-')}")) + return tuple(different) + + +def _same_paths(first: Sequence[str], second: Sequence[str]) -> bool: + return tuple(Path(path).resolve(strict=False) for path in first) == tuple( + Path(path).resolve(strict=False) for path in second + ) + + +def _validate_new_output(path: Path, *, label: str) -> None: + if os.path.lexists(path): + raise ConfigError(f"{label} path already exists and will not be replaced: {path}") + + +def _write_new_text(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x", encoding="utf-8", newline="") as output: + output.write(content) + + def _worker_count(value: str) -> int | str: if value == "auto": return value @@ -414,6 +793,26 @@ def _non_negative_int(value: str) -> int: return parsed +def _positive_int(value: str) -> int: + parsed = _non_negative_int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("value must be at least 1") + return parsed + + +def _worker_candidates(value: str) -> tuple[int, ...]: + candidates: set[int] = set() + for item in value.split(","): + stripped = item.strip() + if not stripped: + raise argparse.ArgumentTypeError("candidate worker counts cannot be empty") + workers = _worker_count(stripped) + if not isinstance(workers, int): + raise argparse.ArgumentTypeError("candidate worker counts must be explicit integers") + candidates.add(workers) + return tuple(sorted(candidates)) + + def _migration_worker_count(value: str) -> int | str: workers = _worker_count(value) if isinstance(workers, int) and workers < 2: diff --git a/src/testenix/config.py b/src/testenix/config.py index f7d42ce..878a1c6 100644 --- a/src/testenix/config.py +++ b/src/testenix/config.py @@ -7,17 +7,34 @@ from __future__ import annotations +import copy import os +import re +import stat +import tempfile import tomllib from collections.abc import Mapping, Sequence +from collections.abc import Set as AbstractSet +from contextlib import suppress from dataclasses import dataclass, replace from math import isfinite from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from testenix.contracts import TestSpec + from testenix.tuning import SpawnMethod DEFAULT_HISTORY_PATH = Path(".testenix/history.sqlite3") +class _ExpectedSourceUnset: + """Sentinel distinguishing an absent file from an omitted snapshot guard.""" + + +_EXPECTED_SOURCE_UNSET = _ExpectedSourceUnset() + + class ConfigError(ValueError): """Raised when Testenix configuration is malformed.""" @@ -41,6 +58,8 @@ class TestenixConfig: json_path: Path | None = None junit_path: Path | None = None history_path: Path | None = DEFAULT_HISTORY_PATH + shard_modules: bool = False + manifest_path: Path | None = None def __post_init__(self) -> None: if self.workers != "auto": @@ -59,10 +78,12 @@ def __post_init__(self) -> None: if not isfinite(timeout) or timeout <= 0: raise ConfigError("timeout must be a finite number greater than zero") object.__setattr__(self, "timeout", timeout) + if not isinstance(self.shard_modules, bool): + raise ConfigError("shard_modules must be a boolean") object.__setattr__(self, "paths", _normalise_paths(self.paths)) object.__setattr__(self, "tags", _normalise_tags(self.tags)) - for field_name in ("json_path", "junit_path", "history_path"): + for field_name in ("json_path", "junit_path", "history_path", "manifest_path"): value = getattr(self, field_name) if value is not None and not isinstance(value, Path): object.__setattr__(self, field_name, Path(value)) @@ -78,12 +99,37 @@ def with_overrides(self, **values: Any) -> TestenixConfig: @property def resolved_workers(self) -> int: - """Concrete local worker count for schedulers and process pools.""" + """Pre-discovery worker capacity retained for compatibility. + + Native execution should prefer :meth:`resolve_workers`, which can see + the scheduler's real module/timeout execution units. Explicit integer + configuration has identical behavior through both APIs. + """ if self.workers == "auto": return max(1, os.cpu_count() or 1) return self.workers + def resolve_workers( + self, + selected_specs: Sequence[TestSpec], + durations: Mapping[str, float], + *, + spawn_method: SpawnMethod = "spawn", + shardable_paths: AbstractSet[str] = frozenset(), + ) -> int: + """Resolve an adaptive count after discovery and history lookup.""" + + from testenix.tuning import resolve_adaptive_workers + + return resolve_adaptive_workers( + self, + selected_specs, + durations, + spawn_method=spawn_method, + shardable_paths=shardable_paths, + ) + # A concise alias is convenient for embedders and keeps the public API flexible. Config = TestenixConfig @@ -125,6 +171,7 @@ def config_from_mapping(raw: Mapping[str, Any]) -> TestenixConfig: "json": "json_path", "junit": "junit_path", "history": "history_path", + "manifest": "manifest_path", } allowed = set(TestenixConfig.__dataclass_fields__) | set(aliases) unknown = sorted(set(raw) - allowed) @@ -142,7 +189,7 @@ def config_from_mapping(raw: Mapping[str, Any]) -> TestenixConfig: values["tags"] = _normalise_tags(values["tags"]) if "paths" in values: values["paths"] = _normalise_paths(values["paths"]) - for name in ("json_path", "junit_path"): + for name in ("json_path", "junit_path", "manifest_path"): if name in values: values[name] = _optional_path(values[name], name) if "history_path" in values: @@ -158,6 +205,147 @@ def config_from_mapping(raw: Mapping[str, Any]) -> TestenixConfig: raise ConfigError(f"invalid [tool.testenix] configuration: {error}") from error +def write_worker_recommendation( + path: str | Path, + workers: int, + *, + expected_source: bytes | None | _ExpectedSourceUnset = _EXPECTED_SOURCE_UNSET, +) -> bool: + """Atomically persist an explicit worker recommendation in ``pyproject.toml``. + + This operation is intentionally separate from loading and tuning. Callers + must expose an explicit user action (the CLI uses ``testenix tune --write``) + before invoking it. ``True`` means bytes changed; an already matching + configuration returns ``False``. When *expected_source* is supplied, it + acts as an optimistic byte-drift guard: ``None`` means the file must still + be absent, while bytes must still match exactly. The transformed content is + checked again immediately before the final atomic replacement. + """ + + if isinstance(workers, bool) or not isinstance(workers, int) or workers < 1: + raise ConfigError("workers must be a positive integer") + config_path = Path(path) + if config_path.is_symlink(): + raise ConfigError(f"refusing to replace symbolic link: {config_path}") + try: + raw_source = config_path.read_bytes() if config_path.exists() else None + if expected_source is not _EXPECTED_SOURCE_UNSET and raw_source != expected_source: + raise ConfigError("configuration changed while tuning; recommendation was not written") + source = (raw_source or b"").decode("utf-8") + except (OSError, UnicodeDecodeError) as error: + raise ConfigError(f"cannot read {config_path}: {error}") from error + + remainder = source.replace("\r\n", "") + if "\r" in remainder or ("\r\n" in source and "\n" in remainder): + raise ConfigError(f"cannot update {config_path}: mixed or unsupported line endings") + newline = "\r\n" if "\r\n" in source else "\n" + normalised = source.replace("\r\n", "\n") + try: + before = tomllib.loads(normalised) if normalised else {} + except tomllib.TOMLDecodeError as error: + raise ConfigError(f"cannot update {config_path}: {error}") from error + + updated_normalised = _set_workers_in_toml(normalised, workers) + try: + parsed = tomllib.loads(updated_normalised) + expected = copy.deepcopy(before) + tool = expected.setdefault("tool", {}) + if not isinstance(tool, dict): + raise ConfigError("[tool] must be a TOML table") + configured = tool.setdefault("testenix", {}) + if not isinstance(configured, dict): + raise ConfigError("[tool.testenix] must be a TOML table") + configured["workers"] = workers + if parsed != expected: + raise ConfigError( + "unsupported TOML layout: refusing an update that would change other values" + ) + loaded = config_from_mapping(configured) + if loaded.workers != workers: + raise ConfigError("worker recommendation was not applied") + except (tomllib.TOMLDecodeError, ConfigError, AttributeError) as error: + raise ConfigError(f"cannot update {config_path}: {error}") from error + updated = updated_normalised if newline == "\n" else updated_normalised.replace("\n", "\r\n") + if source == updated: + current_source = config_path.read_bytes() if config_path.exists() else None + if current_source != raw_source: + raise ConfigError( + "configuration changed while preparing the update; recommendation was not written" + ) + return False + # Always protect the read/transform/write cycle, even for library callers + # which did not provide a longer-lived tuning snapshot. + _atomic_write_text(config_path, updated, expected_source=raw_source) + return True + + +def _set_workers_in_toml(source: str, workers: int) -> str: + table_pattern = re.compile(r"(?m)^[ \t]*\[tool\.testenix\][ \t]*(?:#.*)?$") + next_table_pattern = re.compile(r"(?m)^[ \t]*\[") + workers_pattern = re.compile( + r"(?m)^(?P[ \t]*)workers[ \t]*=[^#\n]*(?P[ \t]*#.*)?$" + ) + table = table_pattern.search(source) + if table is None: + separator = "" if not source else ("" if source.endswith("\n\n") else "\n") + return f"{source}{separator}[tool.testenix]\nworkers = {workers}\n" + + section_start = table.end() + following = next_table_pattern.search(source, section_start) + section_end = len(source) if following is None else following.start() + section = source[section_start:section_end] + existing = workers_pattern.search(section) + if existing is not None: + comment = existing.group("comment") or "" + if comment: + comment = f" {comment.lstrip()}" + replacement = f"{existing.group('indent')}workers = {workers}{comment}" + updated_section = section[: existing.start()] + replacement + section[existing.end() :] + else: + updated_section = f"\nworkers = {workers}" + section + return source[:section_start] + updated_section + source[section_end:] + + +def _atomic_write_text(path: Path, content: str, *, expected_source: bytes | None) -> None: + if path.is_symlink(): + raise ConfigError(f"refusing to replace symbolic link: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + existing_mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else None + temporary_name: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + newline="", + dir=path.parent, + prefix=f".{path.name}.testenix-", + suffix=".tmp", + delete=False, + ) as temporary: + temporary_name = temporary.name + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary_name) + if existing_mode is not None: + temporary_path.chmod(existing_mode) + if path.is_symlink(): + raise ConfigError(f"refusing to replace symbolic link: {path}") + current_source = path.read_bytes() if path.exists() else None + if current_source != expected_source: + raise ConfigError( + "configuration changed while preparing the update; recommendation was not written" + ) + os.replace(temporary_path, path) + temporary_name = None + except OSError as error: + raise ConfigError(f"cannot write {path}: {error}") from error + finally: + if temporary_name is not None: + with suppress(FileNotFoundError): + Path(temporary_name).unlink() + + def _normalise_tags(value: str | Sequence[str]) -> tuple[str, ...]: if isinstance(value, str): candidates = value.split(",") diff --git a/src/testenix/contracts.py b/src/testenix/contracts.py index e5fe584..3e6bd97 100644 --- a/src/testenix/contracts.py +++ b/src/testenix/contracts.py @@ -127,6 +127,8 @@ class RunResult: collection_issues: tuple[CollectionIssue, ...] started_at: float finished_at: float + workers_used: int | None = None + shardable_paths: tuple[str, ...] = () @property def exit_code(self) -> int: diff --git a/src/testenix/discovery.py b/src/testenix/discovery.py index cb7b46a..ba01faa 100644 --- a/src/testenix/discovery.py +++ b/src/testenix/discovery.py @@ -128,6 +128,21 @@ def _expand_paths( return sorted(files, key=lambda value: value.as_posix()), issues +def enumerate_test_files( + paths: str | Path | Iterable[str | Path], +) -> tuple[tuple[Path, ...], tuple[CollectionIssue, ...]]: + """Enumerate collection inputs without importing a Python module. + + This is intentionally a filesystem-only projection used to validate an + explicit trusted collection manifest. Normal discovery still imports + modules so dynamic decorators, fixtures, and collection failures retain + their established semantics. + """ + + files, issues = _expand_paths(paths) + return tuple(files), tuple(issues) + + def _module_identity(path: Path) -> tuple[str, Path]: package_parts: list[str] = [] cursor = path.parent @@ -437,17 +452,11 @@ def _materialise_specs( return specs, issues -def discover( +def _discover( paths: str | Path | Iterable[str | Path] = ".", + *, + function_names: frozenset[str] | None = None, ) -> CollectionResult: - """Discover native tests and fixtures below one or more paths. - - Directories are searched recursively for ``test_*.py``. An explicitly - supplied Python file may have any name, which is useful for focused runs. - Inside a module, ``test_*`` functions and functions decorated with - ``@test`` (or parameterized with ``@case(s)``) are collected. - """ - files, issues = _expand_paths(paths) registry = FixtureRegistry() modules: list[tuple[Path, ModuleType, list[tuple[str, Any]]]] = [] @@ -485,6 +494,17 @@ def discover( for name, function in functions: if get_fixture_metadata(function) is not None: continue + # Locators intentionally store the callable name used in TestSpec. A + # decorator that does not preserve ``__name__`` can make that differ + # from the module binding (``test_original`` -> ``wrapped``). Match + # both identities so the execution-side selected discovery remains + # equivalent to full collection. + if ( + function_names is not None + and name not in function_names + and function.__name__ not in function_names + ): + continue metadata = get_test_metadata(function) explicitly_parameterized = hasattr(function, CASES_METADATA_ATTR) if metadata is None and not name.startswith("test_") and not explicitly_parameterized: @@ -511,6 +531,40 @@ def discover( return CollectionResult(tuple(collected), registry, tuple(issues)) +def discover( + paths: str | Path | Iterable[str | Path] = ".", +) -> CollectionResult: + """Discover native tests and fixtures below one or more paths. + + Directories are searched recursively for ``test_*.py``. An explicitly + supplied Python file may have any name, which is useful for focused runs. + Inside a module, ``test_*`` functions and functions decorated with + ``@test`` (or parameterized with ``@case(s)``) are collected. + """ + + return _discover(paths) + + +def discover_selected( + path: str | Path, + function_names: Iterable[str], +) -> CollectionResult: + """Rediscover only selected functions while registering every fixture. + + Spawned execution workers must import the source module again because user + callables and arbitrary case values are intentionally absent from the + portable ``TestSpec`` contract. For an intra-module shard, however, there + is no reason to rematerialise every unrelated test and case after import. + This projection preserves fixture registration and import failures while + limiting test materialisation to the requested function names. + """ + + selected = frozenset(function_names) + if not selected: + return CollectionResult((), FixtureRegistry()) + return _discover(path, function_names=selected) + + collect = discover @@ -527,6 +581,8 @@ def discover_specs( "CollectionResult", "collect", "discover", + "discover_selected", "discover_specs", + "enumerate_test_files", "load_module", ] diff --git a/src/testenix/reporters/json.py b/src/testenix/reporters/json.py index 6a702e3..0ceaefe 100644 --- a/src/testenix/reporters/json.py +++ b/src/testenix/reporters/json.py @@ -52,8 +52,10 @@ def run_result_to_dict(run: RunResult) -> dict[str, Any]: "format": RESULT_FORMAT, "run_id": run.run_id, "schema_version": EVENT_SCHEMA_VERSION, + "shardable_paths": list(run.shardable_paths), "started_at": run.started_at, "tests": [_test_to_dict(result) for result in sorted(run.tests, key=_test_sort_key)], + "workers_used": run.workers_used, } diff --git a/src/testenix/runner.py b/src/testenix/runner.py index 26fdbfa..147238c 100644 --- a/src/testenix/runner.py +++ b/src/testenix/runner.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import hashlib +import os import statistics import time import uuid @@ -10,6 +12,7 @@ from contextlib import suppress from dataclasses import dataclass, replace from pathlib import Path +from typing import TYPE_CHECKING, cast from testenix.aggregate import finalize_status, reduce_events from testenix.config import TestenixConfig @@ -25,7 +28,7 @@ TestResult, TestSpec, ) -from testenix.discovery import CollectedTest, CollectionResult, discover +from testenix.discovery import CollectedTest, CollectionResult, discover, discover_selected from testenix.events import ( EventFactory, EventSink, @@ -37,8 +40,22 @@ from testenix.executor import NativeExecutionError, execute_tests from testenix.history import HistoryStore from testenix.scheduler import schedule_lpt +from testenix.sharding import ( + CollectionManifestError, + ModuleShardingDecision, + ShardingPolicy, + TrustedCollectionManifest, + assess_collection_sharding, + build_trusted_collection_manifest, + deserialize_trusted_collection_manifest, + validate_trusted_collection_manifest, + verify_trusted_collection_manifest, +) from testenix.worker import ProcessSupervisor, WorkerExecution, WorkItem +if TYPE_CHECKING: + from testenix.tuning import SpawnMethod + _RETRYABLE_TEST_STATUSES = frozenset( { Status.FAIL, @@ -50,6 +67,7 @@ _AUTOMATIC_RECOVERY_STATUSES = frozenset({Status.INFRA_ERROR, Status.CRASH}) _COLLECTION_TIMEOUT = 30.0 _WORKER_STARTUP_TIMEOUT = 30.0 +_NO_SHARDABLE_PATHS: frozenset[str] = frozenset() @dataclass(frozen=True, slots=True) @@ -71,6 +89,7 @@ class _NativeTestLocator: function_name: str case_id: str | None timeout: float | None + expected_sources: tuple[tuple[str, str], ...] = () @dataclass(frozen=True, slots=True) @@ -93,6 +112,9 @@ class _PlannedWork: class _CollectionManifest: tests: tuple[TestSpec, ...] issues: tuple[CollectionIssue, ...] + sharding: tuple[ModuleShardingDecision, ...] = () + expected_source_digests: tuple[tuple[str, str], ...] = () + expected_dependency_digests: tuple[tuple[str, str], ...] = () class _RunEventSink: @@ -149,13 +171,25 @@ def _collection_issue_for_tags(paths: Sequence[str], tags: Sequence[str]) -> Col ) -def _locator(spec: TestSpec) -> _NativeTestLocator: +def _locator( + spec: TestSpec, + expected_source_digests: Mapping[str, str] | None = None, + expected_dependency_digests: Sequence[tuple[str, str]] = (), +) -> _NativeTestLocator: return _NativeTestLocator( id=spec.id, path=spec.path, function_name=spec.function_name, case_id=spec.case_id, timeout=spec.timeout, + expected_sources=( + () + if expected_source_digests is None + else ( + (spec.path, expected_source_digests[spec.path]), + *expected_dependency_digests, + ) + ), ) @@ -167,23 +201,85 @@ def _portable_spec(spec: TestSpec) -> TestSpec: ) -def _discover_manifest(paths: Sequence[str]) -> _CollectionManifest: +def _discover_manifest( + paths: Sequence[str], + analyse_sharding: bool = False, +) -> _CollectionManifest: collection = discover(paths) return _CollectionManifest( tests=tuple(_portable_spec(item.spec) for item in collection.items), issues=collection.issues, + sharding=assess_collection_sharding(collection) if analyse_sharding else (), + ) + + +def _discover_trusted_manifest( + paths: Sequence[str], + project_root: str, +) -> TrustedCollectionManifest: + """Collect and fingerprint inside the same isolated boundary as a run.""" + + # ``paths`` are project-relative by contract. The manifest fingerprinting + # code already resolves them against ``project_root``; collection must use + # that same base when the caller's current working directory is elsewhere. + # This target always runs in a short-lived supervised child process, so the + # directory change cannot leak into the coordinator. + os.chdir(project_root) + collection = discover(paths) + return build_trusted_collection_manifest(paths, collection, project_root=project_root) + + +def collect_trusted_manifest( + paths: Sequence[str] | str, + *, + project_root: str | Path | None = None, +) -> TrustedCollectionManifest: + """Create an explicit collection manifest in a supervised worker process.""" + + effective_paths = (paths,) if isinstance(paths, str) else tuple(paths) + if not effective_paths: + raise CollectionManifestError("at least one collection path is required") + root = Path.cwd().resolve() if project_root is None else Path(project_root).resolve() + supervisor = ProcessSupervisor(start_method="spawn") + execution = supervisor.execute( + WorkItem( + test_id="collection-manifest", + target=_discover_trusted_manifest, + args=(effective_paths, str(root)), + timeout=_COLLECTION_TIMEOUT, + ) ) + if execution.status is Status.CANCELLED: + raise asyncio.CancelledError + if isinstance(execution.value, TrustedCollectionManifest): + manifest = validate_trusted_collection_manifest(execution.value) + if manifest.issues: + details = "; ".join(issue.message for issue in manifest.issues) + raise CollectionManifestError(f"collection reported errors: {details}") + return manifest + error = execution.error + message = error.message if error is not None else "collection worker returned no manifest" + diagnostics = ( + (error.traceback if error is not None else None) + or execution.stderr + or execution.stdout + or "" + ) + suffix = f"\n{diagnostics}" if diagnostics else "" + raise CollectionManifestError(f"isolated manifest collection failed: {message}{suffix}") def _collect_in_worker( paths: Sequence[str], supervisor: ProcessSupervisor, + *, + analyse_sharding: bool = False, ) -> _CollectionManifest: execution = supervisor.execute( WorkItem( test_id="collection", target=_discover_manifest, - args=(tuple(paths),), + args=(tuple(paths), analyse_sharding), timeout=_COLLECTION_TIMEOUT, ) ) @@ -211,18 +307,87 @@ def _collect_in_worker( ) +def _verified_trusted_collection( + trusted_manifest: TrustedCollectionManifest | None, + paths: Sequence[str], +) -> _CollectionManifest | None: + """Project a valid explicit manifest, or request normal collection fallback.""" + + if trusted_manifest is None: + return None + try: + validated = validate_trusted_collection_manifest(trusted_manifest) + except (CollectionManifestError, TypeError, ValueError): + return None + if not verify_trusted_collection_manifest(validated, paths): + return None + test_paths = {spec.path for spec in validated.tests} + return _CollectionManifest( + tests=validated.tests, + issues=validated.issues, + sharding=validated.sharding, + expected_source_digests=tuple( + (fingerprint.path, fingerprint.sha256) for fingerprint in validated.files + ), + expected_dependency_digests=tuple( + (fingerprint.path, fingerprint.sha256) + for fingerprint in validated.files + if fingerprint.path not in test_paths + ), + ) + + +def _source_sha256(path: str) -> str: + digest = hashlib.sha256() + try: + with Path(path).open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as error: + raise NativeExecutionError( + f"cannot verify trusted manifest source {path!r}: {error}" + ) from error + return digest.hexdigest() + + +def _verify_locator_sources( + locators: Sequence[_NativeTestLocator], +) -> None: + expected: dict[str, str] = {} + for locator in locators: + for path, digest in locator.expected_sources: + previous = expected.setdefault(path, digest) + if previous != digest: + raise NativeExecutionError( + f"trusted manifest contains conflicting source digests for {path!r}" + ) + for path, digest in sorted(expected.items()): + if _source_sha256(path) != digest: + raise NativeExecutionError( + f"trusted manifest source digest mismatch for {path!r}; " + "refusing to import changed code" + ) + + def _resolve_locators_in_worker( locators: Sequence[_NativeTestLocator], ) -> tuple[CollectedTest, ...]: """Rediscover tests without pickling arbitrary parameter values.""" + # Validate every selected test source and local collection dependency in + # this execution process before importing any of them. The check is + # repeated for every fresh batch and retry rediscovery. + _verify_locator_sources(locators) + requested_names: dict[str, set[str]] = {} + for locator in locators: + requested_names.setdefault(locator.path, set()).add(locator.function_name) collections: dict[str, CollectionResult] = {} indexes: dict[str, dict[str, CollectedTest]] = {} resolved: list[CollectedTest] = [] for locator in locators: collection = collections.get(locator.path) if collection is None: - collection = discover(locator.path) + collection = discover_selected(locator.path, requested_names[locator.path]) collections[locator.path] = collection indexes[locator.path] = {item.spec.id: item for item in collection.items} if collection.issues: @@ -321,18 +486,23 @@ def _single_deadline(spec: TestSpec) -> float | None: def _execution_units( specs: Sequence[TestSpec], durations: Mapping[str, float], + *, + shardable_paths: frozenset[str] = _NO_SHARDABLE_PATHS, ) -> tuple[_ExecutionUnit, ...]: - """Keep normal modules intact and isolate every hard-timeout test.""" + """Keep normal modules intact unless an opt-in safety decision allows splitting.""" known = tuple(value for value in durations.values() if value >= 0.0) fallback = float(statistics.median(known)) if known else 1.0 modules: dict[str, list[TestSpec]] = {} isolated: list[TestSpec] = [] + sharded: list[TestSpec] = [] for spec in specs: - if spec.timeout is None: - modules.setdefault(spec.path, []).append(spec) - else: + if spec.timeout is not None: isolated.append(spec) + elif spec.path in shardable_paths: + sharded.append(spec) + else: + modules.setdefault(spec.path, []).append(spec) units: list[_ExecutionUnit] = [] for path, module_specs in modules.items(): @@ -344,6 +514,14 @@ def _execution_units( estimated_duration=sum(durations.get(spec.id, fallback) for spec in materialized), ) ) + units.extend( + _ExecutionUnit( + id=f"test:{spec.id}", + specs=(spec,), + estimated_duration=durations.get(spec.id, fallback), + ) + for spec in sharded + ) units.extend( _ExecutionUnit( id=f"isolated:{spec.id}", @@ -361,8 +539,11 @@ def _initial_work_plan( *, worker_count: int, durations: Mapping[str, float], + shardable_paths: frozenset[str] = _NO_SHARDABLE_PATHS, + expected_source_digests: Mapping[str, str] | None = None, + expected_dependency_digests: Sequence[tuple[str, str]] = (), ) -> tuple[tuple[_PlannedWork, ...], ...]: - units = _execution_units(specs, durations) + units = _execution_units(specs, durations, shardable_paths=shardable_paths) if not units: return () unit_durations = {unit.id: unit.estimated_duration for unit in units} @@ -389,7 +570,17 @@ def _initial_work_plan( item=WorkItem( test_id=f"shard-{shard.shard_id}", target=_execute_native_batch, - args=(tuple(_locator(spec) for spec in shared_specs), 1), + args=( + tuple( + _locator( + spec, + expected_source_digests, + expected_dependency_digests, + ) + for spec in shared_specs + ), + 1, + ), stream_callback_arg="_testenix_result_sink", ), ) @@ -404,7 +595,16 @@ def _initial_work_plan( item=WorkItem( test_id=spec.id, target=_execute_native_batch, - args=((_locator(spec),), 1), + args=( + ( + _locator( + spec, + expected_source_digests, + expected_dependency_digests, + ), + ), + 1, + ), timeout=_single_deadline(spec), stream_callback_arg="_testenix_result_sink", ready_callback_arg="_testenix_ready_sink", @@ -591,11 +791,17 @@ def _execute_initial_attempts( worker_count: int, durations: Mapping[str, float], supervisor: ProcessSupervisor, -) -> tuple[TestResult, ...]: + shardable_paths: frozenset[str] = _NO_SHARDABLE_PATHS, + expected_source_digests: Mapping[str, str] | None = None, + expected_dependency_digests: Sequence[tuple[str, str]] = (), +) -> tuple[tuple[TestResult, ...], int]: plan = _initial_work_plan( specs, worker_count=worker_count, durations=durations, + shardable_paths=shardable_paths, + expected_source_digests=expected_source_digests, + expected_dependency_digests=expected_dependency_digests, ) executions = supervisor.execute_shards( tuple(tuple(work.item for work in shard) for shard in plan) @@ -604,7 +810,7 @@ def _execute_initial_attempts( for planned_shard, shard_executions in zip(plan, executions, strict=True): for work, execution in zip(planned_shard, shard_executions, strict=True): results.extend(_results_from_batch_execution(work.specs, 1, execution)) - return tuple(results) + return tuple(results), len(plan) def _execute_retry_attempts( @@ -613,7 +819,9 @@ def _execute_retry_attempts( worker_count: int, durations: Mapping[str, float], supervisor: ProcessSupervisor, -) -> tuple[TestResult, ...]: + expected_source_digests: Mapping[str, str] | None = None, + expected_dependency_digests: Sequence[tuple[str, str]] = (), +) -> tuple[tuple[TestResult, ...], int]: shards = tuple( shard for shard in schedule_lpt( @@ -629,7 +837,14 @@ def _execute_retry_attempts( WorkItem( test_id=item.spec.id, target=_execute_native_one, - args=(_locator(item.spec), item.attempt), + args=( + _locator( + item.spec, + expected_source_digests, + expected_dependency_digests, + ), + item.attempt, + ), attempt=item.attempt, timeout=_single_deadline(item.spec), ready_callback_arg=( @@ -662,7 +877,7 @@ def _execute_retry_attempts( results.append(_merge_outer_output(result, execution)) else: results.append(_failure_for_execution(item.spec, item.attempt, execution)) - return tuple(results) + return tuple(results), len(work_shards) def _emit_attempt(factory: EventFactory, sink: EventSink, result: TestResult) -> None: @@ -702,6 +917,8 @@ def run( config: TestenixConfig | None = None, *, event_sink: EventSink | None = None, + sharding_policy: ShardingPolicy | None = None, + trusted_manifest: TrustedCollectionManifest | None = None, ) -> RunResult: """Discover and execute a native Testenix suite. @@ -710,7 +927,13 @@ def run( reduced after execution. """ - return _run(paths, config, event_sink=event_sink) + return _run( + paths, + config, + event_sink=event_sink, + sharding_policy=sharding_policy, + trusted_manifest=trusted_manifest, + ) def _run( @@ -718,10 +941,28 @@ def _run( config: TestenixConfig | None, *, event_sink: EventSink | None, + sharding_policy: ShardingPolicy | None = None, + trusted_manifest: TrustedCollectionManifest | None = None, supervisor: ProcessSupervisor | None = None, ) -> RunResult: - effective_config = config or TestenixConfig() + policy = ( + ShardingPolicy(intra_module=effective_config.shard_modules) + if sharding_policy is None + else sharding_policy + ) + if not isinstance(policy, ShardingPolicy): + raise TypeError("sharding_policy must be a ShardingPolicy or None") + active_trusted_manifest = trusted_manifest + if active_trusted_manifest is None and effective_config.manifest_path is not None: + try: + active_trusted_manifest = deserialize_trusted_collection_manifest( + effective_config.manifest_path.read_bytes() + ) + except OSError as error: + raise CollectionManifestError( + f"cannot read collection manifest {effective_config.manifest_path}: {error}" + ) from error if paths is None: effective_paths = effective_config.paths elif isinstance(paths, str): @@ -748,7 +989,13 @@ def _run( ) ) sink.emit(factory.create(EventType.COLLECTION_STARTED, payload={"paths": effective_paths})) - collection = _collect_in_worker(effective_paths, active_supervisor) + collection = _verified_trusted_collection(active_trusted_manifest, effective_paths) + if collection is None: + collection = _collect_in_worker( + effective_paths, + active_supervisor, + analyse_sharding=policy.intra_module, + ) selected = _effective_specs(collection.tests, effective_config) issues = list(collection.issues) if not collection.tests and not issues: @@ -805,16 +1052,45 @@ def _run( order = {spec.id: index for index, spec in enumerate(selected)} user_retries_left = {spec.id: effective_config.retries for spec in selected} infrastructure_recovery_used = {spec.id: False for spec in selected} + expected_source_digests = ( + dict(collection.expected_source_digests) if collection.expected_source_digests else None + ) + expected_dependency_digests = collection.expected_dependency_digests + worker_limit = 0 + workers_used = 0 + shardable_paths = _NO_SHARDABLE_PATHS if selected: durations = _duration_history(effective_config, selected) - worker_count = min(len(selected), effective_config.resolved_workers) - first_results = _execute_initial_attempts( + shardable_paths = ( + frozenset(decision.path for decision in collection.sharding if decision.eligible) + if policy.intra_module + else _NO_SHARDABLE_PATHS + ) + schedulable_units = _execution_units( + selected, + durations, + shardable_paths=shardable_paths, + ) + worker_limit = min( + len(schedulable_units), + effective_config.resolve_workers( + selected, + durations, + spawn_method=cast("SpawnMethod", active_supervisor.start_method), + shardable_paths=shardable_paths, + ), + ) + first_results, initial_workers_used = _execute_initial_attempts( selected, - worker_count=worker_count, + worker_count=worker_limit, durations=durations, supervisor=active_supervisor, + shardable_paths=shardable_paths, + expected_source_digests=expected_source_digests, + expected_dependency_digests=expected_dependency_digests, ) + workers_used = max(workers_used, initial_workers_used) first_results = tuple(sorted(first_results, key=lambda result: order[result.test.id])) for result in first_results: attempts_by_test[result.test.id].append(result.attempts[-1]) @@ -837,12 +1113,15 @@ def _run( pending.append(_PendingAttempt(spec, 2)) while pending: - retry_results = _execute_retry_attempts( + retry_results, retry_workers_used = _execute_retry_attempts( pending, - worker_count=worker_count, + worker_count=worker_limit, durations=durations, supervisor=active_supervisor, + expected_source_digests=expected_source_digests, + expected_dependency_digests=expected_dependency_digests, ) + workers_used = max(workers_used, retry_workers_used) retry_results = tuple(sorted(retry_results, key=lambda result: order[result.test.id])) next_pending: list[_PendingAttempt] = [] for result in retry_results: @@ -880,12 +1159,20 @@ def _run( factory.create( EventType.RUN_FINISHED, timestamp=finished_at, - payload={"finished_at": finished_at}, + payload={ + "finished_at": finished_at, + "workers_used": workers_used, + "shardable_paths": tuple(sorted(shardable_paths)), + }, ) ) finally: sink.close() - run_result = reduce_events(memory_sink.events, run_id=run_id) + run_result = replace( + reduce_events(memory_sink.events, run_id=run_id), + workers_used=workers_used, + shardable_paths=tuple(sorted(shardable_paths)), + ) if effective_config.history_path is not None: with HistoryStore(effective_config.history_path) as history: history.record_run(run_result) @@ -897,6 +1184,8 @@ async def run_async( config: TestenixConfig | None = None, *, event_sink: EventSink | None = None, + sharding_policy: ShardingPolicy | None = None, + trusted_manifest: TrustedCollectionManifest | None = None, ) -> RunResult: """Cancellable embedding facade around the process-oriented coordinator.""" @@ -907,6 +1196,8 @@ async def run_async( paths, config, event_sink=event_sink, + sharding_policy=sharding_policy, + trusted_manifest=trusted_manifest, supervisor=supervisor, ) ) @@ -921,4 +1212,4 @@ async def run_async( raise -__all__ = ["run", "run_async"] +__all__ = ["collect_trusted_manifest", "run", "run_async"] diff --git a/src/testenix/scheduler.py b/src/testenix/scheduler.py index c445e64..5d44447 100644 --- a/src/testenix/scheduler.py +++ b/src/testenix/scheduler.py @@ -92,8 +92,11 @@ def schedule_lpt( """Assign items with the LPT greedy algorithm. Items are sorted by decreasing historical duration, then by stable id. Each - item goes to the currently lightest shard, with ``shard_id`` breaking load - ties. The returned plan is therefore independent of input ordering. + item goes to the currently lightest shard. Equal-load shards are ordered by + their item count and then ``shard_id``. The item-count tie-break matters for + legitimate zero-duration estimates: it keeps every feasible worker busy + instead of repeatedly selecting shard zero. The returned plan is therefore + independent of input ordering. ``durations`` is the canonical argument; ``history`` is accepted as an ergonomic alias. If no explicit default is supplied, the median known @@ -136,7 +139,10 @@ def schedule_lpt( shard_items: list[list[T]] = [[] for _ in range(shard_count)] shard_loads = [0.0] * shard_count for estimate, _, item in annotated: - shard_id = min(range(shard_count), key=lambda index: (shard_loads[index], index)) + shard_id = min( + range(shard_count), + key=lambda index: (shard_loads[index], len(shard_items[index]), index), + ) shard_items[shard_id].append(item) shard_loads[shard_id] += estimate diff --git a/src/testenix/sharding.py b/src/testenix/sharding.py new file mode 100644 index 0000000..fe32a09 --- /dev/null +++ b/src/testenix/sharding.py @@ -0,0 +1,1246 @@ +"""Conservative metadata for opt-in intra-module sharding. + +The normal Testenix scheduler keeps every module in one process. That remains +the default because module and session fixtures, import-time lifecycle hooks, +and mutable module globals can make test order and process affinity observable. + +``ShardingPolicy(intra_module=True)`` is an explicit acknowledgement that +unobservable mutable state cannot be proven safe by static analysis. The +analyser still fails closed for hazards Testenix can identify reliably: + +* module- or session-scoped fixtures; +* fixtures imported from outside the collected module source; +* direct writes to module globals through ``global``; +* mutations of obvious nested module containers and mutable class state; and +* executable import-time control flow or lifecycle calls. + +Function-scoped fixtures, including autouse fixtures, are safe to recreate in +each process and therefore do not prevent sharding. +""" + +from __future__ import annotations + +import ast +import hashlib +import json +import math +import re +import sys +from collections import defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any + +from testenix.contracts import CollectionIssue, Scope, TestSpec +from testenix.discovery import CollectionResult, enumerate_test_files + +COLLECTION_MANIFEST_FORMAT = "testenix.collection-manifest" +COLLECTION_MANIFEST_SCHEMA_VERSION = 1 +REDACTED_PARAMETER_VALUE = "" +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") + + +class CollectionManifestError(ValueError): + """A trusted collection manifest is malformed or unsafe to resolve.""" + + +@dataclass(frozen=True, slots=True) +class SourceFingerprint: + """SHA-256 identity of one project-relative collection source file.""" + + path: str + sha256: str + + +@dataclass(frozen=True, slots=True) +class TrustedCollectionManifest: + """Explicit portable collection result that may bypass collection imports. + + This is deliberately not an implicit cache. A caller creates or loads the + manifest and opts into trusting it for a run. Parameter names are retained + for diagnostics, but every parameter value is replaced with the explicit + ```` sentinel. Testenix still compares the complete test-file + inventory plus local import dependencies and every SHA-256 digest before + using it. + Dynamic collection influenced by anything other than source bytes (for + example environment variables) remains the manifest producer's trust + decision. + """ + + collection_roots: tuple[str, ...] + files: tuple[SourceFingerprint, ...] + tests: tuple[TestSpec, ...] + issues: tuple[CollectionIssue, ...] = () + sharding: tuple[ModuleShardingDecision, ...] = () + + +@dataclass(frozen=True, slots=True) +class ShardingPolicy: + """Core scheduling policy independent of CLI/configuration concerns. + + Intra-module sharding is deliberately opt-in. Callers which do not pass a + policy, or pass the default instance, retain the original module-affinity + behaviour exactly. + """ + + intra_module: bool = False + + def __post_init__(self) -> None: + if not isinstance(self.intra_module, bool): + raise TypeError("intra_module must be a boolean") + + +@dataclass(frozen=True, slots=True) +class ModuleShardingDecision: + """Portable result of analysing one collected module.""" + + path: str + module_name: str + eligible: bool + blockers: tuple[str, ...] = () + + +_MUTABLE_FACTORIES = frozenset( + { + "ChainMap", + "Counter", + "OrderedDict", + "defaultdict", + "deque", + "dict", + "list", + "set", + } +) +_MUTATING_METHODS = frozenset( + { + "add", + "append", + "clear", + "discard", + "extend", + "insert", + "pop", + "popitem", + "remove", + "reverse", + "setdefault", + "sort", + "update", + } +) +_SAFE_TESTENIX_DECORATOR_FACTORIES = frozenset( + { + "case", + "cases", + "fixture", + "skip", + "test", + "xfail", + } +) + + +def _qualified_name(node: ast.AST) -> str | None: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + owner = _qualified_name(node.value) + return f"{owner}.{node.attr}" if owner is not None else None + return None + + +def _testenix_decorator_names(tree: ast.Module) -> frozenset[str]: + """Resolve only explicitly imported Testenix decorator factories. + + A bare name is not trusted merely because it happens to be called + ``test`` or ``fixture``. It must originate from Testenix, and any other + top-level binding of that import name makes the analyser fail closed. + """ + + imported: set[str] = set() + rebound: set[str] = set() + for statement in tree.body: + if isinstance(statement, ast.ImportFrom): + for alias in statement.names: + local_name = alias.asname or alias.name + if ( + statement.module in {"testenix", "testenix.api"} + and alias.name in _SAFE_TESTENIX_DECORATOR_FACTORIES + ): + imported.add(alias.asname or alias.name) + elif alias.name != "*": + rebound.add(local_name) + continue + + if isinstance(statement, ast.Import): + for alias in statement.names: + local_name = alias.asname or alias.name.split(".", 1)[0] + if alias.name in {"testenix", "testenix.api"}: + for factory in _SAFE_TESTENIX_DECORATOR_FACTORIES: + imported.add(f"{local_name}.{factory}") + if alias.name == "testenix" and alias.asname is None: + imported.add(f"{local_name}.api.{factory}") + else: + rebound.add(local_name) + continue + + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + rebound.add(statement.name) + elif isinstance(statement, ast.Assign): + for target in statement.targets: + rebound.update(_assigned_names(target)) + root = _root_name(target) + if root is not None: + rebound.add(root) + elif isinstance(statement, (ast.AnnAssign, ast.AugAssign)): + rebound.update(_assigned_names(statement.target)) + root = _root_name(statement.target) + if root is not None: + rebound.add(root) + elif isinstance(statement, ast.Delete): + for target in statement.targets: + root = _root_name(target) + if root is not None: + rebound.add(root) + + return frozenset(name for name in imported if name.split(".", 1)[0] not in rebound) + + +def _fixture_scope_blockers( + statement: ast.stmt, + *, + safe_decorators: frozenset[str], +) -> tuple[str, ...]: + """Require a local fixture's test scope to be evident from its syntax. + + Sharding also runs without a trusted manifest, and dynamic imports can sit + outside the manifest's project-local dependency boundary. Trusting the + runtime value of ``scope=IMPORTED`` would therefore make the safety proof + depend on mutable external state. Only the decorator default and the + literal string ``"test"`` are stable enough for the opt-in proof. + """ + + if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + return () + fixture_names = frozenset( + name for name in safe_decorators if name.rsplit(".", 1)[-1] == "fixture" + ) + blockers: set[str] = set() + for decorator in statement.decorator_list: + decorated_by = decorator.func if isinstance(decorator, ast.Call) else decorator + if _qualified_name(decorated_by) not in fixture_names: + continue + if not isinstance(decorator, ast.Call): + # ``@fixture`` uses the API's immutable test-scope default. + continue + scope_keywords = tuple(keyword for keyword in decorator.keywords if keyword.arg == "scope") + has_dynamic_keywords = any(keyword.arg is None for keyword in decorator.keywords) + scope_is_literal_test = ( + len(scope_keywords) == 1 + and isinstance(scope_keywords[0].value, ast.Constant) + and scope_keywords[0].value.value == Scope.TEST.value + ) + uses_default_scope = not scope_keywords and not has_dynamic_keywords and not decorator.args + if scope_is_literal_test and not has_dynamic_keywords and not decorator.args: + continue + if uses_default_scope: + continue + blockers.add(f"fixture {statement.name!r} does not have a statically guaranteed test scope") + return tuple(sorted(blockers)) + + +class _EagerCallVisitor(ast.NodeVisitor): + """Find calls evaluated while an enclosing expression is constructed.""" + + def __init__(self) -> None: + self.calls: list[ast.Call] = [] + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 - ast visitor API + self.calls.append(node) + self.generic_visit(node) + + def visit_Lambda(self, node: ast.Lambda) -> None: # noqa: N802 - ast visitor API + # Creating a lambda evaluates its defaults, but not its body. + for default in (*node.args.defaults, *node.args.kw_defaults): + if default is not None: + self.visit(default) + + +def _eager_calls(node: ast.AST | None) -> tuple[ast.Call, ...]: + if node is None: + return () + visitor = _EagerCallVisitor() + visitor.visit(node) + return tuple(visitor.calls) + + +def _render_call(call: ast.Call) -> str: + return _qualified_name(call.func) or "" + + +def _definition_time_call_blockers( + statement: ast.stmt, + *, + evaluate_annotations: bool, + safe_decorators: frozenset[str], +) -> tuple[str, ...]: + blockers: set[str] = set() + + def block_calls(node: ast.AST | None, context: str) -> None: + for call in _eager_calls(node): + blockers.add(f"executes import-time {context} call {_render_call(call)}") + + def inspect_decorators( + decorators: Sequence[ast.expr], + ) -> None: + for decorator in decorators: + decorated_by = decorator.func if isinstance(decorator, ast.Call) else decorator + decorator_name = _qualified_name(decorated_by) + if decorator_name not in safe_decorators: + blockers.add(f"executes import-time decorator call {decorator_name or ''}") + for call in _eager_calls(decorator): + name = _qualified_name(call.func) + if name not in safe_decorators: + blockers.add(f"executes import-time decorator call {name or ''}") + + if isinstance(statement, ast.Assign): + for target in statement.targets: + block_calls(target, "assignment target") + block_calls(statement.value, "assignment") + elif isinstance(statement, ast.AnnAssign): + block_calls(statement.target, "assignment target") + block_calls(statement.value, "assignment") + if evaluate_annotations: + block_calls(statement.annotation, "annotation") + elif isinstance(statement, ast.AugAssign): + block_calls(statement.target, "assignment target") + block_calls(statement.value, "assignment") + elif isinstance(statement, ast.Assert): + block_calls(statement.test, "assertion") + block_calls(statement.msg, "assertion") + elif isinstance(statement, ast.Delete): + for target in statement.targets: + block_calls(target, "delete target") + elif isinstance(statement, ast.Expr): + # The outer expression is handled by ``_import_time_blocker``. Walk it + # as well so nested calls can never hide behind another lifecycle call. + block_calls(statement.value, "expression") + elif isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + inspect_decorators(statement.decorator_list) + for default in (*statement.args.defaults, *statement.args.kw_defaults): + block_calls(default, "default") + if evaluate_annotations: + arguments = ( + *statement.args.posonlyargs, + *statement.args.args, + *statement.args.kwonlyargs, + ) + for argument in arguments: + block_calls(argument.annotation, "annotation") + if statement.args.vararg is not None: + block_calls(statement.args.vararg.annotation, "annotation") + if statement.args.kwarg is not None: + block_calls(statement.args.kwarg.annotation, "annotation") + block_calls(statement.returns, "annotation") + elif isinstance(statement, ast.ClassDef): + inspect_decorators(statement.decorator_list) + for base in statement.bases: + block_calls(base, "class base") + for keyword in statement.keywords: + block_calls(keyword.value, "class declaration") + # A class body runs while the module is imported. Apply the same + # checks to its assignments and method/nested-class definitions, but + # never descend into ordinary function bodies. + for child in statement.body: + blocker = _import_time_blocker(child) + if blocker is not None: + blockers.add(blocker) + blockers.update( + _definition_time_call_blockers( + child, + evaluate_annotations=evaluate_annotations, + safe_decorators=safe_decorators, + ) + ) + + return tuple(sorted(blockers)) + + +def _assigned_names(target: ast.AST) -> tuple[str, ...]: + if isinstance(target, ast.Name): + return (target.id,) + if isinstance(target, (ast.Tuple, ast.List)): + return tuple(name for item in target.elts for name in _assigned_names(item)) + return () + + +class _MutableValueVisitor(ast.NodeVisitor): + """Recognise mutable values retained by a module/class binding.""" + + def __init__(self) -> None: + self.found = False + + def visit_List(self, node: ast.List) -> None: # noqa: N802 - ast visitor API + self.found = True + + def visit_Dict(self, node: ast.Dict) -> None: # noqa: N802 - ast visitor API + self.found = True + + def visit_Set(self, node: ast.Set) -> None: # noqa: N802 - ast visitor API + self.found = True + + def visit_ListComp(self, node: ast.ListComp) -> None: # noqa: N802 - ast visitor API + self.found = True + + def visit_DictComp(self, node: ast.DictComp) -> None: # noqa: N802 - ast visitor API + self.found = True + + def visit_SetComp(self, node: ast.SetComp) -> None: # noqa: N802 - ast visitor API + self.found = True + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 - ast visitor API + qualified = _qualified_name(node.func) + if qualified is not None and qualified.rsplit(".", 1)[-1] in _MUTABLE_FACTORIES: + self.found = True + return + self.generic_visit(node) + + def visit_Lambda(self, node: ast.Lambda) -> None: # noqa: N802 - ast visitor API + # A lambda body is lazy, while its defaults are retained immediately and + # can themselves become shared mutable state. + for default in (*node.args.defaults, *node.args.kw_defaults): + if default is not None: + self.visit(default) + + +def _is_mutable_value(node: ast.AST | None) -> bool: + if node is None: + return False + visitor = _MutableValueVisitor() + visitor.visit(node) + return visitor.found + + +def _class_defines_mutable_state(statement: ast.ClassDef) -> bool: + for child in statement.body: + if isinstance(child, (ast.Assign, ast.AnnAssign)) and _is_mutable_value(child.value): + return True + if isinstance(child, ast.ClassDef) and _class_defines_mutable_state(child): + return True + return False + + +def _mutable_class_bindings(tree: ast.Module) -> frozenset[str]: + """Return module class roots that retain obvious mutable class state.""" + + class_names = {statement.name for statement in tree.body if isinstance(statement, ast.ClassDef)} + names = { + statement.name + for statement in tree.body + if isinstance(statement, ast.ClassDef) and _class_defines_mutable_state(statement) + } + for statement in tree.body: + targets: Sequence[ast.expr] + if isinstance(statement, ast.Assign) and _is_mutable_value(statement.value): + targets = statement.targets + elif isinstance(statement, ast.AnnAssign) and _is_mutable_value(statement.value): + targets = (statement.target,) + else: + continue + for target in targets: + root = _root_name(target) + if root in class_names and not isinstance(target, ast.Name): + # Covers both ``State.values = []`` and a nested target such as + # ``Outer.Inner.values = []`` without attempting alias analysis. + names.add(root) + return frozenset(names) + + +def _mutable_module_bindings(tree: ast.Module) -> frozenset[str]: + names: set[str] = set() + for statement in tree.body: + if isinstance(statement, ast.Assign) and _is_mutable_value(statement.value): + for target in statement.targets: + names.update(_assigned_names(target)) + elif isinstance(statement, ast.AnnAssign) and _is_mutable_value(statement.value): + names.update(_assigned_names(statement.target)) + names.update(_mutable_class_bindings(tree)) + return frozenset(names) + + +def _root_name(node: ast.AST) -> str | None: + current = node + while isinstance(current, (ast.Attribute, ast.Subscript)): + current = current.value + return current.id if isinstance(current, ast.Name) else None + + +class _StateHazardVisitor(ast.NodeVisitor): + def __init__(self, mutable_bindings: frozenset[str]) -> None: + self.mutable_bindings = mutable_bindings + self.blockers: set[str] = set() + + def visit_Global(self, node: ast.Global) -> None: # noqa: N802 - ast visitor API + names = ", ".join(sorted(node.names)) + self.blockers.add(f"writes module globals via global: {names}") + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 - ast visitor API + if isinstance(node.func, ast.Attribute): + root = _root_name(node.func.value) + if root in self.mutable_bindings and node.func.attr in _MUTATING_METHODS: + self.blockers.add(f"mutates module-level collection {root!r}") + self.generic_visit(node) + + def _visit_write_target(self, target: ast.AST) -> None: + root = _root_name(target) + if root in self.mutable_bindings and not isinstance(target, ast.Name): + self.blockers.add(f"mutates module-level collection {root!r}") + + def visit_Assign(self, node: ast.Assign) -> None: # noqa: N802 - ast visitor API + for target in node.targets: + self._visit_write_target(target) + self.generic_visit(node) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: # noqa: N802 - ast visitor API + self._visit_write_target(node.target) + self.generic_visit(node) + + def visit_AugAssign(self, node: ast.AugAssign) -> None: # noqa: N802 - ast visitor API + self._visit_write_target(node.target) + self.generic_visit(node) + + def visit_Delete(self, node: ast.Delete) -> None: # noqa: N802 - ast visitor API + for target in node.targets: + self._visit_write_target(target) + self.generic_visit(node) + + +def _import_time_blocker(statement: ast.stmt) -> str | None: + if isinstance(statement, ast.Expr): + if isinstance(statement.value, ast.Constant) and isinstance(statement.value.value, str): + return None + if isinstance(statement.value, ast.Call): + name = _qualified_name(statement.value.func) + return f"executes import-time call {name or ''}" + return "executes an import-time expression" + if isinstance( + statement, + ( + ast.For, + ast.AsyncFor, + ast.While, + ast.If, + ast.With, + ast.AsyncWith, + ast.Try, + ast.Match, + ), + ): + return f"executes import-time {type(statement).__name__.lower()} control flow" + return None + + +def _source_blockers(path: str) -> tuple[str, ...]: + source_path = Path(path) + try: + tree = ast.parse(source_path.read_bytes(), filename=str(source_path)) + except (OSError, SyntaxError, UnicodeError) as error: + return (f"cannot statically inspect source: {type(error).__name__}: {error}",) + + blockers: set[str] = set() + evaluate_annotations = not any( + isinstance(statement, ast.ImportFrom) + and statement.module == "__future__" + and any(alias.name == "annotations" for alias in statement.names) + for statement in tree.body + ) + safe_decorators = _testenix_decorator_names(tree) + for statement in tree.body: + blocker = _import_time_blocker(statement) + if blocker is not None: + blockers.add(blocker) + blockers.update( + _definition_time_call_blockers( + statement, + evaluate_annotations=evaluate_annotations, + safe_decorators=safe_decorators, + ) + ) + blockers.update(_fixture_scope_blockers(statement, safe_decorators=safe_decorators)) + + mutable_class_bindings = _mutable_class_bindings(tree) + blockers.update(f"defines mutable class state on {name!r}" for name in mutable_class_bindings) + visitor = _StateHazardVisitor(_mutable_module_bindings(tree)) + # Module-level assignments establish state; only function/class bodies can + # make that state order-dependent across tests. + for statement in tree.body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + visitor.visit(statement) + blockers.update(visitor.blockers) + return tuple(sorted(blockers)) + + +def assess_collection_sharding( + collection: CollectionResult, +) -> tuple[ModuleShardingDecision, ...]: + """Return deterministic, portable decisions for every collected module.""" + + modules: dict[tuple[str, str], list[str]] = defaultdict(list) + for item in collection.items: + modules[(item.spec.path, item.spec.module_name)].append(item.spec.id) + + decisions: list[ModuleShardingDecision] = [] + definitions = collection.registry.definitions + for (path, module_name), _test_ids in sorted(modules.items()): + blockers = set(_source_blockers(path)) + module_source = Path(path).resolve() + for definition in definitions: + if definition.module_name not in (None, module_name): + continue + code = getattr(definition.function, "__code__", None) + source_name = getattr(code, "co_filename", None) + try: + fixture_source = ( + Path(source_name).resolve() if isinstance(source_name, str) else None + ) + except (OSError, RuntimeError): + fixture_source = None + if not definition.builtin and fixture_source != module_source: + # Keep the sharding proof local even though trusted manifests + # also fingerprint discoverable project-local imports. A + # provider can live outside that boundary or be resolved by + # dynamic import machinery that static dependency discovery + # cannot prove complete. + blockers.add( + f"fixture {definition.name!r} is imported from outside " + "the collected module source" + ) + if definition.scope in {Scope.MODULE, Scope.SESSION}: + blockers.add(f"fixture {definition.name!r} has {definition.scope.value} scope") + rendered = tuple(sorted(blockers)) + decisions.append( + ModuleShardingDecision( + path=path, + module_name=module_name, + eligible=not rendered, + blockers=rendered, + ) + ) + return tuple(decisions) + + +def _project_root(project_root: str | Path | None) -> Path: + return Path.cwd().resolve() if project_root is None else Path(project_root).resolve() + + +def _safe_relative_path(value: object, *, allow_dot: bool = False) -> str: + if not isinstance(value, str) or not value: + raise CollectionManifestError("manifest paths must be non-empty strings") + if value == "." and allow_dot: + return value + if "\\" in value: + raise CollectionManifestError(f"manifest path is not portable: {value!r}") + posix = PurePosixPath(value) + windows = PureWindowsPath(value) + segments = value.split("/") + if ( + posix.is_absolute() + or windows.is_absolute() + or bool(windows.drive) + or value != posix.as_posix() + or any(segment in {"", ".", ".."} for segment in segments) + ): + raise CollectionManifestError(f"manifest path is not a safe relative path: {value!r}") + return value + + +def _relative_to_root(path: str | Path, root: Path, *, allow_dot: bool = False) -> str: + candidate = Path(path) + resolved = (root / candidate).resolve() if not candidate.is_absolute() else candidate.resolve() + try: + relative = resolved.relative_to(root) + except ValueError as error: + raise CollectionManifestError(f"path escapes project root: {path!s}") from error + rendered = relative.as_posix() or "." + return _safe_relative_path(rendered, allow_dot=allow_dot) + + +def _hash_source(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _module_file_candidates(base: Path, module_name: str) -> tuple[Path, ...]: + parts = tuple(part for part in module_name.split(".") if part) + if not parts: + return () + target = base.joinpath(*parts) + candidates = [target.with_suffix(".py"), target / "__init__.py"] + for index in range(1, len(parts)): + candidates.append(base.joinpath(*parts[:index], "__init__.py")) + return tuple(candidates) + + +def _local_import_candidates(source: Path, tree: ast.Module, root: Path) -> tuple[Path, ...]: + search_roots = {source.parent, root} + for raw_path in sys.path: + try: + candidate = Path(raw_path or ".").resolve() + candidate.relative_to(root) + except (OSError, RuntimeError, ValueError): + continue + search_roots.add(candidate) + + candidates: set[Path] = set() + # Imports inside a helper called by a decorator/case factory can execute + # during collection just as top-level imports do. Walking the complete AST + # is deliberately conservative: runtime-only local imports may cause extra + # invalidation, but can never escape the manifest's dependency boundary. + for statement in ast.walk(tree): + if isinstance(statement, ast.Import): + for alias in statement.names: + if alias.name == "testenix" or alias.name.startswith("testenix."): + continue + for search_root in search_roots: + candidates.update(_module_file_candidates(search_root, alias.name)) + continue + if not isinstance(statement, ast.ImportFrom) or statement.module == "__future__": + continue + module_name = statement.module or "" + if statement.level == 0: + if module_name == "testenix" or module_name.startswith("testenix."): + continue + bases = tuple(search_roots) + else: + relative_base = source.parent + for _ in range(statement.level - 1): + relative_base = relative_base.parent + bases = (relative_base,) + for base in bases: + candidates.update(_module_file_candidates(base, module_name)) + for alias in statement.names: + if alias.name != "*": + imported_name = ".".join(part for part in (module_name, alias.name) if part) + candidates.update(_module_file_candidates(base, imported_name)) + return tuple(candidates) + + +def _local_import_dependencies(files: Sequence[Path], root: Path) -> tuple[Path, ...]: + """Find local Python imports whose bytes can influence collection metadata.""" + + initial = {path.resolve() for path in files} + dependencies: set[Path] = set() + pending = list(initial) + inspected: set[Path] = set() + while pending: + source = pending.pop() + if source in inspected: + continue + inspected.add(source) + try: + tree = ast.parse(source.read_bytes(), filename=str(source)) + except (OSError, SyntaxError, UnicodeError): + continue + for candidate in _local_import_candidates(source, tree, root): + try: + resolved = candidate.resolve() + relative = resolved.relative_to(root) + except (OSError, RuntimeError, ValueError): + continue + if ( + not resolved.is_file() + or resolved.suffix != ".py" + or any( + part in {".venv", "venv", "site-packages", "__pycache__"} + for part in relative.parts + ) + ): + continue + if resolved not in initial and resolved not in dependencies: + dependencies.add(resolved) + pending.append(resolved) + return tuple(sorted(dependencies, key=lambda path: path.as_posix())) + + +def _redacted_parameters(parameters: Mapping[str, Any]) -> dict[str, str]: + names = tuple(parameters) + if any(not isinstance(name, str) or not name for name in names): + raise CollectionManifestError("test parameter names must be non-empty strings") + return {name: REDACTED_PARAMETER_VALUE for name in sorted(names)} + + +def _collection_inputs( + paths: str | Path | Iterable[str | Path], + root: Path, +) -> tuple[tuple[str, ...], tuple[Path, ...]]: + supplied = (paths,) if isinstance(paths, (str, Path)) else tuple(paths) + if not supplied: + raise CollectionManifestError("collection_roots must not be empty") + relative: list[str] = [] + resolved: list[Path] = [] + seen: set[str] = set() + for path in supplied: + rendered = _relative_to_root(path, root, allow_dot=True) + if rendered in seen: + raise CollectionManifestError(f"duplicate collection root: {rendered!r}") + seen.add(rendered) + relative.append(rendered) + resolved.append(root if rendered == "." else root / rendered) + return tuple(relative), tuple(resolved) + + +def _portable_spec(spec: TestSpec, root: Path) -> TestSpec: + parameters = _redacted_parameters(spec.parameters) + return TestSpec( + id=spec.id, + path=_relative_to_root(spec.path, root), + module_name=spec.module_name, + function_name=spec.function_name, + display_name=spec.display_name, + parameters=parameters, + case_id=spec.case_id, + tags=frozenset(spec.tags), + skip_reason=spec.skip_reason, + xfail_reason=spec.xfail_reason, + timeout=spec.timeout, + source_line=spec.source_line, + ) + + +def build_trusted_collection_manifest( + paths: str | Path | Iterable[str | Path], + collection: CollectionResult, + *, + project_root: str | Path | None = None, +) -> TrustedCollectionManifest: + """Build an explicit manifest from a completed native collection. + + The returned value contains no executable objects. It is suitable for + deterministic JSON serialization and for a later opt-in ``run`` call. + """ + + root = _project_root(project_root) + collection_roots, inputs = _collection_inputs(paths, root) + files, enumeration_issues = enumerate_test_files(inputs) + if enumeration_issues: + details = "; ".join(issue.message for issue in enumeration_issues) + raise CollectionManifestError(f"cannot fingerprint collection roots: {details}") + dependency_files = _local_import_dependencies(files, root) + fingerprint_files = tuple(sorted({*files, *dependency_files}, key=lambda path: path.as_posix())) + fingerprints = tuple( + SourceFingerprint( + path=_relative_to_root(path, root), + sha256=_hash_source(path), + ) + for path in fingerprint_files + ) + portable_tests = tuple(_portable_spec(item.spec, root) for item in collection.items) + decisions = tuple( + ModuleShardingDecision( + path=_relative_to_root(decision.path, root), + module_name=decision.module_name, + eligible=decision.eligible, + blockers=decision.blockers, + ) + for decision in assess_collection_sharding(collection) + ) + manifest = TrustedCollectionManifest( + collection_roots=collection_roots, + files=fingerprints, + tests=portable_tests, + issues=collection.issues, + sharding=decisions, + ) + # One validation path keeps hand-built and decoded manifests subject to the + # same invariants as manifests produced by this helper. + return trusted_collection_manifest_from_dict(trusted_collection_manifest_to_dict(manifest)) + + +def trusted_collection_manifest_to_dict( + manifest: TrustedCollectionManifest, +) -> dict[str, Any]: + """Return the versioned JSON-compatible representation of a manifest.""" + + return { + "format": COLLECTION_MANIFEST_FORMAT, + "schema_version": COLLECTION_MANIFEST_SCHEMA_VERSION, + "collection_roots": list(manifest.collection_roots), + "files": [ + {"path": fingerprint.path, "sha256": fingerprint.sha256} + for fingerprint in manifest.files + ], + "tests": [ + { + "id": spec.id, + "path": spec.path, + "module_name": spec.module_name, + "function_name": spec.function_name, + "display_name": spec.display_name, + "parameters": _redacted_parameters(spec.parameters), + "case_id": spec.case_id, + "tags": sorted(spec.tags), + "skip_reason": spec.skip_reason, + "xfail_reason": spec.xfail_reason, + "timeout": spec.timeout, + "source_line": spec.source_line, + } + for spec in manifest.tests + ], + "issues": [ + { + "path": issue.path, + "message": issue.message, + "traceback": issue.traceback, + } + for issue in manifest.issues + ], + "sharding": [ + { + "path": decision.path, + "module_name": decision.module_name, + "eligible": decision.eligible, + "blockers": list(decision.blockers), + } + for decision in manifest.sharding + ], + } + + +def validate_trusted_collection_manifest( + manifest: TrustedCollectionManifest, +) -> TrustedCollectionManifest: + """Return a canonical inert copy or raise ``CollectionManifestError``.""" + + return trusted_collection_manifest_from_dict(trusted_collection_manifest_to_dict(manifest)) + + +def serialize_trusted_collection_manifest(manifest: TrustedCollectionManifest) -> str: + """Serialize a trusted collection manifest as deterministic JSON.""" + + validated = validate_trusted_collection_manifest(manifest) + return json.dumps( + trusted_collection_manifest_to_dict(validated), + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise CollectionManifestError(f"duplicate JSON object key: {key!r}") + result[key] = value + return result + + +def _object(value: object, *, name: str, keys: frozenset[str]) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise CollectionManifestError(f"{name} must be an object") + actual = frozenset(value) + if actual != keys: + missing = sorted(keys - actual) + unexpected = sorted(actual - keys) + raise CollectionManifestError( + f"{name} has invalid fields (missing={missing}, unexpected={unexpected})" + ) + if any(not isinstance(key, str) for key in value): + raise CollectionManifestError(f"{name} keys must be strings") + return value + + +def _array(value: object, *, name: str) -> Sequence[Any]: + if not isinstance(value, list): + raise CollectionManifestError(f"{name} must be an array") + return value + + +def _string(value: object, *, name: str, optional: bool = False) -> str | None: + if value is None and optional: + return None + if not isinstance(value, str) or not value: + raise CollectionManifestError(f"{name} must be a non-empty string") + return value + + +def _optional_timeout(value: object, *, name: str) -> float | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CollectionManifestError(f"{name} must be a positive finite number or null") + rendered = float(value) + if not math.isfinite(rendered) or rendered <= 0.0: + raise CollectionManifestError(f"{name} must be a positive finite number or null") + return rendered + + +def _optional_source_line(value: object, *, name: str) -> int | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise CollectionManifestError(f"{name} must be a positive integer or null") + return value + + +def _unique_strings(value: object, *, name: str) -> tuple[str, ...]: + raw = _array(value, name=name) + items: list[str] = [] + seen: set[str] = set() + for index, item in enumerate(raw): + rendered = _string(item, name=f"{name}[{index}]") + assert rendered is not None + if rendered in seen: + raise CollectionManifestError(f"{name} contains duplicate {rendered!r}") + seen.add(rendered) + items.append(rendered) + return tuple(items) + + +_TOP_LEVEL_FIELDS = frozenset( + { + "format", + "schema_version", + "collection_roots", + "files", + "tests", + "issues", + "sharding", + } +) +_FILE_FIELDS = frozenset({"path", "sha256"}) +_TEST_FIELDS = frozenset( + { + "id", + "path", + "module_name", + "function_name", + "display_name", + "parameters", + "case_id", + "tags", + "skip_reason", + "xfail_reason", + "timeout", + "source_line", + } +) +_ISSUE_FIELDS = frozenset({"path", "message", "traceback"}) +_SHARDING_FIELDS = frozenset({"path", "module_name", "eligible", "blockers"}) + + +def trusted_collection_manifest_from_dict(data: Mapping[str, Any]) -> TrustedCollectionManifest: + """Validate and reconstruct an inert trusted collection manifest.""" + + top = _object(data, name="manifest", keys=_TOP_LEVEL_FIELDS) + if top["format"] != COLLECTION_MANIFEST_FORMAT: + raise CollectionManifestError("unsupported collection manifest format") + version = top["schema_version"] + if isinstance(version, bool) or version != COLLECTION_MANIFEST_SCHEMA_VERSION: + raise CollectionManifestError(f"unsupported collection manifest schema: {version!r}") + + roots = tuple( + _safe_relative_path(value, allow_dot=True) + for value in _unique_strings(top["collection_roots"], name="collection_roots") + ) + if not roots: + raise CollectionManifestError("collection_roots must not be empty") + + files: list[SourceFingerprint] = [] + file_paths: set[str] = set() + for index, raw in enumerate(_array(top["files"], name="files")): + item = _object(raw, name=f"files[{index}]", keys=_FILE_FIELDS) + path = _safe_relative_path(item["path"]) + digest = _string(item["sha256"], name=f"files[{index}].sha256") + assert digest is not None + if not _SHA256.fullmatch(digest): + raise CollectionManifestError(f"files[{index}].sha256 is not a lowercase SHA-256") + if path in file_paths: + raise CollectionManifestError(f"duplicate source fingerprint path: {path!r}") + file_paths.add(path) + files.append(SourceFingerprint(path, digest)) + + tests: list[TestSpec] = [] + test_ids: set[str] = set() + for index, raw in enumerate(_array(top["tests"], name="tests")): + item = _object(raw, name=f"tests[{index}]", keys=_TEST_FIELDS) + test_id = _string(item["id"], name=f"tests[{index}].id") + path = _safe_relative_path(item["path"]) + assert test_id is not None + if test_id in test_ids: + raise CollectionManifestError(f"duplicate test id: {test_id!r}") + if path not in file_paths: + raise CollectionManifestError(f"test path has no source fingerprint: {path!r}") + parameters = item["parameters"] + if not isinstance(parameters, Mapping) or any( + not isinstance(key, str) for key in parameters + ): + raise CollectionManifestError(f"tests[{index}].parameters must be an object") + if any(value != REDACTED_PARAMETER_VALUE for value in parameters.values()): + raise CollectionManifestError( + f"tests[{index}].parameters must contain only redacted values" + ) + inert_parameters = _redacted_parameters(parameters) + tags = _unique_strings(item["tags"], name=f"tests[{index}].tags") + module_name = _string(item["module_name"], name=f"tests[{index}].module_name") + function_name = _string(item["function_name"], name=f"tests[{index}].function_name") + display_name = _string(item["display_name"], name=f"tests[{index}].display_name") + assert module_name is not None and function_name is not None and display_name is not None + tests.append( + TestSpec( + id=test_id, + path=path, + module_name=module_name, + function_name=function_name, + display_name=display_name, + parameters=dict(inert_parameters), + case_id=_string(item["case_id"], name=f"tests[{index}].case_id", optional=True), + tags=frozenset(tags), + skip_reason=_string( + item["skip_reason"], name=f"tests[{index}].skip_reason", optional=True + ), + xfail_reason=_string( + item["xfail_reason"], name=f"tests[{index}].xfail_reason", optional=True + ), + timeout=_optional_timeout(item["timeout"], name=f"tests[{index}].timeout"), + source_line=_optional_source_line( + item["source_line"], name=f"tests[{index}].source_line" + ), + ) + ) + test_ids.add(test_id) + + issues: list[CollectionIssue] = [] + for index, raw in enumerate(_array(top["issues"], name="issues")): + item = _object(raw, name=f"issues[{index}]", keys=_ISSUE_FIELDS) + issue_path = _string(item["path"], name=f"issues[{index}].path") + message = _string(item["message"], name=f"issues[{index}].message") + assert issue_path is not None and message is not None + issues.append( + CollectionIssue( + path=issue_path, + message=message, + traceback=_string( + item["traceback"], name=f"issues[{index}].traceback", optional=True + ), + ) + ) + + sharding: list[ModuleShardingDecision] = [] + sharding_paths: set[str] = set() + for index, raw in enumerate(_array(top["sharding"], name="sharding")): + item = _object(raw, name=f"sharding[{index}]", keys=_SHARDING_FIELDS) + path = _safe_relative_path(item["path"]) + if path not in file_paths: + raise CollectionManifestError(f"sharding path has no source fingerprint: {path!r}") + if path in sharding_paths: + raise CollectionManifestError(f"duplicate sharding decision path: {path!r}") + eligible = item["eligible"] + if not isinstance(eligible, bool): + raise CollectionManifestError(f"sharding[{index}].eligible must be a boolean") + blockers = _unique_strings(item["blockers"], name=f"sharding[{index}].blockers") + if eligible == bool(blockers): + raise CollectionManifestError( + f"sharding[{index}] eligibility is inconsistent with its blockers" + ) + module_name = _string(item["module_name"], name=f"sharding[{index}].module_name") + assert module_name is not None + sharding.append(ModuleShardingDecision(path, module_name, eligible, blockers)) + sharding_paths.add(path) + + test_modules = {(spec.path, spec.module_name) for spec in tests} + for decision in sharding: + if (decision.path, decision.module_name) not in test_modules: + raise CollectionManifestError( + f"sharding decision does not identify a collected module: {decision.path!r}" + ) + + return TrustedCollectionManifest( + collection_roots=roots, + files=tuple(files), + tests=tuple(tests), + issues=tuple(issues), + sharding=tuple(sharding), + ) + + +def deserialize_trusted_collection_manifest( + data: str | bytes | bytearray | Mapping[str, Any], +) -> TrustedCollectionManifest: + """Decode and validate trusted collection manifest JSON or mapping data.""" + + if isinstance(data, Mapping): + decoded = data + else: + try: + decoded = json.loads(data, object_pairs_hook=_reject_duplicate_json_keys) + except CollectionManifestError: + raise + except (TypeError, ValueError, UnicodeDecodeError) as error: + raise CollectionManifestError("invalid collection manifest JSON") from error + if not isinstance(decoded, Mapping): + raise CollectionManifestError("manifest must be a JSON object") + return trusted_collection_manifest_from_dict(decoded) + + +def verify_trusted_collection_manifest( + manifest: TrustedCollectionManifest, + paths: str | Path | Iterable[str | Path], + *, + project_root: str | Path | None = None, +) -> bool: + """Return whether roots, inventory, and source hashes still match exactly. + + Any malformed value, missing/added file, unreadable source, or digest + mismatch returns ``False``. The runner can therefore fall back to its + normal isolated collection process instead of turning staleness into a run + failure. + """ + + try: + validated = validate_trusted_collection_manifest(manifest) + root = _project_root(project_root) + collection_roots, inputs = _collection_inputs(paths, root) + if collection_roots != validated.collection_roots: + return False + files, issues = enumerate_test_files(inputs) + if issues: + return False + expected = {fingerprint.path: fingerprint.sha256 for fingerprint in validated.files} + actual_test_paths = {_relative_to_root(path, root) for path in files} + if not actual_test_paths.issubset(expected): + return False + actual: dict[str, str] = {} + for relative_path in expected: + source = (root / relative_path).resolve() + if _relative_to_root(source, root) != relative_path: + return False + actual[relative_path] = _hash_source(source) + return actual == expected + except (CollectionManifestError, OSError, TypeError, ValueError): + return False + + +__all__ = [ + "COLLECTION_MANIFEST_FORMAT", + "COLLECTION_MANIFEST_SCHEMA_VERSION", + "CollectionManifestError", + "ModuleShardingDecision", + "ShardingPolicy", + "SourceFingerprint", + "TrustedCollectionManifest", + "assess_collection_sharding", + "build_trusted_collection_manifest", + "deserialize_trusted_collection_manifest", + "serialize_trusted_collection_manifest", + "trusted_collection_manifest_from_dict", + "trusted_collection_manifest_to_dict", + "validate_trusted_collection_manifest", + "verify_trusted_collection_manifest", +] diff --git a/src/testenix/tuning.py b/src/testenix/tuning.py new file mode 100644 index 0000000..e6b1e20 --- /dev/null +++ b/src/testenix/tuning.py @@ -0,0 +1,1570 @@ +"""Adaptive worker selection and reproducible project-local tuning. + +The fast path in :func:`resolve_adaptive_workers` is a pure cost model. It +uses the execution units that Testenix can actually schedule (normal modules +and individually isolated timeout tests), historical test durations, and a +conservative process-startup estimate. The slower :func:`run_tuning` service +executes an explicit benchmark when a project wants an evidence-based fixed +worker count. +""" + +from __future__ import annotations + +import ctypes +import functools +import hashlib +import json +import math +import os +import signal +import statistics +import subprocess +import sys +import tempfile +import threading +import time +from collections.abc import Callable, Mapping, Sequence, Sized +from collections.abc import Set as AbstractSet +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +from testenix.contracts import CollectionIssue, RunResult, Status, TestResult, TestSpec + +if TYPE_CHECKING: + from testenix.config import TestenixConfig + +SpawnMethod = Literal["fork", "forkserver", "spawn"] +NativeMeasure = Callable[[Sequence[str], "TestenixConfig"], tuple[float, RunResult]] +PytestMeasure = Callable[[Sequence[str]], tuple[float, int]] + +_MARGINAL_SPAWN_FRACTION = 0.25 +_NEAR_BEST_FRACTION = 0.02 +_NEAR_BEST_SECONDS = 0.005 +_COLD_START_WORKERS = 4 +_MINIMUM_HISTORY_COVERAGE = 0.5 +DEFAULT_TUNING_RUN_TIMEOUT = 300.0 +_PROCESS_TERMINATION_GRACE = 2.0 +_PROCESS_TRACKER_INTERVAL = 0.02 +_SOURCE_SUFFIXES = frozenset({".py", ".pyi", ".toml"}) +_SOURCE_SCAN_EXCLUSIONS = frozenset( + { + ".git", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".testenix", + ".tox", + ".venv", + "__pycache__", + "node_modules", + } +) + +_ProcessIdentity = tuple[int, int] | str + + +class TuningError(RuntimeError): + """Raised when a tuning sample cannot produce a trustworthy recommendation.""" + + +@dataclass(slots=True) +class _WindowsKillJob: + """Small kill-on-close Job Object wrapper used only on Windows.""" + + kernel32: Any + handle: Any + closed: bool = False + + def terminate(self, exit_code: int = 1) -> bool: + if self.closed: + return True + try: + return bool(self.kernel32.TerminateJobObject(self.handle, exit_code)) + except Exception: + return False + + def close(self) -> bool: + if self.closed: + return True + try: + closed = bool(self.kernel32.CloseHandle(self.handle)) + except Exception: + return False + if closed: + self.closed = True + return closed + + +@functools.lru_cache(maxsize=1) +def _darwin_child_lister() -> Any | None: + """Return libproc's direct-child query without timing ``ps`` on macOS.""" + + if sys.platform != "darwin": + return None + try: + library = ctypes.CDLL("/usr/lib/libproc.dylib") + function = library.proc_listchildpids + function.argtypes = (ctypes.c_int, ctypes.c_void_p, ctypes.c_int) + function.restype = ctypes.c_int + return function + except (AttributeError, OSError): + return None + + +@functools.lru_cache(maxsize=1) +def _darwin_identity_probe() -> tuple[Any, type[ctypes.Structure]] | None: + if sys.platform != "darwin": + return None + try: + + class _BsdInfo(ctypes.Structure): + _fields_ = [ + ("flags", ctypes.c_uint32), + ("status", ctypes.c_uint32), + ("xstatus", ctypes.c_uint32), + ("pid", ctypes.c_uint32), + ("ppid", ctypes.c_uint32), + ("uid", ctypes.c_uint32), + ("gid", ctypes.c_uint32), + ("ruid", ctypes.c_uint32), + ("rgid", ctypes.c_uint32), + ("svuid", ctypes.c_uint32), + ("svgid", ctypes.c_uint32), + ("reserved", ctypes.c_uint32), + ("comm", ctypes.c_char * 16), + ("name", ctypes.c_char * 32), + ("nfiles", ctypes.c_uint32), + ("pgid", ctypes.c_uint32), + ("pjobc", ctypes.c_uint32), + ("tty_device", ctypes.c_uint32), + ("tty_pgid", ctypes.c_uint32), + ("nice", ctypes.c_int32), + ("start_seconds", ctypes.c_uint64), + ("start_microseconds", ctypes.c_uint64), + ] + + library = ctypes.CDLL("/usr/lib/libproc.dylib") + function = library.proc_pidinfo + function.argtypes = ( + ctypes.c_int, + ctypes.c_int, + ctypes.c_uint64, + ctypes.c_void_p, + ctypes.c_int, + ) + function.restype = ctypes.c_int + return function, _BsdInfo + except (AttributeError, OSError): + return None + + +def _process_identity(pid: int) -> _ProcessIdentity | None: + """Return a creation token so cleanup never signals a recycled PID.""" + + if sys.platform.startswith("linux"): + try: + stat_line = Path(f"/proc/{pid}/stat").read_text(encoding="ascii") + fields_after_name = stat_line.rsplit(")", 1)[1].split() + return fields_after_name[19] + except (IndexError, OSError): + return None + if sys.platform == "darwin": + probe = _darwin_identity_probe() + if probe is None: + return None + function, structure = probe + information = structure() + size = ctypes.sizeof(information) + if function(pid, 3, 0, ctypes.byref(information), size) != size: + return None + return int(information.start_seconds), int(information.start_microseconds) + try: + completed = subprocess.run( + ("ps", "-o", "lstart=", "-p", str(pid)), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + check=False, + timeout=_PROCESS_TERMINATION_GRACE, + ) + except (OSError, subprocess.TimeoutExpired): + return None + token = completed.stdout.strip() + return token if completed.returncode == 0 and token else None + + +def _posix_direct_children(pid: int) -> tuple[int, ...]: + """Read direct children cheaply enough for continuous timing containment.""" + + if sys.platform.startswith("linux"): + children: set[int] = set() + try: + for child_file in Path(f"/proc/{pid}/task").glob("*/children"): + raw = child_file.read_text(encoding="ascii") + children.update(int(value) for value in raw.split()) + except (OSError, ValueError): + pass + return tuple(sorted(children)) + if sys.platform == "darwin": + function = _darwin_child_lister() + if function is not None: + values = (ctypes.c_int * 4096)() + count = function(pid, values, ctypes.sizeof(values)) + if count > 0: + return tuple(values[: min(count, len(values))]) + return () + return _posix_descendant_pids(pid) + + +class _PosixTreeTracker: + """Remember workers before a short-lived coordinator can orphan them.""" + + def __init__(self, root_pid: int) -> None: + self.root_pid = root_pid + self._root_identity = _process_identity(root_pid) + self._identities: dict[int, _ProcessIdentity] = {} + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + self._capture() + self._thread.start() + + def _capture(self) -> None: + with self._lock: + known = { + pid: identity + for pid, identity in self._identities.items() + if _process_identity(pid) == identity + } + root_is_current = ( + self._root_identity is not None + and _process_identity(self.root_pid) == self._root_identity + ) + pending = [*([self.root_pid] if root_is_current else []), *known] + visited: set[int] = set() + discovered: set[int] = set() + while pending: + parent = pending.pop() + if parent in visited: + continue + visited.add(parent) + for child in _posix_direct_children(parent): + if child <= 0 or child == os.getpid(): + continue + if child not in discovered: + discovered.add(child) + pending.append(child) + live = dict(known) + for pid in discovered: + identity = _process_identity(pid) + if identity is not None: + live[pid] = identity + with self._lock: + self._identities = live + + def _run(self) -> None: + while not self._stop.wait(_PROCESS_TRACKER_INTERVAL): + self._capture() + + def stop(self) -> dict[int, _ProcessIdentity]: + self._stop.set() + self._thread.join(timeout=_PROCESS_TERMINATION_GRACE) + self._capture() + with self._lock: + return { + pid: identity + for pid, identity in self._identities.items() + if _process_identity(pid) == identity + } + + +@dataclass(frozen=True, slots=True) +class ExecutionUnitEstimate: + """One independently schedulable native execution unit.""" + + key: str + duration: float + tests: int + isolated: bool = False + + +@dataclass(frozen=True, slots=True) +class WorkerEstimate: + """Predicted makespan for one candidate worker count.""" + + workers: int + workload_seconds: float + startup_seconds: float + + @property + def total_seconds(self) -> float: + return self.workload_seconds + self.startup_seconds + + +@dataclass(frozen=True, slots=True) +class TuningCandidate: + """Measured timings for one explicit worker count.""" + + workers: int + samples: tuple[float, ...] + + @property + def median(self) -> float: + return float(statistics.median(self.samples)) + + @property + def minimum(self) -> float: + return min(self.samples) + + @property + def maximum(self) -> float: + return max(self.samples) + + def to_dict(self) -> dict[str, Any]: + return { + "workers": self.workers, + "samples": list(self.samples), + "median": self.median, + "min": self.minimum, + "max": self.maximum, + } + + +@dataclass(frozen=True, slots=True) +class TuningReport: + """Complete native tuning result with an optional pytest comparison.""" + + paths: tuple[str, ...] + warmups: int + repeats: int + discovered_tests: int + execution_units: int + model_recommendation: int + recommended_workers: int + candidates: tuple[TuningCandidate, ...] + pytest_paths: tuple[str, ...] = () + pytest_samples: tuple[float, ...] = () + shard_modules: bool = False + manifest_used: bool = False + run_timeout: float = DEFAULT_TUNING_RUN_TIMEOUT + + @property + def pytest_median(self) -> float | None: + if not self.pytest_samples: + return None + return float(statistics.median(self.pytest_samples)) + + @property + def native_median(self) -> float: + for candidate in self.candidates: + if candidate.workers == self.recommended_workers: + return candidate.median + raise TuningError("recommended worker count has no measured candidate") + + @property + def pytest_over_native(self) -> float | None: + pytest_median = self.pytest_median + if pytest_median is None or self.native_median <= 0.0: + return None + return pytest_median / self.native_median + + def to_dict(self) -> dict[str, Any]: + document: dict[str, Any] = { + "schema": "testenix.tuning-report", + "schema_version": 1, + "paths": list(self.paths), + "warmups": self.warmups, + "repeats": self.repeats, + "discovered_tests": self.discovered_tests, + "execution_units": self.execution_units, + "model_recommendation": self.model_recommendation, + "recommended_workers": self.recommended_workers, + "execution": { + "fresh_process_wall_clock": True, + "history": "disabled", + "manifest": self.manifest_used, + "shard_modules": self.shard_modules, + "run_timeout_seconds": self.run_timeout, + }, + "candidates": [candidate.to_dict() for candidate in self.candidates], + } + if self.pytest_paths: + document["pytest"] = { + "paths": list(self.pytest_paths), + "samples": list(self.pytest_samples), + "median": self.pytest_median, + "pytest_over_native": self.pytest_over_native, + "inventory_equivalence_verified": False, + } + return document + + def to_json(self) -> str: + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + +def execution_units( + specs: Sequence[TestSpec], + durations: Mapping[str, float], + *, + shardable_paths: AbstractSet[str] = frozenset(), +) -> tuple[ExecutionUnitEstimate, ...]: + """Build the same module/timeout affinity units as the native scheduler.""" + + selected_ids = {spec.id for spec in specs} + known = tuple( + float(value) + for test_id, value in durations.items() + if test_id in selected_ids + if isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(float(value)) + and float(value) >= 0.0 + ) + fallback = float(statistics.median(known)) if known else 1.0 + modules: dict[str, list[TestSpec]] = {} + isolated: list[TestSpec] = [] + sharded: list[TestSpec] = [] + for spec in specs: + if spec.timeout is not None: + isolated.append(spec) + elif spec.path in shardable_paths: + sharded.append(spec) + else: + modules.setdefault(spec.path, []).append(spec) + + units: list[ExecutionUnitEstimate] = [] + for path, module_specs in sorted(modules.items()): + duration = sum(_duration_for(spec.id, durations, fallback) for spec in module_specs) + units.append( + ExecutionUnitEstimate( + key=f"module:{path}", + duration=duration, + tests=len(module_specs), + ) + ) + for spec in sorted(isolated, key=lambda item: item.id): + units.append( + ExecutionUnitEstimate( + key=f"isolated:{spec.id}", + duration=_duration_for(spec.id, durations, fallback), + tests=1, + isolated=True, + ) + ) + for spec in sorted(sharded, key=lambda item: item.id): + units.append( + ExecutionUnitEstimate( + key=f"sharded:{spec.id}", + duration=_duration_for(spec.id, durations, fallback), + tests=1, + ) + ) + return tuple(units) + + +def estimate_spawn_cost( + spawn_method: SpawnMethod = "spawn", + *, + platform: str | None = None, +) -> float: + """Return a conservative per-run process startup estimate in seconds.""" + + if spawn_method == "fork": + return 0.010 + if spawn_method == "forkserver": + return 0.030 + effective_platform = sys.platform if platform is None else platform + if effective_platform.startswith("win"): + return 0.120 + if effective_platform == "darwin": + return 0.080 + return 0.060 + + +def worker_estimates( + units: Sequence[ExecutionUnitEstimate], + *, + maximum_workers: int, + spawn_cost: float, +) -> tuple[WorkerEstimate, ...]: + """Predict every feasible worker count using deterministic LPT placement.""" + + if maximum_workers < 1: + raise ValueError("maximum_workers must be at least 1") + if not math.isfinite(spawn_cost) or spawn_cost < 0.0: + raise ValueError("spawn_cost must be a finite non-negative number") + if not units: + return (WorkerEstimate(1, 0.0, spawn_cost),) + + limit = min(maximum_workers, len(units)) + ordered = sorted(units, key=lambda unit: (-unit.duration, unit.key)) + estimates: list[WorkerEstimate] = [] + for workers in range(1, limit + 1): + loads = [0.0] * workers + item_counts = [0] * workers + for unit in ordered: + shard = min( + range(workers), + key=lambda index: (loads[index], item_counts[index], index), + ) + loads[shard] += unit.duration + item_counts[shard] += 1 + startup = spawn_cost * (1.0 + _MARGINAL_SPAWN_FRACTION * (workers - 1)) + estimates.append(WorkerEstimate(workers, max(loads), startup)) + return tuple(estimates) + + +def resolve_adaptive_workers( + config: TestenixConfig, + selected_specs: Sequence[TestSpec], + durations: Mapping[str, float], + *, + spawn_method: SpawnMethod = "spawn", + spawn_cost: float | None = None, + cpu_count: int | None = None, + shardable_paths: AbstractSet[str] = frozenset(), +) -> int: + """Resolve ``workers = "auto"`` from schedulable work and local costs. + + Explicit integer configuration is returned unchanged. Auto mode is capped + by both CPU capacity and the number of independently schedulable units, then + chooses the smallest count within a narrow tolerance of the predicted best + makespan. That bias avoids starting extra processes for negligible gains. + """ + + if config.workers != "auto": + return config.workers + units = execution_units(selected_specs, durations, shardable_paths=shardable_paths) + if not units: + return 1 + capacity = _schedulable_cpu_capacity(cpu_count) + maximum = min(capacity, len(units)) + if not _has_reliable_durations( + selected_specs, + durations, + shardable_paths=shardable_paths, + ): + # Unknown tests are not one-second measurements. Modelling them as + # such makes tiny/no-op suites look massively parallel and recreates + # the CPU-count oversubscription that adaptive mode is meant to avoid. + return min(_COLD_START_WORKERS, maximum) + cost = estimate_spawn_cost(spawn_method) if spawn_cost is None else spawn_cost + estimates = worker_estimates( + units, + maximum_workers=maximum, + spawn_cost=cost, + ) + best = min(estimate.total_seconds for estimate in estimates) + tolerance = max(_NEAR_BEST_SECONDS, best * _NEAR_BEST_FRACTION) + return min( + estimate.workers for estimate in estimates if estimate.total_seconds <= best + tolerance + ) + + +def default_worker_candidates( + unit_count: int, + *, + model_recommendation: int, + cpu_count: int | None = None, +) -> tuple[int, ...]: + """Return a conservative automatic sweep within the cold-start ceiling. + + Automatic tuning must not turn a large host CPU count into an experiment + that launches hundreds of processes. The adaptive model recommendation is + retained when it is inside the conservative ceiling. Users who have + evidence that a larger process count is useful can still request it + explicitly with ``--candidates``. + """ + + if unit_count < 1: + return (1,) + capacity = _schedulable_cpu_capacity(cpu_count) + limit = min(unit_count, capacity, _COLD_START_WORKERS) + recommendation = min(limit, max(1, model_recommendation)) + candidates = {1, recommendation, limit} + workers = 2 + while workers < limit: + candidates.add(workers) + workers *= 2 + return tuple(sorted(candidates)) + + +def _schedulable_cpu_capacity(cpu_count: int | None = None) -> int: + """Return the process-visible CPU capacity, not only the host total.""" + + if cpu_count is not None: + return max(1, cpu_count) + + detected: list[int] = [] + process_cpu_count = getattr(os, "process_cpu_count", None) + if callable(process_cpu_count): + try: + value = process_cpu_count() + except OSError: + value = None + if isinstance(value, int) and not isinstance(value, bool) and value > 0: + detected.append(value) + + sched_getaffinity = getattr(os, "sched_getaffinity", None) + if callable(sched_getaffinity): + try: + affinity = sched_getaffinity(0) + affinity_count = len(affinity) if isinstance(affinity, Sized) else 0 + except OSError: + affinity_count = 0 + if affinity_count > 0: + detected.append(affinity_count) + + host_count = os.cpu_count() + if host_count is not None and host_count > 0: + detected.append(host_count) + return min(detected, default=1) + + +def run_tuning( + paths: Sequence[str], + config: TestenixConfig, + *, + candidates: Sequence[int] | None = None, + warmups: int = 1, + repeats: int = 5, + run_timeout: float = DEFAULT_TUNING_RUN_TIMEOUT, + pytest_paths: Sequence[str] = (), + native_measure: NativeMeasure | None = None, + pytest_measure: PytestMeasure | None = None, +) -> TuningReport: + """Measure native worker counts without mutating Testenix history. + + A one-worker probe establishes the inventory and supplies fresh duration + estimates to the adaptive model. Candidate order reverses on alternating + rounds to reduce monotonic thermal/load drift. Every measured native run + must preserve the probe inventory and outcomes. + """ + + if warmups < 0: + raise ValueError("warmups must be at least 0") + if repeats < 1: + raise ValueError("repeats must be at least 1") + if not math.isfinite(run_timeout) or run_timeout <= 0.0: + raise ValueError("run_timeout must be a finite number greater than zero") + effective_paths = tuple(paths) if paths else config.paths + effective_pytest_paths = tuple(pytest_paths) + snapshot_paths = (*effective_paths, *effective_pytest_paths) + if config.manifest_path is not None: + snapshot_paths = (*snapshot_paths, str(config.manifest_path)) + source_snapshot: tuple[tuple[str, str], ...] | None + require_source_snapshot = native_measure is None or ( + bool(effective_pytest_paths) and pytest_measure is None + ) + try: + source_snapshot = _tuning_source_snapshot(snapshot_paths) + except TuningError: + if require_source_snapshot: + raise + # Injected measurement callbacks are a library/testing seam and may + # deliberately use virtual paths. Real subprocess measurements never + # bypass source immutability checks. + source_snapshot = None + + def require_unchanged_sources(label: str) -> None: + if source_snapshot is None: + return + if _tuning_source_snapshot(snapshot_paths) != source_snapshot: + raise TuningError( + f"project sources changed during {label}; tuning result was discarded" + ) + + measure_native: NativeMeasure + if native_measure is None: + + def default_native_measure( + measured_paths: Sequence[str], measured_config: TestenixConfig + ) -> tuple[float, RunResult]: + return _measure_native(measured_paths, measured_config, timeout=run_timeout) + + measure_native = default_native_measure + else: + measure_native = native_measure + base = config.with_overrides(history_path=None, json_path=None, junit_path=None) + + _, probe = measure_native(effective_paths, base.with_overrides(workers=1)) + require_unchanged_sources("one-worker probe") + _require_green(probe, "one-worker probe") + specs = tuple(result.test for result in probe.tests) + durations = {result.test.id: result.duration for result in probe.tests} + model_config = base.with_overrides(workers="auto") + shardable_paths = frozenset(probe.shardable_paths) if config.shard_modules else frozenset() + model_recommendation = resolve_adaptive_workers( + model_config, + specs, + durations, + shardable_paths=shardable_paths, + ) + units = execution_units(specs, durations, shardable_paths=shardable_paths) + + if candidates is None: + selected_candidates = default_worker_candidates( + len(units), + model_recommendation=model_recommendation, + ) + else: + requested_candidates = _normalise_candidates(candidates) + unit_limit = max(1, len(units)) + selected_candidates = tuple( + sorted({min(workers, unit_limit) for workers in requested_candidates}) + ) + signature = _result_signature(probe) + + measure_pytest: PytestMeasure + if pytest_measure is None: + + def default_pytest_measure(measured_paths: Sequence[str]) -> tuple[float, int]: + return _measure_pytest(measured_paths, timeout=run_timeout) + + measure_pytest = default_pytest_measure + else: + measure_pytest = pytest_measure + pytest_samples: list[float] = [] + + for warmup in range(warmups): + if effective_pytest_paths and warmup % 2: + elapsed, return_code = measure_pytest(effective_pytest_paths) + require_unchanged_sources("pytest warmup") + _validate_pytest_sample(elapsed, return_code, label="warmup") + for workers in selected_candidates: + _, result = measure_native(effective_paths, base.with_overrides(workers=workers)) + require_unchanged_sources(f"warmup with {workers} workers") + _require_matching(result, signature, f"warmup with {workers} workers") + if effective_pytest_paths and warmup % 2 == 0: + elapsed, return_code = measure_pytest(effective_pytest_paths) + require_unchanged_sources("pytest warmup") + _validate_pytest_sample(elapsed, return_code, label="warmup") + + samples: dict[int, list[float]] = {workers: [] for workers in selected_candidates} + for repeat in range(repeats): + order = selected_candidates if repeat % 2 == 0 else tuple(reversed(selected_candidates)) + if effective_pytest_paths and repeat % 2: + elapsed, return_code = measure_pytest(effective_pytest_paths) + require_unchanged_sources("pytest sample") + _validate_pytest_sample(elapsed, return_code) + pytest_samples.append(elapsed) + for workers in order: + elapsed, result = measure_native( + effective_paths, + base.with_overrides(workers=workers), + ) + require_unchanged_sources(f"sample with {workers} workers") + _require_matching(result, signature, f"sample with {workers} workers") + if not math.isfinite(elapsed) or elapsed < 0.0: + raise TuningError("native timer returned an invalid duration") + samples[workers].append(elapsed) + if effective_pytest_paths and repeat % 2 == 0: + elapsed, return_code = measure_pytest(effective_pytest_paths) + require_unchanged_sources("pytest sample") + _validate_pytest_sample(elapsed, return_code) + pytest_samples.append(elapsed) + + measured = tuple( + TuningCandidate(workers=workers, samples=tuple(samples[workers])) + for workers in selected_candidates + ) + best_median = min(candidate.median for candidate in measured) + tolerance = max(_NEAR_BEST_SECONDS, best_median * _NEAR_BEST_FRACTION) + recommended = min( + candidate.workers for candidate in measured if candidate.median <= best_median + tolerance + ) + + return TuningReport( + paths=effective_paths, + warmups=warmups, + repeats=repeats, + discovered_tests=len(specs), + execution_units=len(units), + model_recommendation=model_recommendation, + recommended_workers=recommended, + candidates=measured, + pytest_paths=effective_pytest_paths, + pytest_samples=tuple(pytest_samples), + shard_modules=config.shard_modules, + manifest_used=config.manifest_path is not None, + run_timeout=run_timeout, + ) + + +def render_tuning_report(report: TuningReport) -> str: + """Render a compact, deterministic human-readable tuning table.""" + + lines = [ + ( + f"Testenix tuning | {report.discovered_tests} tests | " + f"{report.execution_units} execution units" + ), + "Measurement: fresh-process wall clock | history: disabled (--no-history)", + f"Per-run deadline: {report.run_timeout:g}s (platform-aware bounded cleanup)", + ( + "Scheduling: safe intra-module sharding" + if report.shard_modules + else "Scheduling: module affinity" + ), + "", + "workers median min max runs", + ] + for candidate in report.candidates: + marker = " *" if candidate.workers == report.recommended_workers else "" + lines.append( + f"{candidate.workers:>7} {candidate.median:>8.3f}s " + f"{candidate.minimum:>8.3f}s {candidate.maximum:>8.3f}s " + f"{len(candidate.samples):>4}{marker}" + ) + lines.extend( + ( + "", + f"Recommended workers: {report.recommended_workers}", + f"Adaptive model before measurement: {report.model_recommendation}", + ) + ) + if report.pytest_median is not None: + lines.append("pytest comparison: orientation only; inventory equivalence is not verified") + lines.append(f"pytest median: {report.pytest_median:.3f}s") + ratio = report.pytest_over_native + if ratio is not None: + lines.append(f"unverified pytest / recommended native: {ratio:.3f}x") + return "\n".join(lines) + "\n" + + +def _duration_for(test_id: str, durations: Mapping[str, float], fallback: float) -> float: + value = durations.get(test_id, fallback) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return fallback + parsed = float(value) + if not math.isfinite(parsed) or parsed < 0.0: + return fallback + return parsed + + +def _has_reliable_durations( + specs: Sequence[TestSpec], + durations: Mapping[str, float], + *, + shardable_paths: AbstractSet[str] = frozenset(), +) -> bool: + if not specs: + return False + valid_ids: set[str] = set() + for spec in specs: + value = durations.get(spec.id) + if isinstance(value, bool) or not isinstance(value, (int, float)): + continue + parsed = float(value) + if math.isfinite(parsed) and parsed >= 0.0: + valid_ids.add(spec.id) + if len(valid_ids) / len(specs) < _MINIMUM_HISTORY_COVERAGE: + return False + + # Global coverage alone can hide a brand-new, potentially dominant module. + # Require at least one real observation for every schedulable unit before + # using median fallback values inside the cost model. + normal_modules: dict[str, set[str]] = {} + isolated: list[str] = [] + for spec in specs: + if spec.timeout is None and spec.path not in shardable_paths: + normal_modules.setdefault(spec.path, set()).add(spec.id) + else: + isolated.append(spec.id) + return all(test_ids & valid_ids for test_ids in normal_modules.values()) and all( + test_id in valid_ids for test_id in isolated + ) + + +def _normalise_candidates(candidates: Sequence[int]) -> tuple[int, ...]: + if not candidates: + raise ValueError("candidates must contain at least one worker count") + normalised: set[int] = set() + for workers in candidates: + if isinstance(workers, bool) or not isinstance(workers, int) or workers < 1: + raise ValueError("candidate worker counts must be positive integers") + normalised.add(workers) + return tuple(sorted(normalised)) + + +def _tuning_source_snapshot(paths: Sequence[str]) -> tuple[tuple[str, str], ...]: + """Fingerprint project Python/TOML inputs and every explicit file path. + + Measurements are invalid if test bodies, imported project helpers, the + project configuration, or a trusted manifest changes between samples. + Virtual environments and common cache directories are excluded so + the snapshot remains project-local and stable. + """ + + selected: set[Path] = set() + scan_roots: set[Path] = {Path.cwd().resolve()} + for raw_path in paths: + candidate = Path(raw_path).expanduser() + if not candidate.exists() and "::" in raw_path: + # pytest node IDs are valid comparison inputs, while their source + # component remains the filesystem object that must be hashed. + candidate = Path(raw_path.split("::", 1)[0]).expanduser() + try: + resolved = candidate.resolve(strict=True) + except OSError as error: + raise TuningError(f"cannot fingerprint tuning input {candidate}: {error}") from error + if resolved.is_file(): + selected.add(resolved) + elif resolved.is_dir(): + scan_roots.add(resolved) + else: + raise TuningError(f"cannot fingerprint non-file tuning input: {candidate}") + + pending_roots = sorted(scan_roots, key=lambda item: str(item), reverse=True) + visited_roots: set[Path] = set() + while pending_roots: + root = pending_roots.pop().resolve() + if root in visited_roots: + continue + visited_roots.add(root) + try: + + def raise_walk_error(error: OSError) -> None: + raise error + + walker = os.walk(root, onerror=raise_walk_error, followlinks=False) + for directory, directory_names, file_names in walker: + parent = Path(directory) + retained_directories: list[str] = [] + for name in sorted(directory_names): + child = parent / name + if name in _SOURCE_SCAN_EXCLUSIONS or (child / "pyvenv.cfg").is_file(): + continue + if child.is_symlink(): + target = child.resolve(strict=True) + if target.is_dir() and target not in visited_roots: + pending_roots.append(target) + continue + retained_directories.append(name) + directory_names[:] = retained_directories + for name in sorted(file_names): + path = parent / name + if path.suffix in _SOURCE_SUFFIXES: + selected.add(path.resolve(strict=True)) + except OSError as error: + raise TuningError( + f"cannot fingerprint project sources below {root}: {error}" + ) from error + + fingerprints: list[tuple[str, str]] = [] + for source in sorted(selected, key=lambda item: str(item)): + fingerprints.append((str(source), _source_identity_fingerprint(source))) + return tuple(fingerprints) + + +def _source_identity_fingerprint(source: Path) -> str: + """Hash bytes plus immutable-enough metadata and reject an unstable read.""" + + try: + before = source.stat() + digest = hashlib.sha256(source.read_bytes()).hexdigest() + after = source.stat() + except OSError as error: + raise TuningError(f"cannot fingerprint project source {source}: {error}") from error + before_identity = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ) + after_identity = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) + if before_identity != after_identity: + raise TuningError(f"project source changed while being fingerprinted: {source}") + metadata = ":".join(str(value) for value in after_identity) + return f"{digest}:{metadata}" + + +def _result_signature(result: RunResult) -> tuple[tuple[str, str], ...]: + return tuple(sorted((item.test.id, item.status.value) for item in result.tests)) + + +def _require_green(result: RunResult, label: str) -> None: + if result.exit_code != 0: + raise TuningError(f"{label} failed with exit code {result.exit_code}") + + +def _require_matching( + result: RunResult, + signature: tuple[tuple[str, str], ...], + label: str, +) -> None: + _require_green(result, label) + if _result_signature(result) != signature: + raise TuningError(f"{label} produced a different test inventory or outcomes") + + +def _validate_pytest_sample( + elapsed: float, + return_code: int, + *, + label: str = "sample", +) -> None: + if return_code != 0: + raise TuningError(f"pytest {label} failed with exit code {return_code}") + if not math.isfinite(elapsed) or elapsed < 0.0: + raise TuningError("pytest timer returned an invalid duration") + + +def _measure_native( + paths: Sequence[str], + config: TestenixConfig, + *, + timeout: float = DEFAULT_TUNING_RUN_TIMEOUT, +) -> tuple[float, RunResult]: + with tempfile.TemporaryDirectory(prefix="testenix-tune-") as directory: + report_path = Path(directory) / "result.json" + config_path = Path(directory) / "pyproject.toml" + config_path.write_text(_tuning_config_toml(config), encoding="utf-8") + command = [ + sys.executable, + "-m", + "testenix", + "--config", + str(config_path), + "run", + "--no-history", + "--json", + str(report_path), + "--no-color", + "--quiet", + ] + command.extend(paths) + + started = time.perf_counter() + return_code = _run_bounded_process( + command, + env=_tuning_environment(), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=timeout, + label="native Testenix sample", + ) + elapsed = time.perf_counter() - started + try: + document = json.loads(report_path.read_text(encoding="utf-8")) + result = _run_result_from_dict(document) + except (OSError, json.JSONDecodeError, TuningError) as error: + raise TuningError( + f"native timing subprocess failed with exit code {return_code}" + ) from error + if return_code != result.exit_code: + raise TuningError("native timing subprocess exit code does not match its result report") + return elapsed, result + + +def _tuning_config_toml(config: TestenixConfig) -> str: + workers = json.dumps(config.workers) if config.workers == "auto" else str(config.workers) + lines = [ + "[tool.testenix]", + f"workers = {workers}", + f"retries = {config.retries}", + "history = false", + f"shard_modules = {'true' if config.shard_modules else 'false'}", + ] + if config.timeout is not None: + lines.append(f"timeout = {config.timeout!r}") + if config.tags: + rendered_tags = ", ".join(json.dumps(tag, ensure_ascii=True) for tag in config.tags) + lines.append(f"tags = [{rendered_tags}]") + if config.manifest_path is not None: + lines.append(f"manifest = {json.dumps(str(config.manifest_path), ensure_ascii=True)}") + return "\n".join(lines) + "\n" + + +def _measure_pytest( + paths: Sequence[str], + *, + timeout: float = DEFAULT_TUNING_RUN_TIMEOUT, +) -> tuple[float, int]: + command = ( + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + *paths, + ) + started = time.perf_counter() + return_code = _run_bounded_process( + command, + env=_tuning_environment(), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=timeout, + label="pytest sample", + ) + return time.perf_counter() - started, return_code + + +def _run_bounded_process( + command: Sequence[str], + *, + env: Mapping[str, str], + stdout: int, + stderr: int, + timeout: float, + label: str, +) -> int: + """Run one timing command with a bounded, platform-aware cleanup boundary. + + POSIX detached descendants are captured by an immediate identity-aware + snapshot and poller. A child which calls ``setsid()`` and loses its leader + before that first snapshot remains a documented best-effort edge; Windows + uses kernel Job Object containment before the process is resumed. + """ + + if not math.isfinite(timeout) or timeout <= 0.0: + raise ValueError("subprocess timeout must be a finite number greater than zero") + options: dict[str, Any] = { + "env": dict(env), + "stdout": stdout, + "stderr": stderr, + } + if os.name == "nt": + creation_flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200) + creation_flags |= getattr(subprocess, "CREATE_SUSPENDED", 0x00000004) + options["creationflags"] = creation_flags + else: + options["start_new_session"] = True + process = subprocess.Popen(tuple(command), **options) + windows_job: _WindowsKillJob | None = None + tracker: _PosixTreeTracker | None = None + cleanup_started = False + try: + windows_job = _windows_kill_job(process) + tracker = _PosixTreeTracker(process.pid) if os.name == "posix" else None + if os.name == "nt": + if windows_job is None: + raise TuningError( + "cannot place Windows tuning process in a kill-on-close Job Object" + ) + try: + _resume_windows_process(process) + except OSError as error: + raise TuningError( + f"cannot start contained Windows tuning process: {error}" + ) from error + try: + return_code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired as error: + cleanup_started = True + tracked_pids = tracker.stop() if tracker is not None else {} + _terminate_process_tree( + process, + windows_job, + tracked_pids=tracked_pids, + ) + raise TuningError(f"{label} exceeded the {timeout:g}s per-run deadline") from error + + if tracker is not None: + cleanup_started = True + tracked_pids = tracker.stop() + # A successful coordinator must not leave background workers. The + # remembered root PGID also catches an untracked same-session child + # after an unusually fast leader exit. + _posix_signal_tree( + process.pid, + tracked_pids, + signal.SIGKILL, + root_group_owned=False, + ) + elif windows_job is not None: + cleanup_started = True + if not _cleanup_windows_tree(process, windows_job): + raise TuningError("could not verify cleanup of the Windows tuning process tree") + return return_code + except BaseException: + if not cleanup_started: + cleanup_started = True + try: + tracked_pids = tracker.stop() if tracker is not None else {} + except Exception: + tracked_pids = {} + try: + _terminate_process_tree( + process, + windows_job, + tracked_pids=tracked_pids, + ) + except Exception: + with suppress(OSError, ValueError): + process.kill() + with suppress(OSError, subprocess.TimeoutExpired): + process.wait(timeout=_PROCESS_TERMINATION_GRACE) + raise + finally: + if tracker is not None: + tracker.stop() + if windows_job is not None: + windows_job.close() + + +def _windows_kill_job(process: subprocess.Popen[Any]) -> _WindowsKillJob | None: + """Attach a kill-on-close Job Object when the Windows host allows it.""" + + if os.name != "nt": + return None + try: # pragma: no cover - exercised by the Windows CI matrix. + from ctypes import wintypes + + class _BasicLimitInformation(ctypes.Structure): + _fields_ = [ + ("PerProcessUserTimeLimit", ctypes.c_longlong), + ("PerJobUserTimeLimit", ctypes.c_longlong), + ("LimitFlags", wintypes.DWORD), + ("MinimumWorkingSetSize", ctypes.c_size_t), + ("MaximumWorkingSetSize", ctypes.c_size_t), + ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), + ("PriorityClass", wintypes.DWORD), + ("SchedulingClass", wintypes.DWORD), + ] + + class _IoCounters(ctypes.Structure): + _fields_ = [ + ("ReadOperationCount", ctypes.c_ulonglong), + ("WriteOperationCount", ctypes.c_ulonglong), + ("OtherOperationCount", ctypes.c_ulonglong), + ("ReadTransferCount", ctypes.c_ulonglong), + ("WriteTransferCount", ctypes.c_ulonglong), + ("OtherTransferCount", ctypes.c_ulonglong), + ] + + class _ExtendedLimitInformation(ctypes.Structure): + _fields_ = [ + ("BasicLimitInformation", _BasicLimitInformation), + ("IoInfo", _IoCounters), + ("ProcessMemoryLimit", ctypes.c_size_t), + ("JobMemoryLimit", ctypes.c_size_t), + ("PeakProcessMemoryUsed", ctypes.c_size_t), + ("PeakJobMemoryUsed", ctypes.c_size_t), + ] + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) # type: ignore[attr-defined] + kernel32.CreateJobObjectW.argtypes = (ctypes.c_void_p, wintypes.LPCWSTR) + kernel32.CreateJobObjectW.restype = wintypes.HANDLE + kernel32.SetInformationJobObject.argtypes = ( + wintypes.HANDLE, + ctypes.c_int, + ctypes.c_void_p, + wintypes.DWORD, + ) + kernel32.SetInformationJobObject.restype = wintypes.BOOL + kernel32.AssignProcessToJobObject.argtypes = (wintypes.HANDLE, wintypes.HANDLE) + kernel32.AssignProcessToJobObject.restype = wintypes.BOOL + kernel32.TerminateJobObject.argtypes = (wintypes.HANDLE, wintypes.UINT) + kernel32.TerminateJobObject.restype = wintypes.BOOL + kernel32.CloseHandle.argtypes = (wintypes.HANDLE,) + kernel32.CloseHandle.restype = wintypes.BOOL + + handle = kernel32.CreateJobObjectW(None, None) + if not handle: + return None + job = _WindowsKillJob(kernel32, handle) + information = _ExtendedLimitInformation() + information.BasicLimitInformation.LimitFlags = 0x00002000 # KILL_ON_JOB_CLOSE + configured = kernel32.SetInformationJobObject( + handle, + 9, # JobObjectExtendedLimitInformation + ctypes.byref(information), + ctypes.sizeof(information), + ) + raw_process_handle = vars(process).get("_handle") + if raw_process_handle is None: + job.close() + return None + process_handle = wintypes.HANDLE(int(raw_process_handle)) + if not configured or not kernel32.AssignProcessToJobObject(handle, process_handle): + job.close() + return None + return job + except (AttributeError, OSError, TypeError, ValueError): + return None + + +def _resume_windows_process(process: subprocess.Popen[Any]) -> None: + """Resume a suspended process only after Job Object containment exists.""" + + if os.name != "nt": + raise OSError("Windows process resume requested on a non-Windows host") + try: # pragma: no cover - exercised by the Windows CI matrix. + from ctypes import wintypes + + raw_process_handle = vars(process).get("_handle") + if raw_process_handle is None: + raise OSError("subprocess has no Windows process handle") + ntdll = ctypes.WinDLL("ntdll", use_last_error=True) # type: ignore[attr-defined] + ntdll.NtResumeProcess.argtypes = (wintypes.HANDLE,) + ntdll.NtResumeProcess.restype = ctypes.c_long + status = ntdll.NtResumeProcess(wintypes.HANDLE(int(raw_process_handle))) + if status != 0: + raise OSError(f"NtResumeProcess failed with status 0x{status & 0xFFFFFFFF:08x}") + except (AttributeError, TypeError, ValueError) as error: + raise OSError(f"cannot resume contained Windows tuning process: {error}") from error + + +def _terminate_process_tree( + process: subprocess.Popen[Any], + windows_job: _WindowsKillJob | None, + *, + tracked_pids: Mapping[int, _ProcessIdentity] | None = None, +) -> None: + """Best-effort cross-platform cleanup for a spawned timing process tree.""" + + cleaned = True + if os.name == "nt": + cleaned = _cleanup_windows_tree(process, windows_job) + else: + descendants = dict(tracked_pids or {}) + descendants.update(_identity_snapshot(_posix_descendant_pids(process.pid))) + _posix_signal_tree( + process.pid, + descendants, + signal.SIGTERM, + root_group_owned=process.poll() is None, + ) + with suppress(OSError, subprocess.TimeoutExpired): + process.wait(timeout=_PROCESS_TERMINATION_GRACE) + # Testenix workers create their own sessions, so they can escape the + # coordinator's process group. Kill both the snapshotted descendants + # and the group: the former reaches detached workers, while the latter + # catches children created between the process-table snapshot and TERM. + descendants.update(_identity_snapshot(_posix_descendant_pids(process.pid))) + _posix_signal_tree( + process.pid, + descendants, + signal.SIGKILL, + root_group_owned=process.poll() is None, + ) + if process.poll() is None: + with suppress(OSError): + process.kill() + with suppress(OSError, subprocess.TimeoutExpired): + process.wait(timeout=_PROCESS_TERMINATION_GRACE) + if os.name == "nt" and not cleaned: + raise TuningError("could not verify cleanup of the Windows tuning process tree") + + +def _posix_signal_tree( + root_pid: int, + descendants: Mapping[int, _ProcessIdentity], + signum: int, + *, + root_group_owned: bool, +) -> None: + """Signal only live identities, never a PID recycled during a long run.""" + + own_group = os.getpgrp() + groups = {root_pid} if root_group_owned else set() + for pid, identity in descendants.items(): + if _process_identity(pid) != identity: + continue + with suppress(OSError): + group = os.getpgid(pid) + if group != own_group and _process_identity(pid) == identity: + groups.add(group) + groups.discard(own_group) + for group in groups: + with suppress(OSError): + os.killpg(group, signum) + + +def _identity_snapshot(pids: Sequence[int]) -> dict[int, _ProcessIdentity]: + identities: dict[int, _ProcessIdentity] = {} + for pid in pids: + identity = _process_identity(pid) + if identity is not None: + identities[pid] = identity + return identities + + +def _bounded_taskkill(pid: int) -> bool: + try: # pragma: no cover - exercised by the Windows CI matrix. + completed = subprocess.run( + ("taskkill", "/PID", str(pid), "/T", "/F"), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=_PROCESS_TERMINATION_GRACE, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return completed.returncode == 0 + + +def _cleanup_windows_tree( + process: subprocess.Popen[Any], + windows_job: _WindowsKillJob | None, +) -> bool: + if windows_job is not None: + terminated = windows_job.terminate() + closed = windows_job.close() + fallback = False if terminated or closed else _bounded_taskkill(process.pid) + cleaned = terminated or closed or fallback + else: + cleaned = _bounded_taskkill(process.pid) + with suppress(OSError, ValueError): + process.kill() + return cleaned + + +def _posix_descendant_pids(root_pid: int) -> tuple[int, ...]: + """Snapshot descendants deepest-first without a third-party dependency.""" + + try: + completed = subprocess.run( + ("ps", "-axo", "pid=,ppid="), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + check=False, + timeout=_PROCESS_TERMINATION_GRACE, + ) + except (OSError, subprocess.TimeoutExpired): + return () + if completed.returncode != 0: + return () + + children: dict[int, list[int]] = {} + for line in completed.stdout.splitlines(): + fields = line.split() + if len(fields) != 2: + continue + try: + pid, parent = (int(field) for field in fields) + except ValueError: + continue + children.setdefault(parent, []).append(pid) + + depths: dict[int, int] = {} + pending = [(root_pid, 0)] + while pending: + parent, depth = pending.pop() + for child in children.get(parent, ()): + if child in depths: + continue + depths[child] = depth + 1 + pending.append((child, depth + 1)) + return tuple(sorted(depths, key=lambda pid: (-depths[pid], pid))) + + +def _tuning_environment() -> dict[str, str]: + environment = os.environ.copy() + environment.update( + { + "NO_COLOR": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "PYTEST_ADDOPTS": "", + "TERM": "dumb", + } + ) + package_root = str(Path(__file__).resolve().parents[1]) + existing_pythonpath = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = ( + f"{package_root}{os.pathsep}{existing_pythonpath}" if existing_pythonpath else package_root + ) + return environment + + +def _run_result_from_dict(value: object) -> RunResult: + """Read the small stable subset needed by the isolated tuning service.""" + + try: + if not isinstance(value, Mapping): + raise TypeError("result must be an object") + tests: list[TestResult] = [] + raw_tests = value["tests"] + if not isinstance(raw_tests, list): + raise TypeError("tests must be an array") + for item in raw_tests: + if not isinstance(item, Mapping) or not isinstance(item["test"], Mapping): + raise TypeError("test result must be an object") + test = item["test"] + parameters = test.get("parameters", {}) + if not isinstance(parameters, Mapping): + raise TypeError("test parameters must be an object") + spec = TestSpec( + id=str(test["id"]), + path=str(test["path"]), + module_name=str(test["module_name"]), + function_name=str(test["function_name"]), + display_name=str(test["display_name"]), + parameters=dict(parameters), + case_id=None if test.get("case_id") is None else str(test["case_id"]), + tags=frozenset(str(tag) for tag in test.get("tags", ())), + skip_reason=(None if test.get("skip_reason") is None else str(test["skip_reason"])), + xfail_reason=( + None if test.get("xfail_reason") is None else str(test["xfail_reason"]) + ), + timeout=None if test.get("timeout") is None else float(test["timeout"]), + source_line=(None if test.get("source_line") is None else int(test["source_line"])), + ) + tests.append( + TestResult( + test=spec, + status=Status(str(item["status"])), + attempts=(), + duration=float(item["duration"]), + ) + ) + raw_issues = value["collection_issues"] + if not isinstance(raw_issues, list): + raise TypeError("collection_issues must be an array") + issues = tuple( + CollectionIssue( + path=str(issue["path"]), + message=str(issue["message"]), + traceback=(None if issue.get("traceback") is None else str(issue["traceback"])), + ) + for issue in raw_issues + if isinstance(issue, Mapping) + ) + raw_workers = value.get("workers_used") + workers_used = None if raw_workers is None else int(raw_workers) + raw_shardable = value.get("shardable_paths", []) + if not isinstance(raw_shardable, list): + raise TypeError("shardable_paths must be an array") + return RunResult( + run_id=str(value["run_id"]), + tests=tuple(tests), + collection_issues=issues, + started_at=float(value["started_at"]), + finished_at=float(value["finished_at"]), + workers_used=workers_used, + shardable_paths=tuple(str(path) for path in raw_shardable), + ) + except (KeyError, TypeError, ValueError) as error: + raise TuningError("native timing subprocess returned a malformed result") from error + + +__all__ = [ + "ExecutionUnitEstimate", + "TuningCandidate", + "TuningError", + "TuningReport", + "WorkerEstimate", + "default_worker_candidates", + "estimate_spawn_cost", + "execution_units", + "render_tuning_report", + "resolve_adaptive_workers", + "run_tuning", + "worker_estimates", +] diff --git a/tests/test_benchmark_tools.py b/tests/test_benchmark_tools.py new file mode 100644 index 0000000..73f980b --- /dev/null +++ b/tests/test_benchmark_tools.py @@ -0,0 +1,791 @@ +from __future__ import annotations + +import hashlib +import importlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +try: + run_benchmark = importlib.import_module("benchmarks.run_benchmark") + run_migration_benchmark = importlib.import_module("benchmarks.run_migration_benchmark") + process_control = importlib.import_module("benchmarks.process_control") + run_project_benchmark = importlib.import_module("benchmarks.run_project_benchmark") + run_scaling_matrix = importlib.import_module("benchmarks.run_scaling_matrix") +finally: + sys.path.pop(0) + +_module_indexes = run_benchmark._module_indexes +run_bounded_process = process_control.run_bounded_process +_run_migration_process = run_migration_benchmark._run_process +_display_command = run_project_benchmark._display_command +_environment = run_project_benchmark._environment +_explicit_suite_targets = run_project_benchmark._explicit_suite_targets +_migration_gate = run_project_benchmark._migration_gate +_observed_testenix_workers = run_project_benchmark._observed_testenix_workers +_runner_contract = run_project_benchmark._runner_contract +_runners = run_project_benchmark._runners +_validate_output = run_project_benchmark._validate_output +_tree_fingerprint = run_project_benchmark._tree_fingerprint +_testenix_runtime_identity = run_project_benchmark._testenix_runtime_identity +DEFAULT_HISTORIES = run_scaling_matrix.DEFAULT_HISTORIES +DEFAULT_LAYOUTS = run_scaling_matrix.DEFAULT_LAYOUTS +DEFAULT_SHARDING_MODES = run_scaling_matrix.DEFAULT_SHARDING_MODES +DEFAULT_WORKERS = run_scaling_matrix.DEFAULT_WORKERS +_reference_curve = run_scaling_matrix._reference_curve +_validate_coverage = run_scaling_matrix._validate_coverage +build_scenarios = run_scaling_matrix.build_scenarios + + +def test_generated_module_layouts_have_explicit_distribution() -> None: + assert _module_indexes(8, 4, "balanced", 0.5) == (0, 1, 2, 3, 0, 1, 2, 3) + assert _module_indexes(5, 3, "dominant", 0.6) == (0, 0, 0, 1, 2) + assert _module_indexes(4, 9, "single", 0.5) == (0, 0, 0, 0) + + +def test_default_scaling_sweeps_cover_every_required_axis() -> None: + counts = (100, 500, 1_000, 3_000) + scenarios = build_scenarios( + counts=counts, + module_count=16, + workers=DEFAULT_WORKERS, + layouts=DEFAULT_LAYOUTS, + histories=DEFAULT_HISTORIES, + sharding_modes=DEFAULT_SHARDING_MODES, + dominant_fraction=0.5, + full_cross_product=False, + include_duration_skew=False, + ) + + _validate_coverage( + scenarios, + counts=counts, + workers=DEFAULT_WORKERS, + layouts=DEFAULT_LAYOUTS, + histories=DEFAULT_HISTORIES, + sharding_modes=DEFAULT_SHARDING_MODES, + ) + assert len(scenarios) == 13 + assert any(scenario.workers == "auto" for scenario in scenarios) + assert any(scenario.module_layout == "single" for scenario in scenarios) + assert any(scenario.history_mode == "default" for scenario in scenarios) + assert all( + scenario.workers == "auto" for scenario in scenarios if scenario.id.startswith("scale-") + ) + assert { + scenario.module_layout for scenario in scenarios if scenario.sharding_mode == "safe" + } == set(DEFAULT_LAYOUTS) + + +def test_full_cross_product_exposes_a_canonical_reference_curve() -> None: + counts = (100, 500, 1_000, 3_000) + scenarios = build_scenarios( + counts=counts, + module_count=16, + workers=DEFAULT_WORKERS, + layouts=DEFAULT_LAYOUTS, + histories=DEFAULT_HISTORIES, + sharding_modes=DEFAULT_SHARDING_MODES, + dominant_fraction=0.5, + full_cross_product=True, + include_duration_skew=False, + ) + measurement = { + "median": 1.0, + "median_tests_per_second": 100.0, + "observed_workers": [4], + } + results = [ + { + "id": scenario.id, + "scenario": scenario, + "result": {"measurements": {"testenix": measurement}}, + } + for scenario in scenarios + ] + + curve = _reference_curve(results, reference_workers="auto") + + assert [point["test_count"] for point in curve] == list(counts) + assert all(point["workers_requested"] == "auto" for point in curve) + assert all(point["history_mode"] == "disabled" for point in curve) + assert all(point["sharding_mode"] == "disabled" for point in curve) + + +def test_full_cross_product_deduplicates_repeated_axes() -> None: + scenarios = build_scenarios( + counts=(100, 100), + module_count=4, + workers=("auto", "auto"), + layouts=("balanced",), + histories=("disabled",), + sharding_modes=("disabled",), + dominant_fraction=0.5, + full_cross_product=True, + include_duration_skew=False, + ) + + assert len(scenarios) == 1 + assert scenarios[0].id == ( + "tests-100-layout-balanced-workers-auto-history-disabled-sharding-disabled" + ) + + +def test_bounded_process_removes_detached_descendants_after_timeout(tmp_path: Path) -> None: + marker = tmp_path / "orphan-finished" + child_code = ( + "import os, pathlib, sys, time\n" + "if os.name == 'posix':\n" + " os.setsid()\n" + "time.sleep(0.8)\n" + "pathlib.Path(sys.argv[1]).write_text('orphan', encoding='utf-8')\n" + ) + parent_code = ( + "import subprocess, sys, time\n" + f"subprocess.Popen([sys.executable, '-c', {child_code!r}, {str(marker)!r}])\n" + "time.sleep(30)\n" + ) + started = time.monotonic() + + with pytest.raises(subprocess.TimeoutExpired): + run_bounded_process( + (sys.executable, "-c", parent_code), + cwd=tmp_path, + env=os.environ, + timeout=0.3, + ) + + assert time.monotonic() - started < 6.0 + time.sleep(1.0) + assert not marker.exists() + + +def test_bounded_process_tracks_detached_child_before_leader_exits(tmp_path: Path) -> None: + ready = tmp_path / "child-ready" + leader_finished = tmp_path / "leader-finished" + marker = tmp_path / "orphan-finished" + child_code = ( + "import os, pathlib, sys, time\n" + "if os.name == 'posix':\n" + " os.setsid()\n" + "pathlib.Path(sys.argv[1]).write_text('ready', encoding='utf-8')\n" + "time.sleep(1.5)\n" + "pathlib.Path(sys.argv[2]).write_text('orphan', encoding='utf-8')\n" + ) + parent_code = ( + "import pathlib, subprocess, sys, time\n" + f"ready = pathlib.Path({str(ready)!r})\n" + f"subprocess.Popen([sys.executable, '-c', {child_code!r}, str(ready), {str(marker)!r}])\n" + "deadline = time.monotonic() + 2\n" + "while not ready.exists() and time.monotonic() < deadline:\n" + " time.sleep(0.005)\n" + # Keep the leader alive for many 20 ms tracker intervals. The previous + # 50 ms window made this a scheduler-latency assertion on loaded macOS + # CI rather than a process-tree cleanup regression. + "time.sleep(0.3)\n" + f"pathlib.Path({str(leader_finished)!r}).write_text('finished', encoding='utf-8')\n" + ) + + with pytest.raises(subprocess.TimeoutExpired): + run_bounded_process( + (sys.executable, "-c", parent_code), + cwd=tmp_path, + env=os.environ, + timeout=0.75, + ) + + assert ready.exists() + # Prove the leader really exited before the timeout; the test still covers + # cleanup of an already orphaned, session-detached descendant. + assert leader_finished.exists() + time.sleep(1.6) + assert not marker.exists() + + +@pytest.mark.skipif(os.name != "posix", reason="exercises POSIX process-group cleanup") +def test_bounded_process_cleans_background_child_after_success(tmp_path: Path) -> None: + ready = tmp_path / "background-ready" + marker = tmp_path / "background-finished" + child_code = ( + "import pathlib, sys, time\n" + "pathlib.Path(sys.argv[1]).write_text('ready', encoding='utf-8')\n" + "time.sleep(0.8)\n" + "pathlib.Path(sys.argv[2]).write_text('orphan', encoding='utf-8')\n" + ) + parent_code = ( + "import pathlib, subprocess, sys, time\n" + f"ready = pathlib.Path({str(ready)!r})\n" + f"subprocess.Popen([sys.executable, '-c', {child_code!r}, str(ready), " + f"{str(marker)!r}], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n" + "deadline = time.monotonic() + 2\n" + "while not ready.exists() and time.monotonic() < deadline:\n" + " time.sleep(0.005)\n" + "time.sleep(0.05)\n" + ) + + completed = run_bounded_process( + (sys.executable, "-c", parent_code), + cwd=tmp_path, + env=os.environ, + timeout=2.0, + ) + + assert completed.returncode == 0 + assert ready.exists() + time.sleep(1.0) + assert not marker.exists() + + +def test_posix_tracker_discards_a_recycled_descendant_pid( + monkeypatch: pytest.MonkeyPatch, +) -> None: + tokens = {90_001: "root-v1", 90_002: "child-v1"} + children = {90_001: (90_002,), 90_002: ()} + monkeypatch.setattr(process_control, "_TRACKER_INTERVAL_SECONDS", 60.0) + monkeypatch.setattr(process_control, "_process_identity", tokens.get) + monkeypatch.setattr( + process_control, + "_posix_direct_children", + lambda pid: children.get(pid, ()), + ) + tracker = process_control._PosixTreeTracker(90_001) + tokens[90_002] = "child-v2" + children[90_001] = () + + assert tracker.stop() == {} + + +def test_posix_cleanup_does_not_signal_an_unowned_recycled_root_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + signalled: list[int] = [] + monkeypatch.setattr( + process_control, + "os", + SimpleNamespace( + getpgrp=lambda: 100, + getpgid=lambda pid: pytest.fail(f"unexpected POSIX PGID lookup for {pid}"), + killpg=lambda group, _signal: signalled.append(group), + ), + ) + + process_control._posix_signal_tree( + 90_001, + {}, + process_control.signal.SIGTERM, + root_group_owned=False, + ) + + assert signalled == [] + + +def test_windows_cleanup_falls_back_when_job_calls_fail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FailedJob: + def terminate(self) -> bool: + return False + + def close(self) -> bool: + return False + + class FakeProcess: + pid = 1234 + + def __init__(self) -> None: + self.killed = False + + def kill(self) -> None: + self.killed = True + + process = FakeProcess() + observed: list[int] = [] + + def fake_taskkill(pid: int) -> bool: + observed.append(pid) + return True + + monkeypatch.setattr(process_control, "_bounded_taskkill", fake_taskkill) + + assert process_control._cleanup_windows_tree(process, FailedJob()) is True + assert observed == [1234] + assert process.killed is True + + +def test_migration_benchmark_uses_bounded_process_runner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, object] = {} + + def fake_run(command: object, **options: object) -> subprocess.CompletedProcess[str]: + observed["command"] = command + observed.update(options) + return subprocess.CompletedProcess(command, 0, "ok", "") + + monkeypatch.setattr(run_migration_benchmark, "run_bounded_process", fake_run) + + outcome = _run_migration_process( + (sys.executable, "-c", "pass"), + project=tmp_path, + environment={"NO_COLOR": "1"}, + ) + + assert outcome.returncode == 0 + assert outcome.stdout == "ok" + assert observed["cwd"] == tmp_path + assert observed["timeout"] == run_migration_benchmark.COMMAND_TIMEOUT_SECONDS + + +def test_real_project_manifest_commands_are_arrays_and_redactable(tmp_path: Path) -> None: + manifest = { + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": ["{python}", "-m", "pytest", "tests"], + }, + { + "name": "testenix", + "kind": "testenix", + "command": ["{python}", "-m", "testenix", "run", "secret-suite"], + "redact_arguments": [4], + }, + ] + } + + pytest_runner, testenix_runner = _runners(manifest) + + assert pytest_runner.command[0] == sys.executable + rendered = _display_command(testenix_runner, tmp_path) + assert "secret-suite" not in rendered + assert "" in rendered + + +def test_real_project_runner_contract_records_performance_switches() -> None: + manifest = { + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": ["{python}", "-m", "pytest", "-n", "4", "tests"], + }, + { + "name": "testenix", + "kind": "testenix", + "command": [ + "{python}", + "-m", + "testenix", + "run", + "tests_testenix", + "--workers", + "auto", + "--no-history", + "--shard-modules", + ], + }, + ] + } + + pytest_runner, testenix_runner = _runners(manifest) + + assert _runner_contract(pytest_runner)["workers_requested"] == "4" + assert _runner_contract(testenix_runner) == { + "workers_requested": "auto", + "history_mode": "disabled", + "safe_module_sharding": True, + } + assert _observed_testenix_workers("Testenix | 3,000 tests | 16 files | 4 workers\n") == 4 + + +@pytest.mark.parametrize( + ("kind", "command"), + [ + ( + "pytest", + ["{python}", "-m", "pytest", "-n", "2", "--numprocesses=4", "tests"], + ), + ( + "testenix", + [ + "{python}", + "-m", + "testenix", + "run", + "--workers", + "2", + "-w=4", + "tests_testenix", + ], + ), + ], +) +def test_real_project_runner_contract_rejects_duplicate_worker_flags( + kind: str, + command: list[str], +) -> None: + other = ( + {"name": "testenix", "kind": "testenix", "command": ["{python}", "-m", "testenix", "run"]} + if kind == "pytest" + else {"name": "pytest", "kind": "pytest", "command": ["{python}", "-m", "pytest"]} + ) + runner = next( + candidate + for candidate in _runners( + { + "runners": [ + {"name": kind, "kind": kind, "command": command}, + other, + ] + } + ) + if candidate.kind == kind + ) + + with pytest.raises(RuntimeError, match="workers is configured more than once"): + _runner_contract(runner) + + +def test_real_project_runner_contract_rejects_conflicting_history_flags() -> None: + runner = _runners( + { + "runners": [ + {"name": "pytest", "kind": "pytest", "command": ["{python}", "-m", "pytest"]}, + { + "name": "testenix", + "kind": "testenix", + "command": [ + "{python}", + "-m", + "testenix", + "run", + "--history", + "history.sqlite3", + "--no-history", + ], + }, + ] + } + )[1] + + with pytest.raises(RuntimeError, match="history is configured more than once"): + _runner_contract(runner) + + +def test_real_project_manifest_requires_both_runners_and_same_python() -> None: + only_pytest = { + "runners": [ + {"name": "one", "kind": "pytest", "command": ["{python}", "-m", "pytest"]}, + {"name": "two", "kind": "pytest", "command": ["{python}", "-m", "pytest"]}, + ] + } + mixed_interpreters = { + "runners": [ + {"name": "pytest", "kind": "pytest", "command": ["python3", "-m", "pytest"]}, + { + "name": "testenix", + "kind": "testenix", + "command": ["{python}", "-m", "testenix", "run", "tests"], + }, + ] + } + + with pytest.raises(RuntimeError, match="at least one pytest and one testenix"): + _runners(only_pytest) + with pytest.raises(RuntimeError, match=r"must start with \{python\}"): + _runners(mixed_interpreters) + + +def test_real_project_runner_kind_cannot_label_an_arbitrary_script() -> None: + manifest = { + "runners": [ + { + "name": "fake-pytest", + "kind": "pytest", + "command": ["{python}", "fake_pytest.py"], + }, + { + "name": "testenix", + "kind": "testenix", + "command": ["{python}", "-m", "testenix", "run", "tests_testenix"], + }, + ] + } + + with pytest.raises(RuntimeError, match="canonical.*pytest"): + _runners(manifest) + + +def test_real_project_pytest_validation_checks_total_outcomes() -> None: + runner = _runners( + { + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": ["{python}", "-m", "pytest"], + }, + { + "name": "testenix", + "kind": "testenix", + "command": ["{python}", "-m", "testenix", "run", "tests"], + }, + ] + } + )[0] + completed = subprocess.CompletedProcess( + runner.command, + 0, + stdout="117 passed, 1 skipped in 2.70s\n", + stderr="", + ) + + _validate_output(runner, completed, expected_tests=118, expected_passed=117) + with pytest.raises(RuntimeError, match="did not report 119 tests"): + _validate_output(runner, completed, expected_tests=119, expected_passed=117) + + +def test_real_project_runtime_identity_hashes_executed_package() -> None: + environment, _, _ = _environment({"environment": {"NO_COLOR": "1"}}) + + identity = _testenix_runtime_identity(ROOT, environment) + + assert identity["version"] + assert identity["package_files"] > 0 + assert len(identity["package_sha256"]) == 64 + assert isinstance(identity["source_matches_distribution"], bool) + + +def test_real_project_runtime_identity_rejects_unowned_source_override() -> None: + environment, _, _ = _environment({"environment": {"NO_COLOR": "1"}}) + environment["PYTHONPATH"] = str(ROOT / "src") + + identity = _testenix_runtime_identity(ROOT, environment) + + assert identity["source_matches_distribution"] is False + + +def test_tree_fingerprint_records_aggregate_metadata_only(tmp_path: Path) -> None: + suite = tmp_path / "private-tests" + suite.mkdir() + (suite / "test_example.py").write_text("def test_example():\n pass\n", encoding="utf-8") + (suite / "notes.txt").write_text("not benchmark source", encoding="utf-8") + + fingerprint = _tree_fingerprint(tmp_path, "private-tests") + + assert set(fingerprint) == {"sha256", "files", "bytes"} + assert fingerprint["files"] == 1 + assert fingerprint["bytes"] > 0 + + +def test_tree_fingerprint_rejects_empty_python_tree(tmp_path: Path) -> None: + empty = tmp_path / "empty" + empty.mkdir() + + with pytest.raises(RuntimeError, match="contains no Python files"): + _tree_fingerprint(tmp_path, "empty") + + +def _write_verified_migration_report(project: Path) -> tuple[dict[str, object], tuple[object, ...]]: + source = project / "tests" / "test_example.py" + generated = project / "tests_testenix" / "test_example.py" + source.parent.mkdir() + generated.parent.mkdir() + source.write_text("def test_example():\n assert True\n", encoding="utf-8") + generated.write_text("def test_example():\n assert True\n", encoding="utf-8") + test_id = "tests/test_example.py::test_example" + + def summary(runner: str) -> dict[str, object]: + return { + "runner": runner, + "tests": 1, + "passed": 1, + "failed": 0, + "errors": 0, + "skipped": 0, + "xfailed": 0, + "xpassed": 0, + "outcomes": {test_id: "pass"}, + } + + report = { + "format": "testenix.migration-report", + "schema_version": 1, + "framework": "pytest", + "status": "published", + "published": True, + "converted_tests": 1, + "originals_modified": False, + "sources": ["tests"], + "output": "tests_testenix", + "source_hashes": {"tests/test_example.py": hashlib.sha256(source.read_bytes()).hexdigest()}, + "generated_files": ["test_example.py"], + "mappings": [ + { + "source_id": test_id, + "target_file": "test_example.py", + "target_function": "test_example", + "case_id": None, + } + ], + "baseline": summary("pytest"), + "native_serial": summary("testenix-serial"), + "native_parallel": summary("testenix-parallel"), + } + reports = project / "reports" + reports.mkdir() + (reports / "migration.json").write_text(json.dumps(report), encoding="utf-8") + manifest: dict[str, object] = {"migration_report": "reports/migration.json"} + runners = _runners( + { + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": ["{python}", "-m", "pytest", "--", "tests"], + }, + { + "name": "testenix", + "kind": "testenix", + "command": [ + "{python}", + "-m", + "testenix", + "run", + "--", + "tests_testenix", + ], + }, + ] + } + ) + return manifest, runners + + +def test_real_project_publication_gate_verifies_exact_migration_outcomes( + tmp_path: Path, +) -> None: + manifest, runners = _write_verified_migration_report(tmp_path) + + gate = _migration_gate(manifest, tmp_path, 1, 1, runners) + + assert gate is not None + assert gate["runner_paths_verified"] is True + assert gate["source_files_verified"] == 1 + assert gate["generated_files_verified"] == 1 + + +def test_real_project_publication_gate_rejects_per_test_outcome_mismatch( + tmp_path: Path, +) -> None: + manifest, runners = _write_verified_migration_report(tmp_path) + report_path = tmp_path / "reports" / "migration.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + report["native_parallel"]["outcomes"] = {"tests/test_example.py::test_example": "fail"} + report_path.write_text(json.dumps(report), encoding="utf-8") + + with pytest.raises(RuntimeError, match="per-test outcomes are not equivalent"): + _migration_gate(manifest, tmp_path, 1, 1, runners) + + +def test_real_project_publication_gate_rejects_added_source_support_file( + tmp_path: Path, +) -> None: + manifest, runners = _write_verified_migration_report(tmp_path) + (tmp_path / "tests" / "conftest.py").write_text( + "def pytest_configure(config):\n pass\n", + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="source Python inventory is stale"): + _migration_gate(manifest, tmp_path, 1, 1, runners) + + +def test_real_project_publication_gate_rejects_added_native_file(tmp_path: Path) -> None: + manifest, runners = _write_verified_migration_report(tmp_path) + (tmp_path / "tests_testenix" / "test_extra.py").write_text( + "def test_extra():\n pass\n", + encoding="utf-8", + ) + + with pytest.raises(RuntimeError, match="generated Python inventory is stale"): + _migration_gate(manifest, tmp_path, 1, 1, runners) + + +def test_real_project_publication_gate_binds_runner_paths_to_migration( + tmp_path: Path, +) -> None: + manifest, _ = _write_verified_migration_report(tmp_path) + unrelated = tmp_path / "other" + unrelated.mkdir() + runners = _runners( + { + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": [ + "{python}", + "-m", + "pytest", + "-k", + "tests", + "--", + "other", + ], + }, + { + "name": "testenix", + "kind": "testenix", + "command": [ + "{python}", + "-m", + "testenix", + "run", + "--tag", + "tests_testenix", + "--", + "other", + ], + }, + ] + } + ) + + with pytest.raises(RuntimeError, match="paths do not match"): + _migration_gate(manifest, tmp_path, 1, 1, runners) + + pytest_runner, testenix_runner = runners + assert _explicit_suite_targets(pytest_runner, tmp_path) == (unrelated.resolve(),) + assert _explicit_suite_targets(testenix_runner, tmp_path) == (unrelated.resolve(),) + + +def test_real_project_publication_targets_require_explicit_delimiter(tmp_path: Path) -> None: + runner = _runners( + { + "runners": [ + { + "name": "pytest", + "kind": "pytest", + "command": ["{python}", "-m", "pytest", "tests"], + }, + { + "name": "testenix", + "kind": "testenix", + "command": ["{python}", "-m", "testenix", "run", "tests_testenix"], + }, + ] + } + )[0] + + with pytest.raises(RuntimeError, match="exactly one '--' delimiter"): + _explicit_suite_targets(runner, tmp_path) diff --git a/tests/test_cli_reports.py b/tests/test_cli_reports.py index 5f1bc94..41b169b 100644 --- a/tests/test_cli_reports.py +++ b/tests/test_cli_reports.py @@ -114,6 +114,8 @@ def test_load_config_and_validate_cli_options(tmp_path: Path) -> None: json = "reports/results.json" junit = "reports/junit.xml" history = false +shard_modules = true +manifest = ".testenix/collection.json" """, encoding="utf-8", ) @@ -129,6 +131,8 @@ def test_load_config_and_validate_cli_options(tmp_path: Path) -> None: json_path=Path("reports/results.json"), junit_path=Path("reports/junit.xml"), history_path=None, + shard_modules=True, + manifest_path=Path(".testenix/collection.json"), ) with pytest.raises(ConfigError, match="workers"): TestenixConfig(workers=0) @@ -298,6 +302,7 @@ def fake_runner(paths: tuple[str, ...], config: TestenixConfig) -> RunResult: str(junit_path), "--history", str(history_path), + "--shard-modules", "tests/unit", ] ) @@ -307,6 +312,7 @@ def fake_runner(paths: tuple[str, ...], config: TestenixConfig) -> RunResult: config = captured["config"] assert isinstance(config, TestenixConfig) assert (config.workers, config.retries, config.timeout, config.tags) == (3, 1, 2.0, ("unit",)) + assert config.shard_modules is True assert json_path.exists() assert junit_path.exists() with HistoryStore(history_path) as history: diff --git a/tests/test_runner_sharding.py b/tests/test_runner_sharding.py new file mode 100644 index 0000000..1ebd9bf --- /dev/null +++ b/tests/test_runner_sharding.py @@ -0,0 +1,1166 @@ +from __future__ import annotations + +import copy +import json +import os +import textwrap +from pathlib import Path + +import pytest + +import testenix.runner as runner_module +from testenix.cli import main +from testenix.config import TestenixConfig +from testenix.contracts import Scope, Status +from testenix.discovery import discover, discover_selected +from testenix.runner import collect_trusted_manifest, run +from testenix.sharding import ( + CollectionManifestError, + ShardingPolicy, + TrustedCollectionManifest, + assess_collection_sharding, + build_trusted_collection_manifest, + deserialize_trusted_collection_manifest, + serialize_trusted_collection_manifest, + trusted_collection_manifest_to_dict, + verify_trusted_collection_manifest, +) + + +def _suite(directory: Path, source: str, *, name: str = "test_sample.py") -> Path: + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(source), encoding="utf-8") + return path + + +def _config(workers: int) -> TestenixConfig: + return TestenixConfig(workers=workers, retries=0, history_path=None) + + +def test_sharding_policy_requires_an_explicit_boolean() -> None: + assert ShardingPolicy().intra_module is False + assert ShardingPolicy(intra_module=True).intra_module is True + with pytest.raises(TypeError, match="boolean"): + ShardingPolicy(intra_module=1) # type: ignore[arg-type] + + +def test_trusted_manifest_round_trips_as_deterministic_portable_json( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + _suite( + tmp_path / "tests", + """ + from testenix import case, cases + + @cases(case(id="one", value={"answer": 42})) + def test_value(value): + assert value["answer"] == 42 + """, + ) + collection = discover("tests") + + manifest = build_trusted_collection_manifest("tests", collection) + encoded = serialize_trusted_collection_manifest(manifest) + restored = deserialize_trusted_collection_manifest(encoded) + + assert restored == manifest + assert serialize_trusted_collection_manifest(restored) == encoded + assert restored.collection_roots == ("tests",) + assert restored.files[0].path == "tests/test_sample.py" + assert restored.tests[0].parameters == {"value": ""} + assert len(restored.files[0].sha256) == 64 + assert verify_trusted_collection_manifest(restored, "tests") + + +def test_manifest_redacts_dynamic_parameter_secrets_and_still_executes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + secret = "testenix-secret-must-not-reach-manifest" + monkeypatch.setenv("TESTENIX_CASE_SECRET", secret) + _suite( + tmp_path / "tests", + """ + import os + + from testenix import case + + @case(token=os.environ["TESTENIX_CASE_SECRET"]) + def test_secret(token): + assert token == os.environ["TESTENIX_CASE_SECRET"] + """, + ) + collection = discover("tests") + assert collection.items[0].spec.parameters == {"token": secret} + + encoded = serialize_trusted_collection_manifest( + build_trusted_collection_manifest("tests", collection) + ) + manifest = deserialize_trusted_collection_manifest(encoded) + result = run("tests", _config(1), trusted_manifest=manifest) + + assert secret not in encoded + assert manifest.tests[0].parameters == {"token": ""} + assert result.tests[0].test.parameters == {"token": ""} + assert result.tests[0].status is Status.PASS + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda data: data.__setitem__("schema_version", 999), "unsupported"), + ( + lambda data: data["files"][0].__setitem__("path", "../escape.py"), + "safe relative", + ), + ( + lambda data: data["files"].append(copy.deepcopy(data["files"][0])), + "duplicate source", + ), + ( + lambda data: data["tests"].append(copy.deepcopy(data["tests"][0])), + "duplicate test", + ), + ( + lambda data: data["sharding"].append(copy.deepcopy(data["sharding"][0])), + "duplicate sharding", + ), + ], +) +def test_trusted_manifest_rejects_bad_schema_traversal_and_duplicates( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutation: object, + message: str, +) -> None: + monkeypatch.chdir(tmp_path) + _suite(tmp_path / "tests", "def test_ok():\n assert True\n") + manifest = build_trusted_collection_manifest("tests", discover("tests")) + data = trusted_collection_manifest_to_dict(manifest) + + assert callable(mutation) + mutation(data) + with pytest.raises(CollectionManifestError, match=message): + deserialize_trusted_collection_manifest(data) + + +def test_trusted_manifest_rejects_duplicate_json_keys() -> None: + with pytest.raises(CollectionManifestError, match="duplicate JSON object key"): + deserialize_trusted_collection_manifest('{"format":"a","format":"b"}') + + +def test_manifest_verification_fails_closed_for_changed_added_and_removed_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + original = _suite(tests, "def test_one():\n assert True\n", name="test_one.py") + manifest = build_trusted_collection_manifest("tests", discover("tests")) + + original.write_text("def test_one():\n assert False\n", encoding="utf-8") + assert not verify_trusted_collection_manifest(manifest, "tests") + + original.write_text("def test_one():\n assert True\n", encoding="utf-8") + added = _suite(tests, "def test_two():\n assert True\n", name="test_two.py") + assert not verify_trusted_collection_manifest(manifest, "tests") + + added.unlink() + original.unlink() + assert not verify_trusted_collection_manifest(manifest, "tests") + + +def test_execution_worker_rechecks_manifest_digest_before_import( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + source = _suite(tmp_path / "tests", "def test_one():\n assert True\n") + manifest = build_trusted_collection_manifest("tests", discover("tests")) + imported_changed_source = tmp_path / "changed-source-imported" + verify = runner_module.verify_trusted_collection_manifest + + def verify_then_replace( + candidate: TrustedCollectionManifest, + paths: tuple[str, ...], + ) -> bool: + verified = verify(candidate, paths) + assert verified + source.write_text( + textwrap.dedent( + f""" + from pathlib import Path + + Path({str(imported_changed_source)!r}).write_text("unsafe", encoding="utf-8") + + def test_one(): + assert True + """ + ), + encoding="utf-8", + ) + return True + + monkeypatch.setattr(runner_module, "verify_trusted_collection_manifest", verify_then_replace) + + result = run("tests", _config(1), trusted_manifest=manifest) + + assert not imported_changed_source.exists() + assert result.tests[0].status is Status.INFRA_ERROR + assert all(attempt.status is Status.INFRA_ERROR for attempt in result.tests[0].attempts) + assert any( + "source digest mismatch" in (phase.message or "") + for attempt in result.tests[0].attempts + for phase in attempt.phases + ) + + +def test_manifest_fingerprints_imported_case_generator( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + helper = _suite( + tests, + """ + from testenix import case + + def generated_cases(): + return (case(id="one", value=1),) + """, + name="case_helper.py", + ) + _suite( + tests, + """ + from case_helper import generated_cases + from testenix import cases + + @cases(*generated_cases()) + def test_value(value): + assert value > 0 + """, + ) + manifest = build_trusted_collection_manifest("tests", discover("tests")) + + assert verify_trusted_collection_manifest(manifest, "tests") + assert any(fingerprint.path == "tests/case_helper.py" for fingerprint in manifest.files) + + helper.write_text( + textwrap.dedent( + """ + from testenix import case + + def generated_cases(): + return ( + case(id="one", value=1), + case(id="two", value=2), + ) + """ + ), + encoding="utf-8", + ) + + assert not verify_trusted_collection_manifest(manifest, "tests") + + result = run("tests", _config(1), trusted_manifest=manifest) + + assert len(result.tests) == 2 + assert {test.status for test in result.tests} == {Status.PASS} + + +def test_manifest_fingerprints_collection_import_nested_inside_helper( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + nested = _suite( + tests, + """ + from testenix import case + + def build_cases(): + return (case(id="one", value=1),) + """, + name="nested_case_helper.py", + ) + _suite( + tests, + """ + def generated_cases(): + from nested_case_helper import build_cases + + return build_cases() + """, + name="case_helper.py", + ) + _suite( + tests, + """ + from case_helper import generated_cases + from testenix import cases + + @cases(*generated_cases()) + def test_value(value): + assert value > 0 + """, + ) + + manifest = build_trusted_collection_manifest("tests", discover("tests")) + + assert any(fingerprint.path == "tests/nested_case_helper.py" for fingerprint in manifest.files) + nested.write_text( + "from testenix import case\n\ndef build_cases(): return (case(id='two', value=2),)\n", + encoding="utf-8", + ) + assert not verify_trusted_collection_manifest(manifest, "tests") + + +def test_execution_worker_rechecks_imported_case_generator_digest( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + helper = _suite( + tests, + """ + from testenix import case + + def generated_cases(): + return (case(id="one", value=1),) + """, + name="case_helper.py", + ) + _suite( + tests, + """ + from case_helper import generated_cases + from testenix import cases + + @cases(*generated_cases()) + def test_value(value): + assert value == 1 + """, + ) + manifest = build_trusted_collection_manifest("tests", discover("tests")) + imported_changed_helper = tmp_path / "changed-helper-imported" + verify = runner_module.verify_trusted_collection_manifest + + def verify_then_replace( + candidate: TrustedCollectionManifest, + paths: tuple[str, ...], + ) -> bool: + verified = verify(candidate, paths) + assert verified + helper.write_text( + textwrap.dedent( + f""" + from pathlib import Path + + from testenix import case + + Path({str(imported_changed_helper)!r}).write_text("unsafe", encoding="utf-8") + + def generated_cases(): + return (case(id="one", value=1),) + """ + ), + encoding="utf-8", + ) + return True + + monkeypatch.setattr(runner_module, "verify_trusted_collection_manifest", verify_then_replace) + + result = run("tests", _config(1), trusted_manifest=manifest) + + assert not imported_changed_helper.exists() + assert result.tests[0].status is Status.INFRA_ERROR + assert any( + "case_helper.py" in (phase.message or "") + and "source digest mismatch" in (phase.message or "") + for attempt in result.tests[0].attempts + for phase in attempt.phases + ) + + +def test_configured_trusted_manifest_skips_the_full_collection_import( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + counter = tmp_path / "imports.txt" + path = _suite( + tmp_path / "tests", + f""" + from pathlib import Path + + with Path({str(counter)!r}).open("a", encoding="utf-8") as output: + output.write("imported\\n") + + def test_ok(): + assert True + """, + ) + collection = discover("tests") + manifest = build_trusted_collection_manifest("tests", collection) + manifest_path = tmp_path / "collection.json" + manifest_path.write_text(serialize_trusted_collection_manifest(manifest), encoding="utf-8") + assert counter.read_text(encoding="utf-8").splitlines() == ["imported"] + + result = run( + "tests", + TestenixConfig(workers=1, history_path=None, manifest_path=manifest_path), + ) + + assert [test.status for test in result.tests] == [Status.PASS] + # One producer import + one execution-worker import. A normal run would + # also import the whole suite in its isolated collection worker. + assert counter.read_text(encoding="utf-8").splitlines() == ["imported", "imported"] + assert result.tests[0].test.path == path.relative_to(tmp_path).as_posix() + + +def test_stale_trusted_manifest_falls_back_to_isolated_collection( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + path = _suite(tmp_path / "tests", "def test_one():\n assert True\n") + manifest = build_trusted_collection_manifest("tests", discover("tests")) + path.write_text( + "def test_one():\n assert True\n\ndef test_two():\n assert True\n", + encoding="utf-8", + ) + + result = run("tests", _config(1), trusted_manifest=manifest) + + assert len(result.tests) == 2 + assert {test.status for test in result.tests} == {Status.PASS} + + +def test_programmatic_manifest_path_rejects_malformed_json(tmp_path: Path) -> None: + path = _suite(tmp_path, "def test_ok(): pass\n") + manifest_path = tmp_path / "broken.json" + manifest_path.write_text("not-json", encoding="utf-8") + + with pytest.raises(CollectionManifestError, match="invalid collection manifest JSON"): + run( + str(path), + TestenixConfig(workers=1, history_path=None, manifest_path=manifest_path), + ) + + +def test_selected_execution_supports_decorators_that_change_function_name( + tmp_path: Path, +) -> None: + path = _suite( + tmp_path, + """ + def rename(function): + def wrapped(): + function() + return wrapped + + @rename + def test_original(): + assert True + """, + ) + + result = run(str(path), _config(1)) + + assert len(result.tests) == 1 + assert result.tests[0].test.function_name == "wrapped" + assert result.tests[0].status is Status.PASS + + +def test_default_module_affinity_remains_unchanged(tmp_path: Path) -> None: + path = _suite( + tmp_path, + """ + def test_one(): pass + def test_two(): pass + def test_three(): pass + def test_four(): pass + def test_five(): pass + def test_six(): pass + """, + ) + + result = run(str(path), _config(3)) + + assert {test.status for test in result.tests} == {Status.PASS} + assert len({test.attempts[0].worker_id for test in result.tests}) == 1 + assert result.workers_used == 1 + + +def test_opt_in_shards_an_eligible_module_across_workers(tmp_path: Path) -> None: + path = _suite( + tmp_path, + """ + def test_one(): pass + def test_two(): pass + def test_three(): pass + def test_four(): pass + def test_five(): pass + def test_six(): pass + """, + ) + + result = run( + str(path), + _config(3), + sharding_policy=ShardingPolicy(intra_module=True), + ) + + assert {test.status for test in result.tests} == {Status.PASS} + assert len({test.attempts[0].worker_id for test in result.tests}) > 1 + assert result.workers_used == 3 + + +def test_workers_used_reports_executed_non_empty_shards( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = _suite( + tmp_path, + "\n".join(f"def test_{index}(): pass" for index in range(6)), + ) + original_schedule = runner_module.schedule_lpt + + def collapse_plan_to_one_shard(*args, **kwargs): # type: ignore[no-untyped-def] + return original_schedule(args[0], 1, *args[2:], **kwargs) + + monkeypatch.setattr(runner_module, "schedule_lpt", collapse_plan_to_one_shard) + + result = runner_module.run( + str(path), + _config(3), + sharding_policy=ShardingPolicy(intra_module=True), + ) + + assert result.workers_used == 1 + assert len({test.attempts[0].worker_id for test in result.tests}) == 1 + + +def test_auto_workers_follow_real_units_before_and_after_opt_in_sharding(tmp_path: Path) -> None: + path = _suite( + tmp_path, + "\n".join(f"def test_{index}(): pass" for index in range(12)), + ) + config = TestenixConfig(workers="auto", history_path=None) + + affinity = run(str(path), config) + sharded = run( + str(path), + TestenixConfig(workers="auto", history_path=None, shard_modules=True), + ) + + assert affinity.workers_used == 1 + assert sharded.workers_used == min(4, os.cpu_count() or 1, 12) + assert sharded.shardable_paths == (path.as_posix(),) + + +def test_function_autouse_fixture_does_not_block_opt_in_sharding(tmp_path: Path) -> None: + path = _suite( + tmp_path, + """ + from testenix import fixture + + @fixture(scope="test", autouse=True) + def isolated_setup(tmp_path): + assert tmp_path.is_dir() + + def test_one(): pass + def test_two(): pass + def test_three(): pass + def test_four(): pass + """, + ) + + collection = discover(str(path)) + assert collection.fixtures[0].scope is Scope.TEST + assert assess_collection_sharding(collection)[0].eligible + + result = run( + str(path), + _config(2), + sharding_policy=ShardingPolicy(intra_module=True), + ) + assert len({test.attempts[0].worker_id for test in result.tests}) == 2 + + +def test_imported_local_fixture_scope_fails_closed_across_manifest_reuse( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + scope_source = _suite(tests, 'SCOPE = "test"\n', name="fixture_config.py") + _suite( + tests, + """ + from fixture_config import SCOPE + from testenix import fixture + + @fixture(scope=SCOPE) + def shared(): + return 42 + + def test_one(shared): + assert shared == 42 + + def test_two(shared): + assert shared == 42 + """, + ) + manifest = build_trusted_collection_manifest("tests", discover("tests")) + + assert len(manifest.sharding) == 1 + assert not manifest.sharding[0].eligible + assert any( + "statically guaranteed test scope" in blocker for blocker in manifest.sharding[0].blockers + ) + assert any(fingerprint.path == "tests/fixture_config.py" for fingerprint in manifest.files) + + scope_source.write_text('SCOPE = "session"\n', encoding="utf-8") + assert not verify_trusted_collection_manifest(manifest, "tests") + + result = run( + "tests", + TestenixConfig(workers=2, history_path=None, shard_modules=True), + trusted_manifest=manifest, + ) + + assert {test.status for test in result.tests} == {Status.PASS} + assert result.workers_used == 1 + assert result.shardable_paths == () + + +def test_imported_fixture_provider_fails_closed_across_manifest_reuse( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + tests = tmp_path / "tests" + provider = _suite( + tests, + """ + from testenix import fixture + + @fixture + def shared(): + return 42 + """, + name="external_fixture_provider.py", + ) + _suite( + tests, + """ + from external_fixture_provider import shared + + def test_one(shared): + assert shared == 42 + + def test_two(shared): + assert shared == 42 + """, + ) + manifest = build_trusted_collection_manifest("tests", discover("tests")) + + assert len(manifest.sharding) == 1 + assert not manifest.sharding[0].eligible + assert any( + "outside the collected module source" in blocker + for blocker in manifest.sharding[0].blockers + ) + assert any( + fingerprint.path == "tests/external_fixture_provider.py" for fingerprint in manifest.files + ) + + provider.write_text( + textwrap.dedent( + """ + from testenix import fixture + + @fixture(scope="session") + def shared(): + return 42 + """ + ), + encoding="utf-8", + ) + # Local imports that can influence collection are fingerprinted alongside + # selected test files, so the stale manifest is rejected before scheduling. + assert not verify_trusted_collection_manifest(manifest, "tests") + + result = run( + "tests", + TestenixConfig(workers=2, history_path=None, shard_modules=True), + trusted_manifest=manifest, + ) + + assert {test.status for test in result.tests} == {Status.PASS} + assert result.workers_used == 1 + assert result.shardable_paths == () + + +@pytest.mark.parametrize("scope", ["module", "session"]) +def test_wide_fixture_scope_falls_back_to_module_affinity(tmp_path: Path, scope: str) -> None: + path = _suite( + tmp_path, + f""" + from testenix import fixture + + @fixture(scope={scope!r}) + def shared(): + return 42 + + def test_one(shared): assert shared == 42 + def test_two(shared): assert shared == 42 + def test_three(shared): assert shared == 42 + """, + ) + + result = run( + str(path), + _config(3), + sharding_policy=ShardingPolicy(intra_module=True), + ) + + assert {test.status for test in result.tests} == {Status.PASS} + assert len({test.attempts[0].worker_id for test in result.tests}) == 1 + + +def test_obvious_mutable_global_and_import_lifecycle_are_conservative_blockers( + tmp_path: Path, +) -> None: + path = _suite( + tmp_path, + """ + events = [] + print("import lifecycle") + + def test_one(): + events.append("one") + """, + ) + + decision = assess_collection_sharding(discover(str(path)))[0] + + assert not decision.eligible + assert any("module-level collection" in blocker for blocker in decision.blockers) + assert any("import-time call" in blocker for blocker in decision.blockers) + + +@pytest.mark.parametrize( + ("source", "expected_blocker"), + [ + ( + """ + STATE = ([],) + + def test_one(): + STATE[0].append("changed") + """, + "module-level collection 'STATE'", + ), + ( + """ + class State: + values = [] + + def test_one(): + State.values.append("changed") + """, + "mutable class state on 'State'", + ), + ( + """ + class State: + pass + + State.values = [] + + def test_one(): + State.values.append("changed") + """, + "mutable class state on 'State'", + ), + ( + """ + class Outer: + class Inner: + values = [] + + def test_one(): + Outer.Inner.values.append("changed") + """, + "mutable class state on 'Outer'", + ), + ], +) +def test_nested_and_class_mutable_state_block_module_sharding( + tmp_path: Path, + source: str, + expected_blocker: str, +) -> None: + path = _suite(tmp_path, source) + + decision = assess_collection_sharding(discover(str(path)))[0] + + assert not decision.eligible + assert any(expected_blocker in blocker for blocker in decision.blockers) + + +@pytest.mark.parametrize( + ("source", "expected_blocker"), + [ + ( + """ + def connect(): + return object() + + CLIENT = connect() + + def test_one(): + assert CLIENT is not None + """, + "assignment call connect", + ), + ( + """ + HANDLE: object = open(__file__, encoding="utf-8") + HANDLE.close() + + def test_one(): + assert HANDLE.closed + """, + "assignment call open", + ), + ( + """ + def register(): + def decorate(function): + return function + return decorate + + @register() + def test_one(): + assert True + """, + "decorator call register", + ), + ( + """ + def register(function): + function.registered = True + return function + + @register + def test_one(): + assert test_one.registered + """, + "decorator call register", + ), + ( + """ + def factory(): + return 42 + + def helper(default=factory()): + return default + + def test_one(): + assert helper() == 42 + """, + "default call factory", + ), + ( + """ + def connect(): + return int + + def test_one(value: connect() = 1) -> connect(): + assert value == 1 + """, + "annotation call connect", + ), + ( + """ + def connect(): + return int + + VALUE: connect() = 1 + + def test_one(): + assert VALUE == 1 + """, + "annotation call connect", + ), + ( + """ + class Base: + pass + + def factory(): + return Base + + class Derived(factory()): + pass + + def test_one(): + assert issubclass(Derived, Base) + """, + "class base call factory", + ), + ( + """ + def connect(): + return None + + class Helper: + connect() + + def test_one(): + assert Helper is not None + """, + "import-time call connect", + ), + ( + """ + def validate(): + return True + + class Helper: + assert validate() + + def test_one(): + assert Helper is not None + """, + "assertion call validate", + ), + ( + """ + STATE = 0 + + def factory(): + return 1 + + STATE += factory() + + def test_one(): + assert STATE == 1 + """, + "assignment call factory", + ), + ( + """ + import sys + + ORIGINAL = sys.path[0] + + def register_path(): + return ORIGINAL + + sys.path.insert(0, register_path()) + assert sys.path.pop(0) == ORIGINAL + + def test_one(): + assert True + """, + "expression call register_path", + ), + ], +) +def test_definition_time_calls_block_safe_module_sharding( + tmp_path: Path, + source: str, + expected_blocker: str, +) -> None: + path = _suite(tmp_path, source) + + decision = assess_collection_sharding(discover(str(path)))[0] + + assert not decision.eligible + assert any(expected_blocker in blocker for blocker in decision.blockers) + + +def test_plain_constants_and_testenix_decorator_factories_remain_shardable( + tmp_path: Path, +) -> None: + path = _suite( + tmp_path, + """ + from testenix import case, cases, fixture, skip, test, xfail + + CONSTANT = ("plain", 42) + + @fixture(autouse=True) + def isolated_setup(): + assert CONSTANT[1] == 42 + + @test(tags={"unit"}, timeout=1.0) + @cases(case(id="one", value=1), case(id="two", value=2)) + @skip("not skipped", when=False) + @xfail("not expected to fail", when=False) + def test_value(value): + assert value in {1, 2} + + @test + def test_bare_decorator(): + assert CONSTANT[0] == "plain" + """, + ) + + decision = assess_collection_sharding(discover(str(path)))[0] + + assert decision.eligible + assert decision.blockers == () + + +def test_postponed_annotations_do_not_create_false_import_time_blockers( + tmp_path: Path, +) -> None: + path = _suite( + tmp_path, + """ + from __future__ import annotations + + from testenix import test + + def connect(): + return int + + VALUE: connect() = 1 + + @test + def test_one(value: connect() = VALUE) -> connect(): + assert value == 1 + """, + ) + + decision = assess_collection_sharding(discover(str(path)))[0] + + assert decision.eligible + assert decision.blockers == () + + +def test_worker_crash_recovery_semantics_survive_intra_module_sharding(tmp_path: Path) -> None: + state = tmp_path / "crash-once" + path = _suite( + tmp_path, + f""" + import os + from pathlib import Path + + def test_crashes_once(): + state = Path({str(state)!r}) + if not state.exists(): + state.write_text("crashed", encoding="utf-8") + os._exit(19) + + def test_stays_green(): + assert True + """, + ) + + result = run( + str(path), + _config(2), + sharding_policy=ShardingPolicy(intra_module=True), + ) + by_name = {test.test.function_name: test for test in result.tests} + + assert by_name["test_stays_green"].status is Status.PASS + recovered = by_name["test_crashes_once"] + assert recovered.status is Status.FLAKY + assert [attempt.status for attempt in recovered.attempts] == [Status.CRASH, Status.PASS] + + +def test_selected_rediscovery_materialises_only_requested_tests_and_all_fixtures( + tmp_path: Path, +) -> None: + path = _suite( + tmp_path, + """ + from testenix import case, cases, fixture + + @fixture + def value(): return 42 + + def test_selected(value): assert value == 42 + + @cases(case(id="one", number=1), case(id="two", number=2)) + def test_unrelated(number): assert number > 0 + """, + ) + + collection = discover_selected(path, {"test_selected"}) + + assert [item.function_name for item in collection.tests] == ["test_selected"] + assert any(fixture.name == "value" for fixture in collection.fixtures) + + +def test_manifest_json_is_plain_inert_data( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + _suite(tmp_path / "tests", "def test_ok(): pass\n") + encoded = serialize_trusted_collection_manifest( + build_trusted_collection_manifest("tests", discover("tests")) + ) + + decoded = json.loads(encoded) + assert decoded["format"] == "testenix.collection-manifest" + assert "__reduce__" not in encoded + + +def test_manifest_cli_uses_isolated_collection_and_never_overwrites( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.chdir(tmp_path) + _suite(tmp_path / "tests", "def test_ok(): pass\n") + output = tmp_path / ".testenix" / "collection.json" + + assert main(["manifest", "tests", "--output", str(output)]) == 0 + manifest = deserialize_trusted_collection_manifest(output.read_bytes()) + assert manifest == collect_trusted_manifest("tests") + + before = output.read_bytes() + assert main(["manifest", "tests", "--output", str(output)]) == 2 + assert output.read_bytes() == before + assert "will not be replaced" in capsys.readouterr().err + + +def test_public_manifest_collection_resolves_paths_from_explicit_project_root( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_root = tmp_path / "project" + outside = tmp_path / "outside" + outside.mkdir() + _suite( + project_root / "tests", + """ + def test_from_project_root(): + assert True + """, + ) + monkeypatch.chdir(outside) + + manifest = collect_trusted_manifest("tests", project_root=project_root) + + assert manifest.collection_roots == ("tests",) + assert [fingerprint.path for fingerprint in manifest.files] == ["tests/test_sample.py"] + assert [test.function_name for test in manifest.tests] == ["test_from_project_root"] + assert [test.path for test in manifest.tests] == ["tests/test_sample.py"] diff --git a/tests/test_runtime_core.py b/tests/test_runtime_core.py index 7a80a61..aa8483e 100644 --- a/tests/test_runtime_core.py +++ b/tests/test_runtime_core.py @@ -326,6 +326,18 @@ def test_lpt_scheduler_has_deterministic_unknown_duration_and_empty_shards() -> assert [shard.estimated_duration for shard in plan] == [1.0, 1.0, 0.0] +def test_lpt_scheduler_spreads_zero_duration_items_deterministically() -> None: + tests = tuple(_spec(name) for name in ("e", "d", "c", "b", "a")) + history = {test.id: 0.0 for test in tests} + + plan = schedule_lpt(tests, 3, history) + reversed_plan = schedule_lpt(tuple(reversed(tests)), 3, history) + + assert [shard.test_ids for shard in plan] == [("a", "d"), ("b", "e"), ("c",)] + assert [shard.estimated_duration for shard in plan] == [0.0, 0.0, 0.0] + assert plan == reversed_plan + + def test_history_uses_last_substantive_attempt_and_ignores_pure_infra( tmp_path: Path, ) -> None: diff --git a/tests/test_tuning.py b/tests/test_tuning.py new file mode 100644 index 0000000..d2daadf --- /dev/null +++ b/tests/test_tuning.py @@ -0,0 +1,1079 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +import testenix.tuning as tuning_module +from testenix.cli import main +from testenix.config import TestenixConfig, load_config, write_worker_recommendation +from testenix.contracts import RunResult, Status, TestResult, TestSpec +from testenix.tuning import ( + TuningCandidate, + TuningError, + TuningReport, + default_worker_candidates, + execution_units, + resolve_adaptive_workers, + run_tuning, +) + + +def _spec( + name: str, + *, + path: str | None = None, + timeout: float | None = None, +) -> TestSpec: + effective_path = path or f"tests/test_{name}.py" + return TestSpec( + id=f"{effective_path}::test_{name}", + path=effective_path, + module_name=f"test_{name}", + function_name=f"test_{name}", + display_name=f"test_{name}", + timeout=timeout, + ) + + +def _run(*specs: TestSpec, status: Status = Status.PASS) -> RunResult: + return RunResult( + run_id="tuning-run", + tests=tuple( + TestResult(test=spec, status=status, attempts=(), duration=float(index + 1)) + for index, spec in enumerate(specs) + ), + collection_issues=(), + started_at=1.0, + finished_at=2.0, + ) + + +def _report(recommended_workers: int = 2) -> TuningReport: + return TuningReport( + paths=("tests_testenix",), + warmups=1, + repeats=3, + discovered_tests=4, + execution_units=2, + model_recommendation=2, + recommended_workers=recommended_workers, + candidates=( + TuningCandidate(1, (2.0, 2.1, 2.0)), + TuningCandidate(2, (1.0, 1.1, 1.0)), + ), + ) + + +def test_adaptive_workers_preserve_explicit_configuration() -> None: + specs = (_spec("first"), _spec("second")) + + assert resolve_adaptive_workers(TestenixConfig(workers=7), specs, {}) == 7 + assert TestenixConfig(workers=7).resolve_workers(specs, {}) == 7 + + +def test_adaptive_workers_use_module_units_history_and_spawn_cost() -> None: + dominant = _spec("dominant") + specs = (dominant, _spec("small_a"), _spec("small_b"), _spec("small_c")) + durations = { + dominant.id: 10.0, + specs[1].id: 1.0, + specs[2].id: 1.0, + specs[3].id: 1.0, + } + + assert ( + resolve_adaptive_workers( + TestenixConfig(), + specs, + durations, + spawn_cost=0.1, + cpu_count=32, + ) + == 2 + ) + assert ( + resolve_adaptive_workers( + TestenixConfig(), + specs, + durations, + spawn_cost=20.0, + cpu_count=32, + ) + == 1 + ) + + +def test_adaptive_workers_do_not_split_a_normal_module_but_isolate_timeouts() -> None: + shared_path = "tests/test_shared.py" + specs = ( + _spec("first", path=shared_path), + _spec("second", path=shared_path), + _spec("timed_a", path=shared_path, timeout=1.0), + _spec("timed_b", path=shared_path, timeout=2.0), + ) + + units = execution_units(specs, {}) + + assert len(units) == 3 + assert [(unit.tests, unit.isolated) for unit in units] == [ + (2, False), + (1, True), + (1, True), + ] + assert resolve_adaptive_workers(TestenixConfig(), specs[:2], {}, cpu_count=64) == 1 + + +def test_adaptive_workers_see_only_explicitly_shardable_module_tests_as_units() -> None: + shared_path = "tests/test_shared.py" + specs = tuple(_spec(f"case_{index}", path=shared_path) for index in range(20)) + + assert len(execution_units(specs, {})) == 1 + assert len(execution_units(specs, {}, shardable_paths={shared_path})) == 20 + assert resolve_adaptive_workers(TestenixConfig(), specs, {}, cpu_count=64) == 1 + assert ( + resolve_adaptive_workers( + TestenixConfig(), + specs, + {}, + cpu_count=64, + shardable_paths={shared_path}, + ) + == 4 + ) + + +def test_equal_independent_units_scale_when_spawn_is_free() -> None: + specs = tuple(_spec(f"case_{index}") for index in range(4)) + + assert ( + resolve_adaptive_workers( + TestenixConfig(), + specs, + {spec.id: 1.0 for spec in specs}, + spawn_cost=0.0, + cpu_count=4, + ) + == 4 + ) + assert default_worker_candidates(13, model_recommendation=2, cpu_count=14) == (1, 2, 4) + + +def test_default_tuning_candidates_do_not_expand_to_a_large_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("testenix.tuning.os.cpu_count", lambda: 192) + monkeypatch.setattr("testenix.tuning.os.process_cpu_count", lambda: 192, raising=False) + monkeypatch.setattr( + "testenix.tuning.os.sched_getaffinity", + lambda _process: frozenset(range(192)), + raising=False, + ) + + assert default_worker_candidates(3_000, model_recommendation=4) == (1, 2, 4) + + +def test_cold_start_caps_large_unknown_suites_at_four_workers() -> None: + synthetic_100k = tuple( + _spec(f"case_{index}", path=f"tests/test_generated_{index % 16}.py") + for index in range(100_000) + ) + skewed_118 = tuple( + _spec(f"real_{index}", path=f"tests/test_real_{min(index // 9, 12)}.py") + for index in range(118) + ) + + assert ( + resolve_adaptive_workers( + TestenixConfig(), + synthetic_100k, + {}, + cpu_count=64, + ) + == 4 + ) + assert ( + resolve_adaptive_workers( + TestenixConfig(), + skewed_118, + {}, + cpu_count=64, + ) + == 4 + ) + + +def test_partial_history_does_not_guess_duration_for_a_new_module() -> None: + specs = tuple(_spec(f"known_{index}") for index in range(6)) + (_spec("new_module"),) + durations = {spec.id: 0.01 for spec in specs[:-1]} + + assert ( + resolve_adaptive_workers( + TestenixConfig(), + specs, + durations, + spawn_cost=20.0, + cpu_count=64, + ) + == 4 + ) + + +def test_run_tuning_counterbalances_candidates_and_selects_measured_median() -> None: + specs = (_spec("first"), _spec("second"), _spec("third"), _spec("fourth")) + calls: list[int] = [] + elapsed = {1: 4.0, 2: 2.0, 4: 3.0} + + def measure(paths: tuple[str, ...], config: TestenixConfig) -> tuple[float, RunResult]: + assert paths == ("tests_testenix",) + assert isinstance(config.workers, int) + assert config.history_path is None + calls.append(config.workers) + return elapsed[config.workers], _run(*specs) + + report = run_tuning( + ("tests_testenix",), + TestenixConfig(), + candidates=(4, 1, 2, 2), + warmups=1, + repeats=3, + native_measure=measure, + ) + + assert report.recommended_workers == 2 + assert [candidate.workers for candidate in report.candidates] == [1, 2, 4] + assert [candidate.samples for candidate in report.candidates] == [ + (4.0, 4.0, 4.0), + (2.0, 2.0, 2.0), + (3.0, 3.0, 3.0), + ] + # Probe, forward warmups, then forward/reverse/forward measured rounds. + assert calls == [1, 1, 2, 4, 1, 2, 4, 4, 2, 1, 1, 2, 4] + + +def test_run_tuning_prefers_fewer_workers_within_measurement_tolerance() -> None: + specs = tuple(_spec(f"case_{index}") for index in range(4)) + elapsed = {1: 2.0, 2: 1.004, 4: 1.0} + + def measure(paths: tuple[str, ...], config: TestenixConfig) -> tuple[float, RunResult]: + del paths + assert isinstance(config.workers, int) + return elapsed[config.workers], _run(*specs) + + report = run_tuning( + ("tests",), + TestenixConfig(), + candidates=(1, 2, 4), + warmups=0, + repeats=3, + native_measure=measure, + ) + + assert report.recommended_workers == 2 + + +def test_run_tuning_rejects_changed_outcomes() -> None: + spec = _spec("unstable") + calls = 0 + + def measure(paths: tuple[str, ...], config: TestenixConfig) -> tuple[float, RunResult]: + nonlocal calls + del paths, config + calls += 1 + status = Status.PASS if calls == 1 else Status.FAIL + return 1.0, _run(spec, status=status) + + with pytest.raises(TuningError, match="failed with exit code"): + run_tuning( + ("tests",), + TestenixConfig(), + candidates=(1,), + warmups=1, + repeats=1, + native_measure=measure, + ) + + +def test_run_tuning_discards_result_when_project_sources_change( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + suite = tmp_path / "tests_testenix" + suite.mkdir() + test_source = suite / "test_sample.py" + test_source.write_text("def test_sample(): assert True\n", encoding="utf-8") + helper = tmp_path / "project_helper.py" + helper.write_text("VALUE = 1\n", encoding="utf-8") + spec = _spec("sample", path=str(test_source)) + calls = 0 + + def measure(paths: tuple[str, ...], config: TestenixConfig) -> tuple[float, RunResult]: + nonlocal calls + del paths, config + calls += 1 + if calls == 2: + helper.write_text("VALUE = 2\n", encoding="utf-8") + return 1.0, _run(spec) + + with pytest.raises(TuningError, match="project sources changed"): + run_tuning( + ("tests_testenix",), + TestenixConfig(), + candidates=(1,), + warmups=0, + repeats=1, + native_measure=measure, + ) + + +def test_run_tuning_detects_source_changed_and_restored_inside_sample( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + suite = tmp_path / "tests_testenix" + suite.mkdir() + test_source = suite / "test_sample.py" + test_source.write_text("def test_sample(): assert True\n", encoding="utf-8") + helper = tmp_path / "project_helper.py" + original = "VALUE = 1\n" + helper.write_text(original, encoding="utf-8") + spec = _spec("sample", path=str(test_source)) + calls = 0 + + def measure(paths: tuple[str, ...], config: TestenixConfig) -> tuple[float, RunResult]: + nonlocal calls + del paths, config + calls += 1 + if calls == 2: + helper.write_text("VALUE = 2\n", encoding="utf-8") + assert helper.read_text(encoding="utf-8") == "VALUE = 2\n" + helper.write_text(original, encoding="utf-8") + return 1.0, _run(spec) + + with pytest.raises(TuningError, match="project sources changed"): + run_tuning( + ("tests_testenix",), + TestenixConfig(), + candidates=(1,), + warmups=0, + repeats=1, + native_measure=measure, + ) + + +def test_tuning_source_snapshot_follows_source_directory_symlinks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project = tmp_path / "project" + external = tmp_path / "external" + project.mkdir() + external.mkdir() + helper = external / "helper.py" + helper.write_text("VALUE = 1\n", encoding="utf-8") + linked = project / "linked_src" + try: + linked.symlink_to(external, target_is_directory=True) + except OSError: + pytest.skip("directory symbolic links are unavailable") + monkeypatch.chdir(project) + + before = tuning_module._tuning_source_snapshot(("linked_src",)) + helper.write_text("VALUE = 2\n", encoding="utf-8") + + assert tuning_module._tuning_source_snapshot(("linked_src",)) != before + + +def test_run_tuning_can_compare_pytest_without_cache() -> None: + spec = _spec("ok") + pytest_calls: list[tuple[str, ...]] = [] + + def measure_native(paths: tuple[str, ...], config: TestenixConfig) -> tuple[float, RunResult]: + del paths, config + return 1.0, _run(spec) + + def measure_pytest(paths: tuple[str, ...]) -> tuple[float, int]: + pytest_calls.append(paths) + return 2.0, 0 + + report = run_tuning( + ("tests_testenix",), + TestenixConfig(), + candidates=(1,), + warmups=1, + repeats=3, + pytest_paths=("tests",), + native_measure=measure_native, + pytest_measure=measure_pytest, + ) + + assert report.pytest_samples == (2.0, 2.0, 2.0) + assert report.pytest_over_native == 2.0 + assert pytest_calls == [("tests",)] * 4 + + +def test_run_tuning_measures_native_candidates_in_fresh_cli_processes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.chdir(tmp_path) + suite = tmp_path / "tests_testenix" + suite.mkdir() + (suite / "test_sample.py").write_text( + "def test_one(): assert True\ndef test_two(): assert 2 + 2 == 4\n", + encoding="utf-8", + ) + (tmp_path / "pyproject.toml").write_text( + """ +[tool.testenix] +shard_modules = true +json = "must-not-be-written.json" +junit = "must-not-be-written.xml" +""".lstrip(), + encoding="utf-8", + ) + + report = run_tuning( + ("tests_testenix",), + TestenixConfig(history_path=None), + candidates=(1,), + warmups=0, + repeats=1, + ) + + assert report.discovered_tests == 2 + assert report.execution_units == 1 + assert report.recommended_workers == 1 + assert report.candidates[0].median > 0.0 + assert not (tmp_path / "must-not-be-written.json").exists() + assert not (tmp_path / "must-not-be-written.xml").exists() + + +def test_tuning_subprocess_timeout_terminates_its_process_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posix_signals = SimpleNamespace(SIGTERM=15, SIGKILL=9) + + class FakeProcess: + pid = 4242 + + def __init__(self) -> None: + self.waits = 0 + self.reaped = False + + def wait(self, *, timeout: float) -> int: + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired(("python",), timeout) + self.reaped = True + return -15 + + def poll(self) -> int | None: + return -15 if self.reaped else None + + def kill(self) -> None: + raise AssertionError("process-group cleanup should reap the process") + + process = FakeProcess() + popen_options: dict[str, object] = {} + killed_groups: list[tuple[int, int]] = [] + killed_processes: list[tuple[int, int]] = [] + + def fake_popen(command: object, **options: object) -> FakeProcess: + assert command == ("python", "suite.py") + popen_options.update(options) + return process + + class FakeTracker: + def __init__(self, pid: int) -> None: + assert pid == 4242 + + def stop(self) -> dict[int, str]: + return {5001: "worker-a", 5000: "worker-b"} + + monkeypatch.setattr(tuning_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr(tuning_module, "signal", posix_signals) + monkeypatch.setattr(tuning_module, "_PosixTreeTracker", FakeTracker) + monkeypatch.setattr(tuning_module, "_posix_descendant_pids", lambda pid: (5001, 5000)) + monkeypatch.setattr( + tuning_module, + "_process_identity", + lambda pid: {5001: "worker-a", 5000: "worker-b"}.get(pid), + ) + monkeypatch.setattr( + tuning_module, + "os", + SimpleNamespace( + name="posix", + getpgid=lambda pid: 4242, + getpgrp=lambda: 9999, + kill=lambda pid, sig: killed_processes.append((pid, sig)), + killpg=lambda pid, sig: killed_groups.append((pid, sig)), + ), + ) + + with pytest.raises(TuningError, match="2s per-run deadline"): + tuning_module._run_bounded_process( + ("python", "suite.py"), + env={}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=2.0, + label="native sample", + ) + + assert popen_options["start_new_session"] is True + assert killed_groups == [ + (4242, posix_signals.SIGTERM), + (4242, posix_signals.SIGKILL), + ] + assert killed_processes == [] + + +def test_posix_descendant_snapshot_is_deepest_first( + monkeypatch: pytest.MonkeyPatch, +) -> None: + completed = SimpleNamespace( + returncode=0, + stdout="1 0\n10 1\n11 10\n12 1\n99 77\ninvalid\n", + ) + monkeypatch.setattr(tuning_module.subprocess, "run", lambda *args, **kwargs: completed) + + assert tuning_module._posix_descendant_pids(1) == (11, 10, 12) + + +def test_posix_cleanup_never_signals_a_recycled_descendant_or_root_group( + monkeypatch: pytest.MonkeyPatch, +) -> None: + killed_groups: list[tuple[int, int]] = [] + monkeypatch.setattr(tuning_module, "_process_identity", lambda pid: "new-process") + monkeypatch.setattr( + tuning_module, + "os", + SimpleNamespace( + getpgrp=lambda: 9999, + getpgid=lambda pid: pytest.fail(f"recycled PID {pid} must not be resolved"), + killpg=lambda group, sig: killed_groups.append((group, sig)), + ), + ) + + tuning_module._posix_signal_tree( + 4242, + {5001: "old-process"}, + tuning_module.signal.SIGTERM, + root_group_owned=False, + ) + + assert killed_groups == [] + + +def test_windows_tuning_timeout_closes_kill_job_before_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeProcess: + pid = 4343 + + def __init__(self) -> None: + self.waits = 0 + self.reaped = False + + def wait(self, *, timeout: float) -> int: + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired(("python",), timeout) + self.reaped = True + return -9 + + def poll(self) -> int | None: + return -9 if self.reaped else None + + def kill(self) -> None: + self.reaped = True + + class FakeJob: + def __init__(self) -> None: + self.closed = False + + def terminate(self) -> bool: + self.closed = True + return True + + def close(self) -> bool: + self.closed = True + return True + + process = FakeProcess() + job = FakeJob() + popen_options: dict[str, object] = {} + + def fake_popen(command: object, **options: object) -> FakeProcess: + del command + popen_options.update(options) + return process + + monkeypatch.setattr(tuning_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr( + tuning_module.subprocess, + "run", + lambda *args, **kwargs: pytest.fail("taskkill fallback must not run with a Job Object"), + ) + monkeypatch.setattr(tuning_module, "_windows_kill_job", lambda candidate: job) + monkeypatch.setattr(tuning_module, "_resume_windows_process", lambda candidate: None) + monkeypatch.setattr(tuning_module, "os", SimpleNamespace(name="nt")) + + with pytest.raises(TuningError, match="per-run deadline"): + tuning_module._run_bounded_process( + ("python", "suite.py"), + env={}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=2.0, + label="windows sample", + ) + + assert job.closed + assert popen_options["creationflags"] == ( + getattr(tuning_module.subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200) + | getattr(tuning_module.subprocess, "CREATE_SUSPENDED", 0x00000004) + ) + + +def test_windows_tuning_fails_closed_before_resuming_without_job( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class FakeProcess: + pid = 4545 + + def __init__(self) -> None: + self.killed = False + + def kill(self) -> None: + self.killed = True + + def wait(self, *, timeout: float) -> int: + assert timeout == tuning_module._PROCESS_TERMINATION_GRACE + return -9 + + process = FakeProcess() + popen_options: dict[str, object] = {} + + def fake_popen(command: object, **options: object) -> FakeProcess: + del command + popen_options.update(options) + return process + + monkeypatch.setattr(tuning_module.subprocess, "Popen", fake_popen) + monkeypatch.setattr(tuning_module, "_windows_kill_job", lambda candidate: None) + monkeypatch.setattr(tuning_module, "os", SimpleNamespace(name="nt")) + + with pytest.raises(TuningError, match="kill-on-close Job Object"): + tuning_module._run_bounded_process( + ("python", "suite.py"), + env={}, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=2.0, + label="windows sample", + ) + + assert process.killed + assert int(popen_options["creationflags"]) & 0x00000004 + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX process-tree regression") +def test_tuning_timeout_kills_a_descendant_that_created_its_own_session( + tmp_path: Path, +) -> None: + child_pid_path = tmp_path / "child.pid" + child_source = "import os, time; os.setsid(); time.sleep(30)" + parent_source = ( + "import pathlib, subprocess, sys, time; " + f"child = subprocess.Popen([sys.executable, '-c', {child_source!r}]); " + f"pathlib.Path({str(child_pid_path)!r}).write_text(str(child.pid)); " + "time.sleep(30)" + ) + + with pytest.raises(TuningError, match="per-run deadline"): + tuning_module._run_bounded_process( + (sys.executable, "-c", parent_source), + env=os.environ, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=1.0, + label="detached-child sample", + ) + + child_pid = int(child_pid_path.read_text(encoding="utf-8")) + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline: + try: + os.kill(child_pid, 0) + except ProcessLookupError: + break + time.sleep(0.05) + else: + pytest.fail(f"detached timing descendant {child_pid} survived timeout cleanup") + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX process-tree regression") +def test_tuning_cleans_detached_child_after_fast_successful_leader( + tmp_path: Path, +) -> None: + ready = tmp_path / "child-ready" + marker = tmp_path / "orphan-finished" + child_source = ( + "import os, pathlib, sys, time\n" + "os.setsid()\n" + "pathlib.Path(sys.argv[1]).write_text('ready', encoding='utf-8')\n" + "time.sleep(0.8)\n" + "pathlib.Path(sys.argv[2]).write_text('orphan', encoding='utf-8')\n" + ) + parent_source = ( + "import pathlib, subprocess, sys, time\n" + f"ready = pathlib.Path({str(ready)!r})\n" + f"subprocess.Popen([sys.executable, '-c', {child_source!r}, str(ready), " + f"{str(marker)!r}])\n" + "deadline = time.monotonic() + 2\n" + "while not ready.exists() and time.monotonic() < deadline:\n" + " time.sleep(0.005)\n" + "time.sleep(0.05)\n" + ) + + assert ( + tuning_module._run_bounded_process( + (sys.executable, "-c", parent_source), + env=os.environ, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=2.0, + label="fast-leader sample", + ) + == 0 + ) + + assert ready.exists() + time.sleep(1.0) + assert not marker.exists() + + +def test_run_tuning_rejects_invalid_per_run_deadline() -> None: + with pytest.raises(ValueError, match="run_timeout"): + run_tuning(("tests",), TestenixConfig(), run_timeout=0.0) + + +def test_worker_recommendation_updates_only_the_testenix_table(tmp_path: Path) -> None: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + """ +[project] +name = "sample" + +[tool.testenix] +workers = 2 # measured locally +paths = ["tests_testenix"] + +[tool.ruff] +line-length = 100 +""".lstrip(), + encoding="utf-8", + ) + + assert write_worker_recommendation(pyproject, 4) + assert load_config(pyproject).workers == 4 + contents = pyproject.read_text(encoding="utf-8") + assert contents.count("workers = 4") == 1 + assert "workers = 4 # measured locally" in contents + assert "[tool.ruff]\nline-length = 100" in contents + assert not write_worker_recommendation(pyproject, 4) + + +def test_worker_recommendation_fails_closed_on_multiline_toml(tmp_path: Path) -> None: + pyproject = tmp_path / "pyproject.toml" + original = '[tool.testenix]\ntags = ["""\nworkers = 99\n"""]\n' + pyproject.write_text(original, encoding="utf-8") + + with pytest.raises(ValueError, match="refusing an update"): + write_worker_recommendation(pyproject, 4) + + assert pyproject.read_text(encoding="utf-8") == original + + +def test_worker_recommendation_preserves_crlf_and_rejects_symlinks(tmp_path: Path) -> None: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_bytes(b"[tool.testenix]\r\nworkers = 2\r\n") + + assert write_worker_recommendation(pyproject, 3) + assert pyproject.read_bytes() == b"[tool.testenix]\r\nworkers = 3\r\n" + + link = tmp_path / "linked.toml" + try: + link.symlink_to(pyproject) + except OSError: + pytest.skip("symbolic links are unavailable") + with pytest.raises(ValueError, match="symbolic link"): + write_worker_recommendation(link, 4) + assert load_config(pyproject).workers == 3 + + +def test_worker_recommendation_compare_and_swap_preserves_concurrent_edit( + tmp_path: Path, +) -> None: + pyproject = tmp_path / "pyproject.toml" + original = b'[tool.testenix]\npaths = ["tests_testenix"]\n' + concurrent = b'[tool.testenix]\npaths = ["changed_elsewhere"]\n' + pyproject.write_bytes(original) + pyproject.write_bytes(concurrent) + + with pytest.raises(ValueError, match="configuration changed while tuning"): + write_worker_recommendation(pyproject, 4, expected_source=original) + + assert pyproject.read_bytes() == concurrent + + +def test_tune_cli_never_writes_config_without_explicit_flag( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + pyproject = tmp_path / "pyproject.toml" + original = '[tool.testenix]\npaths = ["tests_testenix"]\n' + pyproject.write_text(original, encoding="utf-8") + monkeypatch.setattr("testenix.tuning.run_tuning", lambda *args, **kwargs: _report()) + + assert main(["tune", "--config", str(pyproject), "--repeats", "3"]) == 0 + assert pyproject.read_text(encoding="utf-8") == original + assert "Recommended workers: 2" in capsys.readouterr().out + + assert main(["tune", "--config", str(pyproject), "--write"]) == 0 + assert load_config(pyproject).workers == 2 + assert "Wrote workers = 2" in capsys.readouterr().out + + +def test_tune_write_refuses_configuration_changed_during_measurement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + pyproject = tmp_path / "pyproject.toml" + original = '[tool.testenix]\npaths = ["tests_testenix"]\n' + changed = '[tool.testenix]\npaths = ["other_tests"]\n' + pyproject.write_text(original, encoding="utf-8") + + def measure(*args: object, **kwargs: object) -> TuningReport: + del args, kwargs + pyproject.write_text(changed, encoding="utf-8") + return _report() + + monkeypatch.setattr("testenix.tuning.run_tuning", measure) + + assert main(["tune", "--config", str(pyproject), "--write"]) == 3 + assert pyproject.read_text(encoding="utf-8") == changed + assert "configuration changed while tuning" in capsys.readouterr().err + + +def test_tune_write_refuses_edit_in_the_final_write_window( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + pyproject = tmp_path / "pyproject.toml" + original = b'[tool.testenix]\npaths = ["tests_testenix"]\n' + concurrent = b'[tool.testenix]\npaths = ["changed_elsewhere"]\n' + pyproject.write_bytes(original) + monkeypatch.setattr("testenix.tuning.run_tuning", lambda *args, **kwargs: _report()) + real_write = write_worker_recommendation + + def edit_then_write( + path: str | Path, + workers: int, + *, + expected_source: bytes | None, + ) -> bool: + pyproject.write_bytes(concurrent) + return real_write(path, workers, expected_source=expected_source) + + monkeypatch.setattr("testenix.config.write_worker_recommendation", edit_then_write) + + assert main(["tune", "--config", str(pyproject), "--write"]) == 3 + assert pyproject.read_bytes() == concurrent + assert "configuration changed while tuning" in capsys.readouterr().err + + +def test_tune_cli_passes_per_run_deadline_to_service( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: list[float] = [] + + def measure(*args: object, **kwargs: object) -> TuningReport: + del args + observed.append(float(kwargs["run_timeout"])) + return _report() + + monkeypatch.setattr("testenix.tuning.run_tuning", measure) + + assert main(["tune", "tests_testenix", "--run-timeout", "12.5"]) == 0 + assert observed == [12.5] + + +def test_tune_cli_json_stdout_is_machine_readable( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr("testenix.tuning.run_tuning", lambda *args, **kwargs: _report()) + + assert main(["tune", "--json", "-", "tests_testenix"]) == 0 + captured = capsys.readouterr() + document = json.loads(captured.out) + assert document["schema"] == "testenix.tuning-report" + assert document["recommended_workers"] == 2 + assert "Recommended workers: 2" in captured.err + + +def test_benchmark_alias_runs_the_same_tuning_service( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setattr("testenix.tuning.run_tuning", lambda *args, **kwargs: _report()) + + assert main(["benchmark", "tests_testenix", "--candidates", "1,2"]) == 0 + assert "Recommended workers: 2" in capsys.readouterr().out + + +def test_tune_write_rejects_positional_suite_different_from_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.testenix]\npaths = ["configured"]\n', encoding="utf-8") + called = False + + def should_not_run(*args: object, **kwargs: object) -> TuningReport: + nonlocal called + called = True + return _report() + + monkeypatch.setattr("testenix.tuning.run_tuning", should_not_run) + + assert main(["tune", "other", "--config", str(pyproject), "--write"]) == 2 + assert not called + assert pyproject.read_text(encoding="utf-8") == ('[tool.testenix]\npaths = ["configured"]\n') + + +@pytest.mark.parametrize( + ("configuration", "arguments"), + [ + ('paths = ["tests_testenix"]\n', ("--shard-modules",)), + ( + 'paths = ["tests_testenix"]\nshard_modules = true\n', + ("--no-shard-modules",), + ), + ('paths = ["tests_testenix"]\n', ("--manifest", "transient.json")), + ( + 'paths = ["tests_testenix"]\nmanifest = "configured.json"\n', + ("--manifest", "transient.json"), + ), + ], +) +def test_tune_write_rejects_transient_execution_profiles( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + configuration: str, + arguments: tuple[str, ...], +) -> None: + pyproject = tmp_path / "pyproject.toml" + original = f"[tool.testenix]\n{configuration}" + pyproject.write_text(original, encoding="utf-8") + called = False + + def should_not_run(*args: object, **kwargs: object) -> TuningReport: + nonlocal called + called = True + return _report() + + monkeypatch.setattr("testenix.tuning.run_tuning", should_not_run) + + assert main(["tune", "--config", str(pyproject), *arguments, "--write"]) == 2 + assert not called + assert pyproject.read_text(encoding="utf-8") == original + message = capsys.readouterr().err + assert "--write refuses a workers-only recommendation" in message + assert "transient execution-profile override" in message + + +def test_tune_write_allows_matching_explicit_execution_profile( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[tool.testenix]\npaths = ["tests_testenix"]\nshard_modules = true\n', + encoding="utf-8", + ) + measured_configs: list[TestenixConfig] = [] + + def measure(*args: object, **kwargs: object) -> TuningReport: + measured_configs.append(args[1]) + return _report() + + monkeypatch.setattr("testenix.tuning.run_tuning", measure) + + assert main(["tune", "--config", str(pyproject), "--shard-modules", "--write"]) == 0 + assert measured_configs[0].shard_modules is True + assert load_config(pyproject).workers == 2 + + +def test_tune_json_refuses_existing_and_configuration_targets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject = tmp_path / "pyproject.toml" + original = '[tool.testenix]\npaths = ["tests_testenix"]\n' + pyproject.write_text(original, encoding="utf-8") + monkeypatch.setattr("testenix.tuning.run_tuning", lambda *args, **kwargs: _report()) + + assert main(["tune", "--config", str(pyproject), "--json", str(pyproject)]) == 2 + assert pyproject.read_text(encoding="utf-8") == original + + existing = tmp_path / "report.json" + existing.write_text("keep", encoding="utf-8") + assert main(["tune", "--config", str(pyproject), "--json", str(existing)]) == 2 + assert existing.read_text(encoding="utf-8") == "keep" + + +def test_tune_cli_contains_unexpected_exceptions( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + def explode(*args: object, **kwargs: object) -> TuningReport: + raise RuntimeError("boom") + + monkeypatch.setattr("testenix.tuning.run_tuning", explode) + + assert main(["tune", "tests_testenix", "--candidates", "1"]) == 3 + assert "tuning error: boom" in capsys.readouterr().err + + +@pytest.mark.parametrize( + "arguments", + [ + ("--candidates", "1,auto"), + ("--candidates", "1,,2"), + ("--repeats", "0"), + ("--warmups", "-1"), + ("--run-timeout", "0"), + ], +) +def test_tune_cli_rejects_invalid_measurement_options(arguments: tuple[str, ...]) -> None: + with pytest.raises(SystemExit) as exit_info: + main(["tune", *arguments]) + + assert exit_info.value.code == 2