From 637e18f3db36b79092106017ef29e9e13ff943f8 Mon Sep 17 00:00:00 2001 From: Dominik Franczyk Date: Tue, 21 Jul 2026 09:17:23 +0200 Subject: [PATCH 1/3] Improve terminal test output --- CHANGELOG.md | 13 + README.md | 37 ++- docs/getting-started.md | 14 +- docs/guides/pytest-compatibility.md | 17 +- docs/guides/reports.md | 51 ++- docs/llms-full.txt | 143 +++++++- docs/reference/cli.md | 42 ++- llms-full.txt | 143 +++++++- src/testenix/cli.py | 68 +++- src/testenix/reporters/console.py | 494 +++++++++++++++++++++++++--- tests/test_cli_reports.py | 135 ++++++++ tests/test_console_reporter.py | 387 ++++++++++++++++++++++ 12 files changed, 1453 insertions(+), 91 deletions(-) create mode 100644 tests/test_console_reporter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e5fa335..999e5e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- Native console controls for quiet output, one- or two-level verbosity, skipped/expected-failure + reasons, slow-test duration lists, and explicit automatic/forced/disabled ANSI color handling. + +### Changed + +- `testenix run` now defaults to a compact per-file report while retaining complete collection and + failure diagnostics plus the final summary. Console rendering remains deterministic and is + emitted after execution rather than presented as live progress. +- Documentation now distinguishes native Testenix rendering from the unchanged pytest output + produced by the transparent `testenix pytest` compatibility bridge. + ## [0.2.0] - 2026-07-20 ### Added diff --git a/README.md b/README.md index b59f5f0..31f525b 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ If a supported pytest (`>=8.3,<10`) is already installed in the same environment `testenix` package is sufficient. Testenix requires Python 3.11 or newer. The project is currently an alpha; pin the version before -using it in CI. Until the first PyPI release is visible, use the GitHub installation below. +using it in CI. Use the GitHub installation below only when you intentionally want unreleased +changes from the current development branch. To try the current checkout before publication: @@ -61,12 +62,14 @@ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataengin Yes. Testenix provides a transparent compatibility bridge for existing pytest projects: ```bash -testenix pytest -q tests +testenix pytest -q --tb=short tests ``` Everything after `testenix pytest` is forwarded unchanged to the same interpreter as `python -m pytest`. This preserves pytest collection, `conftest.py`, fixtures, parametrization, markers, assertion rewriting, plugins, configuration, output, node IDs, and exit codes. +Consequently, output produced by `uv run pytest tests` or `testenix pytest ...` is pytest's UI, +not the native Testenix console reporter. ```bash testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 @@ -164,6 +167,29 @@ Run it with: testenix run tests ``` +The native command uses a compact, file-level report by default and still prints complete failure +details and a final summary. A typical failing run looks like this (the run ID and timings vary): + +```console +$ testenix run tests +Testenix | 4 tests | 2 files | 2 workers + +PASS tests/test_multiplication.py 2 passed [8ms] +FAIL tests/test_checkout.py 1 passed, 1 failed [12ms] + +Problems (1) +FAIL tests/test_checkout.py::test_rejects_expired_card + attempt 1, call: expected status 402, got 200 + +4 tests, 3 passed, 1 failed in 0.084s +``` + +Use `-q` to hide the header and file table while retaining collection errors, failure details, and +the final summary. `-v` prints one result row per test; `-vv` also exposes worker, attempt, and +phase metadata. `--show-skips` includes skip and expected-failure reasons, `--durations N` lists +the `N` slowest tests (`--durations 0` lists all), and `--color auto|always|never` controls ANSI +styling. + Plain `test_*` functions are collected without `@test`; the decorator is useful for descriptions, tags, and per-test timeouts. @@ -184,11 +210,14 @@ Command-line options override this table: ```text testenix run [PATH ...] [--workers auto|N] [--retries N] [--timeout SECONDS] [--tag TAG ...] [--json FILE] [--junit FILE] - [--history FILE | --no-history] + [--history FILE | --no-history] [-q | -v | -vv] + [--color auto|always|never | --no-color] + [--show-skips] [--durations N] ``` Repeated `--tag` options use AND semantics: a selected test must contain every requested tag. -The console report is always printed. JSON preserves the complete run/test/attempt/phase model, +The console report is always printed after execution; it is deterministic output, not a live +progress display. JSON preserves the complete run/test/attempt/phase model, JUnit targets CI systems, and SQLite history supplies duration estimates to later runs. History is enabled at `.testenix/history.sqlite3` by default; use `--no-history` for a side-effect-free run. diff --git a/docs/getting-started.md b/docs/getting-started.md index 9cad010..6e2727c 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -33,7 +33,7 @@ $ python -m pip install "testenix[pytest]" $ uv add --dev "testenix[pytest]" ``` -Until the first PyPI release is visible, install directly from the protected `main` branch: +To evaluate unreleased development changes, install directly from the protected `main` branch: ```console $ python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" @@ -46,7 +46,7 @@ $ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataeng Run an unchanged pytest suite through its real engine: ```console -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests ``` Use `testenix run` for native Testenix tests and the built-in scheduler, retries, history, and @@ -85,6 +85,16 @@ Run the suite: $ testenix run tests ``` +The default console output is a compact per-file report followed by complete failure details and a +final summary. It is rendered deterministically after the run rather than updated as live progress. +Use `-q` to omit the header and file table, `-v` for one row per test, or `-vv` for worker, attempt, +and phase metadata. Collection errors and failure details remain visible with `-q`. + +```console +$ testenix run -q tests +4 tests, 3 passed, 1 skipped in 0.071s +``` + The process exits with code `0` when every selected test has a non-gating terminal status. ## Add native metadata diff --git a/docs/guides/pytest-compatibility.md b/docs/guides/pytest-compatibility.md index 240ec89..5d38adc 100644 --- a/docs/guides/pytest-compatibility.md +++ b/docs/guides/pytest-compatibility.md @@ -10,7 +10,7 @@ configuration: ```console $ python -m pip install "testenix[pytest]" -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests ``` For the supported static subset, `testenix migrate pytest tests` can instead create a validated @@ -36,6 +36,14 @@ working directory, environment, terminal, standard streams, and pytest's signal therefore remains responsible for collection, execution, configuration, plugin loading, output, descendants such as pytest-xdist workers, and exit status. +This also explains the visual style: output from `uv run pytest tests` is rendered by pytest, and +`testenix pytest` deliberately preserves that same renderer. It does not activate Testenix's +compact native console. For shorter pytest output with concise tracebacks, use: + +```console +$ testenix pytest -q --tb=short tests +``` + ```console $ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 $ testenix pytest -m "unit and not slow" tests @@ -84,12 +92,17 @@ timeouts, tags, history, event model, JSON reporter, or JUnit reporter. Pass the pytest or plugin options after the subcommand. For example, use pytest's `--junitxml`, not Testenix's native `--junit`. +Native presentation options are not interpreted by the bridge either. Arguments such as `-q`, +`-v`, `--color`, `--show-skips`, and `--durations` go straight to pytest and follow pytest's syntax +and semantics. For example, pytest uses `-rs` for skipped reasons, `--durations=N` for its slowest +tests list, and `--color=yes|no|auto`; Testenix does not translate the native spellings. + Pytest and every required plugin must be installed beside the `testenix` executable in the same interpreter environment. For uv-managed projects, prefer: ```console $ uv add --dev "testenix[pytest]" -$ uv run testenix pytest -q tests +$ uv run testenix pytest -q --tb=short tests ``` An isolated `uv tool install testenix` environment does not automatically see pytest plugins from diff --git a/docs/guides/reports.md b/docs/guides/reports.md index 9399b78..d61fb25 100644 --- a/docs/guides/reports.md +++ b/docs/guides/reports.md @@ -5,13 +5,60 @@ console and red in JSON because both are derived from one model. ## Console -The console report is always enabled: +The console report is always enabled. Its default mode is compact: Testenix prints an aggregate row +for each source file, then complete failure details and the final summary. ```console $ testenix run tests +Testenix | 5 tests | 2 files | 2 workers + +PASS tests/test_accounts.py 3 passed [9ms] +FAIL tests/test_checkout.py 1 passed, 1 failed [12ms] + +Problems (1) +FAIL tests/test_checkout.py::test_rejects_expired_card + attempt 1, call: expected status 402, got 200 + +5 tests, 4 passed, 1 failed in 0.091s +``` + +Run IDs and timings naturally vary. The report is assembled in stable source order when execution +finishes; it is not a live-progress display. + +Choose the amount of terminal detail without changing execution semantics: + +| Mode | Output | +| --- | --- | +| default | Run header, compact per-file table, collection/failure details, final summary. | +| `-q`, `--quiet` | No header or file table; collection/failure details and final summary remain. | +| `-v` | One stable result row per test, including duration. | +| `-vv` | Per-test rows plus worker, attempt, and phase metadata, including captured output. | + +Skipped and expected-failure reasons are hidden unless they are requested explicitly: + +```console +$ testenix run --show-skips tests +``` + +List the slowest tests with `--durations N`; use `0` to list every test. The duration section is +printed immediately before the final summary: + +```console +$ testenix run --durations 10 tests +$ testenix run --durations 0 tests +``` + +ANSI styling defaults to `--color auto`, which considers the output terminal plus `NO_COLOR`, +`FORCE_COLOR`, `CI`, and `TERM=dumb`. Force it with `--color always`, or produce plain output with +either `--color never` or `--no-color`: + +```console +$ testenix run --no-color tests ``` -It prints test outcomes, failure details, and a final summary suitable for local development. +These flags affect the native `testenix run` console only. The transparent `testenix pytest` +bridge preserves pytest's renderer and argument meanings; a concise bridge command is +`testenix pytest -q --tb=short tests`. ## JSON diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 00276b5..75cabfe 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -223,7 +223,7 @@ $ python -m pip install "testenix[pytest]" $ uv add --dev "testenix[pytest]" ``` -Until the first PyPI release is visible, install directly from the protected `main` branch: +To evaluate unreleased development changes, install directly from the protected `main` branch: ```console $ python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" @@ -236,7 +236,7 @@ $ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataeng Run an unchanged pytest suite through its real engine: ```console -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests ``` Use `testenix run` for native Testenix tests and the built-in scheduler, retries, history, and @@ -275,6 +275,16 @@ Run the suite: $ testenix run tests ``` +The default console output is a compact per-file report followed by complete failure details and a +final summary. It is rendered deterministically after the run rather than updated as live progress. +Use `-q` to omit the header and file table, `-v` for one row per test, or `-vv` for worker, attempt, +and phase metadata. Collection errors and failure details remain visible with `-q`. + +```console +$ testenix run -q tests +4 tests, 3 passed, 1 skipped in 0.071s +``` + The process exits with code `0` when every selected test has a non-gating terminal status. ## Add native metadata @@ -363,7 +373,7 @@ configuration: ```console $ python -m pip install "testenix[pytest]" -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests ``` For the supported static subset, `testenix migrate pytest tests` can instead create a validated @@ -389,6 +399,14 @@ working directory, environment, terminal, standard streams, and pytest's signal therefore remains responsible for collection, execution, configuration, plugin loading, output, descendants such as pytest-xdist workers, and exit status. +This also explains the visual style: output from `uv run pytest tests` is rendered by pytest, and +`testenix pytest` deliberately preserves that same renderer. It does not activate Testenix's +compact native console. For shorter pytest output with concise tracebacks, use: + +```console +$ testenix pytest -q --tb=short tests +``` + ```console $ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 $ testenix pytest -m "unit and not slow" tests @@ -437,12 +455,17 @@ timeouts, tags, history, event model, JSON reporter, or JUnit reporter. Pass the pytest or plugin options after the subcommand. For example, use pytest's `--junitxml`, not Testenix's native `--junit`. +Native presentation options are not interpreted by the bridge either. Arguments such as `-q`, +`-v`, `--color`, `--show-skips`, and `--durations` go straight to pytest and follow pytest's syntax +and semantics. For example, pytest uses `-rs` for skipped reasons, `--durations=N` for its slowest +tests list, and `--color=yes|no|auto`; Testenix does not translate the native spellings. + Pytest and every required plugin must be installed beside the `testenix` executable in the same interpreter environment. For uv-managed projects, prefer: ```console $ uv add --dev "testenix[pytest]" -$ uv run testenix pytest -q tests +$ uv run testenix pytest -q --tb=short tests ``` An isolated `uv tool install testenix` environment does not automatically see pytest plugins from @@ -1157,13 +1180,60 @@ console and red in JSON because both are derived from one model. ## Console -The console report is always enabled: +The console report is always enabled. Its default mode is compact: Testenix prints an aggregate row +for each source file, then complete failure details and the final summary. ```console $ testenix run tests +Testenix | 5 tests | 2 files | 2 workers + +PASS tests/test_accounts.py 3 passed [9ms] +FAIL tests/test_checkout.py 1 passed, 1 failed [12ms] + +Problems (1) +FAIL tests/test_checkout.py::test_rejects_expired_card + attempt 1, call: expected status 402, got 200 + +5 tests, 4 passed, 1 failed in 0.091s ``` -It prints test outcomes, failure details, and a final summary suitable for local development. +Run IDs and timings naturally vary. The report is assembled in stable source order when execution +finishes; it is not a live-progress display. + +Choose the amount of terminal detail without changing execution semantics: + +| Mode | Output | +| --- | --- | +| default | Run header, compact per-file table, collection/failure details, final summary. | +| `-q`, `--quiet` | No header or file table; collection/failure details and final summary remain. | +| `-v` | One stable result row per test, including duration. | +| `-vv` | Per-test rows plus worker, attempt, and phase metadata, including captured output. | + +Skipped and expected-failure reasons are hidden unless they are requested explicitly: + +```console +$ testenix run --show-skips tests +``` + +List the slowest tests with `--durations N`; use `0` to list every test. The duration section is +printed immediately before the final summary: + +```console +$ testenix run --durations 10 tests +$ testenix run --durations 0 tests +``` + +ANSI styling defaults to `--color auto`, which considers the output terminal plus `NO_COLOR`, +`FORCE_COLOR`, `CI`, and `TERM=dumb`. Force it with `--color always`, or produce plain output with +either `--color never` or `--no-color`: + +```console +$ testenix run --no-color tests +``` + +These flags affect the native `testenix run` console only. The transparent `testenix pytest` +bridge preserves pytest's renderer and argument meanings; a concise bridge command is +`testenix pytest -q --tb=short tests`. ## JSON @@ -1272,6 +1342,10 @@ testenix run [PATH ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] + [-q | -v | -vv] + [--color {auto,always,never} | --no-color] + [--show-skips] + [--durations N] ``` | Argument | Default | Description | @@ -1285,9 +1359,26 @@ testenix run [PATH ...] | `--junit` | none | Write a JUnit XML report. | | `--history` | `.testenix/history.sqlite3` | Override the duration-history database. | | `--no-history` | off | Disable reading and writing history. | +| `-q`, `--quiet` | off | Hide the run header and compact per-file table. Collection errors, failure details, and the final summary remain visible. | +| `-v`, `--verbose` | off | Print one result row per test in the stable detailed format. Repeat for `-vv`. | +| `-vv` | off | Add worker, attempt, and phase metadata, including captured output. | +| `--color auto\|always\|never` | `auto` | Select automatic ANSI styling, force styling, or disable it. | +| `--no-color` | off | Alias for `--color never`, useful in logs and snapshots. | +| `--show-skips` | off | Include reasons for skipped and expected-failure tests. | +| `--durations N` | none | List the `N` slowest tests before the summary; `0` lists every test. | CLI options override `[tool.testenix]` values for the current run. +With no presentation flags, Testenix prints a compact row for each source file, complete +collection/failure diagnostics, and a final summary. Console output is assembled in stable source +order after execution completes; these modes do not promise live progress updates. Presentation +flags change only terminal rendering, not selection, scheduling, result statuses, JSON, JUnit, or +exit codes. + +In `auto` color mode, Testenix requires a terminal, respects `NO_COLOR`, allows `FORCE_COLOR`, and +disables styling for a truthy `CI` value or `TERM=dumb`. Explicit `always` or `never` takes +precedence. + ## `testenix pytest` ```text @@ -1296,10 +1387,12 @@ testenix pytest [PYTEST_ARGS ...] This compatibility command hands the current CLI process to pytest from the same interpreter and forwards every argument without translation. It preserves pytest configuration, collection, -fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. +fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. The +compact native Testenix renderer is not involved: output is pytest's own output, just as with +`uv run pytest tests`. For a concise bridge invocation, use `testenix pytest -q --tb=short tests`. ```console -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests $ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 $ testenix pytest -n auto tests $ testenix pytest --junitxml=reports/pytest.xml tests @@ -1310,10 +1403,13 @@ Pytest and its plugins must be installed in the same Python environment as Teste consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. `[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, -`--json`, `--junit`, and `--history` do not affect this command. Pass pytest or plugin options -instead. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately -follow `testenix`. See [pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full -boundary. +`--json`, `--junit`, `--history`, `--show-skips`, and `--durations` do not affect this command. +Flags such as `-q`, `-v`, and `--color` are forwarded to pytest and have pytest's meaning, not the +native reporter's meaning. Pytest's equivalent spellings include `-rs`, `--durations=N`, and +`--color=yes|no|auto`; Testenix does not translate the native spellings. Pass pytest or plugin +options instead. In particular, +`testenix --config PATH pytest ...` is rejected; `pytest` must immediately follow `testenix`. See +[pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full boundary. ## `testenix migrate` @@ -1362,9 +1458,11 @@ $ testenix run $ testenix run tests/unit tests/integration --workers 4 $ testenix run --tag unit --tag fast $ testenix run --retries 1 --timeout 10 +$ testenix run -v --show-skips --durations 10 +$ testenix run --color never $ testenix run --json reports/run.json --junit reports/junit.xml $ testenix --config config/pyproject.toml run -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests $ testenix migrate pytest tests --dry-run $ testenix migrate auto tests --check --report-json reports/migration.json $ testenix migrate unittest tests --output tests_testenix @@ -2382,6 +2480,19 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- Native console controls for quiet output, one- or two-level verbosity, skipped/expected-failure + reasons, slow-test duration lists, and explicit automatic/forced/disabled ANSI color handling. + +### Changed + +- `testenix run` now defaults to a compact per-file report while retaining complete collection and + failure diagnostics plus the final summary. Console rendering remains deterministic and is + emitted after execution rather than presented as live progress. +- Documentation now distinguishes native Testenix rendering from the unchanged pytest output + produced by the transparent `testenix pytest` compatibility bridge. + ## [0.2.0] - 2026-07-20 ### Added @@ -2593,7 +2704,7 @@ Complete audit record for a migration attempt. ## `testenix.MigrationStatus` ```text -MigrationStatus(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) +MigrationStatus(*values) ``` Values: ANALYZED='analyzed', VALIDATED='validated', PUBLISHED='published', UNSUPPORTED='unsupported', VALIDATION_FAILED='validation_failed', SAFETY_ERROR='safety_error' @@ -2642,7 +2753,7 @@ RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tu ## `testenix.Scope` ```text -Scope(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) +Scope(*values) ``` Values: TEST='test', MODULE='module', SESSION='session' @@ -2652,7 +2763,7 @@ Lifetime of a fixture instance. ## `testenix.Status` ```text -Status(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) +Status(*values) ``` Values: PASS='pass', FAIL='fail', ERROR_SETUP='error_setup', ERROR_TEARDOWN='error_teardown', SKIP='skip', XFAIL='xfail', XPASS='xpass', TIMEOUT='timeout', CRASH='crash', INFRA_ERROR='infra_error', CANCELLED='cancelled', NOT_RUN='not_run', FLAKY='flaky', CACHED_PASS='cached_pass' diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d18a63c..7d38156 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -27,6 +27,10 @@ testenix run [PATH ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] + [-q | -v | -vv] + [--color {auto,always,never} | --no-color] + [--show-skips] + [--durations N] ``` | Argument | Default | Description | @@ -40,9 +44,26 @@ testenix run [PATH ...] | `--junit` | none | Write a JUnit XML report. | | `--history` | `.testenix/history.sqlite3` | Override the duration-history database. | | `--no-history` | off | Disable reading and writing history. | +| `-q`, `--quiet` | off | Hide the run header and compact per-file table. Collection errors, failure details, and the final summary remain visible. | +| `-v`, `--verbose` | off | Print one result row per test in the stable detailed format. Repeat for `-vv`. | +| `-vv` | off | Add worker, attempt, and phase metadata, including captured output. | +| `--color auto\|always\|never` | `auto` | Select automatic ANSI styling, force styling, or disable it. | +| `--no-color` | off | Alias for `--color never`, useful in logs and snapshots. | +| `--show-skips` | off | Include reasons for skipped and expected-failure tests. | +| `--durations N` | none | List the `N` slowest tests before the summary; `0` lists every test. | CLI options override `[tool.testenix]` values for the current run. +With no presentation flags, Testenix prints a compact row for each source file, complete +collection/failure diagnostics, and a final summary. Console output is assembled in stable source +order after execution completes; these modes do not promise live progress updates. Presentation +flags change only terminal rendering, not selection, scheduling, result statuses, JSON, JUnit, or +exit codes. + +In `auto` color mode, Testenix requires a terminal, respects `NO_COLOR`, allows `FORCE_COLOR`, and +disables styling for a truthy `CI` value or `TERM=dumb`. Explicit `always` or `never` takes +precedence. + ## `testenix pytest` ```text @@ -51,10 +72,12 @@ testenix pytest [PYTEST_ARGS ...] This compatibility command hands the current CLI process to pytest from the same interpreter and forwards every argument without translation. It preserves pytest configuration, collection, -fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. +fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. The +compact native Testenix renderer is not involved: output is pytest's own output, just as with +`uv run pytest tests`. For a concise bridge invocation, use `testenix pytest -q --tb=short tests`. ```console -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests $ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 $ testenix pytest -n auto tests $ testenix pytest --junitxml=reports/pytest.xml tests @@ -65,10 +88,13 @@ Pytest and its plugins must be installed in the same Python environment as Teste consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. `[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, -`--json`, `--junit`, and `--history` do not affect this command. Pass pytest or plugin options -instead. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately -follow `testenix`. See [pytest compatibility](../guides/pytest-compatibility.md) for the full -boundary. +`--json`, `--junit`, `--history`, `--show-skips`, and `--durations` do not affect this command. +Flags such as `-q`, `-v`, and `--color` are forwarded to pytest and have pytest's meaning, not the +native reporter's meaning. Pytest's equivalent spellings include `-rs`, `--durations=N`, and +`--color=yes|no|auto`; Testenix does not translate the native spellings. Pass pytest or plugin +options instead. In particular, +`testenix --config PATH pytest ...` is rejected; `pytest` must immediately follow `testenix`. See +[pytest compatibility](../guides/pytest-compatibility.md) for the full boundary. ## `testenix migrate` @@ -117,9 +143,11 @@ $ testenix run $ testenix run tests/unit tests/integration --workers 4 $ testenix run --tag unit --tag fast $ testenix run --retries 1 --timeout 10 +$ testenix run -v --show-skips --durations 10 +$ testenix run --color never $ testenix run --json reports/run.json --junit reports/junit.xml $ testenix --config config/pyproject.toml run -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests $ testenix migrate pytest tests --dry-run $ testenix migrate auto tests --check --report-json reports/migration.json $ testenix migrate unittest tests --output tests_testenix diff --git a/llms-full.txt b/llms-full.txt index 00276b5..75cabfe 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -223,7 +223,7 @@ $ python -m pip install "testenix[pytest]" $ uv add --dev "testenix[pytest]" ``` -Until the first PyPI release is visible, install directly from the protected `main` branch: +To evaluate unreleased development changes, install directly from the protected `main` branch: ```console $ python -m pip install "testenix @ git+https://github.com/polishdataengineer/testenix.git@main" @@ -236,7 +236,7 @@ $ python -m pip install "testenix[pytest] @ git+https://github.com/polishdataeng Run an unchanged pytest suite through its real engine: ```console -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests ``` Use `testenix run` for native Testenix tests and the built-in scheduler, retries, history, and @@ -275,6 +275,16 @@ Run the suite: $ testenix run tests ``` +The default console output is a compact per-file report followed by complete failure details and a +final summary. It is rendered deterministically after the run rather than updated as live progress. +Use `-q` to omit the header and file table, `-v` for one row per test, or `-vv` for worker, attempt, +and phase metadata. Collection errors and failure details remain visible with `-q`. + +```console +$ testenix run -q tests +4 tests, 3 passed, 1 skipped in 0.071s +``` + The process exits with code `0` when every selected test has a non-gating terminal status. ## Add native metadata @@ -363,7 +373,7 @@ configuration: ```console $ python -m pip install "testenix[pytest]" -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests ``` For the supported static subset, `testenix migrate pytest tests` can instead create a validated @@ -389,6 +399,14 @@ working directory, environment, terminal, standard streams, and pytest's signal therefore remains responsible for collection, execution, configuration, plugin loading, output, descendants such as pytest-xdist workers, and exit status. +This also explains the visual style: output from `uv run pytest tests` is rendered by pytest, and +`testenix pytest` deliberately preserves that same renderer. It does not activate Testenix's +compact native console. For shorter pytest output with concise tracebacks, use: + +```console +$ testenix pytest -q --tb=short tests +``` + ```console $ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 $ testenix pytest -m "unit and not slow" tests @@ -437,12 +455,17 @@ timeouts, tags, history, event model, JSON reporter, or JUnit reporter. Pass the pytest or plugin options after the subcommand. For example, use pytest's `--junitxml`, not Testenix's native `--junit`. +Native presentation options are not interpreted by the bridge either. Arguments such as `-q`, +`-v`, `--color`, `--show-skips`, and `--durations` go straight to pytest and follow pytest's syntax +and semantics. For example, pytest uses `-rs` for skipped reasons, `--durations=N` for its slowest +tests list, and `--color=yes|no|auto`; Testenix does not translate the native spellings. + Pytest and every required plugin must be installed beside the `testenix` executable in the same interpreter environment. For uv-managed projects, prefer: ```console $ uv add --dev "testenix[pytest]" -$ uv run testenix pytest -q tests +$ uv run testenix pytest -q --tb=short tests ``` An isolated `uv tool install testenix` environment does not automatically see pytest plugins from @@ -1157,13 +1180,60 @@ console and red in JSON because both are derived from one model. ## Console -The console report is always enabled: +The console report is always enabled. Its default mode is compact: Testenix prints an aggregate row +for each source file, then complete failure details and the final summary. ```console $ testenix run tests +Testenix | 5 tests | 2 files | 2 workers + +PASS tests/test_accounts.py 3 passed [9ms] +FAIL tests/test_checkout.py 1 passed, 1 failed [12ms] + +Problems (1) +FAIL tests/test_checkout.py::test_rejects_expired_card + attempt 1, call: expected status 402, got 200 + +5 tests, 4 passed, 1 failed in 0.091s ``` -It prints test outcomes, failure details, and a final summary suitable for local development. +Run IDs and timings naturally vary. The report is assembled in stable source order when execution +finishes; it is not a live-progress display. + +Choose the amount of terminal detail without changing execution semantics: + +| Mode | Output | +| --- | --- | +| default | Run header, compact per-file table, collection/failure details, final summary. | +| `-q`, `--quiet` | No header or file table; collection/failure details and final summary remain. | +| `-v` | One stable result row per test, including duration. | +| `-vv` | Per-test rows plus worker, attempt, and phase metadata, including captured output. | + +Skipped and expected-failure reasons are hidden unless they are requested explicitly: + +```console +$ testenix run --show-skips tests +``` + +List the slowest tests with `--durations N`; use `0` to list every test. The duration section is +printed immediately before the final summary: + +```console +$ testenix run --durations 10 tests +$ testenix run --durations 0 tests +``` + +ANSI styling defaults to `--color auto`, which considers the output terminal plus `NO_COLOR`, +`FORCE_COLOR`, `CI`, and `TERM=dumb`. Force it with `--color always`, or produce plain output with +either `--color never` or `--no-color`: + +```console +$ testenix run --no-color tests +``` + +These flags affect the native `testenix run` console only. The transparent `testenix pytest` +bridge preserves pytest's renderer and argument meanings; a concise bridge command is +`testenix pytest -q --tb=short tests`. ## JSON @@ -1272,6 +1342,10 @@ testenix run [PATH ...] [--json FILE] [--junit FILE] [--history FILE | --no-history] + [-q | -v | -vv] + [--color {auto,always,never} | --no-color] + [--show-skips] + [--durations N] ``` | Argument | Default | Description | @@ -1285,9 +1359,26 @@ testenix run [PATH ...] | `--junit` | none | Write a JUnit XML report. | | `--history` | `.testenix/history.sqlite3` | Override the duration-history database. | | `--no-history` | off | Disable reading and writing history. | +| `-q`, `--quiet` | off | Hide the run header and compact per-file table. Collection errors, failure details, and the final summary remain visible. | +| `-v`, `--verbose` | off | Print one result row per test in the stable detailed format. Repeat for `-vv`. | +| `-vv` | off | Add worker, attempt, and phase metadata, including captured output. | +| `--color auto\|always\|never` | `auto` | Select automatic ANSI styling, force styling, or disable it. | +| `--no-color` | off | Alias for `--color never`, useful in logs and snapshots. | +| `--show-skips` | off | Include reasons for skipped and expected-failure tests. | +| `--durations N` | none | List the `N` slowest tests before the summary; `0` lists every test. | CLI options override `[tool.testenix]` values for the current run. +With no presentation flags, Testenix prints a compact row for each source file, complete +collection/failure diagnostics, and a final summary. Console output is assembled in stable source +order after execution completes; these modes do not promise live progress updates. Presentation +flags change only terminal rendering, not selection, scheduling, result statuses, JSON, JUnit, or +exit codes. + +In `auto` color mode, Testenix requires a terminal, respects `NO_COLOR`, allows `FORCE_COLOR`, and +disables styling for a truthy `CI` value or `TERM=dumb`. Explicit `always` or `never` takes +precedence. + ## `testenix pytest` ```text @@ -1296,10 +1387,12 @@ testenix pytest [PYTEST_ARGS ...] This compatibility command hands the current CLI process to pytest from the same interpreter and forwards every argument without translation. It preserves pytest configuration, collection, -fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. +fixtures, parametrization, markers, classes, hooks, plugins, output, and normal exit status. The +compact native Testenix renderer is not involved: output is pytest's own output, just as with +`uv run pytest tests`. For a concise bridge invocation, use `testenix pytest -q --tb=short tests`. ```console -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests $ testenix pytest tests/test_api.py::test_health -k smoke --maxfail=1 $ testenix pytest -n auto tests $ testenix pytest --junitxml=reports/pytest.xml tests @@ -1310,10 +1403,13 @@ Pytest and its plugins must be installed in the same Python environment as Teste consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. `[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, -`--json`, `--junit`, and `--history` do not affect this command. Pass pytest or plugin options -instead. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately -follow `testenix`. See [pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full -boundary. +`--json`, `--junit`, `--history`, `--show-skips`, and `--durations` do not affect this command. +Flags such as `-q`, `-v`, and `--color` are forwarded to pytest and have pytest's meaning, not the +native reporter's meaning. Pytest's equivalent spellings include `-rs`, `--durations=N`, and +`--color=yes|no|auto`; Testenix does not translate the native spellings. Pass pytest or plugin +options instead. In particular, +`testenix --config PATH pytest ...` is rejected; `pytest` must immediately follow `testenix`. See +[pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full boundary. ## `testenix migrate` @@ -1362,9 +1458,11 @@ $ testenix run $ testenix run tests/unit tests/integration --workers 4 $ testenix run --tag unit --tag fast $ testenix run --retries 1 --timeout 10 +$ testenix run -v --show-skips --durations 10 +$ testenix run --color never $ testenix run --json reports/run.json --junit reports/junit.xml $ testenix --config config/pyproject.toml run -$ testenix pytest -q tests +$ testenix pytest -q --tb=short tests $ testenix migrate pytest tests --dry-run $ testenix migrate auto tests --check --report-json reports/migration.json $ testenix migrate unittest tests --output tests_testenix @@ -2382,6 +2480,19 @@ project intends to use Semantic Versioning once its public API reaches stability ## [Unreleased] +### Added + +- Native console controls for quiet output, one- or two-level verbosity, skipped/expected-failure + reasons, slow-test duration lists, and explicit automatic/forced/disabled ANSI color handling. + +### Changed + +- `testenix run` now defaults to a compact per-file report while retaining complete collection and + failure diagnostics plus the final summary. Console rendering remains deterministic and is + emitted after execution rather than presented as live progress. +- Documentation now distinguishes native Testenix rendering from the unchanged pytest output + produced by the transparent `testenix pytest` compatibility bridge. + ## [0.2.0] - 2026-07-20 ### Added @@ -2593,7 +2704,7 @@ Complete audit record for a migration attempt. ## `testenix.MigrationStatus` ```text -MigrationStatus(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) +MigrationStatus(*values) ``` Values: ANALYZED='analyzed', VALIDATED='validated', PUBLISHED='published', UNSUPPORTED='unsupported', VALIDATION_FAILED='validation_failed', SAFETY_ERROR='safety_error' @@ -2642,7 +2753,7 @@ RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tu ## `testenix.Scope` ```text -Scope(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) +Scope(*values) ``` Values: TEST='test', MODULE='module', SESSION='session' @@ -2652,7 +2763,7 @@ Lifetime of a fixture instance. ## `testenix.Status` ```text -Status(value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) +Status(*values) ``` Values: PASS='pass', FAIL='fail', ERROR_SETUP='error_setup', ERROR_TEARDOWN='error_teardown', SKIP='skip', XFAIL='xfail', XPASS='xpass', TIMEOUT='timeout', CRASH='crash', INFRA_ERROR='infra_error', CANCELLED='cancelled', NOT_RUN='not_run', FLAKY='flaky', CACHED_PASS='cached_pass' diff --git a/src/testenix/cli.py b/src/testenix/cli.py index f770d81..5e72b1e 100644 --- a/src/testenix/cli.py +++ b/src/testenix/cli.py @@ -67,6 +67,46 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="run tests with this tag; repeat for multiple tags", ) + verbosity_group = run_parser.add_mutually_exclusive_group() + verbosity_group.add_argument( + "-q", + "--quiet", + action="store_true", + help="show failures and the final summary only", + ) + verbosity_group.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="increase detail; repeat for full captured output", + ) + color_group = run_parser.add_mutually_exclusive_group() + color_group.add_argument( + "--color", + choices=("auto", "always", "never"), + default="auto", + help="color output: auto, always, or never (default: auto)", + ) + color_group.add_argument( + "--no-color", + dest="color", + action="store_const", + const="never", + help="alias for --color never", + ) + run_parser.add_argument( + "--show-skips", + action="store_true", + help="show details for skipped and expected-failure tests", + ) + run_parser.add_argument( + "--durations", + type=_non_negative_int, + default=None, + metavar="N", + help="show the N slowest tests; 0 shows all", + ) run_parser.add_argument("--json", dest="json_path", type=Path, default=None) run_parser.add_argument("--junit", dest="junit_path", type=Path, default=None) history_group = run_parser.add_mutually_exclusive_group() @@ -188,7 +228,15 @@ def _run_command(arguments: argparse.Namespace) -> int: print("testenix: runner returned an invalid result", file=sys.stderr) return EXIT_INTERNAL_ERROR - ConsoleReporter().write(result) + verbosity = -1 if arguments.quiet else min(arguments.verbose, 2) + workers = _reporter_worker_count(result, config) + ConsoleReporter( + verbosity=verbosity, + color=arguments.color, + show_skips=arguments.show_skips, + durations=arguments.durations, + workers=workers, + ).write(result) try: if config.json_path is not None: JsonReporter(config.json_path).write(result) @@ -208,6 +256,14 @@ def _call_runner(paths: Sequence[str], config: TestenixConfig) -> RunResult: return run(paths, config) +def _reporter_worker_count(result: RunResult, config: TestenixConfig) -> int: + """Mirror the native runner's initial module/timeout execution units.""" + + shared_modules = {test.test.path for test in result.tests if test.test.timeout is None} + isolated_tests = sum(test.test.timeout is not None for test in result.tests) + return min(config.resolved_workers, len(shared_modules) + isolated_tests) + + def _pytest_command(arguments: Sequence[str]) -> int: from testenix.pytest_adapter import PytestInvocationError, PytestUnavailableError @@ -348,6 +404,16 @@ def _positive_float(value: str) -> float: return parsed +def _non_negative_int(value: str) -> int: + try: + parsed = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("value must be an integer") from error + if parsed < 0: + raise argparse.ArgumentTypeError("value must be at least 0") + return parsed + + def _migration_worker_count(value: str) -> int | str: workers = _worker_count(value) if isinstance(workers, int) and workers < 2: diff --git a/src/testenix/reporters/console.py b/src/testenix/reporters/console.py index 326cba5..fced856 100644 --- a/src/testenix/reporters/console.py +++ b/src/testenix/reporters/console.py @@ -1,14 +1,19 @@ -"""Deterministic, colour-free terminal reporting.""" +"""Deterministic terminal reporting with compact and diagnostic views.""" from __future__ import annotations +import os +import shutil import sys from collections import Counter +from collections.abc import Mapping from io import StringIO -from typing import TextIO +from typing import Literal, TextIO from testenix.contracts import PhaseResult, RunResult, Status, TestResult +ColorMode = Literal["auto", "always", "never"] + _SUMMARY_LABELS = { Status.PASS: "passed", Status.FAIL: "failed", @@ -26,45 +31,287 @@ Status.CACHED_PASS: "cached", } +_NON_PROBLEM_STATUSES = { + Status.PASS, + Status.CACHED_PASS, + Status.SKIP, + Status.XFAIL, +} +_SKIP_STATUSES = {Status.SKIP, Status.XFAIL} +_ANSI_RESET = "\x1b[0m" +_ANSI_BY_STATUS = { + Status.PASS: "\x1b[32m", + Status.CACHED_PASS: "\x1b[32m", + Status.SKIP: "\x1b[33m", + Status.XFAIL: "\x1b[33m", + Status.XPASS: "\x1b[33m", + Status.FLAKY: "\x1b[33m", + Status.FAIL: "\x1b[31m", + Status.ERROR_SETUP: "\x1b[35m", + Status.ERROR_TEARDOWN: "\x1b[35m", + Status.TIMEOUT: "\x1b[35m", + Status.CRASH: "\x1b[35m", + Status.INFRA_ERROR: "\x1b[35m", + Status.CANCELLED: "\x1b[35m", + Status.NOT_RUN: "\x1b[35m", +} +_ANSI_COLLECT = "\x1b[31m" +_ANSI_HEADER = "\x1b[1;36m" +_DEFAULT_WIDTH = 100 +_MAX_WIDTH = 120 + class ConsoleReporter: - """Render results in manifest order reconstructed from source locations.""" + """Render a run in quiet, compact, legacy, or diagnostic form. + + ``verbosity=1`` deliberately retains Testenix's original deterministic, + colour-free output when all other arguments use their defaults. Compact + output groups successful results by file, so its size is independent of + the number of passing tests in a file. + """ + + def __init__( + self, + *, + verbosity: int = 1, + color: ColorMode = "never", + show_skips: bool = False, + durations: int | None = None, + workers: int | None = None, + width: int | None = None, + ) -> None: + if isinstance(verbosity, bool) or verbosity not in {-1, 0, 1, 2}: + raise ValueError("verbosity must be one of -1, 0, 1, or 2") + if color not in {"auto", "always", "never"}: + raise ValueError("color must be 'auto', 'always', or 'never'") + if not isinstance(show_skips, bool): + raise TypeError("show_skips must be a bool") + if durations is not None and ( + isinstance(durations, bool) or not isinstance(durations, int) or durations < 0 + ): + raise ValueError("durations must be None or a non-negative integer") + if workers is not None and ( + isinstance(workers, bool) or not isinstance(workers, int) or workers < 0 + ): + raise ValueError("workers must be None or a non-negative integer") + if width is not None and ( + isinstance(width, bool) or not isinstance(width, int) or width < 1 + ): + raise ValueError("width must be None or a positive integer") + + self.verbosity: int = verbosity + self.color: ColorMode = color + self.show_skips: bool = show_skips + self.durations: int | None = durations + self.workers: int | None = workers + self.width: int | None = width def render(self, run: RunResult) -> str: + """Return deterministic text. + + ``auto`` colour is intentionally plain here because no destination is + known. ``write`` resolves it against the actual stream and environment. + """ + + use_color = self.color == "always" + width = min(self.width or _DEFAULT_WIDTH, _MAX_WIDTH) + return self._render(run, use_color=use_color, width=width) + + def write(self, run: RunResult, stream: TextIO | None = None) -> None: + """Write a run, resolving automatic colour and terminal width.""" + + target = stream if stream is not None else sys.stdout + use_color = _should_use_color(self.color, target, os.environ) + width = min(self.width or _stream_width(target), _MAX_WIDTH) + target.write(self._render(run, use_color=use_color, width=width)) + + report = write + + def _render(self, run: RunResult, *, use_color: bool, width: int) -> str: output = StringIO() - output.write(f"Testenix run {run.run_id}\n") - for issue in sorted(run.collection_issues, key=lambda item: (item.path, item.message)): - output.write(f"COLLECT {issue.path}\n") - for line in issue.message.splitlines() or [""]: - output.write(f" {line}\n") - if issue.traceback: - for line in issue.traceback.rstrip().splitlines(): - output.write(f" {line}\n") + if self.verbosity == 0: + file_count = len({result.test.path for result in run.tests}) + test_label = "test" if len(run.tests) == 1 else "tests" + file_label = "file" if file_count == 1 else "files" + header = f"Testenix | {len(run.tests)} {test_label} | {file_count} {file_label}" + if self.workers is not None: + worker_label = "worker" if self.workers == 1 else "workers" + header = f"{header} | {self.workers} {worker_label}" + output.write(f"{_paint_header(header, use_color)}\n\n") + elif self.verbosity >= 1: + header = f"Testenix run {run.run_id}" + if self.verbosity == 2 and self.workers is not None: + header = f"{header} [workers={self.workers}]" + output.write(f"{header}\n") + + _write_collection_issues(output, run, use_color=use_color) + + ordered_tests = sorted(run.tests, key=_test_sort_key) + if self.verbosity == 0: + if run.collection_issues and ordered_tests: + _write_section_break(output) + self._write_compact_rows(output, ordered_tests, use_color=use_color, width=width) + if _has_problems(ordered_tests): + _write_section_break(output) + self._write_problem_section(output, ordered_tests, use_color=use_color, width=width) + elif self.verbosity == 1: + self._write_legacy_rows(output, ordered_tests, use_color=use_color) + elif self.verbosity == 2: + if run.collection_issues and ordered_tests: + _write_section_break(output) + self._write_debug_rows(output, ordered_tests, use_color=use_color) + else: + if run.collection_issues and _has_problems(ordered_tests): + _write_section_break(output) + self._write_problem_section(output, ordered_tests, use_color=use_color, width=width) + + if self.show_skips and any(result.status in _SKIP_STATUSES for result in ordered_tests): + if self.verbosity != 1 or ordered_tests: + _write_section_break(output) + self._write_skips_section(output, ordered_tests, use_color=use_color, width=width) + if self.durations is not None and ordered_tests: + _write_section_break(output) + self._write_durations_section(output, ordered_tests, width=width) + + # Keep this line plain and stable: benchmark tooling and shell users + # intentionally parse it from the start of a line. + if self.verbosity != 1: + _write_section_break(output) + output.write(_summary_line(run)) + return output.getvalue() - for result in sorted(run.tests, key=_test_sort_key): + def _write_legacy_rows( + self, + output: StringIO, + tests: list[TestResult], + *, + use_color: bool, + ) -> None: + for result in tests: + status = f"{result.status.value.upper():<9}" output.write( - f"{result.status.value.upper():<9} {result.test.id} [{result.duration:.3f}s]\n" + f"{_paint_status(status, result.status, use_color)} " + f"{result.test.id} [{result.duration:.3f}s]\n" ) + for line in _failure_details(result, include_not_run=False): + output.write(f" {line}\n") + + def _write_compact_rows( + self, + output: StringIO, + tests: list[TestResult], + *, + use_color: bool, + width: int, + ) -> None: + grouped: dict[str, list[TestResult]] = {} + for result in tests: + grouped.setdefault(result.test.path, []).append(result) + + rows: list[tuple[str, str, Status, str, str]] = [] + for path, path_tests in grouped.items(): + counts = Counter(result.status for result in path_tests) + label, label_status = _group_label(counts) + duration = sum(max(0.0, result.duration) for result in path_tests) + rows.append( + ( + path, + label, + label_status, + _counts_text(counts), + f"[{_format_duration(duration)}]", + ) + ) + + counts_width = max((len(row[3]) for row in rows), default=0) + duration_width = max((len(row[4]) for row in rows), default=0) + for path, label, label_status, counts_text, duration_text in rows: + prefix = f"{label:<5} " + suffix_width = counts_width + 1 + duration_width + path_width = max(0, width - len(prefix) - suffix_width - 2) + fitted_path = _fit_path(path, path_width).ljust(path_width) + suffix = f"{counts_text:<{counts_width}} {duration_text:>{duration_width}}" + output.write( + f"{_paint_status(prefix, label_status, use_color)}{fitted_path} {suffix}\n" + ) + + def _write_problem_section( + self, + output: StringIO, + tests: list[TestResult], + *, + use_color: bool, + width: int, + ) -> None: + problems = [result for result in tests if result.status not in _NON_PROBLEM_STATUSES] + if not problems: + return + + output.write(f"Problems ({len(problems)})\n") + for result in problems: + status = f"{result.status.value.upper():<9}" + identifier = _fit_path(result.test.id, width - len(status) - 2) + output.write(f"{_paint_status(status, result.status, use_color)} {identifier}\n") for line in _failure_details(result): output.write(f" {line}\n") - counts = Counter(test.status for test in run.tests) - parts = [f"{len(run.tests)} tests"] - parts.extend( - f"{counts[status]} {_SUMMARY_LABELS[status]}" for status in Status if counts[status] - ) - if run.collection_issues: - parts.append(f"{len(run.collection_issues)} collection errors") - duration = max(0.0, run.finished_at - run.started_at) - output.write(f"{', '.join(parts)} in {duration:.3f}s\n") - return output.getvalue() + def _write_debug_rows( + self, + output: StringIO, + tests: list[TestResult], + *, + use_color: bool, + ) -> None: + for result in tests: + status = f"{result.status.value.upper():<9}" + output.write( + f"{_paint_status(status, result.status, use_color)} " + f"{result.test.id} [{_format_duration(result.duration)}]\n" + ) + for line in _debug_details(result): + output.write(f" {line}\n") - def write(self, run: RunResult, stream: TextIO | None = None) -> None: - target = stream if stream is not None else sys.stdout - target.write(self.render(run)) + def _write_skips_section( + self, + output: StringIO, + tests: list[TestResult], + *, + use_color: bool, + width: int, + ) -> None: + skipped = [result for result in tests if result.status in _SKIP_STATUSES] + if not skipped: + return - report = write + output.write(f"Skipped tests ({len(skipped)})\n") + for result in skipped: + reason = _skip_reason(result) + suffix = f" - {reason}" if reason else "" + status = f"{result.status.value.upper():<6}" + identifier = _fit_path(result.test.id, width - len(status) - len(suffix) - 1) + output.write( + f"{_paint_status(status, result.status, use_color)} {identifier}{suffix}\n" + ) + + def _write_durations_section( + self, + output: StringIO, + tests: list[TestResult], + *, + width: int, + ) -> None: + ordered = sorted(tests, key=lambda result: (-result.duration, _test_sort_key(result))) + selected = ordered if self.durations == 0 else ordered[: self.durations] + if not selected: + return + + title = "Durations (all)" if self.durations == 0 else f"Slowest durations ({len(selected)})" + output.write(f"{title}\n") + for result in selected: + duration = _format_duration(result.duration) + identifier = _fit_path(result.test.id, width - len(duration) - 2) + output.write(f"{duration:>9} {identifier}\n") def _test_sort_key(result: TestResult) -> tuple[str, int, str]: @@ -72,14 +319,25 @@ def _test_sort_key(result: TestResult) -> tuple[str, int, str]: return (result.test.path, line, result.test.id) -def _failure_details(result: TestResult) -> tuple[str, ...]: - if result.status in { - Status.PASS, - Status.CACHED_PASS, - Status.SKIP, - Status.XFAIL, - Status.NOT_RUN, - }: +def _write_collection_issues(output: StringIO, run: RunResult, *, use_color: bool) -> None: + for issue in sorted(run.collection_issues, key=lambda item: (item.path, item.message)): + label = f"{_ANSI_COLLECT}COLLECT{_ANSI_RESET}" if use_color else "COLLECT" + output.write(f"{label} {issue.path}\n") + for line in issue.message.splitlines() or [""]: + output.write(f" {line}\n") + if issue.traceback: + for line in issue.traceback.rstrip().splitlines(): + output.write(f" {line}\n") + + +def _failure_details( + result: TestResult, + *, + include_not_run: bool = True, +) -> tuple[str, ...]: + if result.status in _NON_PROBLEM_STATUSES or ( + result.status is Status.NOT_RUN and not include_not_run + ): return () lines: list[str] = [] @@ -103,10 +361,164 @@ def _failure_details(result: TestResult) -> tuple[str, ...]: return tuple(lines) +def _debug_details(result: TestResult) -> tuple[str, ...]: + lines: list[str] = [] + for attempt in sorted(result.attempts, key=lambda item: item.attempt): + lines.append( + f"attempt {attempt.attempt}: {attempt.status.value}, " + f"worker={attempt.worker_id}, duration={_format_duration(attempt.duration)}" + ) + for phase in attempt.phases: + heading = ( + f" {phase.phase.value}: {phase.status.value}, " + f"duration={_format_duration(phase.duration)}" + ) + if phase.message: + heading = f"{heading}: {phase.message}" + lines.append(heading) + if phase.traceback: + lines.extend(f" {line}" for line in phase.traceback.rstrip().splitlines()) + if phase.stdout: + lines.append(f" [{phase.phase.value} stdout]") + lines.extend(f" {line}" for line in phase.stdout.rstrip().splitlines()) + if phase.stderr: + lines.append(f" [{phase.phase.value} stderr]") + lines.extend(f" {line}" for line in phase.stderr.rstrip().splitlines()) + return tuple(lines) + + def _is_problem_phase(phase: PhaseResult) -> bool: - return phase.status not in { - Status.PASS, - Status.CACHED_PASS, - Status.SKIP, - Status.XFAIL, - } + return phase.status not in _NON_PROBLEM_STATUSES + + +def _group_label(counts: Counter[Status]) -> tuple[str, Status]: + problems = [ + status for status in Status if counts[status] and status not in _NON_PROBLEM_STATUSES + ] + if problems: + return "FAIL", problems[0] + if counts[Status.PASS] or counts[Status.CACHED_PASS]: + return "PASS", Status.PASS + if counts[Status.SKIP] or counts[Status.XFAIL]: + status = Status.SKIP if counts[Status.SKIP] else Status.XFAIL + return "SKIP", status + return "PASS", Status.PASS + + +def _has_problems(tests: list[TestResult]) -> bool: + return any(result.status not in _NON_PROBLEM_STATUSES for result in tests) + + +def _counts_text(counts: Counter[Status]) -> str: + return ", ".join( + f"{counts[status]} {_SUMMARY_LABELS[status]}" for status in Status if counts[status] + ) + + +def _summary_line(run: RunResult) -> str: + counts = Counter(test.status for test in run.tests) + parts = [f"{len(run.tests)} tests"] + parts.extend( + f"{counts[status]} {_SUMMARY_LABELS[status]}" for status in Status if counts[status] + ) + if run.collection_issues: + parts.append(f"{len(run.collection_issues)} collection errors") + duration = max(0.0, run.finished_at - run.started_at) + return f"{', '.join(parts)} in {duration:.3f}s\n" + + +def _skip_reason(result: TestResult) -> str | None: + if result.status is Status.SKIP and result.test.skip_reason: + return result.test.skip_reason + if result.status is Status.XFAIL and result.test.xfail_reason: + return result.test.xfail_reason + for attempt in sorted(result.attempts, key=lambda item: item.attempt): + for phase in attempt.phases: + if phase.message: + return phase.message + return None + + +def _format_duration(seconds: float) -> str: + seconds = max(0.0, seconds) + if seconds >= 1.0: + return f"{seconds:.3g}s" + if seconds >= 0.001: + return f"{seconds * 1_000:.3g}ms" + return f"{seconds * 1_000_000:.3g}us" + + +def _fit_path(value: str, available: int) -> str: + if available <= 0: + return "" + if len(value) <= available: + return value + if available <= 3: + return "." * available + + remaining = available - 3 + left = remaining // 3 + right = remaining - left + return f"{value[:left]}...{value[-right:]}" + + +def _paint_status(text: str, status: Status, enabled: bool) -> str: + if not enabled: + return text + return f"{_ANSI_BY_STATUS[status]}{text}{_ANSI_RESET}" + + +def _paint_header(text: str, enabled: bool) -> str: + if not enabled: + return text + return f"{_ANSI_HEADER}{text}{_ANSI_RESET}" + + +def _write_section_break(output: StringIO) -> None: + position = output.tell() + if position == 0: + return + output.seek(max(0, position - 2)) + tail = output.read() + output.seek(position) + if not tail.endswith("\n\n"): + output.write("\n") + + +def _stream_width(stream: TextIO) -> int: + if not _stream_is_tty(stream): + return _DEFAULT_WIDTH + detected = shutil.get_terminal_size(fallback=(_DEFAULT_WIDTH, 24)).columns + return min(_MAX_WIDTH, max(1, detected)) + + +def _stream_is_tty(stream: TextIO) -> bool: + isatty = getattr(stream, "isatty", None) + if not callable(isatty): + return False + try: + return bool(isatty()) + except (OSError, ValueError): + return False + + +def _should_use_color(mode: ColorMode, stream: TextIO, environ: Mapping[str, str]) -> bool: + if mode == "always": + return True + if mode == "never": + return False + if "NO_COLOR" in environ: + return False + if _env_enabled(environ.get("FORCE_COLOR")): + return True + if _env_enabled(environ.get("CI")): + return False + if environ.get("TERM", "").lower() == "dumb": + return False + return _stream_is_tty(stream) + + +def _env_enabled(value: str | None) -> bool: + if value is None: + return False + return value.strip().lower() not in {"", "0", "false", "no", "off"} diff --git a/tests/test_cli_reports.py b/tests/test_cli_reports.py index 862b9b8..5f1bc94 100644 --- a/tests/test_cli_reports.py +++ b/tests/test_cli_reports.py @@ -314,6 +314,140 @@ def fake_runner(paths: tuple[str, ...], config: TestenixConfig) -> RunResult: assert "PASS" in capsys.readouterr().out +@pytest.mark.parametrize( + ("reporter_arguments", "expected"), + [ + ((), (0, "auto", False, None)), + (("-q",), (-1, "auto", False, None)), + (("--quiet",), (-1, "auto", False, None)), + (("-v",), (1, "auto", False, None)), + (("-vv",), (2, "auto", False, None)), + (("-vvv",), (2, "auto", False, None)), + (("--verbose", "--verbose"), (2, "auto", False, None)), + (("--color", "always"), (0, "always", False, None)), + (("--color", "never"), (0, "never", False, None)), + (("--no-color",), (0, "never", False, None)), + (("--show-skips", "--durations", "0"), (0, "auto", True, 0)), + (("--durations", "7"), (0, "auto", False, 7)), + ], +) +def test_run_cli_maps_console_reporter_options( + reporter_arguments: tuple[str, ...], + expected: tuple[int, str, bool, int | None], + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = _run_result( + _test_result( + "suite::first", + path="tests/test_suite.py", + line=1, + status=Status.PASS, + duration=0.1, + ), + _test_result( + "suite::second", + path="tests/test_suite.py", + line=2, + status=Status.PASS, + duration=0.2, + ), + ) + captured: dict[str, object] = {} + + class RecordingConsoleReporter: + def __init__( + self, + *, + verbosity: int, + color: str, + show_skips: bool, + durations: int | None, + workers: int, + ) -> None: + captured["options"] = (verbosity, color, show_skips, durations) + captured["workers"] = workers + + def write(self, result: RunResult) -> None: + captured["result"] = result + + monkeypatch.setattr("testenix.cli._call_runner", lambda paths, config: run) + monkeypatch.setattr("testenix.cli.ConsoleReporter", RecordingConsoleReporter) + + exit_code = main(["run", "--workers", "8", *reporter_arguments, "tests"]) + + assert exit_code == 0 + assert captured == { + "options": expected, + "workers": 1, + "result": run, + } + + +def test_run_cli_reports_zero_workers_for_an_empty_result( + monkeypatch: pytest.MonkeyPatch, +) -> None: + run = _run_result() + captured: dict[str, object] = {} + + class RecordingConsoleReporter: + def __init__(self, **options: object) -> None: + captured.update(options) + + def write(self, result: RunResult) -> None: + captured["result"] = result + + monkeypatch.setattr("testenix.cli._call_runner", lambda paths, config: run) + monkeypatch.setattr("testenix.cli.ConsoleReporter", RecordingConsoleReporter) + + assert main(["run", "--workers", "8", "tests"]) == 0 + assert captured["workers"] == 0 + assert captured["result"] is run + + +@pytest.mark.parametrize( + "arguments", + [ + ("-q", "-v"), + ("--quiet", "--verbose"), + ("--color", "always", "--no-color"), + ("--durations", "-1"), + ("--durations", "not-an-integer"), + ], +) +def test_run_cli_rejects_conflicting_or_invalid_console_options( + arguments: tuple[str, ...], +) -> None: + with pytest.raises(SystemExit) as exit_info: + main(["run", *arguments]) + + assert exit_info.value.code == 2 + + +def test_pytest_bridge_forwards_native_console_flag_names_unchanged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: list[tuple[str, ...]] = [] + + def fake_pytest(arguments: tuple[str, ...]) -> int: + captured.append(arguments) + return 5 + + monkeypatch.setattr("testenix.cli._call_pytest", fake_pytest) + forwarded = ( + "-q", + "-vv", + "--color", + "always", + "--no-color", + "--show-skips", + "--durations", + "0", + ) + + assert main(["pytest", *forwarded]) == 5 + assert captured == [forwarded] + + def test_run_help_describes_configured_default_paths( capsys: pytest.CaptureFixture[str], ) -> None: @@ -323,3 +457,4 @@ def test_run_help_describes_configured_default_paths( assert exit_info.value.code == 0 help_text = " ".join(capsys.readouterr().out.split()) assert "default: [tool.testenix].paths, otherwise tests" in help_text + assert "show failures and the final summary only" in help_text diff --git a/tests/test_console_reporter.py b/tests/test_console_reporter.py new file mode 100644 index 0000000..590413e --- /dev/null +++ b/tests/test_console_reporter.py @@ -0,0 +1,387 @@ +from __future__ import annotations + +from io import StringIO + +import pytest + +from testenix.contracts import ( + AttemptResult, + CollectionIssue, + Phase, + PhaseResult, + RunResult, + Status, + TestResult, + TestSpec, +) +from testenix.reporters.console import ConsoleReporter + + +class _Stream(StringIO): + def __init__(self, *, tty: bool) -> None: + super().__init__() + self._tty = tty + + def isatty(self) -> bool: + return self._tty + + +def _result( + test_id: str, + *, + path: str = "tests/test_sample.py", + line: int = 1, + status: Status = Status.PASS, + duration: float = 0.001, + message: str | None = None, + stdout: str = "", + stderr: str = "", + skip_reason: str | None = None, + xfail_reason: str | None = None, + worker: str = "worker-1", + attempt_number: int = 1, +) -> TestResult: + phase = PhaseResult( + phase=Phase.CALL, + status=status, + duration=duration, + message=message, + exception_type="AssertionError" if message else None, + traceback="Traceback line\nAssertionError: boom" if message else None, + stdout=stdout, + stderr=stderr, + ) + attempt = AttemptResult( + test_id=test_id, + attempt=attempt_number, + worker_id=worker, + status=status, + duration=duration, + phases=(phase,), + started_at=10.0, + finished_at=10.0 + duration, + ) + spec = TestSpec( + id=test_id, + path=path, + module_name="test_sample", + function_name=test_id.rsplit("::", 1)[-1], + display_name=test_id.rsplit("::", 1)[-1], + skip_reason=skip_reason, + xfail_reason=xfail_reason, + source_line=line, + ) + return TestResult(test=spec, status=status, attempts=(attempt,), duration=duration) + + +def _run( + *tests: TestResult, + issues: tuple[CollectionIssue, ...] = (), + duration: float = 2.5, +) -> RunResult: + return RunResult( + run_id="run-1", + tests=tuple(tests), + collection_issues=issues, + started_at=10.0, + finished_at=10.0 + duration, + ) + + +def test_default_reporter_preserves_the_legacy_plain_format() -> None: + passed = _result("tests/test_sample.py::test_ok", line=10) + failed = _result( + "tests/test_sample.py::test_bad", + line=20, + status=Status.FAIL, + duration=0.25, + message="boom", + stdout="captured stdout", + stderr="captured stderr", + ) + + assert ConsoleReporter().render(_run(failed, passed)) == ( + "Testenix run run-1\n" + "PASS tests/test_sample.py::test_ok [0.001s]\n" + "FAIL tests/test_sample.py::test_bad [0.250s]\n" + " attempt 1, call: boom\n" + " Traceback line\n" + " AssertionError: boom\n" + " [attempt 1 call stdout]\n" + " captured stdout\n" + " [attempt 1 call stderr]\n" + " captured stderr\n" + "2 tests, 1 passed, 1 failed in 2.500s\n" + ) + + +def test_compact_groups_by_path_and_keeps_complete_problem_details() -> None: + passed = _result("tests/unit/test_api.py::test_ok", path="tests/unit/test_api.py", line=1) + failed = _result( + "tests/unit/test_api.py::test_bad", + path="tests/unit/test_api.py", + line=2, + status=Status.FAIL, + duration=0.025, + message="boom", + stdout="out", + stderr="err", + ) + skipped = _result( + "tests/unit/test_auth.py::test_optional", + path="tests/unit/test_auth.py", + status=Status.SKIP, + duration=0.000004, + message="needs service", + ) + + rendered = ConsoleReporter(verbosity=0).render(_run(skipped, failed, passed)) + + lines = rendered.splitlines() + assert lines[:2] == ["Testenix | 3 tests | 2 files", ""] + assert rendered.count("PASS ") == 0 + api_line = next(line for line in lines if line.startswith("FAIL tests/unit/test_api.py")) + auth_line = next(line for line in lines if line.startswith("SKIP tests/unit/test_auth.py")) + assert api_line.endswith("1 passed, 1 failed [26ms]") + assert "1 skipped" in auth_line and auth_line.endswith("[4us]") + assert api_line.index("1 passed") == auth_line.index("1 skipped") + assert "Problems (1)" in rendered + assert "\n\nProblems (1)\n" in rendered + assert "tests/unit/test_api.py::test_bad" in rendered + assert "Traceback line" in rendered + assert "[attempt 1 call stdout]" in rendered + assert "out" in rendered + assert "[attempt 1 call stderr]" in rendered + assert "err" in rendered + assert "Skipped tests" not in rendered + + +def test_quiet_hides_header_and_successes_but_not_problems_or_summary() -> None: + passed = _result("tests/test_sample.py::test_ok") + failed = _result( + "tests/test_sample.py::test_bad", + line=2, + status=Status.FAIL, + message="boom", + ) + + rendered = ConsoleReporter(verbosity=-1).render(_run(passed, failed)) + + assert "Testenix run" not in rendered + assert "test_ok" not in rendered + assert "Problems (1)" in rendered + assert "test_bad" in rendered + assert "AssertionError: boom" in rendered + assert rendered.endswith("2 tests, 1 passed, 1 failed in 2.500s\n") + + +def test_debug_includes_worker_attempt_phase_capture_and_adaptive_duration() -> None: + failed = _result( + "tests/test_sample.py::test_bad", + status=Status.FAIL, + duration=0.000125, + message="boom", + stdout="debug out", + stderr="debug err", + worker="worker-7", + attempt_number=2, + ) + + rendered = ConsoleReporter(verbosity=2, workers=8).render(_run(failed)) + + assert "Testenix run run-1 [workers=8]" in rendered + assert "[125us]" in rendered + assert "attempt 2: fail, worker=worker-7, duration=125us" in rendered + assert "call: fail, duration=125us: boom" in rendered + assert "Traceback line" in rendered + assert "[call stdout]" in rendered and "debug out" in rendered + assert "[call stderr]" in rendered and "debug err" in rendered + + +def test_show_skips_reports_native_reasons_in_source_order() -> None: + xfailed = _result( + "tests/test_sample.py::test_later", + line=20, + status=Status.XFAIL, + xfail_reason="known bug", + ) + skipped = _result( + "tests/test_sample.py::test_earlier", + line=10, + status=Status.SKIP, + skip_reason="linux only", + ) + + rendered = ConsoleReporter(verbosity=-1, show_skips=True).render(_run(xfailed, skipped)) + + assert "Skipped tests (2)" in rendered + assert rendered.index("test_earlier") < rendered.index("test_later") + assert "test_earlier - linux only" in rendered + assert "test_later - known bug" in rendered + + +def test_durations_selects_slowest_or_all_with_adaptive_units() -> None: + slow = _result("suite::slow", line=1, duration=2.5) + medium = _result("suite::medium", line=2, duration=0.025) + fast = _result("suite::fast", line=3, duration=0.000004) + + slowest = ConsoleReporter(verbosity=-1, durations=2).render(_run(fast, slow, medium)) + all_durations = ConsoleReporter(verbosity=-1, durations=0).render(_run(fast, slow, medium)) + + assert "Slowest durations (2)" in slowest + assert slowest.index("2.5s") < slowest.index("25ms") + assert "suite::fast" not in slowest + assert "Durations (all)" in all_durations + assert "4us" in all_durations + assert all_durations.index("suite::slow") < all_durations.index("suite::medium") + assert all_durations.index("suite::medium") < all_durations.index("suite::fast") + + +@pytest.mark.parametrize( + ("environment", "tty", "expected_color"), + [ + ({}, True, True), + ({}, False, False), + ({"NO_COLOR": "1", "FORCE_COLOR": "1"}, True, False), + ({"FORCE_COLOR": "1", "CI": "1", "TERM": "dumb"}, False, True), + ({"CI": "1"}, True, False), + ({"TERM": "dumb"}, True, False), + ], +) +def test_auto_color_precedence_and_fake_tty( + monkeypatch: pytest.MonkeyPatch, + environment: dict[str, str], + tty: bool, + expected_color: bool, +) -> None: + for name in ("NO_COLOR", "FORCE_COLOR", "CI", "TERM"): + monkeypatch.delenv(name, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + stream = _Stream(tty=tty) + + ConsoleReporter(verbosity=0, color="auto").write(_run(_result("suite::ok")), stream=stream) + rendered = stream.getvalue() + + assert ("\x1b[" in rendered) is expected_color + assert "\x1b[" not in rendered.splitlines()[-1] + + +def test_explicit_color_overrides_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NO_COLOR", "1") + always = _Stream(tty=False) + ConsoleReporter(color="always").write(_run(_result("suite::ok")), always) + + monkeypatch.setenv("FORCE_COLOR", "1") + never = _Stream(tty=True) + ConsoleReporter(color="never").write(_run(_result("suite::ok")), never) + + assert "\x1b[" in always.getvalue() + assert "\x1b[" not in never.getvalue() + assert "\x1b[" not in always.getvalue().splitlines()[-1] + + +def test_compact_truncates_a_long_path_to_requested_width() -> None: + long_path = "tests/" + "deeply_nested/" * 8 + "test_terminal_output.py" + rendered = ConsoleReporter(verbosity=0, width=60).render( + _run(_result(f"{long_path}::test_ok", path=long_path)) + ) + group_line = rendered.splitlines()[2] + + assert "..." in group_line + assert long_path not in group_line + assert len(group_line) <= 60 + assert group_line.endswith("1 passed [1ms]") + + max_width_line = ( + ConsoleReporter(verbosity=0, width=1_000) + .render(_run(_result(f"{long_path}::test_ok", path=long_path))) + .splitlines()[2] + ) + assert len(max_width_line) == 120 + + +def test_render_is_deterministic_and_sorts_files_then_source_lines() -> None: + later_file = _result("z.py::test_z", path="z.py", line=1) + later_line = _result("a.py::test_later", path="a.py", line=20) + earlier_line = _result("a.py::test_earlier", path="a.py", line=10, status=Status.FAIL) + run = _run(later_file, later_line, earlier_line) + reporter = ConsoleReporter(verbosity=0) + + first = reporter.render(run) + second = reporter.render(run) + + assert first == second + assert first.index("a.py") < first.index("z.py") + assert "Problems (1)" in first + assert "a.py::test_earlier" in first + + +def test_compact_output_is_bounded_for_100k_passing_tests_in_one_file() -> None: + passed = _result("tests/test_bulk.py::test_case", path="tests/test_bulk.py") + run = _run(*(passed,) * 100_000) + + rendered = ConsoleReporter(verbosity=0).render(run) + + assert rendered.count("tests/test_bulk.py") == 1 + assert rendered.count("\n") == 5 + assert len(rendered) < 250 + assert "100000 tests, 100000 passed" in rendered + + +def test_collection_issues_are_visible_even_in_quiet_mode() -> None: + issue = CollectionIssue( + path="tests/test_broken.py", + message="import failed", + traceback="Traceback\nRuntimeError: broken", + ) + + rendered = ConsoleReporter(verbosity=-1).render(_run(issues=(issue,))) + + assert rendered.startswith("COLLECT tests/test_broken.py\n") + assert "RuntimeError: broken" in rendered + assert rendered.endswith("0 tests, 1 collection errors in 2.500s\n") + + +def test_every_status_can_be_rendered_with_color() -> None: + results = tuple( + _result( + f"tests/test_status.py::test_{status.value}", + line=index, + status=status, + message="detail" if status not in {Status.PASS, Status.CACHED_PASS} else None, + ) + for index, status in enumerate(Status) + ) + + rendered = ConsoleReporter(verbosity=2, color="always").render(_run(*results)) + + assert rendered.count("\x1b[") >= len(Status) + assert "1 passed" in rendered + assert "1 infra errors" in rendered + assert "1 cached" in rendered + assert "\x1b[" not in rendered.splitlines()[-1] + + +@pytest.mark.parametrize( + "kwargs", + [ + {"verbosity": 3}, + {"verbosity": True}, + {"color": "sometimes"}, + {"show_skips": 1}, + {"durations": -1}, + {"durations": True}, + {"workers": -1}, + {"width": 0}, + ], +) +def test_rejects_invalid_options(kwargs: dict[str, object]) -> None: + with pytest.raises((TypeError, ValueError)): + ConsoleReporter(**kwargs) # type: ignore[arg-type] + + +def test_debug_accepts_zero_workers_for_an_empty_run() -> None: + rendered = ConsoleReporter(verbosity=2, workers=0).render(_run()) + + assert "Testenix run run-1 [workers=0]" in rendered From e533751b0e2b54c033dff88301a58800cb6bc74a Mon Sep 17 00:00:00 2001 From: Dominik Franczyk Date: Tue, 21 Jul 2026 09:22:34 +0200 Subject: [PATCH 2/3] Stabilize generated API docs across Python versions --- docs/llms-full.txt | 6 +++--- llms-full.txt | 6 +++--- scripts/generate_docs_assets.py | 20 +++++++++++++++----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 75cabfe..35808e7 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -2704,7 +2704,7 @@ Complete audit record for a migration attempt. ## `testenix.MigrationStatus` ```text -MigrationStatus(*values) +MigrationStatus(value) ``` Values: ANALYZED='analyzed', VALIDATED='validated', PUBLISHED='published', UNSUPPORTED='unsupported', VALIDATION_FAILED='validation_failed', SAFETY_ERROR='safety_error' @@ -2753,7 +2753,7 @@ RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tu ## `testenix.Scope` ```text -Scope(*values) +Scope(value) ``` Values: TEST='test', MODULE='module', SESSION='session' @@ -2763,7 +2763,7 @@ Lifetime of a fixture instance. ## `testenix.Status` ```text -Status(*values) +Status(value) ``` Values: PASS='pass', FAIL='fail', ERROR_SETUP='error_setup', ERROR_TEARDOWN='error_teardown', SKIP='skip', XFAIL='xfail', XPASS='xpass', TIMEOUT='timeout', CRASH='crash', INFRA_ERROR='infra_error', CANCELLED='cancelled', NOT_RUN='not_run', FLAKY='flaky', CACHED_PASS='cached_pass' diff --git a/llms-full.txt b/llms-full.txt index 75cabfe..35808e7 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -2704,7 +2704,7 @@ Complete audit record for a migration attempt. ## `testenix.MigrationStatus` ```text -MigrationStatus(*values) +MigrationStatus(value) ``` Values: ANALYZED='analyzed', VALIDATED='validated', PUBLISHED='published', UNSUPPORTED='unsupported', VALIDATION_FAILED='validation_failed', SAFETY_ERROR='safety_error' @@ -2753,7 +2753,7 @@ RunResult(run_id: 'str', tests: 'tuple[TestResult, ...]', collection_issues: 'tu ## `testenix.Scope` ```text -Scope(*values) +Scope(value) ``` Values: TEST='test', MODULE='module', SESSION='session' @@ -2763,7 +2763,7 @@ Lifetime of a fixture instance. ## `testenix.Status` ```text -Status(*values) +Status(value) ``` Values: PASS='pass', FAIL='fail', ERROR_SETUP='error_setup', ERROR_TEARDOWN='error_teardown', SKIP='skip', XFAIL='xfail', XPASS='xpass', TIMEOUT='timeout', CRASH='crash', INFRA_ERROR='infra_error', CANCELLED='cancelled', NOT_RUN='not_run', FLAKY='flaky', CACHED_PASS='cached_pass' diff --git a/scripts/generate_docs_assets.py b/scripts/generate_docs_assets.py index 18c1f1d..a9507eb 100644 --- a/scripts/generate_docs_assets.py +++ b/scripts/generate_docs_assets.py @@ -592,14 +592,24 @@ def _public_api_snapshot() -> str: for name in testenix.__all__: value = getattr(testenix, name) sections.extend((f"## `testenix.{name}`", "")) - try: - signature = inspect.signature(value) - except (TypeError, ValueError): - signature = None + is_enum = inspect.isclass(value) and issubclass(value, Enum) + if is_enum: + # EnumMeta exposes a different introspected signature across supported + # Python versions. An enum's stable public constructor is its value. + signature = inspect.Signature( + parameters=( + inspect.Parameter("value", inspect.Parameter.POSITIONAL_OR_KEYWORD), + ) + ) + else: + try: + signature = inspect.signature(value) + except (TypeError, ValueError): + signature = None if signature is not None: sections.extend(("```text", f"{name}{signature}", "```", "")) - if inspect.isclass(value) and issubclass(value, Enum): + if is_enum: members = ", ".join(f"{item.name}={item.value!r}" for item in value) sections.extend((f"Values: {members}", "")) elif inspect.isclass(value) and is_dataclass(value): From 9c7edb1a3901457e46063b5a5ec5ef6b6f72046f Mon Sep 17 00:00:00 2001 From: Dominik Franczyk Date: Tue, 21 Jul 2026 09:28:45 +0200 Subject: [PATCH 3/3] Preserve diagnostics in compact terminal output --- docs/llms-full.txt | 12 +++--- docs/reference/cli.md | 12 +++--- llms-full.txt | 12 +++--- scripts/generate_docs_assets.py | 12 +++--- src/testenix/reporters/console.py | 61 ++++++++++++++++++++++++++----- tests/test_console_reporter.py | 59 ++++++++++++++++++++++++++++++ 6 files changed, 134 insertions(+), 34 deletions(-) diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 35808e7..d66c7e9 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -1402,12 +1402,12 @@ Pytest and its plugins must be installed in the same Python environment as Teste `testenix[pytest]` extra installs a supported pytest (`>=8.3,<10`) when needed. Testenix does not consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. -`[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, -`--json`, `--junit`, `--history`, `--show-skips`, and `--durations` do not affect this command. -Flags such as `-q`, `-v`, and `--color` are forwarded to pytest and have pytest's meaning, not the -native reporter's meaning. Pytest's equivalent spellings include `-rs`, `--durations=N`, and -`--color=yes|no|auto`; Testenix does not translate the native spellings. Pass pytest or plugin -options instead. In particular, +`[tool.testenix]` does not configure this command. Arguments after `pytest` are never interpreted as +native Testenix options, even when their names overlap: `-q`, `-v`, `--color`, `--show-skips`, +`--durations`, `--workers`, and every other token are forwarded unchanged. They can therefore be +handled by pytest or a plugin, or rejected by pytest if unsupported. Pytest's equivalent spellings +include `-rs`, `--durations=N`, and `--color=yes|no|auto`; Testenix does not translate native +spellings. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately follow `testenix`. See [pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full boundary. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7d38156..0751f72 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -87,12 +87,12 @@ Pytest and its plugins must be installed in the same Python environment as Teste `testenix[pytest]` extra installs a supported pytest (`>=8.3,<10`) when needed. Testenix does not consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. -`[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, -`--json`, `--junit`, `--history`, `--show-skips`, and `--durations` do not affect this command. -Flags such as `-q`, `-v`, and `--color` are forwarded to pytest and have pytest's meaning, not the -native reporter's meaning. Pytest's equivalent spellings include `-rs`, `--durations=N`, and -`--color=yes|no|auto`; Testenix does not translate the native spellings. Pass pytest or plugin -options instead. In particular, +`[tool.testenix]` does not configure this command. Arguments after `pytest` are never interpreted as +native Testenix options, even when their names overlap: `-q`, `-v`, `--color`, `--show-skips`, +`--durations`, `--workers`, and every other token are forwarded unchanged. They can therefore be +handled by pytest or a plugin, or rejected by pytest if unsupported. Pytest's equivalent spellings +include `-rs`, `--durations=N`, and `--color=yes|no|auto`; Testenix does not translate native +spellings. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately follow `testenix`. See [pytest compatibility](../guides/pytest-compatibility.md) for the full boundary. diff --git a/llms-full.txt b/llms-full.txt index 35808e7..d66c7e9 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1402,12 +1402,12 @@ Pytest and its plugins must be installed in the same Python environment as Teste `testenix[pytest]` extra installs a supported pytest (`>=8.3,<10`) when needed. Testenix does not consume a leading `--`: pytest receives it unchanged, including its normal end-of-options meaning. -`[tool.testenix]` and native options such as `--workers`, `--retries`, `--timeout`, `--tag`, -`--json`, `--junit`, `--history`, `--show-skips`, and `--durations` do not affect this command. -Flags such as `-q`, `-v`, and `--color` are forwarded to pytest and have pytest's meaning, not the -native reporter's meaning. Pytest's equivalent spellings include `-rs`, `--durations=N`, and -`--color=yes|no|auto`; Testenix does not translate the native spellings. Pass pytest or plugin -options instead. In particular, +`[tool.testenix]` does not configure this command. Arguments after `pytest` are never interpreted as +native Testenix options, even when their names overlap: `-q`, `-v`, `--color`, `--show-skips`, +`--durations`, `--workers`, and every other token are forwarded unchanged. They can therefore be +handled by pytest or a plugin, or rejected by pytest if unsupported. Pytest's equivalent spellings +include `-rs`, `--durations=N`, and `--color=yes|no|auto`; Testenix does not translate native +spellings. In particular, `testenix --config PATH pytest ...` is rejected; `pytest` must immediately follow `testenix`. See [pytest compatibility](https://polishdataengineer.github.io/testenix/guides/pytest-compatibility/) for the full boundary. diff --git a/scripts/generate_docs_assets.py b/scripts/generate_docs_assets.py index a9507eb..3f92762 100644 --- a/scripts/generate_docs_assets.py +++ b/scripts/generate_docs_assets.py @@ -592,14 +592,12 @@ def _public_api_snapshot() -> str: for name in testenix.__all__: value = getattr(testenix, name) sections.extend((f"## `testenix.{name}`", "")) - is_enum = inspect.isclass(value) and issubclass(value, Enum) - if is_enum: + enum_type = value if inspect.isclass(value) and issubclass(value, Enum) else None + if enum_type is not None: # EnumMeta exposes a different introspected signature across supported # Python versions. An enum's stable public constructor is its value. signature = inspect.Signature( - parameters=( - inspect.Parameter("value", inspect.Parameter.POSITIONAL_OR_KEYWORD), - ) + parameters=(inspect.Parameter("value", inspect.Parameter.POSITIONAL_OR_KEYWORD),) ) else: try: @@ -609,8 +607,8 @@ def _public_api_snapshot() -> str: if signature is not None: sections.extend(("```text", f"{name}{signature}", "```", "")) - if is_enum: - members = ", ".join(f"{item.name}={item.value!r}" for item in value) + if enum_type is not None: + members = ", ".join(f"{item.name}={item.value!r}" for item in enum_type) sections.extend((f"Values: {members}", "")) elif inspect.isclass(value) and is_dataclass(value): sections.extend(("Fields:", "")) diff --git a/src/testenix/reporters/console.py b/src/testenix/reporters/console.py index fced856..ecd9996 100644 --- a/src/testenix/reporters/console.py +++ b/src/testenix/reporters/console.py @@ -5,6 +5,7 @@ import os import shutil import sys +import textwrap from collections import Counter from collections.abc import Mapping from io import StringIO @@ -59,6 +60,7 @@ _ANSI_HEADER = "\x1b[1;36m" _DEFAULT_WIDTH = 100 _MAX_WIDTH = 120 +_MIN_COMPACT_PATH_WIDTH = 12 class ConsoleReporter: @@ -154,7 +156,7 @@ def _render(self, run: RunResult, *, use_color: bool, width: int) -> str: self._write_compact_rows(output, ordered_tests, use_color=use_color, width=width) if _has_problems(ordered_tests): _write_section_break(output) - self._write_problem_section(output, ordered_tests, use_color=use_color, width=width) + self._write_problem_section(output, ordered_tests, use_color=use_color) elif self.verbosity == 1: self._write_legacy_rows(output, ordered_tests, use_color=use_color) elif self.verbosity == 2: @@ -164,7 +166,7 @@ def _render(self, run: RunResult, *, use_color: bool, width: int) -> str: else: if run.collection_issues and _has_problems(ordered_tests): _write_section_break(output) - self._write_problem_section(output, ordered_tests, use_color=use_color, width=width) + self._write_problem_section(output, ordered_tests, use_color=use_color) if self.show_skips and any(result.status in _SKIP_STATUSES for result in ordered_tests): if self.verbosity != 1 or ordered_tests: @@ -230,11 +232,28 @@ def _write_compact_rows( prefix = f"{label:<5} " suffix_width = counts_width + 1 + duration_width path_width = max(0, width - len(prefix) - suffix_width - 2) - fitted_path = _fit_path(path, path_width).ljust(path_width) suffix = f"{counts_text:<{counts_width}} {duration_text:>{duration_width}}" - output.write( - f"{_paint_status(prefix, label_status, use_color)}{fitted_path} {suffix}\n" - ) + minimum_path_width = min(len(path), _MIN_COMPACT_PATH_WIDTH) + if path_width >= minimum_path_width: + fitted_path = _fit_path(path, path_width).ljust(path_width) + output.write( + f"{_paint_status(prefix, label_status, use_color)}{fitted_path} {suffix}\n" + ) + continue + + # Keep the file identifiable when an unusually varied status summary + # cannot share a terminal row with it. Continuation lines retain every + # count instead of trading correctness for a hard truncation. + layout_width = max(width, len(prefix) + 1) + fitted_path = _fit_path(path, layout_width - len(prefix)) + output.write(f"{_paint_status(prefix, label_status, use_color)}{fitted_path}\n") + indentation = " " * len(prefix) + for line in _wrap_compact_details( + counts_text, + duration_text, + max(1, layout_width - len(indentation)), + ): + output.write(f"{indentation}{line}\n") def _write_problem_section( self, @@ -242,7 +261,6 @@ def _write_problem_section( tests: list[TestResult], *, use_color: bool, - width: int, ) -> None: problems = [result for result in tests if result.status not in _NON_PROBLEM_STATUSES] if not problems: @@ -251,8 +269,7 @@ def _write_problem_section( output.write(f"Problems ({len(problems)})\n") for result in problems: status = f"{result.status.value.upper():<9}" - identifier = _fit_path(result.test.id, width - len(status) - 2) - output.write(f"{_paint_status(status, result.status, use_color)} {identifier}\n") + output.write(f"{_paint_status(status, result.status, use_color)} {result.test.id}\n") for line in _failure_details(result): output.write(f" {line}\n") @@ -462,6 +479,32 @@ def _fit_path(value: str, available: int) -> str: return f"{value[:left]}...{value[-right:]}" +def _wrap_compact_details(counts_text: str, duration_text: str, width: int) -> tuple[str, ...]: + counts = counts_text.split(", ") + items = [f"{count}," for count in counts[:-1]] + items.append(f"{counts[-1]} {duration_text}") + lines: list[str] = [] + current = "" + for item in items: + candidate = item if not current else f"{current} {item}" + if len(candidate) <= width: + current = candidate + continue + if current: + lines.append(current) + wrapped = textwrap.wrap( + item, + width=width, + break_long_words=True, + break_on_hyphens=False, + ) + lines.extend(wrapped[:-1]) + current = wrapped[-1] + if current: + lines.append(current) + return tuple(lines) + + def _paint_status(text: str, status: Status, enabled: bool) -> str: if not enabled: return text diff --git a/tests/test_console_reporter.py b/tests/test_console_reporter.py index 590413e..1aab0a3 100644 --- a/tests/test_console_reporter.py +++ b/tests/test_console_reporter.py @@ -301,6 +301,65 @@ def test_compact_truncates_a_long_path_to_requested_width() -> None: assert len(max_width_line) == 120 +def test_compact_wraps_varied_status_counts_without_losing_the_file() -> None: + statuses = ( + Status.PASS, + Status.FAIL, + Status.ERROR_SETUP, + Status.ERROR_TEARDOWN, + Status.SKIP, + Status.XFAIL, + Status.TIMEOUT, + ) + path = "tests/test_mix.py" + results = tuple( + _result( + f"{path}::test_{status.value}", + path=path, + line=index, + status=status, + message=None if status is Status.PASS else "detail", + ) + for index, status in enumerate(statuses) + ) + + rendered = ConsoleReporter(verbosity=0, width=80).render(_run(*results)) + compact_table = rendered.partition("\n\nProblems")[0] + + assert path in compact_table + for label in ( + "1 passed", + "1 failed", + "1 setup errors", + "1 teardown errors", + "1 skipped", + "1 xfailed", + "1 timed out", + ): + assert label in compact_table + assert max(map(len, compact_table.splitlines())) <= 80 + + +@pytest.mark.parametrize("verbosity", [0, -1]) +def test_problem_section_preserves_a_long_node_id(verbosity: int) -> None: + long_id = ( + "tests/integration/test_checkout.py::test_rejects_expired_card[" + + "customer-with-a-very-long-parametrized-case-id-" * 4 + + "]" + ) + failed = _result( + long_id, + path="tests/integration/test_checkout.py", + status=Status.FAIL, + message="boom", + ) + + rendered = ConsoleReporter(verbosity=verbosity, width=60).render(_run(failed)) + problems = rendered.partition("Problems (1)\n")[2] + + assert long_id in problems + + def test_render_is_deterministic_and_sorts_files_then_source_lines() -> None: later_file = _result("z.py::test_z", path="z.py", line=1) later_line = _result("a.py::test_later", path="a.py", line=20)