diff --git a/Cargo.lock b/Cargo.lock index be388f8..be7f6ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2383,6 +2383,7 @@ dependencies = [ "tracing-subscriber", "uuid", "walkdir", + "windows-sys 0.61.2", "zip", ] diff --git a/Cargo.toml b/Cargo.toml index f08b650..1c7ff72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ reqwest = { version = "0.12", default-features = false, features = ["blocking", [target.'cfg(windows)'.dependencies] process-wrap = { version = "9.1.0", default-features = false, features = ["std", "job-object", "creation-flags"] } +windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } [dev-dependencies] assert_cmd = "2" diff --git a/SKILL/SKILL.md b/SKILL/SKILL.md index 394883c..8fa5d99 100644 --- a/SKILL/SKILL.md +++ b/SKILL/SKILL.md @@ -95,7 +95,8 @@ v8-runner init - Do not delete or recreate an infobase, workspace, temp directory, or generated state unless the user explicitly asks or the command itself is the documented recovery path. - Do not invent raw `1cv8`, `ibcmd`, or `1cedtcli` flags; prefer the `v8-runner` command surface. - Check `git status` before `dump` when the result may overwrite or mix with existing source changes. -- Preserve failed test artifacts under `workPath/temp//runs//` for diagnosis instead of cleaning them immediately. +- Preserve successful and failed test artifacts under `workPath/temp//runs//` for diagnosis; remove them only explicitly. +- The artifact inventory includes at most 100 regular diagnostic files shared by `error-details/` then `screenshots/`; a truncated category is represented by its directory path with the same diagnostic kind and role. - Report missing local 1C utilities as environment/setup issues, not as project source failures. - Keep final answers concrete: command run, result, relevant artifact path, and any follow-up command. diff --git a/SKILL/references/testing.md b/SKILL/references/testing.md index 5bab3b7..6df32e4 100644 --- a/SKILL/references/testing.md +++ b/SKILL/references/testing.md @@ -87,10 +87,23 @@ v8-runner syntax edt ## Artifacts -Preserve failed test artifacts under: +Each YaXUnit or Vanessa Automation test run retains its diagnostics under: ```text workPath/temp//runs// ``` +Supported current/latest native runners produce JUnit XML and Allure results together. In +`--json-message`, read the summary from `data.execution.metrics` and every existing typed path +from `data.execution.artifacts.items`; `data.retained_paths` is only a compatibility projection. +Missing, empty, or malformed JUnit and missing or empty Allure results are `invalid_output` +infrastructure failures. Successful and failed run directories remain until explicitly removed; +internal cleanup markers are never public artifacts. A missing file-infobase rejected by +`--no-build` preflight creates no run directory because validation precedes artifact preparation. +Optional runner diagnostics may appear under `error-details/` and `screenshots/` in the run +directory. These directories are not pre-created; existing regular files are inventoried as +`error_details` and `screenshot`, respectively. Their inventory has one shared cap of 100 regular +files, scanning `error-details/` before `screenshots/`; when a category is truncated, its directory +path is retained as the fallback artifact with that category's kind and role. + In final answers, include the command, pass/fail result, and artifact path when present. diff --git a/docs/CAPABILITIES.md b/docs/CAPABILITIES.md index e460ee4..e6bf827 100644 --- a/docs/CAPABILITIES.md +++ b/docs/CAPABILITIES.md @@ -214,6 +214,22 @@ v8-runner test [--no-build] va --feature login --filter-tag @smoke `test va` или MCP `run_all_tests` с `runner=vanessa`, а не дефолтный YaXUnit-runner. - `--full` включает полный вывод успешных кейсов и расширенные stack traces. - `tests.*.timeouts.total_ms` остаётся активным пользовательским контрактом таймаутов. +- Поддерживаемые current/latest YaXUnit и Vanessa Automation одновременно создают JUnit и + Allure results; вручную подставленный старый runner без обоих отчётов считается несовместимым. +- Каждый запуск, дошедший до подготовки runner, сохраняется в + `workPath/temp//runs//`. Успешные и неуспешные результаты остаются там до + явного удаления пользователем. Ошибка file-infobase preflight в `--no-build` возникает раньше + и поэтому не создаёт run directory. +- В `--json-message` summary находится в `data.execution.metrics`, а все существующие пути и их + точные kinds — в `data.execution.artifacts.items`; `data.retained_paths` остаётся compatibility + projection основных путей. +- Runner может дополнительно сохранить диагностические файлы в `error-details/` и снимки экрана + в `screenshots/` внутри run directory. Эти каталоги не создаются заранее; в inventory публикуются + не более 100 обычных файлов суммарно для обеих категорий, сначала `error-details`, затем + `screenshots`. При усечении категории публикуется и путь к её каталогу с тем же kind и role, + чтобы оставшиеся файлы были доступны без раздувания inventory. +- Отсутствующий, пустой или некорректный JUnit либо отсутствующий/пустой Allure — это + инфраструктурная ошибка `invalid_output`, а не успешный или обычный test failure. ### `syntax` diff --git a/docs/superpowers/plans/2026-07-26-test-result-artifacts.md b/docs/superpowers/plans/2026-07-26-test-result-artifacts.md new file mode 100644 index 0000000..644d82a --- /dev/null +++ b/docs/superpowers/plans/2026-07-26-test-result-artifacts.md @@ -0,0 +1,454 @@ +# Stable Test Result Artifacts Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Return retained, typed JUnit, Allure, and log artifacts with a report-authoritative summary for both YaXUnit and Vanessa Automation. + +**Architecture:** Extend the existing `ExecutionOutcome< TestReport >` and `ArtifactSet` contracts instead of adding a second result envelope. Both native runners write into one unique per-run layout; a shared collector validates and inventories their outputs, aggregates all JUnit XML files, and applies one exhaustive terminal-classification policy. + +**Tech Stack:** Rust, serde, quick-xml through the existing JUnit parser, clap CLI integration tests, MCP service unit tests. + +## Global Constraints + +- Base commit is canonical `alkoleft/master` at `d612e2d07e0702c6c829f3981d083022611f6373`. +- Both JUnit and Allure are mandatory for supported current/latest native runners. +- Allure validity means the results directory contains at least one regular file recursively; native files remain opaque. +- Native JUnit summary overrides a nonzero process exit when the report contains failures or errors. +- Successful and failed terminal runs retain their unique run directory. +- Public artifact inventory contains only paths that exist when the outcome is built. +- CLI and MCP expose the same canonical `ExecutionOutcome` artifact and metrics semantics. +- Follow TDD: every production behavior starts with a test that fails for the intended missing behavior. +- Apply `rust-expert-best-practices-code-review`; use typed enums, exhaustive matches, borrowed `&Path`, and `Result` propagation without new `unwrap`/`expect` in production paths. +- Update `SKILL/SKILL.md` or `SKILL/references/testing.md` for the changed external test workflow. +- Do not fix unrelated baseline failures; compare the final full-suite failure set with the recorded 656-pass/48-fail sandbox baseline. + +--- + +### Task 1: Typed test artifact and error vocabulary + +**Files:** +- Modify: `src/domain/artifact.rs` +- Modify: `src/domain/test.rs` +- Modify: `src/domain/runner.rs` +- Modify: `src/use_cases/request.rs` +- Modify: `src/cli/execute.rs` +- Modify: `src/mcp/service.rs` + +**Interfaces:** +- Produces `ArtifactKind::{JunitXml, AllureResults, ErrorDetails, Screenshot}`. +- Produces roles `junit_xml`, `allure_results`, `error_details`, and `screenshot`. +- Produces `RunnerOutputFormat::AllureResults`. +- Produces `TestErrorKind::{AllureNotProduced, AllureEmpty}` mapped to `ExecutionStatus::InvalidOutput`. +- Keeps `RetainedPaths.junit_xml` as the first-report compatibility projection and adds `allure_results`. + +- [ ] **Step 1: Write failing domain serialization and status tests** + +Add literal assertions that: + +```rust +assert_eq!( + serde_json::to_value(ArtifactKind::JunitXml).unwrap(), + serde_json::json!("junit_xml") +); +assert_eq!( + serde_json::to_value(ArtifactKind::AllureResults).unwrap(), + serde_json::json!("allure_results") +); +assert_eq!( + test_execution_status(Some(TestErrorKind::AllureNotProduced), false), + ExecutionStatus::InvalidOutput +); +``` + +Extend the retained-path roundtrip test with an Allure directory and verify +that repeated JUnit artifacts keep the first sorted report as the compatibility +`junit_xml`. + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: + +```bash +cargo test domain::artifact domain::test use_cases::request cli::execute::tests::maps_vanessa_request_from_configured_profile mcp::service::tests::run_all_tests_maps_vanessa_request_with_profile_overrides -- --nocapture +``` + +Expected: compilation/test failures because the new variants and fields do not +exist. + +- [ ] **Step 3: Implement the minimal typed vocabulary** + +Add the enum variants and constants. Add: + +```rust +pub fn get_all_by_role<'a>(&'a self, role: &'a str) -> impl Iterator + 'a +``` + +Use it to select a deterministic first JUnit report in `RetainedPaths`. +Add Allure to YaXUnit and Vanessa `RunnerProfile.output_formats`, and set +`retain_artifacts_on_success: true` for both CLI and MCP request builders. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run the Step 2 command and require zero failures. + +- [ ] **Step 5: Self-review and commit** + +Check exhaustive `TestErrorKind` matches and JSON names, then commit: + +```text +feat(test): add typed result artifact vocabulary + +- distinguish JUnit and Allure outputs +- retain supported test artifacts on success +``` + +--- + +### Task 2: Generate simultaneous native reports and inventory existing outputs + +**Files:** +- Modify: `src/use_cases/run_tests.rs` +- Modify: `src/use_cases/vanessa.rs` + +**Interfaces:** +- `RunArtifacts` owns `junit_dir`, primary YaXUnit `junit_xml`, and `allure_results_dir`. +- `build_yaxunit_config` serializes `reports: Vec`. +- `VanessaTestArtifacts` carries the Allure directory. +- `collect_run_artifacts(&RunArtifacts) -> ArtifactSet` returns only existing paths. +- A `Drop` implementation removes `run.inprogress` without deleting the run directory. + +- [ ] **Step 1: Write failing YaXUnit and Vanessa configuration tests** + +Replace the legacy single-format assertion with literal JSON behavior: + +```rust +assert_eq!(json["reports"][0]["format"], "jUnit"); +assert_eq!(json["reports"][0]["path"], artifacts.junit_xml.display().to_string()); +assert_eq!(json["reports"][1]["format"], "allure"); +assert_eq!( + json["reports"][1]["path"], + artifacts.allure_results_dir.display().to_string() +); +assert!(json.get("reportFormat").is_none()); +``` + +Add a Vanessa overlay test asserting: + +```rust +assert_eq!(payload["ДелатьОтчетВФорматеjUnit"], true); +assert_eq!(payload["ДелатьОтчетВФорматеАллюр"], true); +assert_eq!( + payload["КаталогВыгрузкиAllure"], + artifacts.allure_results_dir.display().to_string() +); +``` + +Add inventory tests proving a missing expected JUnit path is omitted while +existing config/log/Allure paths are present, and proving the sentinel is +removed when `RunArtifacts` is dropped. + +- [ ] **Step 2: Run focused tests and verify RED** + +Run: + +```bash +cargo test use_cases::run_tests use_cases::vanessa -- --nocapture +``` + +Expected: failures because simultaneous configuration, Allure layout, filtered +inventory, and sentinel cleanup do not exist. + +- [ ] **Step 3: Implement simultaneous native configuration** + +Use: + +```rust +#[derive(Debug, Serialize)] +struct YaXUnitReportConfig { + format: &'static str, + path: String, +} +``` + +Create both output directories before launch. Add Vanessa keys +`ДелатьОтчетВФорматеАллюр` and `КаталогВыгрузкиAllure`. Do not enable +environment-dependent screenshot capture. + +- [ ] **Step 4: Implement existing-path inventory** + +Build `ArtifactSet` from the run directory, generated config, discovered JUnit +files, Allure directory, runner log, platform log, and optional diagnostics. +Call `metadata`/`is_file`/`is_dir` before inserting each item. Sort repeated +items by path. Remove the sentinel in `Drop for RunArtifacts`. + +- [ ] **Step 5: Run focused tests and verify GREEN** + +Run the Step 2 command and require zero failures. + +- [ ] **Step 6: Self-review and commit** + +Verify borrowed path arguments, safe recursive traversal, and no deletion of +the run directory, then commit: + +```text +feat(test): generate JUnit and Allure artifacts + +- configure simultaneous native reports +- inventory only materialized run outputs +``` + +--- + +### Task 3: Aggregate native reports and classify terminal results + +**Files:** +- Modify: `src/use_cases/run_tests.rs` +- Modify: `src/use_cases/run_tests/coordinator.rs` +- Modify: `src/use_cases/run_tests/helpers.rs` +- Modify: `src/parsers/junit.rs` only if a parser-level aggregation helper is cleaner + +**Interfaces:** +- `discover_junit_reports(&Path) -> std::io::Result>` is recursive and sorted. +- `parse_junit_reports(&[PathBuf]) -> NormalizedParse` aggregates every report. +- `validate_allure_results(&Path) -> Result<(), TestErrorKind>` checks recursive non-emptiness. +- `classify_test_completion(&TestSummary, i32) -> TestErrorKind/ExecutionStatus decision` encodes report-first precedence. + +- [ ] **Step 1: Write failing discovery and aggregation tests** + +Create two nested JUnit fixtures in reverse filesystem order. Assert the +returned paths are sorted and the aggregate literal summary is: + +```rust +TestSummary { + total: 3, + passed: 1, + failed: 1, + skipped: 0, + errors: 1, +} +``` + +Add a malformed second report and assert the aggregate is invalid output rather +than silently using the first report. + +- [ ] **Step 2: Write failing Allure validation and classification table tests** + +Use table rows: + +```rust +// summary failed, exit 1 => TestFailures +// summary errors, exit 2 => TestFailures +// green summary, exit 1 => EnterpriseExitedNonZero +// green summary, exit 0 => success/no error +``` + +Add empty/missing Allure cases mapping to `AllureEmpty` and +`AllureNotProduced`. + +- [ ] **Step 3: Run focused tests and verify RED** + +Run: + +```bash +cargo test use_cases::run_tests parsers::junit -- --nocapture +``` + +Expected: failures for missing collection, aggregation, validation, and +classification behavior. + +- [ ] **Step 4: Implement deterministic aggregation and validation** + +Traverse directories without following symlinks. Sort paths before parsing. +Sum counters with `saturating_add`, append suites and extracted errors, and +return all parse errors with the offending path in details. Treat any invalid +JUnit file as invalid native output. + +- [ ] **Step 5: Implement report-authoritative coordinator flow** + +After native process completion: + +1. materialize the Vanessa runner log; +2. collect and parse every JUnit report; +3. validate Allure; +4. parse runner logs; +5. classify from summary before considering nonzero exit; +6. attach the existing-path artifact inventory to every terminal outcome, + including success. + +Replace `expect("junit parse error")` with explicit `Result`/fallback handling. +Do not call `cleanup_run_dir`. + +- [ ] **Step 6: Run focused tests and verify GREEN** + +Run the Step 3 command and require zero failures. + +- [ ] **Step 7: Self-review and commit** + +Check the classification matrix, overflow behavior, path diagnostics, and +successful retention, then commit: + +```text +feat(test): classify results from native reports + +- aggregate deterministic JUnit summaries +- preserve test failures across nonzero exits +``` + +--- + +### Task 4: Public contract integration, documentation, and regression coverage + +**Files:** +- Modify: `tests/cli_test.rs` +- Modify: `tests/snapshots/cli_test__test_module_compact_json.snap` +- Modify: `tests/snapshots/cli_test__test_module_full_json.snap` +- Modify: `src/command_envelope.rs` if compatibility projection needs adjustment +- Modify: `docs/CAPABILITIES.md` +- Modify: `SKILL/references/testing.md` +- Modify: `SKILL/SKILL.md` only if its concise top-level guidance must change + +**Interfaces:** +- Fake YaXUnit and Vanessa executables read the new native configuration and materialize both report types. +- CLI JSON returns `execution.artifacts.items[]` with exact kind/path and `execution.metrics`. +- MCP continues to serialize the same `ExecutionOutcome`. + +- [ ] **Step 1: Update fake native runners and write failing CLI contract tests** + +Make each fake runner: + +- read JUnit and Allure destinations from its generated native config; +- create a JUnit XML and at least one Allure result file; +- create engine and platform logs. + +Add assertions for both YaXUnit and Vanessa success: + +```rust +assert_eq!(payload["data"]["execution"]["metrics"]["total"], 1); +assert!(artifact_items.iter().any(|item| item["kind"] == "junit_xml")); +assert!(artifact_items.iter().any(|item| item["kind"] == "allure_results")); +for item in artifact_items { + assert!(Path::new(item["path"].as_str().unwrap()).exists()); +} +``` + +Add failures for: + +- nonzero exit plus failing JUnit returns `test_failures` with summary; +- missing JUnit returns `invalid_output`; +- missing/empty Allure returns `invalid_output`; +- two runs return distinct retained roots. + +- [ ] **Step 2: Run CLI tests and verify RED** + +Run: + +```bash +cargo test --test cli_test -- --nocapture +``` + +Expected: contract failures until the fake runners and public projection match +the new behavior. + +- [ ] **Step 3: Complete CLI/MCP projection and snapshots** + +Keep `execution.artifacts` canonical. Preserve the legacy compatibility +projection where its required primary paths exist. Update snapshots with +stable scrubbed run paths and exact artifact kinds. + +Add or update MCP service tests to assert `RunnerOutputFormat::AllureResults` +and success retention policy for both runner kinds. + +- [ ] **Step 4: Update external documentation and repo-local skill** + +Document: + +- retained location `workPath/temp//runs//`; +- simultaneous JUnit and Allure; +- summary and artifact locations in `--json-message`; +- invalid native reports as infrastructure failure; +- current/latest native runner compatibility; +- storage lifecycle: successful results remain until explicitly removed. + +Keep `SKILL/references/testing.md` concise and actionable. + +- [ ] **Step 5: Run integration and documentation checks** + +Run: + +```bash +cargo test --test cli_test -- --nocapture +cargo test mcp::service::tests -- --nocapture +cargo test generated_schema_artifacts_are_current -- --nocapture +cargo fmt --all -- --check +cargo clippy --all-targets --all-features -- -D warnings +git diff --check +``` + +Require zero issue-related failures. Pre-existing warnings must be reported and +must not be silently waived. + +- [ ] **Step 6: Self-review and commit** + +Check every issue acceptance criterion against a named test, then commit: + +```text +feat(test): expose stable native result artifacts + +- cover CLI and MCP artifact contracts +- document retained test diagnostics +``` + +--- + +### Task 5: Independent final verification and review + +**Files:** +- Review all changes since `d612e2d07e0702c6c829f3981d083022611f6373`. + +**Interfaces:** +- Produces independent tester, general reviewer, and Rust-expert reports. +- Produces final verification evidence without modifying unrelated code. + +- [ ] **Step 1: Run the focused verification matrix** + +Run: + +```bash +cargo test parsers::junit -- --nocapture +cargo test use_cases::run_tests -- --nocapture +cargo test --test cli_test -- --nocapture +cargo test mcp::service::tests -- --nocapture +cargo fmt --all -- --check +cargo clippy --all-targets --all-features -- -D warnings +``` + +- [ ] **Step 2: Run the complete suite and compare baseline** + +Run: + +```bash +cargo test --all-targets +``` + +Any new failing test relative to the recorded 48-failure baseline is blocking. +Existing sandbox-only failures are reported with exact counts. + +- [ ] **Step 3: Dispatch independent reviews** + +The general reviewer checks issue criteria, architecture, CLI/MCP compatibility, +and test completeness. A separate Rust expert explicitly applies +`rust-expert-best-practices-code-review` to type safety, error handling, API +design, filesystem traversal, and performance. Every finding is fixed or +recorded as an accepted waiver with a concise technical reason. + +- [ ] **Step 4: Verify repository state and prepare final commit if needed** + +Run: + +```bash +git status --short +git diff --check +git log --oneline d612e2d07e0702c6c829f3981d083022611f6373..HEAD +``` + +Ensure no generated scratch files or unrelated changes remain. diff --git a/docs/superpowers/plans/2026-07-27-artifact-review-followups.md b/docs/superpowers/plans/2026-07-27-artifact-review-followups.md new file mode 100644 index 0000000..30d82cb --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-artifact-review-followups.md @@ -0,0 +1,275 @@ +# Artifact Review Follow-ups Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Resolve the still-valid PR #52 artifact review findings while keeping optional diagnostics bounded and test fixtures resistant to silent drift. + +**Architecture:** Artifact collection will share one remaining-file budget across error details and screenshots and will add a category root entry when files are omitted. Embedded YaXUnit scripts will resolve reports by `format`; Vanessa fixture rewrites will use a checked replacement helper. Small duplicated test helpers will be collapsed without changing observable behavior. + +**Tech Stack:** Rust 2021, serde JSON fixtures, shell/Python embedded test scripts, Cargo test/Clippy. + +## Global Constraints + +- Keep at most 100 regular diagnostic files across `error-details/` and `screenshots/`. +- A truncated category adds its root directory as an artifact, so optional diagnostic inventory is bounded to 102 entries. +- Preserve deterministic category and path ordering. +- Do not change `ArtifactSet::get_all_by_role`; document the rejected lifetime finding in the PR response. +- Update external-facing documentation for the bounded inventory contract. + +--- + +### Task 1: Bound Optional Diagnostic Inventory + +**Files:** +- Modify: `src/use_cases/run_tests.rs:780-870` +- Test: `src/use_cases/run_tests.rs:1190-1285` +- Modify: `docs/CAPABILITIES.md:220-232` +- Modify: `SKILL/SKILL.md` + +**Interfaces:** +- Consumes: `collect_regular_files(root: &Path) -> std::io::Result>` +- Produces: `const OPTIONAL_DIAGNOSTIC_FILE_LIMIT: usize = 100` +- Produces: `push_optional_diagnostics(set, kind, role, root, remaining: &mut usize)` + +- [ ] **Step 1: Write a failing shared-limit test** + +Add a unit test that creates 101 sorted error-detail files and one screenshot, +calls `collect_run_artifacts`, and asserts: + +```rust +assert_eq!(diagnostic_files.len(), OPTIONAL_DIAGNOSTIC_FILE_LIMIT); +assert!(collected.items.iter().any(|artifact| { + artifact.kind == ArtifactKind::ErrorDetails + && artifact.path == artifacts.error_details_dir +})); +assert!(collected.items.iter().any(|artifact| { + artifact.kind == ArtifactKind::Screenshot + && artifact.path == artifacts.screenshots_dir +})); +assert!(!collected.items.iter().any(|artifact| artifact.path == screenshot)); +``` + +The production mutation caught is independently budgeting each category or +serializing every discovered file. + +- [ ] **Step 2: Run the new test and verify RED** + +Run: + +```bash +cargo test --locked collect_run_artifacts_bounds_optional_diagnostics_across_categories -- --nocapture +``` + +Expected: FAIL because all 102 files are currently serialized and no fallback +directory entry exists. + +- [ ] **Step 3: Implement the shared budget** + +Define: + +```rust +const OPTIONAL_DIAGNOSTIC_FILE_LIMIT: usize = 100; +``` + +In `collect_run_artifacts`, create one mutable remaining counter and pass it to +both category calls in existing error-details-then-screenshots order. Change +`push_optional_diagnostics` to take the first `remaining` sorted paths, reduce +the counter by the number added, and add the category root with the same kind +and role when `paths.len() > added`. + +- [ ] **Step 4: Verify GREEN and existing ordering** + +Run: + +```bash +cargo test --locked collect_run_artifacts_ -- --nocapture +``` + +Expected: the new limit test and existing ordering/symlink tests PASS. + +- [ ] **Step 5: Document the contract** + +Update `docs/CAPABILITIES.md` and the test-artifact guidance in `SKILL/SKILL.md` +to state the exact 100-file shared cap and category-directory fallback. + +- [ ] **Step 6: Commit** + +```bash +git add src/use_cases/run_tests.rs docs/CAPABILITIES.md SKILL/SKILL.md +git commit -m "fix(test): bound optional artifact inventory" \ + -m "- cap diagnostic files across error details and screenshots +- retain truncated category directories and document the contract" +``` + +### Task 2: Select YaXUnit Reports by Format + +**Files:** +- Modify: `tests/cli_test.rs:80-125` +- Modify: `tests/cli_test.rs:840-885` +- Modify: `tests/cli_test.rs:1015-1055` +- Modify: `tests/mcp_stdio.rs:340-365` + +**Interfaces:** +- Consumes: YaXUnit JSON `reports` entries with `format` and `path` +- Produces: embedded Python selection using `next(report["path"] for report in reports if report["format"] == "...")` + +- [ ] **Step 1: Replace positional selection in every fixture** + +For each embedded Python block, replace `reports[0]`/`reports[1]` with: + +```python +reports = json.load(fh)['reports'] +print(next(report['path'] for report in reports if report['format'] == 'jUnit')) +``` + +and: + +```python +reports = json.load(fh)['reports'] +print(next(report['path'] for report in reports if report['format'] == 'allure')) +``` + +Use the exact format strings produced by `src/use_cases/run_tests.rs`. + +- [ ] **Step 2: Verify fixture behavior** + +Run: + +```bash +cargo test --locked --test cli_test test_yaxunit +cargo test --locked --test mcp_stdio mcp_stdio_test +``` + +Expected: all matching CLI and MCP tests PASS with unchanged outputs. + +- [ ] **Step 3: Commit** + +```bash +git add tests/cli_test.rs tests/mcp_stdio.rs +git commit -m "test(yaxunit): select reports by format" \ + -m "- remove positional assumptions from CLI and MCP fixtures" +``` + +### Task 3: Harden and Simplify CLI Test Helpers + +**Files:** +- Modify: `tests/cli_test.rs:175-225` +- Modify: `tests/cli_test.rs:255-325` +- Modify: `tests/cli_test.rs:460-490` + +**Interfaces:** +- Produces: `replace_once(body: &str, from: &str, to: &str) -> String` +- Removes: `setup_project_with_additional_launch_keys` + +- [ ] **Step 1: Add a checked replacement helper** + +Add: + +```rust +fn replace_once(body: &str, from: &str, to: &str) -> String { + let replaced = body.replacen(from, to, 1); + assert_ne!(replaced, body, "VA script fixture pattern not found: {from}"); + replaced +} +``` + +Use it for both preparatory rewrites and every `NativeReportFixture` rewrite in +`write_va_test_script`. For `MissingAllure`, apply the two checked replacements +sequentially. + +- [ ] **Step 2: Run Vanessa fixture tests** + +Run: + +```bash +cargo test --locked --test cli_test test_va_ +``` + +Expected: complete, missing, and empty native-report fixtures PASS. + +- [ ] **Step 3: Remove the empty setup delegate** + +Change `setup_project` and the explicit additional-launch-keys caller to invoke +`setup_project_with_native_reports` directly, then delete +`setup_project_with_additional_launch_keys`. + +- [ ] **Step 4: Collapse retained-path normalization** + +Replace the five repeated `is_string` blocks with: + +```rust +for key in [ + "config_json", + "junit_xml", + "allure_results", + "yaxunit_log", + "platform_log", +] { + if value["data"]["retained_paths"][key].is_string() { + value["data"]["retained_paths"][key] = Value::String(format!("<{key}>")); + } +} +``` + +- [ ] **Step 5: Verify the CLI integration suite** + +Run: + +```bash +cargo test --locked --test cli_test +``` + +Expected: PASS with unchanged snapshots. + +- [ ] **Step 6: Commit** + +```bash +git add tests/cli_test.rs +git commit -m "test(cli): harden artifact fixtures" \ + -m "- fail fast when Vanessa fixture templates drift +- remove duplicated setup and snapshot normalization" +``` + +### Task 4: Independent Review and Full Verification + +**Files:** +- Review: all files changed since `b5e5dff` + +**Interfaces:** +- Consumes: Tasks 1-3 commits +- Produces: review findings resolved or explicitly waived + +- [ ] **Step 1: Format and inspect the diff** + +Run: + +```bash +cargo fmt --all -- --check +git diff --check b5e5dff..HEAD +``` + +Expected: both commands exit successfully. + +- [ ] **Step 2: Run static and full test verification** + +Run: + +```bash +cargo clippy --locked --all-targets -- -D warnings +cargo test --locked +``` + +Expected: both commands exit successfully with no warnings or failures. + +- [ ] **Step 3: Run independent reviewer and Rust-expert checks** + +The reviewer checks architecture, public contract, test fidelity, and scope. +The Rust expert independently applies the complete +`rust-expert-best-practices-code-review` checklist. Resolve every finding or +record an accepted waiver with a reason. + +- [ ] **Step 4: Push and report in the PR** + +Push `feat/issue-26-test-artifacts`, report the verified fixes, and state that +the lifetime nitpick was skipped because its claimed temporary-string failure +does not reproduce and the iterator necessarily captures the role reference. diff --git a/docs/superpowers/specs/2026-07-26-test-result-artifacts-design.md b/docs/superpowers/specs/2026-07-26-test-result-artifacts-design.md new file mode 100644 index 0000000..73f5cd6 --- /dev/null +++ b/docs/superpowers/specs/2026-07-26-test-result-artifacts-design.md @@ -0,0 +1,143 @@ +# Stable Test Result Artifacts + +## Goal + +Make `test yaxunit` and `test va` return the same stable machine-readable +artifact contract for successful test runs, test failures, and infrastructure +failures. Native JUnit results are the source of truth for the test summary; +the native process exit code is supporting infrastructure evidence. + +## Supported Native Runners + +This contract targets the current releases downloaded by `v8-runner`: + +- YaXUnit versions that support the `reports` array with simultaneous `jUnit` + and `allure` outputs; +- Vanessa Automation versions that support independent JUnit and Allure + output settings. + +Older manually supplied runners that cannot produce both formats are outside +the compatibility guarantee and fail with an invalid-output infrastructure +result. + +## Per-run Layout + +Every invocation that reaches runner preparation allocates its existing unique directory: + +```text +workPath/temp//runs/--/ +├── config.json | va-params.json +├── junit/ +│ └── one-or-more.xml +├── allure-results/ +├── runner.log +└── enterprise.out.log +``` + +YaXUnit writes its JUnit report into `junit/report.xml`. Vanessa may create one +or more XML files below `junit/`. Both runners write raw Allure result files +below `allure-results/`. + +The run directory is retained for every terminal result after allocation. A +`--no-build` file-infobase preflight failure happens before allocation, as +required by the no-build contract, and therefore has no run directory. The internal +`run.inprogress` sentinel is removed when the in-process `RunArtifacts` guard +is dropped and is not part of the public artifact inventory. + +## Artifact Contract + +`ExecutionOutcome.artifacts` remains the canonical artifact container. Extend +`ArtifactKind` with exact test-result kinds: + +- `junit_xml` for every discovered JUnit file; +- `allure_results` for the Allure results directory; +- `runner_log` and `platform_log` for engine and platform logs; +- `error_details` and `screenshot` for optional discovered diagnostic files. + +The run directory and generated native configuration remain artifacts. Only +paths that exist at inventory time are returned. Artifact items are sorted by +kind/role/path so JSON is deterministic. + +`TestEnvelopeData.retained_paths` remains a compatibility projection. It may +expose the first JUnit XML but does not replace the canonical multi-artifact +list. + +The explicit summary stays in `report.summary` and +`execution.metrics` (`total`, `passed`, `failed`, `errors`, `skipped`). + +## Native Configuration + +YaXUnit receives: + +```json +{ + "reports": [ + { "format": "jUnit", "path": "/junit/report.xml" }, + { "format": "allure", "path": "/allure-results" } + ] +} +``` + +Vanessa receives both independent sets of keys: + +```json +{ + "ДелатьОтчетВФорматеjUnit": true, + "КаталогВыгрузкиJUnit": "/junit", + "ДелатьОтчетВФорматеАллюр": true, + "КаталогВыгрузкиAllure": "/allure-results" +} +``` + +The runner does not enable screenshot capture because Vanessa screenshot +capture requires environment-specific external tooling. Existing screenshots +and error-detail files produced by the native runner are inventoried when +present. + +## Report Validation and Aggregation + +JUnit discovery recursively collects every `.xml` file below `junit/`, sorts +paths, parses each file with the existing parser, and aggregates suites, +extracted errors, and summary counters. + +JUnit is invalid infrastructure output when no XML exists, any XML is empty, +or any XML is malformed. + +Allure is valid when `allure-results/` exists and contains at least one regular +file recursively. The runner treats Allure files as opaque native results; it +does not impose one Allure schema because YaXUnit and Vanessa may emit +different native result encodings. + +## Terminal Classification + +Classification is exhaustive and ordered: + +1. cancellation or timeout preserves its terminal status; +2. missing, empty, or malformed JUnit, or missing/empty Allure, is + `invalid_output` and an infrastructure error; +3. a valid JUnit summary with `failed > 0` or `errors > 0` is + `test_failures`, even when the native process exit code is nonzero; +4. valid green reports plus a nonzero native exit code is + `enterprise_exited_non_zero`; +5. valid green reports plus exit code zero is success. + +A nonzero process exit remains in diagnostics when JUnit proves test +failures; it does not replace the report-authoritative classification. + +## Tests + +Use TDD for: + +- YaXUnit and Vanessa simultaneous output configuration; +- deterministic multiple-JUnit discovery and summary aggregation; +- missing, empty, and malformed JUnit; +- missing and empty Allure directories; +- the complete report-validity × summary × process-exit classification matrix; +- existing paths only in success and failure artifact inventories; +- distinct run directories; +- CLI JSON success/failure envelopes for both runners; +- CLI/MCP semantic parity through the shared execution outcome. + +Targeted Rust suites must pass. The final full-suite result is compared with +the recorded baseline of 656 passed and 48 environment-sensitive failures in +the current macOS sandbox. diff --git a/docs/superpowers/specs/2026-07-27-artifact-review-followups-design.md b/docs/superpowers/specs/2026-07-27-artifact-review-followups-design.md new file mode 100644 index 0000000..0de67ff --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-artifact-review-followups-design.md @@ -0,0 +1,59 @@ +# Artifact Review Follow-ups Design + +## Scope + +Address the still-valid CodeRabbit nitpicks on PR #52 without changing the +stable test-artifact contract beyond bounding optional diagnostic inventory. +The already-addressed Windows coverage, retained inventory assertion, and +Allure text output remain unchanged. + +The `ArtifactSet::get_all_by_role` lifetime suggestion is not implemented. +The iterator captures `role`, so the role must live for the iterator's use; +the current signature already accepts an inline temporary when the iterator is +consumed in the same statement. The suggested lifetime bound does not enable +the claimed temporary-string use case. + +## Changes + +### Bound optional diagnostics + +Keep at most 100 regular diagnostic files across `error-details/` and +`screenshots/`, preserving the existing deterministic category and path order. +If a category is truncated, add one artifact for that category's root +directory. The directory entry tells consumers where the omitted files remain +available while keeping the serialized `ArtifactSet` bounded to at most 102 +optional diagnostic entries. + +Only optional diagnostics are limited. JUnit, Allure, logs, configuration, and +the run-directory entries retain their current behavior. + +### Harden test fixtures + +- Select YaXUnit JUnit and Allure report paths by their `format` field instead + of array position in CLI and MCP embedded scripts. +- Make every Vanessa fixture-specific textual replacement assert that its + expected source fragment exists before replacement. +- Remove the redundant `setup_project_with_additional_launch_keys` delegate. +- Collapse repeated retained-path snapshot normalization into a deterministic + key loop. + +These changes affect test infrastructure only and preserve fixture output. + +## Testing + +Use test-first coverage for the diagnostic bound: + +- more than 100 files never produce more than 100 file entries; +- truncation adds the appropriate category directory entry; +- the limit is shared across both diagnostic categories; +- existing small inventories keep their exact stable ordering. + +Run targeted unit and integration tests for artifact collection, CLI fixtures, +and MCP stdio fixtures, followed by formatting, Clippy, and the full repository +test suite. Windows-specific behavior remains covered by the existing PR CI. + +## Documentation + +Update `docs/CAPABILITIES.md` and `SKILL/SKILL.md` to describe the bounded +optional diagnostic inventory and directory fallback because it is visible to +external CLI/MCP consumers. diff --git a/scripts/test/README.md b/scripts/test/README.md index 797ac9b..9e50060 100644 --- a/scripts/test/README.md +++ b/scripts/test/README.md @@ -43,7 +43,7 @@ live-mcp-http.py | Файл | Роль | Назначение | Зона ответственности | | --- | --- | --- | --- | -| `ci-rust.sh` | CI entrypoint | Диспетчер CI-контуров по `V8_RUNNER_CI_SCOPE` | Выбрать нужный scope и передать управление в `cargo test`, Windows contract `cargo check`, или `ci-happy-path.sh` | +| `ci-rust.sh` | CI entrypoint | Диспетчер CI-контуров по `V8_RUNNER_CI_SCOPE` | Выбрать нужный scope и передать управление в `cargo test`, Windows contract `cargo check` с targeted platform tests, или `ci-happy-path.sh` | | `ci-happy-path.sh` | CI helper | Canonical happy-path для trusted CI | Собрать бинарь, выполнить `cargo check`, опционально `cargo test`, затем запустить обязательный packaging/live contour | | `ci-platform-install.sh` | CI helper | Установить 1С platform bundle на GitHub-hosted runner | Скачать secret-backed bundle, проверить checksum, распаковать и отдать `tools.platform.path`/`ibsrv` paths | | `ci-designer-config.sh` | CI helper | Материализовать dedicated live config для mandatory CI smoke | Подготовить `format=DESIGNER`, `builder=DESIGNER`, file `infobase.connection`, required source-set'ы и `tools.platform.path` | @@ -163,7 +163,7 @@ V8_RUNNER_CI_SCOPE=happy-path bash scripts/test/ci-rust.sh `.github/workflows/ci.yml` запускает install/config/ibsrv/upload helpers only when `live_available=true`, то есть когда для текущей matrix OS настроены `V8TR_PLATFORM_BUNDLE_URL_*` и `V8TR_PLATFORM_BUNDLE_SHA256_*`. Без этой пары secrets happy-path остаётся blocking для Rust build/check, получает `V8TR_CI_SKIP_DUPLICATE_RUST_TESTS=1`, полагается на contract job для Rust test coverage и soft-skips real 1C smoke. -Windows contract scope currently runs `cargo check --locked --all-targets`; full `cargo test --locked` remains Linux-owned until the Windows-specific test suite is hardened. Exit criterion: switch Windows contract back to `cargo test --locked` after fixing the tracked path separator, fake-binary, ACL, and process-lifecycle failures in `spec/acceptance/real-environment-validation.md`. +Windows contract scope runs `cargo check --locked --all-targets` plus targeted Vanessa runner-log materialization and atomic-replacement tests; full `cargo test --locked` remains Linux-owned until the Windows-specific test suite is hardened. Exit criterion: switch Windows contract back to `cargo test --locked` after fixing the tracked path separator, fake-binary, ACL, and process-lifecycle failures in `spec/acceptance/real-environment-validation.md`. ```bash bash scripts/test/ci-platform-install.sh diff --git a/scripts/test/ci-rust.sh b/scripts/test/ci-rust.sh index 5316309..5f45922 100755 --- a/scripts/test/ci-rust.sh +++ b/scripts/test/ci-rust.sh @@ -12,8 +12,10 @@ case "$CI_SCOPE" in contract) case "$TARGET_OS_LABEL" in Windows|MINGW*|MSYS*|CYGWIN*) - echo "Windows contract scope runs compile/check smoke; full cargo test remains Linux-owned until the Windows test suite is hardened." + echo "Windows contract scope runs compile/check smoke plus targeted platform-specific tests; full cargo test remains Linux-owned until the Windows test suite is hardened." cargo check --locked --all-targets + cargo test --locked --bin v8-runner materialize_vanessa_runner_log + cargo test --locked --bin v8-runner windows_atomic_replace_supports_extended_length_paths ;; *) cargo test --locked diff --git a/src/cli/execute.rs b/src/cli/execute.rs index 79f9bfb..d27ee81 100644 --- a/src/cli/execute.rs +++ b/src/cli/execute.rs @@ -1036,6 +1036,7 @@ fn build_vanessa_execution( kind: RunnerKind::Vanessa, output_formats: vec![ RunnerOutputFormat::JunitXml, + RunnerOutputFormat::AllureResults, RunnerOutputFormat::PlainTextLog, ], backend_hint: Some("enterprise".to_owned()), @@ -1047,7 +1048,7 @@ fn build_vanessa_execution( ), policy: ExecutionPolicy { retain_artifacts_on_failure: true, - retain_artifacts_on_success: false, + retain_artifacts_on_success: true, }, launch: LaunchOptions::default(), }; @@ -1877,21 +1878,30 @@ fn append_retained_test_artifacts(details: &mut Vec, result: &TestRunRes details, format!("[artifact] run_dir -> {}", paths.run_dir.display()), ); - push_unique_detail( - details, - format!("[artifact] report -> {}", paths.junit_xml.display()), - ); - push_unique_detail( - details, - format!("[artifact] runner_log -> {}", paths.yaxunit_log.display()), - ); - push_unique_detail( - details, - format!( - "[diagnostic] platform_log -> {}", - paths.platform_log.display() - ), - ); + if let Some(junit_xml) = paths.junit_xml { + push_unique_detail( + details, + format!("[artifact] report -> {}", junit_xml.display()), + ); + } + if let Some(allure_results) = paths.allure_results { + push_unique_detail( + details, + format!("[artifact] allure_results -> {}", allure_results.display()), + ); + } + if let Some(yaxunit_log) = paths.yaxunit_log { + push_unique_detail( + details, + format!("[artifact] runner_log -> {}", yaxunit_log.display()), + ); + } + if let Some(platform_log) = paths.platform_log { + push_unique_detail( + details, + format!("[diagnostic] platform_log -> {}", platform_log.display()), + ); + } } fn should_hide_success_test_diagnostic(diagnostic: &str) -> bool { @@ -2483,7 +2493,7 @@ mod tests { use crate::domain::load::{ CompatibilityState, LoadExecutionMetadata, LoadMode, LoadResult, LoadTargetKind, }; - use crate::domain::runner::{LaunchOptions, RunnerKind}; + use crate::domain::runner::{LaunchOptions, RunnerKind, RunnerOutputFormat}; use crate::output::presenter::{ColorMode, Presenter}; use crate::support::fs::acquire_advisory_lock; use crate::support::temp::platform_logs_dir; @@ -2616,6 +2626,12 @@ mod tests { assert_eq!(request.execution.profile.kind, RunnerKind::Vanessa); assert_eq!(request.execution.profile.id, "smoke"); + assert!(request + .execution + .profile + .output_formats + .contains(&RunnerOutputFormat::AllureResults)); + assert!(request.execution.policy.retain_artifacts_on_success); assert_eq!(request.build_policy, TestBuildPolicy::BuildFirst); assert_eq!(request.scope, TestScopeRequest::All); assert_eq!(request.execution.timeouts.total_ms, Some(300_000)); diff --git a/src/domain/artifact.rs b/src/domain/artifact.rs index dd0d48a..09731ed 100644 --- a/src/domain/artifact.rs +++ b/src/domain/artifact.rs @@ -4,12 +4,21 @@ use serde::{Deserialize, Serialize}; pub const ARTIFACT_ROLE_RUN_DIR: &str = "run_dir"; pub const ARTIFACT_ROLE_CONFIG: &str = "config"; +#[allow(dead_code)] pub const ARTIFACT_ROLE_REPORT: &str = "report"; +pub const ARTIFACT_ROLE_JUNIT_XML: &str = "junit_xml"; +pub const ARTIFACT_ROLE_ALLURE_RESULTS: &str = "allure_results"; +#[allow(dead_code)] +pub const ARTIFACT_ROLE_ERROR_DETAILS: &str = "error_details"; +#[allow(dead_code)] +pub const ARTIFACT_ROLE_SCREENSHOT: &str = "screenshot"; pub const ARTIFACT_ROLE_RUNNER_LOG: &str = "runner_log"; pub const ARTIFACT_ROLE_PLATFORM_LOG: &str = "platform_log"; -pub const ARTIFACT_ROLE_SENTINEL: &str = "sentinel"; pub const ARTIFACT_ROLE_PACKAGE_FILE: &str = "package_file"; pub const ARTIFACT_ROLE_STAGE_FILE: &str = "stage_file"; +#[allow(dead_code)] +#[deprecated(note = "sentinels are internal cleanup markers and are no longer emitted")] +pub const ARTIFACT_ROLE_SENTINEL: &str = "sentinel"; /// Stable artifact classification for runner/package outputs. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -19,8 +28,13 @@ pub enum ArtifactKind { Config, Package, Report, + JunitXml, + AllureResults, + ErrorDetails, + Screenshot, RunnerLog, PlatformLog, + #[deprecated(note = "sentinels are internal cleanup markers and are no longer emitted")] Sentinel, Other(String), } @@ -79,4 +93,45 @@ impl ArtifactSet { .find(|item| item.role.as_deref() == Some(role)) .map(|item| item.path.as_path()) } + + pub fn get_all_by_role<'a>(&'a self, role: &'a str) -> impl Iterator + 'a { + self.items + .iter() + .filter(move |item| item.role.as_deref() == Some(role)) + .map(|item| item.path.as_path()) + } +} + +#[cfg(test)] +#[allow(deprecated)] +mod tests { + use super::{ + ArtifactKind, ARTIFACT_ROLE_ALLURE_RESULTS, ARTIFACT_ROLE_ERROR_DETAILS, + ARTIFACT_ROLE_JUNIT_XML, ARTIFACT_ROLE_SCREENSHOT, ARTIFACT_ROLE_SENTINEL, + }; + + #[test] + fn serializes_typed_test_artifact_kinds_and_roles() { + assert_eq!( + serde_json::to_value(ArtifactKind::JunitXml).unwrap(), + serde_json::json!("junit_xml") + ); + assert_eq!( + serde_json::to_value(ArtifactKind::AllureResults).unwrap(), + serde_json::json!("allure_results") + ); + assert_eq!(ARTIFACT_ROLE_JUNIT_XML, "junit_xml"); + assert_eq!(ARTIFACT_ROLE_ALLURE_RESULTS, "allure_results"); + assert_eq!(ARTIFACT_ROLE_ERROR_DETAILS, "error_details"); + assert_eq!(ARTIFACT_ROLE_SCREENSHOT, "screenshot"); + } + + #[test] + fn deserializes_legacy_sentinel_artifact_kind() { + assert_eq!( + serde_json::from_value::(serde_json::json!("sentinel")).unwrap(), + ArtifactKind::Sentinel + ); + assert_eq!(ARTIFACT_ROLE_SENTINEL, "sentinel"); + } } diff --git a/src/domain/runner.rs b/src/domain/runner.rs index aae4ce7..4180a2f 100644 --- a/src/domain/runner.rs +++ b/src/domain/runner.rs @@ -87,6 +87,7 @@ pub enum RunnerKind { #[serde(rename_all = "snake_case")] pub enum RunnerOutputFormat { JunitXml, + AllureResults, PlainTextLog, Json, Binary, diff --git a/src/domain/test.rs b/src/domain/test.rs index f7829d8..f35e396 100644 --- a/src/domain/test.rs +++ b/src/domain/test.rs @@ -1,9 +1,12 @@ use serde::{Deserialize, Serialize}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +#[cfg(test)] +use crate::domain::artifact::{ArtifactKind, ArtifactRef}; use crate::domain::artifact::{ - ArtifactKind, ArtifactRef, ArtifactSet, ARTIFACT_ROLE_CONFIG, ARTIFACT_ROLE_PLATFORM_LOG, - ARTIFACT_ROLE_REPORT, ARTIFACT_ROLE_RUNNER_LOG, ARTIFACT_ROLE_RUN_DIR, ARTIFACT_ROLE_SENTINEL, + ArtifactSet, ARTIFACT_ROLE_ALLURE_RESULTS, ARTIFACT_ROLE_CONFIG, ARTIFACT_ROLE_JUNIT_XML, + ARTIFACT_ROLE_PLATFORM_LOG, ARTIFACT_ROLE_REPORT, ARTIFACT_ROLE_RUNNER_LOG, + ARTIFACT_ROLE_RUN_DIR, }; use crate::domain::execution::{ ExecutionError, ExecutionMetrics, ExecutionOutcome, ExecutionStatus, StepResult, @@ -23,6 +26,8 @@ pub const TEST_ERROR_CODE_TEST_FAILURES: &str = "test_failures"; pub const TEST_ERROR_CODE_JUNIT_NOT_PRODUCED: &str = "junit_not_produced"; pub const TEST_ERROR_CODE_JUNIT_EMPTY: &str = "junit_empty"; pub const TEST_ERROR_CODE_JUNIT_MALFORMED: &str = "junit_malformed"; +pub const TEST_ERROR_CODE_ALLURE_NOT_PRODUCED: &str = "allure_not_produced"; +pub const TEST_ERROR_CODE_ALLURE_EMPTY: &str = "allure_empty"; pub const TEST_RUNNER_ID: &str = "yaxunit"; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -56,6 +61,8 @@ pub enum TestErrorKind { JunitNotProduced, JunitEmpty, JunitMalformed, + AllureNotProduced, + AllureEmpty, } impl TestErrorKind { @@ -75,6 +82,8 @@ impl TestErrorKind { Self::JunitNotProduced => TEST_ERROR_CODE_JUNIT_NOT_PRODUCED, Self::JunitEmpty => TEST_ERROR_CODE_JUNIT_EMPTY, Self::JunitMalformed => TEST_ERROR_CODE_JUNIT_MALFORMED, + Self::AllureNotProduced => TEST_ERROR_CODE_ALLURE_NOT_PRODUCED, + Self::AllureEmpty => TEST_ERROR_CODE_ALLURE_EMPTY, } } @@ -94,6 +103,8 @@ impl TestErrorKind { TEST_ERROR_CODE_JUNIT_NOT_PRODUCED => Self::JunitNotProduced, TEST_ERROR_CODE_JUNIT_EMPTY => Self::JunitEmpty, TEST_ERROR_CODE_JUNIT_MALFORMED => Self::JunitMalformed, + TEST_ERROR_CODE_ALLURE_NOT_PRODUCED => Self::AllureNotProduced, + TEST_ERROR_CODE_ALLURE_EMPTY => Self::AllureEmpty, _ => return None, }) } @@ -102,50 +113,76 @@ impl TestErrorKind { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct RetainedPaths { pub run_dir: PathBuf, - pub config_json: PathBuf, - pub junit_xml: PathBuf, - pub yaxunit_log: PathBuf, - pub platform_log: PathBuf, - pub sentinel: PathBuf, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub config_json: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub junit_xml: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub allure_results: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub yaxunit_log: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub platform_log: Option, } impl RetainedPaths { + #[cfg(test)] pub fn into_artifact_set(self) -> ArtifactSet { let mut set = ArtifactSet::with_root(self.run_dir.clone()); set.push( ArtifactRef::new(ArtifactKind::RunDirectory, self.run_dir) .with_role(ARTIFACT_ROLE_RUN_DIR), ); - set.push( - ArtifactRef::new(ArtifactKind::Config, self.config_json) - .with_role(ARTIFACT_ROLE_CONFIG), - ); - set.push( - ArtifactRef::new(ArtifactKind::Report, self.junit_xml).with_role(ARTIFACT_ROLE_REPORT), - ); - set.push( - ArtifactRef::new(ArtifactKind::RunnerLog, self.yaxunit_log) - .with_role(ARTIFACT_ROLE_RUNNER_LOG), - ); - set.push( - ArtifactRef::new(ArtifactKind::PlatformLog, self.platform_log) - .with_role(ARTIFACT_ROLE_PLATFORM_LOG), - ); - set.push( - ArtifactRef::new(ArtifactKind::Sentinel, self.sentinel) - .with_role(ARTIFACT_ROLE_SENTINEL), - ); + if let Some(config_json) = self.config_json { + set.push( + ArtifactRef::new(ArtifactKind::Config, config_json).with_role(ARTIFACT_ROLE_CONFIG), + ); + } + if let Some(junit_xml) = self.junit_xml { + set.push( + ArtifactRef::new(ArtifactKind::JunitXml, junit_xml) + .with_role(ARTIFACT_ROLE_JUNIT_XML), + ); + } + if let Some(allure_results) = self.allure_results { + set.push( + ArtifactRef::new(ArtifactKind::AllureResults, allure_results) + .with_role(ARTIFACT_ROLE_ALLURE_RESULTS), + ); + } + if let Some(yaxunit_log) = self.yaxunit_log { + set.push( + ArtifactRef::new(ArtifactKind::RunnerLog, yaxunit_log) + .with_role(ARTIFACT_ROLE_RUNNER_LOG), + ); + } + if let Some(platform_log) = self.platform_log { + set.push( + ArtifactRef::new(ArtifactKind::PlatformLog, platform_log) + .with_role(ARTIFACT_ROLE_PLATFORM_LOG), + ); + } set } pub fn from_artifact_set(set: &ArtifactSet) -> Option { Some(Self { run_dir: set.get_by_role(ARTIFACT_ROLE_RUN_DIR)?.to_path_buf(), - config_json: set.get_by_role(ARTIFACT_ROLE_CONFIG)?.to_path_buf(), - junit_xml: set.get_by_role(ARTIFACT_ROLE_REPORT)?.to_path_buf(), - yaxunit_log: set.get_by_role(ARTIFACT_ROLE_RUNNER_LOG)?.to_path_buf(), - platform_log: set.get_by_role(ARTIFACT_ROLE_PLATFORM_LOG)?.to_path_buf(), - sentinel: set.get_by_role(ARTIFACT_ROLE_SENTINEL)?.to_path_buf(), + config_json: set.get_by_role(ARTIFACT_ROLE_CONFIG).map(Path::to_path_buf), + junit_xml: set + .get_all_by_role(ARTIFACT_ROLE_JUNIT_XML) + .min() + .or_else(|| set.get_all_by_role(ARTIFACT_ROLE_REPORT).min()) + .map(Path::to_path_buf), + allure_results: set + .get_by_role(ARTIFACT_ROLE_ALLURE_RESULTS) + .map(Path::to_path_buf), + yaxunit_log: set + .get_by_role(ARTIFACT_ROLE_RUNNER_LOG) + .map(Path::to_path_buf), + platform_log: set + .get_by_role(ARTIFACT_ROLE_PLATFORM_LOG) + .map(Path::to_path_buf), }) } } @@ -289,7 +326,9 @@ pub fn test_execution_status(kind: Option, ok: bool) -> Execution Some( TestErrorKind::JunitMalformed | TestErrorKind::JunitEmpty - | TestErrorKind::JunitNotProduced, + | TestErrorKind::JunitNotProduced + | TestErrorKind::AllureNotProduced + | TestErrorKind::AllureEmpty, ) => ExecutionStatus::InvalidOutput, Some( TestErrorKind::BuildFailed @@ -313,6 +352,10 @@ mod tests { test_execution_error, RetainedPaths, TestErrorKind, TestOutputMode, TestReport, TestRunResult, TestSummary, TestTarget, }; + use crate::domain::artifact::{ + ArtifactKind, ArtifactRef, ArtifactSet, ARTIFACT_ROLE_JUNIT_XML, ARTIFACT_ROLE_REPORT, + ARTIFACT_ROLE_RUN_DIR, + }; use crate::domain::execution::{ExecutionMetrics, ExecutionOutcome, ExecutionStatus}; use std::path::PathBuf; @@ -320,18 +363,107 @@ mod tests { fn retained_paths_roundtrip_to_artifact_set() { let retained = RetainedPaths { run_dir: PathBuf::from("/tmp/run"), - config_json: PathBuf::from("/tmp/config.json"), - junit_xml: PathBuf::from("/tmp/report.xml"), - yaxunit_log: PathBuf::from("/tmp/yaxunit.log"), - platform_log: PathBuf::from("/tmp/platform.log"), - sentinel: PathBuf::from("/tmp/sentinel"), + config_json: Some(PathBuf::from("/tmp/config.json")), + junit_xml: Some(PathBuf::from("/tmp/z-report.xml")), + allure_results: Some(PathBuf::from("/tmp/allure-results")), + yaxunit_log: Some(PathBuf::from("/tmp/yaxunit.log")), + platform_log: Some(PathBuf::from("/tmp/platform.log")), + }; + + let first_junit = PathBuf::from("/tmp/a-report.xml"); + let mut expected = retained.clone(); + expected.junit_xml = Some(first_junit.clone()); + let mut set = retained.into_artifact_set(); + set.push( + ArtifactRef::new(ArtifactKind::JunitXml, first_junit) + .with_role(ARTIFACT_ROLE_JUNIT_XML), + ); + + assert_eq!(RetainedPaths::from_artifact_set(&set), Some(expected)); + } + + #[test] + fn retained_paths_falls_back_to_legacy_report_role() { + let mut set = ArtifactSet::with_root("/tmp/run"); + set.push( + ArtifactRef::new(ArtifactKind::RunDirectory, "/tmp/run") + .with_role(ARTIFACT_ROLE_RUN_DIR), + ); + set.push( + ArtifactRef::new(ArtifactKind::Report, "/tmp/legacy-report.xml") + .with_role(ARTIFACT_ROLE_REPORT), + ); + + let projected = RetainedPaths::from_artifact_set(&set).expect("retained paths"); + + assert_eq!( + projected.junit_xml, + Some(PathBuf::from("/tmp/legacy-report.xml")) + ); + } + + #[test] + fn retained_paths_prefers_modern_junit_role_over_legacy_report() { + let mut set = ArtifactSet::with_root("/tmp/run"); + set.push( + ArtifactRef::new(ArtifactKind::RunDirectory, "/tmp/run") + .with_role(ARTIFACT_ROLE_RUN_DIR), + ); + set.push( + ArtifactRef::new(ArtifactKind::Report, "/tmp/a-legacy-report.xml") + .with_role(ARTIFACT_ROLE_REPORT), + ); + set.push( + ArtifactRef::new(ArtifactKind::JunitXml, "/tmp/z-modern-report.xml") + .with_role(ARTIFACT_ROLE_JUNIT_XML), + ); + + let projected = RetainedPaths::from_artifact_set(&set).expect("retained paths"); + + assert_eq!( + projected.junit_xml, + Some(PathBuf::from("/tmp/z-modern-report.xml")) + ); + } + + #[test] + fn retained_paths_omit_allure_artifact_until_runner_produces_it() { + let retained = RetainedPaths { + run_dir: PathBuf::from("/tmp/run"), + config_json: Some(PathBuf::from("/tmp/config.json")), + junit_xml: Some(PathBuf::from("/tmp/report.xml")), + allure_results: None, + yaxunit_log: Some(PathBuf::from("/tmp/yaxunit.log")), + platform_log: Some(PathBuf::from("/tmp/platform.log")), }; let set = retained.clone().into_artifact_set(); + assert_eq!( + set.get_by_role(crate::domain::artifact::ARTIFACT_ROLE_ALLURE_RESULTS), + None + ); assert_eq!(RetainedPaths::from_artifact_set(&set), Some(retained)); } + #[test] + fn retained_paths_do_not_require_or_serialize_internal_sentinel() { + let retained = RetainedPaths { + run_dir: PathBuf::from("/tmp/run"), + config_json: Some(PathBuf::from("/tmp/config.json")), + junit_xml: Some(PathBuf::from("/tmp/report.xml")), + allure_results: Some(PathBuf::from("/tmp/allure-results")), + yaxunit_log: Some(PathBuf::from("/tmp/yaxunit.log")), + platform_log: Some(PathBuf::from("/tmp/platform.log")), + }; + let set = retained.into_artifact_set(); + + let projected = RetainedPaths::from_artifact_set(&set).expect("retained paths"); + let json = serde_json::to_value(projected).expect("json"); + + assert!(json.get("sentinel").is_none()); + } + #[test] fn test_error_kind_codes_roundtrip_for_setup_and_process_failures() { let kinds = [ @@ -369,15 +501,27 @@ mod tests { } } + #[test] + fn allure_output_errors_are_invalid_output() { + assert_eq!( + super::test_execution_status(Some(TestErrorKind::AllureNotProduced), false), + ExecutionStatus::InvalidOutput + ); + assert_eq!( + super::test_execution_status(Some(TestErrorKind::AllureEmpty), false), + ExecutionStatus::InvalidOutput + ); + } + #[test] fn test_run_result_derives_read_model_from_outcome() { let retained = RetainedPaths { run_dir: PathBuf::from("/tmp/run"), - config_json: PathBuf::from("/tmp/config.json"), - junit_xml: PathBuf::from("/tmp/report.xml"), - yaxunit_log: PathBuf::from("/tmp/yaxunit.log"), - platform_log: PathBuf::from("/tmp/platform.log"), - sentinel: PathBuf::from("/tmp/sentinel"), + config_json: Some(PathBuf::from("/tmp/config.json")), + junit_xml: Some(PathBuf::from("/tmp/report.xml")), + allure_results: Some(PathBuf::from("/tmp/allure-results")), + yaxunit_log: Some(PathBuf::from("/tmp/yaxunit.log")), + platform_log: Some(PathBuf::from("/tmp/platform.log")), }; let outcome = ExecutionOutcome::new(ExecutionStatus::Failed) .with_diagnostics(vec!["diag".to_owned()]) diff --git a/src/mcp/service.rs b/src/mcp/service.rs index 1478316..beacd9a 100644 --- a/src/mcp/service.rs +++ b/src/mcp/service.rs @@ -519,6 +519,7 @@ fn build_vanessa_run_all_execution( kind: RunnerKind::Vanessa, output_formats: vec![ RunnerOutputFormat::JunitXml, + RunnerOutputFormat::AllureResults, RunnerOutputFormat::PlainTextLog, ], backend_hint: Some("enterprise".to_owned()), @@ -530,7 +531,7 @@ fn build_vanessa_run_all_execution( ), policy: ExecutionPolicy { retain_artifacts_on_failure: true, - retain_artifacts_on_success: false, + retain_artifacts_on_success: true, }, launch: LaunchOptions { c: Some("StartFeaturePlayer;VAParams={params_path}".to_owned()), @@ -999,7 +1000,7 @@ mod tests { use crate::domain::launch::{ LaunchMode, LaunchResult, PlatformResolution, PlatformResolutionSource, }; - use crate::domain::runner::RunnerKind; + use crate::domain::runner::{RunnerKind, RunnerOutputFormat}; use crate::domain::syntax::{SyntaxCheckResult, SyntaxCheckStatus, SyntaxIssueSummary}; use crate::domain::test::{ RetainedPaths, TestCase, TestOutputMode, TestReport, TestRunResult, TestStatus, TestSuite, @@ -1022,6 +1023,17 @@ mod tests { }; use crate::use_cases::result::{UseCaseError, UseCaseErrorKind, UseCaseFailure, UseCaseResult}; + fn assert_serialized_test_execution_contract(data: &serde_json::Value) { + assert_eq!(data["execution"]["metrics"]["total"], 3); + let artifacts = data["execution"]["artifacts"]["items"] + .as_array() + .expect("artifact items"); + assert!(artifacts.iter().any(|item| item["kind"] == "junit_xml")); + assert!(artifacts + .iter() + .any(|item| item["kind"] == "allure_results")); + } + #[derive(Default)] struct StubPort { build_result: RefCell>>, @@ -1187,7 +1199,7 @@ mod tests { assert_eq!(requests.len(), 1); assert_eq!(requests[0].0.command(), CommandName::Build); assert_eq!(requests[0].0.transport(), ExecutionTransport::McpStdio); - assert_eq!(requests[0].1.full_rebuild, true); + assert!(requests[0].1.full_rebuild); assert_eq!(requests[0].1.source_set.as_deref(), Some("main")); } @@ -1261,10 +1273,18 @@ mod tests { assert_eq!(response.data["report"]["summary"]["passed"], 2); assert_eq!(response.data["report"]["summary"]["failed"], 1); assert!(response.error.is_none()); + assert_serialized_test_execution_contract(&response.data); let requests = service.port.test_requests.borrow(); - assert_eq!(requests[0].1.full, true); + assert!(requests[0].1.full); assert_eq!(requests[0].1.scope, TestScopeRequest::All); assert_eq!(requests[0].1.execution.profile.kind, RunnerKind::YaXUnit); + assert!(requests[0] + .1 + .execution + .profile + .output_formats + .contains(&RunnerOutputFormat::AllureResults)); + assert!(requests[0].1.execution.policy.retain_artifacts_on_success); } #[test] @@ -1296,11 +1316,19 @@ mod tests { .expect("success"); assert!(response.ok); + assert_serialized_test_execution_contract(&response.data); let requests = service.port.test_requests.borrow(); - assert_eq!(requests[0].1.full, true); + assert!(requests[0].1.full); assert_eq!(requests[0].1.scope, TestScopeRequest::All); assert_eq!(requests[0].1.execution.profile.kind, RunnerKind::Vanessa); assert_eq!(requests[0].1.execution.profile.id, "acceptance"); + assert!(requests[0] + .1 + .execution + .profile + .output_formats + .contains(&RunnerOutputFormat::AllureResults)); + assert!(requests[0].1.execution.policy.retain_artifacts_on_success); assert_eq!( requests[0].1.execution.launch.c.as_deref(), Some("StartFeaturePlayer;VAParams={params_path}") @@ -2557,11 +2585,11 @@ mod tests { fn sample_test_result(ok: bool) -> TestRunResult { let retained = RetainedPaths { run_dir: PathBuf::from("/tmp/run"), - config_json: PathBuf::from("/tmp/config.json"), - junit_xml: PathBuf::from("/tmp/junit.xml"), - yaxunit_log: PathBuf::from("/tmp/yaxunit.log"), - platform_log: PathBuf::from("/tmp/platform.log"), - sentinel: PathBuf::from("/tmp/sentinel"), + config_json: Some(PathBuf::from("/tmp/config.json")), + junit_xml: Some(PathBuf::from("/tmp/junit.xml")), + allure_results: Some(PathBuf::from("/tmp/allure-results")), + yaxunit_log: Some(PathBuf::from("/tmp/yaxunit.log")), + platform_log: Some(PathBuf::from("/tmp/platform.log")), }; let report = TestReport { summary: TestSummary { diff --git a/src/use_cases/request.rs b/src/use_cases/request.rs index 6abbf25..772d6e2 100644 --- a/src/use_cases/request.rs +++ b/src/use_cases/request.rs @@ -74,6 +74,7 @@ impl TestRequest { kind: RunnerKind::YaXUnit, output_formats: vec![ RunnerOutputFormat::JunitXml, + RunnerOutputFormat::AllureResults, RunnerOutputFormat::PlainTextLog, ], backend_hint: Some("enterprise".to_owned()), @@ -82,7 +83,7 @@ impl TestRequest { timeouts: ExecutionTimeouts::default(), policy: ExecutionPolicy { retain_artifacts_on_failure: true, - retain_artifacts_on_success: false, + retain_artifacts_on_success: true, }, launch: LaunchOptions::default(), } @@ -660,8 +661,9 @@ mod tests { use super::{ DesignerClientScope, DesignerClientScopes, DesignerConfigCheck, DesignerConfigChecks, DesignerConfigSyntaxRequest, DesignerModulesSyntaxRequest, ExtendedModulesDetail, - ExtendedModulesPolicy, SyntaxExtensionScope, + ExtendedModulesPolicy, SyntaxExtensionScope, TestRequest, }; + use crate::domain::runner::RunnerOutputFormat; use crate::use_cases::result::UseCaseErrorKind; #[test] @@ -693,6 +695,21 @@ mod tests { assert!(request.extension_scope().includes_all_extensions()); } + #[test] + fn default_test_execution_retains_junit_and_allure_results() { + let execution = TestRequest::default_execution(); + + assert_eq!( + execution.profile.output_formats, + vec![ + RunnerOutputFormat::JunitXml, + RunnerOutputFormat::AllureResults, + RunnerOutputFormat::PlainTextLog, + ] + ); + assert!(execution.policy.retain_artifacts_on_success); + } + #[test] fn extended_modules_policy_rejects_invalid_dependency_combinations() { let error = diff --git a/src/use_cases/run_tests.rs b/src/use_cases/run_tests.rs index 370c8cd..1acb124 100644 --- a/src/use_cases/run_tests.rs +++ b/src/use_cases/run_tests.rs @@ -8,7 +8,11 @@ use serde::Serialize; use uuid::Uuid; use crate::config::model::AppConfig; -use crate::domain::artifact::ArtifactSet; +use crate::domain::artifact::{ + ArtifactKind, ArtifactRef, ArtifactSet, ARTIFACT_ROLE_ALLURE_RESULTS, ARTIFACT_ROLE_CONFIG, + ARTIFACT_ROLE_ERROR_DETAILS, ARTIFACT_ROLE_JUNIT_XML, ARTIFACT_ROLE_PLATFORM_LOG, + ARTIFACT_ROLE_RUNNER_LOG, ARTIFACT_ROLE_RUN_DIR, ARTIFACT_ROLE_SCREENSHOT, +}; use crate::domain::execution::{ ExecutionMetrics, ExecutionOutcome, ExecutionStatus, ExecutionStepKind, StepResult, }; @@ -29,6 +33,7 @@ use crate::use_cases::vanessa::{self, VanessaTestArtifacts}; use tracing::debug; const STACK_TRACE_LIMIT: usize = 500; +const OPTIONAL_DIAGNOSTIC_FILE_LIMIT: usize = 100; mod coordinator; mod helpers; @@ -58,10 +63,7 @@ pub fn execute( struct YaXUnitConfig { #[serde(skip_serializing_if = "Option::is_none")] filter: Option, - #[serde(rename = "reportFormat")] - report_format: &'static str, - #[serde(rename = "reportPath")] - report_path: String, + reports: Vec, #[serde(rename = "closeAfterTests")] close_after_tests: bool, #[serde(rename = "showReport")] @@ -69,6 +71,12 @@ struct YaXUnitConfig { logging: YaXUnitLogging, } +#[derive(Debug, Serialize)] +struct YaXUnitReportConfig { + format: &'static str, + path: String, +} + #[derive(Debug, Serialize)] struct YaXUnitFilter { modules: Vec, @@ -87,11 +95,24 @@ struct RunArtifacts { config_json: PathBuf, junit_xml: PathBuf, junit_dir: PathBuf, + allure_results_dir: PathBuf, + error_details_dir: PathBuf, + screenshots_dir: PathBuf, runner_log: PathBuf, platform_log: PathBuf, sentinel: PathBuf, } +impl Drop for RunArtifacts { + fn drop(&mut self) { + if let Err(error) = fs::remove_file(&self.sentinel) { + if self.sentinel.exists() { + debug!(path = %self.sentinel.display(), %error, "failed to remove test run sentinel"); + } + } + } +} + enum PreparedRun { YaXUnit, Vanessa { @@ -118,8 +139,16 @@ fn build_yaxunit_config(target: &TestTarget, artifacts: &RunArtifacts) -> YaXUni modules: vec![name.clone()], }), }, - report_format: "jUnit", - report_path: artifacts.junit_xml.display().to_string(), + reports: vec![ + YaXUnitReportConfig { + format: "jUnit", + path: artifacts.junit_xml.display().to_string(), + }, + YaXUnitReportConfig { + format: "allure", + path: artifacts.allure_results_dir.display().to_string(), + }, + ], close_after_tests: true, show_report: false, logging: YaXUnitLogging { @@ -151,15 +180,23 @@ fn create_run_artifacts(config: &AppConfig, runner_id: &str) -> std::io::Result< fs::write(&sentinel, &run_id)?; set_file_permissions(&sentinel)?; + let junit_dir = run_dir.join("junit"); let artifacts = RunArtifacts { run_dir: run_dir.clone(), config_json: run_dir.join("config.json"), - junit_xml: run_dir.join("report.xml"), - junit_dir: run_dir.join("junit"), + junit_xml: junit_dir.join("report.xml"), + junit_dir, + allure_results_dir: run_dir.join("allure-results"), + error_details_dir: run_dir.join("error-details"), + screenshots_dir: run_dir.join("screenshots"), runner_log: run_dir.join("runner.log"), platform_log: run_dir.join("enterprise.out.log"), sentinel, }; + fs::create_dir_all(&artifacts.junit_dir)?; + set_dir_permissions(&artifacts.junit_dir)?; + fs::create_dir_all(&artifacts.allure_results_dir)?; + set_dir_permissions(&artifacts.allure_results_dir)?; Ok(artifacts) } @@ -180,6 +217,7 @@ fn prepare_vanessa_run( VanessaTestArtifacts { run_dir: &artifacts.run_dir, junit_dir: &artifacts.junit_dir, + allure_results_dir: &artifacts.allure_results_dir, runner_log: &artifacts.runner_log, }, )?; @@ -192,18 +230,190 @@ fn prepare_vanessa_run( } fn materialize_vanessa_runner_log(artifacts: &RunArtifacts) -> Result<(), String> { - if artifacts - .runner_log - .metadata() - .is_ok_and(|metadata| metadata.len() > 0) - { - return Ok(()); + match fs::symlink_metadata(&artifacts.runner_log) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "failed to materialize Vanessa runner log: destination '{}' is a symlink", + artifacts.runner_log.display() + )); + } + Ok(metadata) if !metadata.is_file() => { + return Err(format!( + "failed to materialize Vanessa runner log: destination '{}' is not a regular file", + artifacts.runner_log.display() + )); + } + Ok(metadata) if metadata.len() > 0 => return Ok(()), + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(format!( + "failed to inspect Vanessa runner log destination '{}': {error}", + artifacts.runner_log.display() + )); + } } - fs::copy(&artifacts.platform_log, &artifacts.runner_log).map_err(|error| { - format!("failed to materialize Vanessa runner log from enterprise output: {error}") + + let source_metadata = fs::symlink_metadata(&artifacts.platform_log).map_err(|error| { + format!( + "failed to materialize Vanessa runner log from enterprise output '{}': {error}", + artifacts.platform_log.display() + ) })?; - set_file_permissions(&artifacts.runner_log) - .map_err(|error| format!("failed to chmod Vanessa runner log: {error}")) + if source_metadata.file_type().is_symlink() { + return Err(format!( + "failed to materialize Vanessa runner log: source '{}' is a symlink", + artifacts.platform_log.display() + )); + } + if !source_metadata.is_file() { + return Err(format!( + "failed to materialize Vanessa runner log: source '{}' is not a regular file", + artifacts.platform_log.display() + )); + } + + let temp_path = artifacts + .run_dir + .join(format!(".runner-log-{}.tmp", Uuid::new_v4().simple())); + let result = (|| -> std::io::Result<()> { + let mut source = open_file_no_follow(&artifacts.platform_log)?; + let mut temp = create_private_file(&temp_path)?; + std::io::copy(&mut source, &mut temp)?; + set_open_file_permissions(&temp)?; + temp.sync_all()?; + drop(temp); + replace_file(&temp_path, &artifacts.runner_log) + })(); + if result.is_err() { + let _ = fs::remove_file(&temp_path); + } + result.map_err(|error| { + format!("failed to materialize Vanessa runner log from enterprise output: {error}") + }) +} + +#[cfg(unix)] +fn open_file_no_follow(path: &Path) -> std::io::Result { + use std::os::unix::fs::OpenOptionsExt; + + let mut options = fs::OpenOptions::new(); + options.read(true); + options.custom_flags(libc::O_NOFOLLOW); + options.open(path) +} + +#[cfg(windows)] +fn open_file_no_follow(path: &Path) -> std::io::Result { + use std::os::windows::fs::OpenOptionsExt; + use windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OPEN_REPARSE_POINT; + + let file = fs::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_OPEN_REPARSE_POINT) + .open(path)?; + ensure_windows_handle_is_not_reparse_point(&file)?; + Ok(file) +} + +#[cfg(windows)] +fn ensure_windows_handle_is_not_reparse_point(file: &fs::File) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{ + GetFileInformationByHandle, BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_REPARSE_POINT, + }; + + let mut information = BY_HANDLE_FILE_INFORMATION::default(); + // SAFETY: `file` owns a valid live handle and `information` points to writable storage + // of the exact structure expected by `GetFileInformationByHandle`. + let succeeded = unsafe { GetFileInformationByHandle(file.as_raw_handle(), &mut information) }; + if succeeded == 0 { + return Err(std::io::Error::last_os_error()); + } + if information.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "source is a Windows reparse point", + )); + } + Ok(()) +} + +fn create_private_file(path: &Path) -> std::io::Result { + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + options.open(path) +} + +fn set_open_file_permissions(file: &fs::File) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = file.metadata()?.permissions(); + permissions.set_mode(0o600); + file.set_permissions(permissions)?; + } + Ok(()) +} + +#[cfg(unix)] +fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> { + fs::rename(source, destination) +} + +#[cfg(windows)] +fn replace_file(source: &Path, destination: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::{ + MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, + }; + + fn wide_path(path: &Path) -> std::io::Result> { + let parent = path.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("Windows path has no parent: {}", path.display()), + ) + })?; + let file_name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("Windows path has no file name: {}", path.display()), + ) + })?; + let normalized = fs::canonicalize(parent)?.join(file_name); + let mut wide = normalized.as_os_str().encode_wide().collect::>(); + if wide.contains(&0) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Windows path contains a NUL code unit", + )); + } + wide.push(0); + Ok(wide) + } + + let source = wide_path(source)?; + let destination = wide_path(destination)?; + // SAFETY: both vectors are NUL-terminated, contain no interior NULs, and remain alive + // for the duration of the synchronous `MoveFileExW` call. + let succeeded = unsafe { + MoveFileExW( + source.as_ptr(), + destination.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + }; + if succeeded == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } } fn parse_runner_log( @@ -218,7 +428,7 @@ fn parse_runner_log( PreparedRun::YaXUnit => match yaxunit_log::normalize_file(runner_log_path) { Ok(parsed) => { if let Some(errors) = parsed.payload { - report.extracted_errors = errors; + report.extracted_errors.extend(errors); } warnings.extend(parsed.warnings); steps.push( @@ -250,7 +460,7 @@ fn parse_runner_log( PreparedRun::Vanessa { .. } => match vanessa_log::normalize_file(runner_log_path) { Ok(parsed) => { if let Some(errors) = parsed.payload { - report.extracted_errors = errors; + report.extracted_errors.extend(errors); } warnings.extend(parsed.warnings); steps.push( @@ -282,78 +492,228 @@ fn parse_runner_log( } } -fn resolve_vanessa_junit_path(artifacts: &mut RunArtifacts) { - if artifacts.junit_xml.exists() { - return; - } - if let Some(path) = discover_junit_report(&artifacts.junit_dir) { - artifacts.junit_xml = path; - } -} - -fn discover_junit_report(root: &Path) -> Option { - let entries = fs::read_dir(root).ok()?; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() - && path - .extension() +fn discover_junit_reports(root: &Path) -> std::io::Result> { + Ok(collect_regular_files(root)? + .into_iter() + .filter(|path| { + path.extension() .and_then(|value| value.to_str()) - .is_some_and(|ext| ext.eq_ignore_ascii_case("xml")) - { - return Some(path); - } - if path.is_dir() { - if let Some(found) = discover_junit_report(&path) { - return Some(found); - } - } - } - None + .is_some_and(|extension| extension.eq_ignore_ascii_case("xml")) + }) + .collect()) } -fn parse_junit_report(artifacts: &RunArtifacts) -> crate::parsers::NormalizedParse { - if !artifacts.junit_xml.exists() { +fn parse_junit_reports(reports: &[PathBuf]) -> crate::parsers::NormalizedParse { + if reports.is_empty() { return crate::parsers::NormalizedParse::default().with_errors(vec![test_execution_error( TestErrorKind::JunitNotProduced, "JUnit report was not produced", )]); } - if fs::metadata(&artifacts.junit_xml) - .map(|meta| meta.len() == 0) - .unwrap_or(false) - { - return crate::parsers::NormalizedParse::default().with_errors(vec![test_execution_error( - TestErrorKind::JunitEmpty, - "JUnit report is empty", - )]); + + let mut report = TestReport { + summary: crate::domain::test::TestSummary { + total: 0, + passed: 0, + failed: 0, + skipped: 0, + errors: 0, + }, + suites: Vec::new(), + extracted_errors: Vec::new(), + }; + let mut errors = Vec::new(); + + for path in reports { + match fs::File::open(path) { + Ok(file) => { + let normalized = junit::parse_normalized(BufReader::new(file)); + if let Some(parsed) = normalized.payload { + report.summary.total = + report.summary.total.saturating_add(parsed.summary.total); + report.summary.passed = + report.summary.passed.saturating_add(parsed.summary.passed); + report.summary.failed = + report.summary.failed.saturating_add(parsed.summary.failed); + report.summary.skipped = report + .summary + .skipped + .saturating_add(parsed.summary.skipped); + report.summary.errors = + report.summary.errors.saturating_add(parsed.summary.errors); + report.suites.extend(parsed.suites); + report.extracted_errors.extend(parsed.extracted_errors); + } + errors.extend(normalized.errors.into_iter().map(|error| { + let mut details = vec![format!("JUnit report: {}", path.display())]; + details.extend(error.details.clone()); + match error.code.as_str() { + "junit_empty" => { + test_execution_error(TestErrorKind::JunitEmpty, error.message) + .with_details(details) + } + "junit_malformed" => { + test_execution_error(TestErrorKind::JunitMalformed, error.message) + .with_details(details) + } + _ => error.with_details(details), + } + })); + } + Err(error) => errors.push( + test_execution_error(TestErrorKind::JunitNotProduced, error.to_string()) + .with_details(vec![format!("JUnit report: {}", path.display())]), + ), + } + } + + if !errors.is_empty() { + return crate::parsers::NormalizedParse::default().with_errors(errors); } - let file = fs::File::open(&artifacts.junit_xml).map_err(|error| error.to_string()); - let file = match file { - Ok(file) => file, + + crate::parsers::NormalizedParse::default() + .with_metrics(ExecutionMetrics::from(&report.summary)) + .with_payload(report) +} + +fn parse_junit_report(artifacts: &RunArtifacts) -> crate::parsers::NormalizedParse { + match discover_junit_reports(&artifacts.junit_dir) { + Ok(reports) if reports.is_empty() => crate::parsers::NormalizedParse::default() + .with_errors(vec![test_execution_error( + TestErrorKind::JunitNotProduced, + "JUnit report was not produced", + ) + .with_details(vec![format!( + "JUnit report directory: {}", + artifacts.junit_dir.display() + )])]), + Ok(reports) => parse_junit_reports(&reports), Err(error) => { - return crate::parsers::NormalizedParse::default().with_errors(vec![ - test_execution_error(TestErrorKind::JunitNotProduced, error), - ]); + crate::parsers::NormalizedParse::default().with_errors(vec![test_execution_error( + TestErrorKind::JunitNotProduced, + error.to_string(), + ) + .with_details(vec![format!( + "JUnit report directory: {}", + artifacts.junit_dir.display() + )])]) + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct AllureValidationFailure { + kind: TestErrorKind, + message: String, + details: Vec, +} + +fn validate_allure_results(root: &Path) -> Result<(), AllureValidationFailure> { + let metadata = match fs::symlink_metadata(root) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(AllureValidationFailure { + kind: TestErrorKind::AllureNotProduced, + message: "Allure results directory was not produced".to_owned(), + details: vec![format!("Allure results directory: {}", root.display())], + }); + } + Err(error) => { + return Err(allure_io_failure(root, error)); } }; - let reader = BufReader::new(file); - let mut normalized = junit::parse_normalized(reader); - if normalized.errors.is_empty() { - return normalized; + if !metadata.is_dir() { + return Err(AllureValidationFailure { + kind: TestErrorKind::AllureNotProduced, + message: "Allure results directory was not produced".to_owned(), + details: vec![format!("Allure results directory: {}", root.display())], + }); } - normalized.errors = normalized - .errors - .into_iter() - .map(|error| match error.code.as_str() { - "junit_empty" => test_execution_error(TestErrorKind::JunitEmpty, error.message) - .with_details(error.details), - "junit_malformed" => test_execution_error(TestErrorKind::JunitMalformed, error.message) - .with_details(error.details), - _ => error, + let files = collect_regular_files(root).map_err(|error| allure_io_failure(root, error))?; + if files.is_empty() { + Err(AllureValidationFailure { + kind: TestErrorKind::AllureEmpty, + message: "Allure results directory is empty".to_owned(), + details: vec![format!("Allure results directory: {}", root.display())], }) - .collect(); - normalized + } else { + Ok(()) + } +} + +fn allure_io_failure(root: &Path, error: std::io::Error) -> AllureValidationFailure { + AllureValidationFailure { + kind: TestErrorKind::TestSetupFailed, + message: "failed to inspect Allure results".to_owned(), + details: vec![ + format!("Allure results directory: {}", root.display()), + error.to_string(), + ], + } +} + +fn collect_regular_files(root: &Path) -> std::io::Result> { + let root_metadata = fs::symlink_metadata(root).map_err(|error| { + std::io::Error::new( + error.kind(), + format!("failed to inspect '{}': {error}", root.display()), + ) + })?; + if !root_metadata.is_dir() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("'{}' is not a directory", root.display()), + )); + } + + let mut pending = vec![root.to_path_buf()]; + let mut files = Vec::new(); + while let Some(directory) = pending.pop() { + let entries = fs::read_dir(&directory).map_err(|error| { + std::io::Error::new( + error.kind(), + format!( + "failed to read directory '{}': {error}", + directory.display() + ), + ) + })?; + for entry in entries { + let entry = entry.map_err(|error| { + std::io::Error::new( + error.kind(), + format!("failed to read entry in '{}': {error}", directory.display()), + ) + })?; + let path = entry.path(); + let metadata = fs::symlink_metadata(&path).map_err(|error| { + std::io::Error::new( + error.kind(), + format!("failed to inspect '{}': {error}", path.display()), + ) + })?; + if metadata.is_file() { + files.push(path); + } else if metadata.is_dir() { + pending.push(path); + } + } + } + files.sort(); + Ok(files) +} + +fn classify_test_completion( + summary: &crate::domain::test::TestSummary, + exit_code: i32, +) -> Option { + if summary.failed > 0 || summary.errors > 0 { + Some(TestErrorKind::TestFailures) + } else if exit_code != 0 { + Some(TestErrorKind::EnterpriseExitedNonZero) + } else { + None + } } fn compact_report(report: &TestReport) -> TestReport { @@ -392,20 +752,146 @@ fn retain_run_artifacts( _config: &AppConfig, artifacts: &RunArtifacts, ) -> std::io::Result { - Ok(crate::domain::test::RetainedPaths { - run_dir: artifacts.run_dir.clone(), - config_json: artifacts.config_json.clone(), - junit_xml: artifacts.junit_xml.clone(), - yaxunit_log: artifacts.runner_log.clone(), - platform_log: artifacts.platform_log.clone(), - sentinel: artifacts.sentinel.clone(), + Ok(collect_run_artifacts(artifacts)) +} + +fn collect_run_artifacts(artifacts: &RunArtifacts) -> ArtifactSet { + let mut collected = if is_existing_dir(&artifacts.run_dir) { + let mut set = ArtifactSet::with_root(artifacts.run_dir.clone()); + set.push( + ArtifactRef::new(ArtifactKind::RunDirectory, artifacts.run_dir.clone()) + .with_role(ARTIFACT_ROLE_RUN_DIR), + ); + set + } else { + ArtifactSet::default() + }; + + push_existing_file( + &mut collected, + ArtifactKind::Config, + ARTIFACT_ROLE_CONFIG, + &artifacts.config_json, + ); + + let mut junit_reports = collect_existing_junit_reports(&artifacts.junit_dir); + if is_existing_file(&artifacts.junit_xml) && !junit_reports.contains(&artifacts.junit_xml) { + junit_reports.push(artifacts.junit_xml.clone()); + } + junit_reports.sort(); + for report in junit_reports { + push_existing_file( + &mut collected, + ArtifactKind::JunitXml, + ARTIFACT_ROLE_JUNIT_XML, + &report, + ); } - .into_artifact_set()) + + push_existing_dir( + &mut collected, + ArtifactKind::AllureResults, + ARTIFACT_ROLE_ALLURE_RESULTS, + &artifacts.allure_results_dir, + ); + let mut remaining_optional_diagnostics = OPTIONAL_DIAGNOSTIC_FILE_LIMIT; + push_optional_diagnostics( + &mut collected, + ArtifactKind::ErrorDetails, + ARTIFACT_ROLE_ERROR_DETAILS, + &artifacts.error_details_dir, + &mut remaining_optional_diagnostics, + ); + push_optional_diagnostics( + &mut collected, + ArtifactKind::Screenshot, + ARTIFACT_ROLE_SCREENSHOT, + &artifacts.screenshots_dir, + &mut remaining_optional_diagnostics, + ); + push_existing_file( + &mut collected, + ArtifactKind::RunnerLog, + ARTIFACT_ROLE_RUNNER_LOG, + &artifacts.runner_log, + ); + push_existing_file( + &mut collected, + ArtifactKind::PlatformLog, + ARTIFACT_ROLE_PLATFORM_LOG, + &artifacts.platform_log, + ); + + collected.items.sort_by(|left, right| { + artifact_kind_sort_key(&left.kind) + .cmp(artifact_kind_sort_key(&right.kind)) + .then_with(|| left.role.cmp(&right.role)) + .then_with(|| left.path.cmp(&right.path)) + }); + + collected } -fn cleanup_run_dir(artifacts: &RunArtifacts) { - let _ = fs::remove_file(&artifacts.sentinel); - let _ = fs::remove_dir_all(&artifacts.run_dir); +#[allow(deprecated)] +fn artifact_kind_sort_key(kind: &ArtifactKind) -> &str { + match kind { + ArtifactKind::RunDirectory => "run_directory", + ArtifactKind::Config => "config", + ArtifactKind::Package => "package", + ArtifactKind::Report => "report", + ArtifactKind::JunitXml => "junit_xml", + ArtifactKind::AllureResults => "allure_results", + ArtifactKind::ErrorDetails => "error_details", + ArtifactKind::Screenshot => "screenshot", + ArtifactKind::RunnerLog => "runner_log", + ArtifactKind::PlatformLog => "platform_log", + ArtifactKind::Sentinel => "sentinel", + ArtifactKind::Other(value) => value, + } +} + +fn collect_existing_junit_reports(root: &Path) -> Vec { + discover_junit_reports(root).unwrap_or_default() +} + +fn push_optional_diagnostics( + set: &mut ArtifactSet, + kind: ArtifactKind, + role: &str, + root: &Path, + remaining: &mut usize, +) { + let Ok(paths) = collect_regular_files(root) else { + return; + }; + let added = paths.len().min(*remaining); + for path in paths.iter().take(added) { + set.push(ArtifactRef::new(kind.clone(), path.clone()).with_role(role)); + } + *remaining -= added; + if paths.len() > added { + set.push(ArtifactRef::new(kind, root.to_path_buf()).with_role(role)); + } +} + +fn push_existing_file(set: &mut ArtifactSet, kind: ArtifactKind, role: &str, path: &Path) { + if is_existing_file(path) { + set.push(ArtifactRef::new(kind, path).with_role(role)); + } +} + +fn push_existing_dir(set: &mut ArtifactSet, kind: ArtifactKind, role: &str, path: &Path) { + if is_existing_dir(path) { + set.push(ArtifactRef::new(kind, path).with_role(role)); + } +} + +fn is_existing_file(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_file()) +} + +fn is_existing_dir(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok_and(|metadata| metadata.is_dir()) } fn sanitize_text(text: &str, config: &AppConfig) -> String { @@ -514,26 +1000,36 @@ fn set_file_permissions(path: &Path) -> std::io::Result<()> { #[cfg(test)] mod tests { use super::{ - build_yaxunit_config, compact_report, create_run_artifacts, materialize_vanessa_runner_log, - parse_junit_report, retain_run_artifacts, run_tests, sanitize_text, sanitize_text_full, - truncate_stack_trace, RunArtifacts, + build_yaxunit_config, classify_test_completion, collect_run_artifacts, compact_report, + create_run_artifacts, discover_junit_reports, materialize_vanessa_runner_log, + parse_junit_report, parse_junit_reports, retain_run_artifacts, run_tests, sanitize_text, + sanitize_text_full, truncate_stack_trace, validate_allure_results, RunArtifacts, }; use crate::config::model::{ AppConfig, BuildConfig, BuilderBackend, PlatformToolConfig, SourceFormat, SourceSetConfig, SourceSetPurpose, TestsConfig, ToolsConfig, VanessaProfileConfig, }; + use crate::domain::artifact::{ + ArtifactKind, ARTIFACT_ROLE_ALLURE_RESULTS, ARTIFACT_ROLE_CONFIG, + ARTIFACT_ROLE_ERROR_DETAILS, ARTIFACT_ROLE_JUNIT_XML, ARTIFACT_ROLE_PLATFORM_LOG, + ARTIFACT_ROLE_RUNNER_LOG, ARTIFACT_ROLE_SCREENSHOT, + }; use crate::domain::execution::{ExecutionStatus, ExecutionTimeouts}; use crate::domain::runner::{ ExecutionPolicy, LaunchClientModeRequest, LaunchOptions, RunnerKind, RunnerProfile, ScenarioExecutionRequest, }; use crate::domain::test::{ - TestCase, TestErrorKind, TestReport, TestStatus, TestSuite, TestSummary, TestTarget, + test_execution_status, TestCase, TestErrorKind, TestReport, TestStatus, TestSuite, + TestSummary, TestTarget, }; use crate::use_cases::context::{CommandName, ExecutionContext}; use crate::use_cases::request::{TestRequest, TestScopeRequest}; use std::path::PathBuf; use tempfile::tempdir; + + #[cfg(windows)] + use super::{open_file_no_follow, replace_file}; use tokio_util::sync::CancellationToken; fn config(work_path: &std::path::Path) -> AppConfig { @@ -568,6 +1064,8 @@ mod tests { let first = create_run_artifacts(&config, "yaxunit").expect("first"); let second = create_run_artifacts(&config, "yaxunit").expect("second"); assert_ne!(first.run_dir, second.run_dir); + assert!(!first.error_details_dir.exists()); + assert!(!first.screenshots_dir.exists()); } #[test] @@ -584,6 +1082,514 @@ mod tests { assert_eq!(json["filter"]["modules"][0], "Foo Бар"); } + #[test] + fn yaxunit_config_serializes_simultaneous_junit_and_allure_reports() { + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + let payload = build_yaxunit_config(&TestTarget::All, &artifacts); + + let json = serde_json::to_value(payload).expect("json"); + + assert_eq!(json["reports"][0]["format"], "jUnit"); + assert_eq!( + json["reports"][0]["path"], + artifacts.junit_xml.display().to_string() + ); + assert_eq!(json["reports"][1]["format"], "allure"); + assert_eq!( + json["reports"][1]["path"], + artifacts.allure_results_dir.display().to_string() + ); + assert!(json.get("reportFormat").is_none()); + } + + #[test] + fn collect_run_artifacts_omits_missing_junit_and_keeps_existing_outputs() { + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(&artifacts.allure_results_dir).expect("allure dir"); + std::fs::write(&artifacts.config_json, b"{}").expect("config"); + std::fs::write(&artifacts.runner_log, b"runner").expect("runner log"); + std::fs::write(&artifacts.platform_log, b"platform").expect("platform log"); + + let collected = collect_run_artifacts(&artifacts); + + assert!(collected.get_by_role(ARTIFACT_ROLE_JUNIT_XML).is_none()); + assert_eq!( + collected.get_by_role(ARTIFACT_ROLE_CONFIG), + Some(artifacts.config_json.as_path()) + ); + assert_eq!( + collected.get_by_role(ARTIFACT_ROLE_ALLURE_RESULTS), + Some(artifacts.allure_results_dir.as_path()) + ); + assert_eq!( + collected.get_by_role(ARTIFACT_ROLE_RUNNER_LOG), + Some(artifacts.runner_log.as_path()) + ); + assert_eq!( + collected.get_by_role(ARTIFACT_ROLE_PLATFORM_LOG), + Some(artifacts.platform_log.as_path()) + ); + } + + #[test] + fn collect_run_artifacts_sorts_public_inventory_by_kind_role_and_path() { + // Break caught: append order leaking implementation details makes JSON artifacts unstable. + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(artifacts.junit_dir.join("nested")).expect("junit dir"); + std::fs::create_dir_all(&artifacts.allure_results_dir).expect("allure dir"); + for path in [ + &artifacts.config_json, + &artifacts.junit_xml, + &artifacts.junit_dir.join("nested").join("second.xml"), + &artifacts.runner_log, + &artifacts.platform_log, + ] { + std::fs::write(path, b"fixture").expect("artifact"); + } + + let collected = collect_run_artifacts(&artifacts); + let inventory: Vec<_> = collected + .items + .iter() + .map(|item| { + ( + item.kind.clone(), + item.role.as_deref().expect("role"), + item.path.clone(), + ) + }) + .collect(); + + assert_eq!( + inventory, + vec![ + ( + ArtifactKind::AllureResults, + ARTIFACT_ROLE_ALLURE_RESULTS, + artifacts.allure_results_dir.clone(), + ), + ( + ArtifactKind::Config, + ARTIFACT_ROLE_CONFIG, + artifacts.config_json.clone(), + ), + ( + ArtifactKind::JunitXml, + ARTIFACT_ROLE_JUNIT_XML, + artifacts.junit_dir.join("nested").join("second.xml"), + ), + ( + ArtifactKind::JunitXml, + ARTIFACT_ROLE_JUNIT_XML, + artifacts.junit_xml.clone(), + ), + ( + ArtifactKind::PlatformLog, + ARTIFACT_ROLE_PLATFORM_LOG, + artifacts.platform_log.clone(), + ), + ( + ArtifactKind::RunDirectory, + "run_dir", + artifacts.run_dir.clone(), + ), + ( + ArtifactKind::RunnerLog, + ARTIFACT_ROLE_RUNNER_LOG, + artifacts.runner_log.clone(), + ), + ] + ); + assert!(collected.items.iter().all(|artifact| { + serde_json::to_value(&artifact.kind).expect("artifact kind") + != serde_json::json!("sentinel") + })); + } + + #[test] + fn collect_run_artifacts_includes_nested_optional_diagnostics_in_stable_order() { + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(artifacts.error_details_dir.join("nested")).expect("error details"); + std::fs::create_dir_all(&artifacts.screenshots_dir).expect("screenshots"); + let later_error = artifacts.error_details_dir.join("z.txt"); + let earlier_error = artifacts.error_details_dir.join("nested/a.txt"); + let screenshot = artifacts.screenshots_dir.join("failure.png"); + std::fs::write(&later_error, "later").expect("later error"); + std::fs::write(&earlier_error, "earlier").expect("earlier error"); + std::fs::write(&screenshot, "png").expect("screenshot"); + + let collected = collect_run_artifacts(&artifacts); + let diagnostics = collected + .items + .iter() + .filter(|artifact| { + matches!( + artifact.kind, + ArtifactKind::ErrorDetails | ArtifactKind::Screenshot + ) + }) + .map(|artifact| { + ( + artifact.kind.clone(), + artifact.role.clone().expect("role"), + artifact.path.clone(), + ) + }) + .collect::>(); + + assert_eq!( + diagnostics, + vec![ + ( + ArtifactKind::ErrorDetails, + ARTIFACT_ROLE_ERROR_DETAILS.to_owned(), + earlier_error, + ), + ( + ArtifactKind::ErrorDetails, + ARTIFACT_ROLE_ERROR_DETAILS.to_owned(), + later_error, + ), + ( + ArtifactKind::Screenshot, + ARTIFACT_ROLE_SCREENSHOT.to_owned(), + screenshot, + ), + ] + ); + } + + #[test] + fn collect_run_artifacts_bounds_optional_diagnostics_across_categories() { + // Break caught: independently budgeting categories or serializing every discovered + // diagnostic file exceeds the shared inventory limit and loses the category fallback. + const OPTIONAL_DIAGNOSTIC_FILE_LIMIT: usize = 100; + + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(&artifacts.error_details_dir).expect("error details"); + std::fs::create_dir_all(&artifacts.screenshots_dir).expect("screenshots"); + for index in 0..=OPTIONAL_DIAGNOSTIC_FILE_LIMIT { + std::fs::write( + artifacts.error_details_dir.join(format!("{index:03}.txt")), + "detail", + ) + .expect("error detail"); + } + let screenshot = artifacts.screenshots_dir.join("failure.png"); + std::fs::write(&screenshot, "png").expect("screenshot"); + + let collected = collect_run_artifacts(&artifacts); + let diagnostic_files = collected + .items + .iter() + .filter(|artifact| { + matches!( + artifact.kind, + ArtifactKind::ErrorDetails | ArtifactKind::Screenshot + ) && artifact.path.is_file() + }) + .collect::>(); + + assert_eq!(diagnostic_files.len(), OPTIONAL_DIAGNOSTIC_FILE_LIMIT); + assert!(collected.items.iter().any(|artifact| { + artifact.kind == ArtifactKind::ErrorDetails + && artifact.path == artifacts.error_details_dir + && artifact.role.as_deref() == Some(ARTIFACT_ROLE_ERROR_DETAILS) + })); + assert!(collected.items.iter().any(|artifact| { + artifact.kind == ArtifactKind::Screenshot + && artifact.path == artifacts.screenshots_dir + && artifact.role.as_deref() == Some(ARTIFACT_ROLE_SCREENSHOT) + })); + assert!(collected + .items + .iter() + .any(|artifact| artifact.path == artifacts.error_details_dir.join("000.txt"))); + assert!(!collected + .items + .iter() + .any(|artifact| artifact.path == artifacts.error_details_dir.join("100.txt"))); + assert!(!collected + .items + .iter() + .any(|artifact| artifact.path == screenshot)); + } + + #[cfg(unix)] + #[test] + fn collect_run_artifacts_skips_symlinked_optional_diagnostics() { + use std::os::unix::fs::symlink; + + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&outside).expect("outside"); + std::fs::write(outside.join("secret.txt"), "secret").expect("outside file"); + std::fs::create_dir_all(&artifacts.error_details_dir).expect("error details"); + std::fs::create_dir_all(&artifacts.screenshots_dir).expect("screenshots"); + symlink(&outside, artifacts.error_details_dir.join("external")).expect("dir symlink"); + symlink( + outside.join("secret.txt"), + artifacts.screenshots_dir.join("external.png"), + ) + .expect("file symlink"); + + let collected = collect_run_artifacts(&artifacts); + + assert!(collected + .get_all_by_role(ARTIFACT_ROLE_ERROR_DETAILS) + .next() + .is_none()); + assert!(collected + .get_all_by_role(ARTIFACT_ROLE_SCREENSHOT) + .next() + .is_none()); + } + + #[test] + fn discovers_sorted_nested_junit_reports_and_aggregates_every_report() { + // Break caught: returning the first report or filesystem iteration order would lose cases. + let dir = tempdir().expect("tempdir"); + let junit_dir = dir.path().join("junit"); + let nested = junit_dir.join("a"); + std::fs::create_dir_all(&nested).expect("nested junit dir"); + + let later_path = junit_dir.join("z-report.xml"); + std::fs::write( + &later_path, + r#""#, + ) + .expect("later report"); + let earlier_path = nested.join("a-report.xml"); + std::fs::write( + &earlier_path, + r#""#, + ) + .expect("earlier report"); + + let reports = discover_junit_reports(&junit_dir).expect("discover reports"); + + assert_eq!(reports, vec![earlier_path, later_path]); + let parsed = parse_junit_reports(&reports); + assert!(parsed.errors.is_empty()); + assert_eq!( + parsed.payload.expect("aggregate report").summary, + TestSummary { + total: 3, + passed: 1, + failed: 1, + skipped: 0, + errors: 1, + } + ); + } + + #[test] + fn rejects_aggregate_when_any_junit_report_is_malformed_with_its_path() { + // Break caught: accepting the first valid report silently hides invalid native output. + let dir = tempdir().expect("tempdir"); + let junit_dir = dir.path().join("junit"); + std::fs::create_dir_all(&junit_dir).expect("junit dir"); + let valid = junit_dir.join("a-valid.xml"); + std::fs::write( + &valid, + r#""#, + ) + .expect("valid report"); + let malformed = junit_dir.join("b-malformed.xml"); + std::fs::write(&malformed, "").expect("malformed report"); + let another_malformed = junit_dir.join("c-malformed.xml"); + std::fs::write(&another_malformed, "") + .expect("another malformed report"); + + let parsed = parse_junit_reports(&[valid, malformed.clone(), another_malformed.clone()]); + + assert!(parsed.payload.is_none()); + assert_eq!(parsed.errors.len(), 2); + for path in [malformed, another_malformed] { + assert!(parsed.errors.iter().any(|error| { + error.code == TestErrorKind::JunitMalformed.code() + && error + .details + .iter() + .any(|detail| detail.contains(&path.display().to_string())) + })); + } + } + + #[test] + fn validates_missing_and_empty_allure_results() { + // Break caught: treating pre-created or missing Allure directories as valid native output. + let dir = tempdir().expect("tempdir"); + let missing = dir.path().join("missing-allure-results"); + let empty = dir.path().join("empty-allure-results"); + std::fs::create_dir_all(empty.join("nested")).expect("empty allure dir"); + + assert_eq!( + validate_allure_results(&missing).expect_err("missing").kind, + TestErrorKind::AllureNotProduced + ); + assert_eq!( + validate_allure_results(&empty).expect_err("empty").kind, + TestErrorKind::AllureEmpty + ); + } + + #[test] + fn accepts_allure_results_with_a_nested_regular_file() { + let dir = tempdir().expect("tempdir"); + let allure = dir.path().join("allure-results"); + std::fs::create_dir_all(allure.join("nested")).expect("allure dir"); + std::fs::write(allure.join("nested/result.json"), "{}").expect("allure result"); + + assert_eq!(validate_allure_results(&allure), Ok(())); + } + + #[cfg(unix)] + #[test] + fn discovery_and_allure_validation_ignore_symlinked_outputs() { + use std::os::unix::fs::symlink; + + // Break caught: following symlinks can escape the run directory or make empty output valid. + let dir = tempdir().expect("tempdir"); + let outside = dir.path().join("outside"); + std::fs::create_dir_all(&outside).expect("outside dir"); + std::fs::write( + outside.join("report.xml"), + "", + ) + .expect("outside junit"); + std::fs::write(outside.join("result.json"), "{}").expect("outside allure"); + + let junit_dir = dir.path().join("junit"); + let allure_dir = dir.path().join("allure-results"); + std::fs::create_dir_all(&junit_dir).expect("junit dir"); + std::fs::create_dir_all(&allure_dir).expect("allure dir"); + symlink(&outside, junit_dir.join("external")).expect("junit symlink"); + symlink( + outside.join("result.json"), + allure_dir.join("external.json"), + ) + .expect("allure symlink"); + + assert!(discover_junit_reports(&junit_dir) + .expect("discover reports") + .is_empty()); + assert_eq!( + validate_allure_results(&allure_dir) + .expect_err("symlinks do not count") + .kind, + TestErrorKind::AllureEmpty + ); + } + + #[cfg(unix)] + #[test] + fn allure_traversal_io_failure_preserves_path_and_os_error() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempdir().expect("tempdir"); + let allure = dir.path().join("allure-results"); + let unreadable = allure.join("unreadable"); + std::fs::create_dir_all(&unreadable).expect("allure dir"); + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o000)) + .expect("restrict"); + + let failure = validate_allure_results(&allure).expect_err("traversal failure"); + std::fs::set_permissions(&unreadable, std::fs::Permissions::from_mode(0o700)) + .expect("restore"); + + assert_eq!(failure.kind, TestErrorKind::TestSetupFailed); + assert!(failure + .details + .iter() + .any(|detail| detail.contains(&unreadable.display().to_string()))); + assert!(failure + .details + .iter() + .any(|detail| detail.contains("Permission denied"))); + } + + #[test] + fn classifies_native_reports_before_process_exit_status() { + // Break caught: nonzero process exits masking report-proven test failures. + let cases = [ + ( + TestSummary { + total: 1, + passed: 0, + failed: 1, + skipped: 0, + errors: 0, + }, + 1, + Some(TestErrorKind::TestFailures), + ExecutionStatus::Failed, + ), + ( + TestSummary { + total: 1, + passed: 0, + failed: 0, + skipped: 0, + errors: 1, + }, + 2, + Some(TestErrorKind::TestFailures), + ExecutionStatus::Failed, + ), + ( + TestSummary { + total: 1, + passed: 1, + failed: 0, + skipped: 0, + errors: 0, + }, + 1, + Some(TestErrorKind::EnterpriseExitedNonZero), + ExecutionStatus::Failed, + ), + ( + TestSummary { + total: 1, + passed: 1, + failed: 0, + skipped: 0, + errors: 0, + }, + 0, + None, + ExecutionStatus::Succeeded, + ), + ]; + + for (summary, exit_code, expected, expected_status) in cases { + let is_success = expected.is_none(); + assert_eq!(classify_test_completion(&summary, exit_code), expected); + assert_eq!(test_execution_status(expected, is_success), expected_status); + } + } + + #[test] + fn dropping_run_artifacts_removes_sentinel_but_keeps_run_directory() { + let dir = tempdir().expect("tempdir"); + let config = config(dir.path()); + let (run_dir, sentinel) = { + let artifacts = create_run_artifacts(&config, "yaxunit").expect("artifacts"); + assert!(artifacts.sentinel.is_file()); + (artifacts.run_dir.clone(), artifacts.sentinel.clone()) + }; + + assert!(run_dir.is_dir()); + assert!(!sentinel.exists()); + } + #[test] fn sanitizer_masks_passwords() { let dir = tempdir().expect("tempdir"); @@ -671,12 +1677,100 @@ mod tests { assert!(warning.contains("failed to materialize Vanessa runner log")); } + #[cfg(windows)] #[test] - fn vanessa_junit_parse_failure_retains_materialized_runner_log() { + fn windows_atomic_replace_supports_extended_length_paths() { + let dir = tempdir().expect("tempdir"); + let mut long_dir = dir.path().to_path_buf(); + while long_dir.as_os_str().len() <= 300 { + long_dir.push("long-path-segment"); + } + std::fs::create_dir_all(&long_dir).expect("long directory"); + let source = long_dir.join("source.tmp"); + let destination = long_dir.join("runner.log"); + std::fs::write(&source, "replacement").expect("source"); + std::fs::write(&destination, "old").expect("destination"); + + replace_file(&source, &destination).expect("atomic replace"); + + assert_eq!( + std::fs::read_to_string(&destination).expect("destination"), + "replacement" + ); + assert!(!source.exists()); + } + + #[cfg(windows)] + #[test] + fn windows_materialize_vanessa_runner_log_rejects_reparse_source() { + use std::os::windows::fs::symlink_file; + + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(&artifacts.run_dir).expect("run dir"); + let outside = dir.path().join("outside.log"); + std::fs::write(&outside, "outside").expect("outside"); + symlink_file(&outside, &artifacts.platform_log).expect("source symlink"); + + let warning = materialize_vanessa_runner_log(&artifacts).expect_err("warning"); + assert!(warning.contains("symlink")); + assert!(!artifacts.runner_log.exists()); + + let error = open_file_no_follow(&artifacts.platform_log).expect_err("reparse point"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + } + + #[cfg(unix)] + #[test] + fn materialize_vanessa_runner_log_rejects_symlink_source() { + use std::os::unix::fs::symlink; + + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(&artifacts.run_dir).expect("run dir"); + let outside = dir.path().join("outside.log"); + std::fs::write(&outside, "outside").expect("outside"); + symlink(&outside, &artifacts.platform_log).expect("source symlink"); + + let warning = materialize_vanessa_runner_log(&artifacts).expect_err("warning"); + + assert!(warning.contains("symlink")); + assert_eq!( + std::fs::read_to_string(&outside).expect("outside"), + "outside" + ); + assert!(!artifacts.runner_log.exists()); + } + + #[cfg(unix)] + #[test] + fn materialize_vanessa_runner_log_rejects_symlink_destination() { + use std::os::unix::fs::symlink; + + let dir = tempdir().expect("tempdir"); + let artifacts = create_artifacts(dir.path()); + std::fs::create_dir_all(&artifacts.run_dir).expect("run dir"); + std::fs::write(&artifacts.platform_log, "platform").expect("platform"); + let outside = dir.path().join("outside.log"); + std::fs::write(&outside, "outside").expect("outside"); + symlink(&outside, &artifacts.runner_log).expect("destination symlink"); + + let warning = materialize_vanessa_runner_log(&artifacts).expect_err("warning"); + + assert!(warning.contains("symlink")); + assert_eq!( + std::fs::read_to_string(&outside).expect("outside"), + "outside" + ); + } + + #[test] + fn vanessa_junit_parse_failure_inventories_materialized_runner_log() { let dir = tempdir().expect("tempdir"); let config = config(dir.path()); let artifacts = create_artifacts(dir.path()); std::fs::create_dir_all(&artifacts.run_dir).expect("run dir"); + std::fs::create_dir_all(&artifacts.junit_dir).expect("junit dir"); std::fs::write(&artifacts.platform_log, b"enterprise /Out").expect("platform log"); materialize_vanessa_runner_log(&artifacts).expect("materialize log"); @@ -686,12 +1780,17 @@ mod tests { junit_parse.errors[0].code, TestErrorKind::JunitNotProduced.code() ); + assert!(junit_parse.errors[0] + .details + .iter() + .any(|detail| detail.contains(&artifacts.junit_dir.display().to_string()))); let retained = retain_run_artifacts(&config, &artifacts).expect("retain artifacts"); - let retained_paths = crate::domain::test::RetainedPaths::from_artifact_set(&retained) - .expect("retained paths"); - assert!(retained_paths.yaxunit_log.exists()); - assert_eq!(retained_paths.yaxunit_log, artifacts.runner_log); + assert_eq!( + retained.get_by_role(ARTIFACT_ROLE_RUNNER_LOG), + Some(artifacts.runner_log.as_path()) + ); + assert!(retained.get_by_role(ARTIFACT_ROLE_JUNIT_XML).is_none()); } #[test] @@ -739,6 +1838,7 @@ mod tests { assert!(result.is_err()); let error = result.err().expect("error"); assert!(error.error.to_string().contains("unsafe path characters")); + assert!(!dir.path().join("temp").exists()); } #[test] @@ -775,14 +1875,19 @@ mod tests { } fn create_artifacts(root: &std::path::Path) -> RunArtifacts { + let run_dir = root.join("run"); + let junit_dir = run_dir.join("junit"); RunArtifacts { - run_dir: root.join("run"), - config_json: root.join("run/config.json"), - junit_xml: root.join("run/report.xml"), + run_dir: run_dir.clone(), + config_json: run_dir.join("config.json"), + junit_xml: junit_dir.join("report.xml"), junit_dir: root.join("run/junit"), - runner_log: root.join("run/yax.log"), - platform_log: root.join("run/platform.log"), - sentinel: root.join("run/run.inprogress"), + allure_results_dir: run_dir.join("allure-results"), + error_details_dir: run_dir.join("error-details"), + screenshots_dir: run_dir.join("screenshots"), + runner_log: run_dir.join("yax.log"), + platform_log: run_dir.join("platform.log"), + sentinel: run_dir.join("run.inprogress"), } } diff --git a/src/use_cases/run_tests/coordinator.rs b/src/use_cases/run_tests/coordinator.rs index e1a5006..9701813 100644 --- a/src/use_cases/run_tests/coordinator.rs +++ b/src/use_cases/run_tests/coordinator.rs @@ -40,7 +40,7 @@ pub(super) fn run_tests( let mut steps = Vec::new(); let mut warnings = Vec::new(); if let Some(failure) = - interrupted_test_failure(context, &target, &mode, &warnings, &steps, started) + interrupted_test_failure(context, &target, &mode, &warnings, &steps, started, None) { return Err(failure); } @@ -65,70 +65,13 @@ pub(super) fn run_tests( } }; - let build_started = Instant::now(); match args.build_policy { - TestBuildPolicy::BuildFirst => { - debug!("running build prerequisite for tests"); - log_live_stage( - "test: build prerequisite", - "[Build] preparing test infobase", - ); - let build_result = match build_project::execute( - context, - config, - &BuildArgs { - full_rebuild: false, - source_set: None, - }, - ) { - Ok(result) => result, - Err(failure) => { - let summary = failure - .payload - .as_ref() - .map(build_summary) - .unwrap_or_else(|| failure.error.to_string()); - steps.push( - failed_step( - "build", - ExecutionStepKind::PlatformCommand, - build_started.elapsed().as_millis() as u64, - summary.clone(), - ) - .with_errors(vec![test_execution_error( - TestErrorKind::BuildFailed, - summary.clone(), - )]), - ); - let outcome = ExecutionOutcome::new(ExecutionStatus::Failed) - .with_diagnostics(vec![summary.clone()]) - .with_errors(vec![test_execution_error( - TestErrorKind::BuildFailed, - summary.clone(), - )]); - let result = make_test_result( - target, - mode, - outcome, - warnings, - steps, - started.elapsed().as_millis() as u64, - ); - return Err(TestExecutionFailure::with_payload(failure.error, result)); - } - }; - steps.push(succeeded_step( - "build", - ExecutionStepKind::PlatformCommand, - build_started.elapsed().as_millis() as u64, - build_summary(&build_result), - )); - } + TestBuildPolicy::BuildFirst => {} TestBuildPolicy::Skip => { steps.push(skipped_step( "build", ExecutionStepKind::PlatformCommand, - build_started.elapsed().as_millis() as u64, + 0, "build prerequisite explicitly skipped by --no-build", )); if let Err(error) = validate_prepared_infobase(config) { @@ -166,7 +109,7 @@ pub(super) fn run_tests( debug!("preparing test run artifacts"); let prepare_artifacts_started = Instant::now(); - let mut artifacts = match create_run_artifacts(config, &runner_id) { + let mut artifacts = match create_run_artifacts(config, runner_id) { Ok(artifacts) => artifacts, Err(error) => { let app_error = @@ -210,10 +153,82 @@ pub(super) fn run_tests( .with_target(artifacts.run_dir.display().to_string()), ); + match args.build_policy { + TestBuildPolicy::BuildFirst => { + debug!("running build prerequisite for tests"); + log_live_stage( + "test: build prerequisite", + "[Build] preparing test infobase", + ); + let build_started = Instant::now(); + let build_result = match build_project::execute( + context, + config, + &BuildArgs { + full_rebuild: false, + source_set: None, + }, + ) { + Ok(result) => result, + Err(failure) => { + let summary = failure + .payload + .as_ref() + .map(build_summary) + .unwrap_or_else(|| failure.error.to_string()); + steps.push( + failed_step( + "build", + ExecutionStepKind::PlatformCommand, + build_started.elapsed().as_millis() as u64, + summary.clone(), + ) + .with_errors(vec![test_execution_error( + TestErrorKind::BuildFailed, + summary.clone(), + )]), + ); + let retained_paths = retain_run_artifacts(config, &artifacts).ok(); + let outcome = with_retained_artifacts( + ExecutionOutcome::new(ExecutionStatus::Failed) + .with_diagnostics(vec![summary.clone()]) + .with_errors(vec![test_execution_error( + TestErrorKind::BuildFailed, + summary.clone(), + )]), + retained_paths, + ); + let result = make_test_result( + target, + mode, + outcome, + warnings, + steps, + started.elapsed().as_millis() as u64, + ); + return Err(TestExecutionFailure::with_payload(failure.error, result)); + } + }; + steps.push(succeeded_step( + "build", + ExecutionStepKind::PlatformCommand, + build_started.elapsed().as_millis() as u64, + build_summary(&build_result), + )); + } + TestBuildPolicy::Skip => {} + } + let prepare_runner_started = Instant::now(); - if let Some(failure) = - interrupted_test_failure(context, &target, &mode, &warnings, &steps, started) - { + if let Some(failure) = interrupted_test_failure( + context, + &target, + &mode, + &warnings, + &steps, + started, + retain_run_artifacts(config, &artifacts).ok(), + ) { return Err(failure); } let prepared_run = match prepare_runner_artifacts(config, args, &target, &mut artifacts) { @@ -319,9 +334,15 @@ pub(super) fn run_tests( } }; - if let Some(failure) = - interrupted_test_failure(context, &target, &mode, &warnings, &steps, started) - { + if let Some(failure) = interrupted_test_failure( + context, + &target, + &mode, + &warnings, + &steps, + started, + retain_run_artifacts(config, &artifacts).ok(), + ) { return Err(failure); } let platform_result = match enterprise.run_launch(&platform_launch) { @@ -383,13 +404,12 @@ pub(super) fn run_tests( }; if matches!(prepared_run, PreparedRun::Vanessa { .. }) { - resolve_vanessa_junit_path(&mut artifacts); if let Err(warning) = materialize_vanessa_runner_log(&artifacts) { warnings.push(warning); } } - debug!(path = %artifacts.junit_xml.display(), "parsing JUnit report"); + debug!(path = %artifacts.junit_dir.display(), "parsing JUnit reports"); let parse_junit_started = Instant::now(); let junit_parse = parse_junit_report(&artifacts); let mut report = match junit_parse.payload { @@ -399,18 +419,25 @@ pub(super) fn run_tests( "parse_junit", ExecutionStepKind::ParseOutput, parse_junit_started.elapsed().as_millis() as u64, - format!("parsed {} test cases", report.summary.total), + format!( + "parsed {} test cases from native JUnit reports", + report.summary.total + ), ) - .with_target(artifacts.junit_xml.display().to_string()), + .with_target(artifacts.junit_dir.display().to_string()), ); report } None => { - let error = junit_parse - .errors - .first() - .cloned() - .expect("junit parse error"); + let errors = if junit_parse.errors.is_empty() { + vec![test_execution_error( + TestErrorKind::JunitMalformed, + "JUnit report parsing returned no report or error", + )] + } else { + junit_parse.errors + }; + let error = errors[0].clone(); let kind = TestErrorKind::from_code(&error.code).unwrap_or(TestErrorKind::JunitMalformed); let message = error.message.clone(); @@ -421,17 +448,15 @@ pub(super) fn run_tests( parse_junit_started.elapsed().as_millis() as u64, message.clone(), ) - .with_target(artifacts.junit_xml.display().to_string()) - .with_errors(vec![error - .clone() - .with_details(junit_parse.diagnostics.clone())]), + .with_target(artifacts.junit_dir.display().to_string()) + .with_errors(errors.clone()), ); let retained_paths = retain_run_artifacts(config, &artifacts).ok(); let diagnostics = collect_diagnostics(&platform_result, vec![message.clone()], config); let outcome = with_retained_artifacts( ExecutionOutcome::new(test_execution_status(Some(kind.clone()), false)) .with_diagnostics(diagnostics) - .with_errors(vec![error.with_details(junit_parse.diagnostics)]), + .with_errors(errors), retained_paths, ); let result = make_test_result( @@ -449,6 +474,53 @@ pub(super) fn run_tests( } }; + let validate_allure_started = Instant::now(); + if let Err(failure) = validate_allure_results(&artifacts.allure_results_dir) { + let kind = failure.kind; + let message = failure.message; + let error = + test_execution_error(kind.clone(), message.clone()).with_details(failure.details); + steps.push( + failed_step( + "validate_allure", + ExecutionStepKind::ParseOutput, + validate_allure_started.elapsed().as_millis() as u64, + message.clone(), + ) + .with_target(artifacts.allure_results_dir.display().to_string()) + .with_errors(vec![error.clone()]), + ); + let retained_paths = retain_run_artifacts(config, &artifacts).ok(); + let diagnostics = collect_diagnostics(&platform_result, vec![message.clone()], config); + let outcome = with_retained_artifacts( + ExecutionOutcome::new(test_execution_status(Some(kind), false)) + .with_diagnostics(diagnostics) + .with_errors(vec![error]), + retained_paths, + ); + let result = make_test_result( + target, + mode, + outcome, + warnings, + steps, + started.elapsed().as_millis() as u64, + ); + return Err(TestExecutionFailure::with_payload( + AppError::Runtime(message), + result, + )); + } + steps.push( + succeeded_step( + "validate_allure", + ExecutionStepKind::ParseOutput, + validate_allure_started.elapsed().as_millis() as u64, + "validated native Allure results", + ) + .with_target(artifacts.allure_results_dir.display().to_string()), + ); + parse_runner_log( &prepared_run, &artifacts.runner_log, @@ -462,35 +534,47 @@ pub(super) fn run_tests( TestOutputMode::Compact => compact_report(&report), }; - let has_test_failures = report.summary.failed > 0 || report.summary.errors > 0; - let process_failed = platform_result.process.exit_code != 0; - let diagnostics = collect_diagnostics(&platform_result, Vec::new(), config); + let classification = + classify_test_completion(&report.summary, platform_result.process.exit_code); + let mut diagnostics = collect_diagnostics(&platform_result, Vec::new(), config); - if process_failed || has_test_failures { + if let Some(kind) = classification { debug!( - process_failed, - has_test_failures, "retaining failed test artifacts" + error_kind = kind.clone().code(), + "retaining failed test artifacts" ); + if matches!(&kind, TestErrorKind::TestFailures) && platform_result.process.exit_code != 0 { + diagnostics.push(format!( + "enterprise test run exited with code {}", + platform_result.process.exit_code + )); + } let retained_paths = retain_run_artifacts(config, &artifacts).ok(); - let kind = if process_failed { - TestErrorKind::EnterpriseExitedNonZero - } else { - TestErrorKind::TestFailures + let message = match &kind { + TestErrorKind::EnterpriseExitedNonZero => format!( + "enterprise test run exited with code {}", + platform_result.process.exit_code + ), + TestErrorKind::TestFailures => "test run reported failures".to_owned(), + TestErrorKind::BuildFailed + | TestErrorKind::InfobaseUnavailable + | TestErrorKind::TestSetupFailed + | TestErrorKind::EnterpriseSpawnFailed + | TestErrorKind::EnterpriseStartupCheckFailed + | TestErrorKind::EnterpriseExitedEarly + | TestErrorKind::EnterpriseStdoutLogIo + | TestErrorKind::EnterpriseStderrLogIo + | TestErrorKind::EnterpriseTimedOut + | TestErrorKind::JunitNotProduced + | TestErrorKind::JunitEmpty + | TestErrorKind::JunitMalformed + | TestErrorKind::AllureNotProduced + | TestErrorKind::AllureEmpty => "test run failed".to_owned(), }; let outcome = with_retained_artifacts( ExecutionOutcome::new(test_execution_status(Some(kind.clone()), false)) .with_diagnostics(diagnostics) - .with_errors(vec![test_execution_error( - kind, - if process_failed { - format!( - "enterprise test run exited with code {}", - platform_result.process.exit_code - ) - } else { - "test run reported failures".to_owned() - }, - )]) + .with_errors(vec![test_execution_error(kind, message.clone())]) .with_metrics(ExecutionMetrics::from(&report.summary)) .with_payload(rendered_report), retained_paths, @@ -504,27 +588,22 @@ pub(super) fn run_tests( started.elapsed().as_millis() as u64, ); return Err(TestExecutionFailure::with_payload( - AppError::Runtime(if process_failed { - format!( - "enterprise test run exited with code {}", - platform_result.process.exit_code - ) - } else { - "test run reported failures".to_owned() - }), + AppError::Runtime(message), result, )); } - debug!(path = %artifacts.run_dir.display(), "cleaning successful test run directory"); - cleanup_run_dir(&artifacts); + let retained_paths = retain_run_artifacts(config, &artifacts).ok(); Ok(make_test_result( target, mode, - ExecutionOutcome::new(ExecutionStatus::Succeeded) - .with_diagnostics(diagnostics) - .with_metrics(ExecutionMetrics::from(&report.summary)) - .with_payload(rendered_report), + with_retained_artifacts( + ExecutionOutcome::new(ExecutionStatus::Succeeded) + .with_diagnostics(diagnostics) + .with_metrics(ExecutionMetrics::from(&report.summary)) + .with_payload(rendered_report), + retained_paths, + ), warnings, steps, started.elapsed().as_millis() as u64, diff --git a/src/use_cases/run_tests/helpers.rs b/src/use_cases/run_tests/helpers.rs index c11ccde..41a7a2d 100644 --- a/src/use_cases/run_tests/helpers.rs +++ b/src/use_cases/run_tests/helpers.rs @@ -38,13 +38,17 @@ pub(super) fn make_test_result( #[cfg(test)] mod tests { use std::path::PathBuf; + use std::time::Instant; - use super::enterprise_error_kind; + use super::{enterprise_error_kind, interrupted_test_failure}; + use crate::domain::artifact::{ArtifactKind, ArtifactRef, ArtifactSet, ARTIFACT_ROLE_RUN_DIR}; use crate::domain::execution::ExecutionStatus; - use crate::domain::test::TestErrorKind; + use crate::domain::test::{TestErrorKind, TestOutputMode, TestTarget}; use crate::platform::enterprise::EnterpriseError; use crate::platform::process::ProcessError; use crate::support::error::AppError; + use crate::use_cases::context::{CommandName, ExecutionContext}; + use tokio_util::sync::CancellationToken; fn assert_process_mapping( process_error: ProcessError, @@ -142,6 +146,44 @@ mod tests { assert!(interruption.is_some()); assert_eq!(status, ExecutionStatus::TimedOut); } + + #[test] + fn cancellation_after_artifact_creation_retains_existing_inventory() { + // Break caught: cancellation after run-directory allocation dropped retained artifacts. + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let context = ExecutionContext::cli(CommandName::Test).with_cancellation(cancellation); + let run_dir = tempfile::tempdir().expect("run dir"); + let mut artifacts = ArtifactSet::with_root(run_dir.path()); + artifacts.push( + ArtifactRef::new(ArtifactKind::RunDirectory, run_dir.path()) + .with_role(ARTIFACT_ROLE_RUN_DIR), + ); + + let failure = interrupted_test_failure( + &context, + &TestTarget::All, + &TestOutputMode::Compact, + &[], + &[], + Instant::now(), + Some(artifacts), + ) + .expect("cancelled failure"); + let result = failure.payload.expect("cancelled result"); + + assert_eq!(result.execution.status, ExecutionStatus::Cancelled); + assert_eq!(result.execution.interruptions.len(), 1); + assert!(result.execution.errors.is_empty()); + let retained = result.execution.artifacts.expect("retained artifacts"); + assert_eq!(retained.root_dir, Some(run_dir.path().to_path_buf())); + assert_eq!(retained.items.len(), 1); + assert_eq!(retained.items[0].kind, ArtifactKind::RunDirectory); + assert_eq!( + retained.items[0].role.as_deref(), + Some(ARTIFACT_ROLE_RUN_DIR) + ); + } } pub(super) fn succeeded_step( @@ -203,16 +245,20 @@ pub(super) fn interrupted_test_failure( warnings: &[String], steps: &[StepResult], started: Instant, + retained_paths: Option, ) -> Option { let interruption = context.interruption()?; let message = interruption_message(context, interruption); - let outcome = ExecutionOutcome::new(command_interruption_status(interruption)) - .with_diagnostics(vec![message.clone()]) - .with_interruptions(vec![command_interruption_details( - interruption, - "command_boundary", - message.clone(), - )]); + let outcome = with_retained_artifacts( + ExecutionOutcome::new(command_interruption_status(interruption)) + .with_diagnostics(vec![message.clone()]) + .with_interruptions(vec![command_interruption_details( + interruption, + "command_boundary", + message.clone(), + )]), + retained_paths, + ); let result = make_test_result( target.clone(), mode.clone(), diff --git a/src/use_cases/vanessa.rs b/src/use_cases/vanessa.rs index db6ce1e..2ef9614 100644 --- a/src/use_cases/vanessa.rs +++ b/src/use_cases/vanessa.rs @@ -19,6 +19,7 @@ pub(crate) struct VanessaLaunch { pub(crate) struct VanessaTestArtifacts<'a> { pub(crate) run_dir: &'a Path, pub(crate) junit_dir: &'a Path, + pub(crate) allure_results_dir: &'a Path, pub(crate) runner_log: &'a Path, } @@ -33,6 +34,13 @@ pub(crate) fn prepare_test_launch( fs::create_dir_all(artifacts.junit_dir) .map_err(|error| AppError::Runtime(format!("failed to create JUnit directory: {error}")))?; + set_dir_permissions(artifacts.junit_dir) + .map_err(|error| AppError::Runtime(format!("failed to chmod JUnit directory: {error}")))?; + fs::create_dir_all(artifacts.allure_results_dir).map_err(|error| { + AppError::Runtime(format!("failed to create Allure directory: {error}")) + })?; + set_dir_permissions(artifacts.allure_results_dir) + .map_err(|error| AppError::Runtime(format!("failed to chmod Allure directory: {error}")))?; let runtime_params_path = artifacts.run_dir.join("va-params.json"); validate_params_payload_path(&runtime_params_path, "test va")?; @@ -230,6 +238,11 @@ fn apply_test_overlay(object: &mut Map, artifacts: VanessaTestArt "КаталогВыгрузкиJUnit".to_owned(), Value::String(artifacts.junit_dir.display().to_string()), ); + object.insert("ДелатьОтчетВФорматеАллюр".to_owned(), Value::Bool(true)); + object.insert( + "КаталогВыгрузкиAllure".to_owned(), + Value::String(artifacts.allure_results_dir.display().to_string()), + ); apply_logging_overlay( object, artifacts.runner_log, @@ -332,3 +345,39 @@ fn set_file_permissions(path: &Path) -> std::io::Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::{apply_test_overlay, VanessaTestArtifacts}; + use serde_json::{Map, Value}; + use std::path::Path; + use tempfile::tempdir; + + #[test] + fn test_overlay_enables_junit_and_allure_reports() { + let dir = tempdir().expect("tempdir"); + let run_dir = dir.path().join("run"); + let junit_dir = run_dir.join("junit"); + let allure_results_dir = run_dir.join("allure-results"); + let runner_log = run_dir.join("runner.log"); + let mut payload = Map::new(); + + apply_test_overlay( + &mut payload, + VanessaTestArtifacts { + run_dir: Path::new(&run_dir), + junit_dir: Path::new(&junit_dir), + allure_results_dir: Path::new(&allure_results_dir), + runner_log: Path::new(&runner_log), + }, + ); + + let payload = Value::Object(payload); + assert_eq!(payload["ДелатьОтчетВФорматеjUnit"], true); + assert_eq!(payload["ДелатьОтчетВФорматеАллюр"], true); + assert_eq!( + payload["КаталогВыгрузкиAllure"], + allure_results_dir.display().to_string() + ); + } +} diff --git a/tests/architecture_guardrails.rs b/tests/architecture_guardrails.rs index 1f5e766..5bc1e6c 100644 --- a/tests/architecture_guardrails.rs +++ b/tests/architecture_guardrails.rs @@ -185,3 +185,36 @@ fn change_checklist_covers_mcp_workspace_lock_and_config_contract() { ); } } + +#[test] +fn windows_runner_log_materialization_keeps_reparse_and_atomic_replace_guards() { + let source = read("src/use_cases/run_tests.rs"); + let open = extract_between(&source, "fn open_file_no_follow", "fn create_private_file"); + for required in [ + "FILE_FLAG_OPEN_REPARSE_POINT", + "GetFileInformationByHandle", + "FILE_ATTRIBUTE_REPARSE_POINT", + ] { + assert!( + open.contains(required), + "Windows no-follow open must use '{required}'" + ); + } + + let replace = extract_between(&source, "fn replace_file", "#[cfg(test)]"); + for required in [ + "MoveFileExW", + "MOVEFILE_REPLACE_EXISTING", + "MOVEFILE_WRITE_THROUGH", + "fs::canonicalize(parent)", + ] { + assert!( + replace.contains(required), + "Windows atomic replacement must use '{required}'" + ); + } + assert!( + !replace.contains("fs::remove_file(destination)"), + "Windows replacement must not expose a remove-then-rename gap" + ); +} diff --git a/tests/cli_test.rs b/tests/cli_test.rs index 9e88e6c..0187f2c 100644 --- a/tests/cli_test.rs +++ b/tests/cli_test.rs @@ -5,7 +5,7 @@ mod support; use std::fs; use std::io::{BufRead, BufReader, ErrorKind}; use std::path::{Path, PathBuf}; -use std::process::Stdio; +use std::process::{Command, Stdio}; use std::sync::mpsc; use std::time::Duration; @@ -28,12 +28,60 @@ const YAXUNIT_LOG_FIXTURE: &str = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/parsers/yaxunit.log" )); +const JUNIT_FAILURE_REPORT_FIXTURE: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/parsers/junit_report.xml" +)); + +#[derive(Clone, Copy)] +enum NativeReportFixture { + Complete, + MissingJunit, + EmptyJunit, + MissingAllure, + EmptyAllure, +} + +struct NativeReportOutput<'a> { + report_xml: &'a str, + fixture: NativeReportFixture, +} + +struct ProjectTestSetup<'a> { + report_xml: &'a str, + yax_log: &'a str, + enterprise_exit: i32, + build_fail: bool, + timeout_seconds: u64, + sleep_seconds: Option, + native_reports: NativeReportFixture, +} + +impl NativeReportFixture { + fn materialization(self, report_xml: &str, junit_file: &str) -> (String, String) { + let junit = match self { + Self::Complete | Self::MissingAllure | Self::EmptyAllure => format!( + "mkdir -p \"$(dirname \"{junit_file}\")\"\ncat <<'XML' > \"{junit_file}\"\n{report_xml}\nXML" + ), + Self::MissingJunit => String::new(), + Self::EmptyJunit => { + format!("mkdir -p \"$(dirname \"{junit_file}\")\"\n: > \"{junit_file}\"") + } + }; + let allure = match self { + Self::Complete | Self::MissingJunit | Self::EmptyJunit => "mkdir -p \"$allure_dir\"\nprintf '%s\\n' '{\"uuid\":\"fixture\",\"name\":\"fixture\",\"status\":\"passed\",\"stage\":\"finished\"}' > \"$allure_dir/fixture-result.json\"".to_owned(), + Self::EmptyAllure => "mkdir -p \"$allure_dir\"".to_owned(), + Self::MissingAllure => "rmdir \"$allure_dir\"".to_owned(), + }; + (junit, allure) + } +} fn write_test_script( path: &Path, calls_log: &Path, captured_config: &Path, - report_xml: &str, + output: NativeReportOutput<'_>, yax_log: &str, exit_code: i32, sleep_seconds: Option, @@ -41,11 +89,14 @@ fn write_test_script( let sleep_branch = sleep_seconds .map(|value| format!("sleep {value}")) .unwrap_or_default(); + let (junit_output, allure_output) = + output.fixture.materialization(output.report_xml, "$report"); let body = format!( - "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\ncp \"$cfg\" '{}'\nreport=$(awk -F '\"' '/reportPath/ {{print $4; exit}}' \"$cfg\")\nylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\nmkdir -p \"$(dirname \"$report\")\" \"$(dirname \"$ylog\")\" \"$(dirname \"$out\")\"\ncat <<'XML' > \"$report\"\n{}\nXML\ncat <<'LOG' > \"$ylog\"\n{}\nLOG\nprintf 'platform /P secret uri http://user:pass@example\\n' > \"$out\"\n{}\nexit {}", + "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\ncp \"$cfg\" '{}'\nreport=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'jUnit'))\nPY\n) || exit $?\n[ -n \"$report\" ] || exit 1\nallure_dir=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'allure'))\nPY\n) || exit $?\n[ -n \"$allure_dir\" ] || exit 1\nylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\n[ -n \"$ylog\" ] || exit 1\nmkdir -p \"$(dirname \"$ylog\")\" \"$(dirname \"$out\")\"\n{}\n{}\ncat <<'LOG' > \"$ylog\"\n{}\nLOG\nprintf 'platform /P secret uri http://user:pass@example\\n' > \"$out\"\n{}\nexit {}", calls_log.display(), captured_config.display(), - report_xml, + junit_output, + allure_output, yax_log, sleep_branch, exit_code @@ -122,12 +173,22 @@ fn write_native_edt_project( .expect("module marker"); } +fn replace_once(body: &str, from: &str, to: &str) -> String { + let replaced = body.replacen(from, to, 1); + assert_ne!( + replaced, body, + "VA script fixture pattern not found: {from}" + ); + replaced +} + fn write_va_test_script( path: &Path, calls_log: &Path, captured_params: &Path, report_xml: &str, exit_code: i32, + native_reports: NativeReportFixture, ) { let body = format!( "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nexecute=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n if [ \"$prev\" = \"/Execute\" ]; then execute=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^StartFeaturePlayer;VAParams=//')\ncp \"$cfg\" '{}'\nreport_dir=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n data = json.load(fh)\nprint(data['КаталогВыгрузкиJUnit'])\nPY\n)\ntext_log=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n data = json.load(fh)\nprint(data['ИмяФайлаЛогВыполненияСценариев'])\nPY\n)\nmkdir -p \"$report_dir\" \"$(dirname \"$out\")\" \"$(dirname \"$text_log\")\"\ncat <<'XML' > \"$report_dir/result.xml\"\n{}\nXML\nprintf 'va execute=%s\\n' \"$execute\" > \"$out\"\nprintf 'INFO ok\\nОшибка VA из текстового лога\\n' > \"$text_log\"\nexit {}", @@ -136,6 +197,45 @@ fn write_va_test_script( report_xml, exit_code ); + let body = replace_once( + &body, + r#"mkdir -p "$report_dir" "$(dirname "$out")" "$(dirname "$text_log")" +cat <<'XML' > "$report_dir/result.xml""#, + r#"allure_dir=$(python3 - <<'PY' "$cfg" +import json, sys +with open(sys.argv[1], 'r', encoding='utf-8') as fh: + data = json.load(fh) +print(data['КаталогВыгрузкиAllure']) +PY +) +mkdir -p "$report_dir" "$allure_dir" "$(dirname "$out")" "$(dirname "$text_log")" +cat <<'XML' > "$report_dir/result.xml""#, + ); + let body = replace_once( + &body, + "XML\nprintf 'va execute=%s\\n'", + "XML\nprintf '%s\\n' '{\"uuid\":\"fixture\",\"name\":\"fixture\",\"status\":\"passed\",\"stage\":\"finished\"}' > \"$allure_dir/fixture-result.json\"\nprintf 'va execute=%s\\n'", + ); + let junit_output = format!("cat <<'XML' > \"$report_dir/result.xml\"\n{report_xml}\nXML\n"); + let allure_output = "printf '%s\\n' '{\"uuid\":\"fixture\",\"name\":\"fixture\",\"status\":\"passed\",\"stage\":\"finished\"}' > \"$allure_dir/fixture-result.json\"\n"; + let body = match native_reports { + NativeReportFixture::Complete => body, + NativeReportFixture::MissingJunit => replace_once(&body, &junit_output, ""), + NativeReportFixture::EmptyJunit => replace_once( + &body, + &junit_output, + "cat <<'XML' > \"$report_dir/result.xml\"\nXML\n", + ), + NativeReportFixture::MissingAllure => { + let body = replace_once( + &body, + "mkdir -p \"$report_dir\" \"$allure_dir\"", + "mkdir -p \"$report_dir\"", + ); + replace_once(&body, allure_output, "rmdir \"$allure_dir\"\n") + } + NativeReportFixture::EmptyAllure => replace_once(&body, allure_output, ""), + }; write_script(path, &body); } @@ -177,14 +277,17 @@ fn setup_project( timeout_seconds: u64, sleep_seconds: Option, ) -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf, PathBuf) { - setup_project_with_additional_launch_keys( + setup_project_with_native_reports( work_dir_name, - report_xml, - yax_log, - enterprise_exit, - build_fail, - timeout_seconds, - sleep_seconds, + ProjectTestSetup { + report_xml, + yax_log, + enterprise_exit, + build_fail, + timeout_seconds, + sleep_seconds, + native_reports: NativeReportFixture::Complete, + }, &[], ) } @@ -219,14 +322,9 @@ fn configure_server_infobase(config_path: &Path) { .expect("updated config"); } -fn setup_project_with_additional_launch_keys( +fn setup_project_with_native_reports( work_dir_name: &str, - report_xml: &str, - yax_log: &str, - enterprise_exit: i32, - build_fail: bool, - timeout_seconds: u64, - sleep_seconds: Option, + setup: ProjectTestSetup<'_>, additional_launch_keys: &[&str], ) -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf, PathBuf) { let dir = temp_workspace(); @@ -249,23 +347,26 @@ fn setup_project_with_additional_launch_keys( write_build_script( &install_dir.join("bin").join("1cv8"), &build_calls, - build_fail, + setup.build_fail, ); write_test_script( &install_dir.join("bin").join("1cv8c"), &test_calls, &captured_config, - report_xml, - yax_log, - enterprise_exit, - sleep_seconds, + NativeReportOutput { + report_xml: setup.report_xml, + fixture: setup.native_reports, + }, + setup.yax_log, + setup.enterprise_exit, + setup.sleep_seconds, ); write_config( &config_path, &base_path, &work_path, &install_dir, - timeout_seconds, + setup.timeout_seconds, additional_launch_keys, ); @@ -276,13 +377,35 @@ fn setup_va_project( report_xml: &str, additional_launch_keys: &[&str], ) -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf, PathBuf) { - setup_va_project_with_work_name(report_xml, additional_launch_keys, "work") + setup_va_project_with_native_reports( + report_xml, + additional_launch_keys, + "work", + 0, + NativeReportFixture::Complete, + ) } fn setup_va_project_with_work_name( report_xml: &str, additional_launch_keys: &[&str], work_dir_name: &str, +) -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf, PathBuf) { + setup_va_project_with_native_reports( + report_xml, + additional_launch_keys, + work_dir_name, + 0, + NativeReportFixture::Complete, + ) +} + +fn setup_va_project_with_native_reports( + report_xml: &str, + additional_launch_keys: &[&str], + work_dir_name: &str, + enterprise_exit: i32, + native_reports: NativeReportFixture, ) -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf, PathBuf) { let dir = temp_workspace(); let base_path = dir.path().join("project"); @@ -316,7 +439,8 @@ fn setup_va_project_with_work_name( &test_calls, &captured_params, report_xml, - 0, + enterprise_exit, + native_reports, ); let additional_launch_keys_block = if additional_launch_keys.is_empty() { @@ -347,11 +471,17 @@ fn setup_va_project_with_work_name( fn scrub_snapshot(value: &mut Value) { value["duration_ms"] = Value::String("".to_owned()); value["data"]["retained_paths"]["run_dir"] = Value::String("".to_owned()); - value["data"]["retained_paths"]["config_json"] = Value::String("".to_owned()); - value["data"]["retained_paths"]["junit_xml"] = Value::String("".to_owned()); - value["data"]["retained_paths"]["yaxunit_log"] = Value::String("".to_owned()); - value["data"]["retained_paths"]["platform_log"] = Value::String("".to_owned()); - value["data"]["retained_paths"]["sentinel"] = Value::String("".to_owned()); + for key in [ + "config_json", + "junit_xml", + "allure_results", + "yaxunit_log", + "platform_log", + ] { + if value["data"]["retained_paths"][key].is_string() { + value["data"]["retained_paths"][key] = Value::String(format!("<{key}>")); + } + } if value["data"]["execution"]["artifacts"]["root_dir"].is_string() { value["data"]["execution"]["artifacts"]["root_dir"] = Value::String("".to_owned()); } @@ -360,10 +490,10 @@ fn scrub_snapshot(value: &mut Value) { let replacement = match item["role"].as_str() { Some("run_dir") => "", Some("config") => "", - Some("report") => "", + Some("junit_xml") => "", + Some("allure_results") => "", Some("runner_log") => "", Some("platform_log") => "", - Some("sentinel") => "", _ => continue, }; item["path"] = Value::String(replacement.to_owned()); @@ -377,7 +507,8 @@ fn scrub_snapshot(value: &mut Value) { Some("prepare_artifacts") => "", Some("prepare_runner") => "", Some("run") => "", - Some("parse_junit") => "", + Some("parse_junit") => "", + Some("validate_allure") => "", Some("parse_log") => "", _ => "", }; @@ -390,8 +521,20 @@ fn scrub_snapshot(value: &mut Value) { } } +fn assert_success_artifact_contract(payload: &Value) { + assert_eq!(payload["data"]["execution"]["metrics"]["total"], 1); + let items = payload["data"]["execution"]["artifacts"]["items"] + .as_array() + .expect("artifact items"); + assert!(items.iter().any(|item| item["kind"] == "junit_xml")); + assert!(items.iter().any(|item| item["kind"] == "allure_results")); + for item in items { + assert!(Path::new(item["path"].as_str().expect("artifact path")).exists()); + } +} + #[test] -fn test_all_full_json_runs_build_first_and_returns_report() { +fn test_all_full_json_builds_before_runner_and_returns_report() { let (_dir, config_path, build_calls, test_calls, _captured_config) = setup_project( "work path", JUNIT_SMOKE_REPORT_FIXTURE, @@ -439,7 +582,11 @@ fn test_all_full_json_runs_build_first_and_returns_report() { payload["data"]["report"]["suites"][0]["cases"][0]["name"], "ok" ); - assert_eq!(payload["data"]["retained_paths"], Value::Null); + let retained = &payload["data"]["retained_paths"]; + assert!(retained["run_dir"].is_string()); + assert!(retained["junit_xml"].is_string()); + assert!(retained["allure_results"].is_string()); + assert_success_artifact_contract(&payload); } #[test] @@ -591,6 +738,10 @@ fn test_no_build_rejects_missing_file_infobase_before_platform_launch() { assert!(!output.status.success()); assert!(!build_calls.exists()); assert!(!test_calls.exists()); + assert!( + !dir.path().join("work").join("temp").exists(), + "no-build preflight failure must not allocate test artifacts" + ); let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); assert_eq!( payload["data"]["execution"]["errors"][0]["code"], @@ -601,14 +752,17 @@ fn test_no_build_rejects_missing_file_infobase_before_platform_launch() { #[test] fn test_run_appends_enterprise_additional_launch_keys() { let (_dir, config_path, _build_calls, test_calls, _captured_config) = - setup_project_with_additional_launch_keys( + setup_project_with_native_reports( "work", - JUNIT_SMOKE_REPORT_FIXTURE, - "12:00:00.000 [INF] ok", - 0, - false, - 5, - None, + ProjectTestSetup { + report_xml: JUNIT_SMOKE_REPORT_FIXTURE, + yax_log: "12:00:00.000 [INF] ok", + enterprise_exit: 0, + build_fail: false, + timeout_seconds: 5, + sleep_seconds: None, + native_reports: NativeReportFixture::Complete, + }, &["/TESTMANAGER", "/TCUser", "ci-user"], ); @@ -700,7 +854,7 @@ fn test_command_streams_enterprise_stage_before_runner_finishes() { write_script( &dir.path().join("platform").join("bin").join("1cv8c"), &format!( - "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\ncp \"$cfg\" '{}'\ntouch '{}'\nwhile [ ! -f '{}' ]; do sleep 0.05; done\nreport=$(awk -F '\"' '/reportPath/ {{print $4; exit}}' \"$cfg\")\nylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\nmkdir -p \"$(dirname \"$report\")\" \"$(dirname \"$ylog\")\" \"$(dirname \"$out\")\"\ncat <<'XML' > \"$report\"\n{}\nXML\ncat <<'LOG' > \"$ylog\"\n12:00:00.000 [INF] ok\nLOG\nprintf 'platform ok\\n' > \"$out\"\nexit 0", + "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\ncp \"$cfg\" '{}'\ntouch '{}'\nwhile [ ! -f '{}' ]; do sleep 0.05; done\nreport=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'jUnit'))\nPY\n) || exit $?\n[ -n \"$report\" ] || exit 1\nallure_dir=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'allure'))\nPY\n) || exit $?\n[ -n \"$allure_dir\" ] || exit 1\nylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\n[ -n \"$ylog\" ] || exit 1\nmkdir -p \"$(dirname \"$report\")\" \"$allure_dir\" \"$(dirname \"$ylog\")\" \"$(dirname \"$out\")\"\ncat <<'XML' > \"$report\"\n{}\nXML\nprintf '%s\\n' '{{\"uuid\":\"fixture\",\"name\":\"fixture\",\"status\":\"passed\",\"stage\":\"finished\"}}' > \"$allure_dir/fixture-result.json\"\ncat <<'LOG' > \"$ylog\"\n12:00:00.000 [INF] ok\nLOG\nprintf 'platform ok\\n' > \"$out\"\nexit 0", test_calls.display(), captured_config.display(), runner_started.display(), @@ -783,7 +937,28 @@ fn test_text_output_surfaces_failure_code_and_retained_artifacts() { assert!(stdout.contains("✗ enterprise run: runtime error: enterprise test run timed out")); assert!(stdout.contains("[warning] enterprise test run timed out")); assert!(stdout.contains("[artifact] run_dir -> ")); - assert!(stdout.contains("[diagnostic] platform_log -> ")); + let run_dir = stdout + .lines() + .find_map(|line| { + line.split_once("[artifact] run_dir -> ") + .map(|(_, path)| path.trim()) + }) + .expect("retained run directory"); + let expected_platform_log = Path::new(run_dir).join("enterprise.out.log"); + let rendered_platform_log = stdout.lines().find_map(|line| { + line.split_once("[diagnostic] platform_log -> ") + .map(|(_, path)| Path::new(path.trim())) + }); + match rendered_platform_log { + Some(platform_log) => { + assert_eq!(platform_log, expected_platform_log); + assert!(expected_platform_log.is_file()); + } + None => assert!( + !expected_platform_log.is_file(), + "an existing platform log must be rendered as a diagnostic" + ), + } } #[test] @@ -814,6 +989,23 @@ fn test_text_output_surfaces_success_log_findings_without_full_step_noise() { let stdout = String::from_utf8_lossy(&output.stdout); assert!(stdout.contains("● Tests completed with warnings")); assert!(stdout.contains("[error:test_report]")); + let run_dir = stdout + .lines() + .find_map(|line| { + line.split_once("[artifact] run_dir -> ") + .map(|(_, path)| path.trim()) + }) + .expect("retained run directory"); + let allure_results = stdout + .lines() + .find_map(|line| { + line.split_once("[artifact] allure_results -> ") + .map(|(_, path)| Path::new(path.trim())) + }) + .expect("retained Allure results"); + let expected_allure_results = Path::new(run_dir).join("allure-results"); + assert_eq!(allure_results, expected_allure_results); + assert!(expected_allure_results.is_dir()); assert!(!stdout.contains("prepare artifacts")); } @@ -831,7 +1023,7 @@ fn test_accepts_explicit_client_mode_for_vanessa_and_yaxunit() { write_script( &dir.path().join("platform").join("bin").join("1cv8"), &format!( - "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\nif printf '%s' \"$payload\" | grep -F -q -- 'RunUnitTests='; then\n cfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\n report=$(awk -F '\"' '/reportPath/ {{print $4; exit}}' \"$cfg\")\n ylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\n mkdir -p \"$(dirname \"$report\")\" \"$(dirname \"$ylog\")\"\n cat <<'XML' > \"$report\"\n{}\nXML\n cat <<'LOG' > \"$ylog\"\n12:00:00.000 [INF] ok\nLOG\n if [ -n \"$out\" ]; then mkdir -p \"$(dirname \"$out\")\" && : > \"$out\"; fi\n exit 0\nfi\nif [ -n \"$out\" ]; then printf 'build /P secret\\n' > \"$out\"; fi\nexit 0", + "printf '%s\\n' \"$*\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\nif printf '%s' \"$payload\" | grep -F -q -- 'RunUnitTests='; then\n cfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\n report=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'jUnit'))\nPY\n) || exit $?\n[ -n \"$report\" ] || exit 1\nallure_dir=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'allure'))\nPY\n) || exit $?\n[ -n \"$allure_dir\" ] || exit 1\n ylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\n[ -n \"$ylog\" ] || exit 1\n mkdir -p \"$(dirname \"$report\")\" \"$allure_dir\" \"$(dirname \"$ylog\")\"\n cat <<'XML' > \"$report\"\n{}\nXML\nprintf '%s\\n' '{{\"uuid\":\"fixture\",\"name\":\"fixture\",\"status\":\"passed\",\"stage\":\"finished\"}}' > \"$allure_dir/fixture-result.json\"\n cat <<'LOG' > \"$ylog\"\n12:00:00.000 [INF] ok\nLOG\n if [ -n \"$out\" ]; then mkdir -p \"$(dirname \"$out\")\" && : > \"$out\"; fi\n exit 0\nfi\nif [ -n \"$out\" ]; then printf 'build /P secret\\n' > \"$out\"; fi\nexit 0", test_calls.display(), JUNIT_SMOKE_REPORT_FIXTURE ), @@ -992,6 +1184,350 @@ fn test_va_builds_vanessa_command_and_overlay() { payload["data"]["report"]["extracted_errors"][0], "Ошибка VA из текстового лога" ); + assert_success_artifact_contract(&payload); +} + +#[test] +fn test_yaxunit_nonzero_exit_with_failed_junit_reports_test_failures() { + // Break caught: nonzero process exits must not mask report-proven test failures. + let (_dir, config_path, _build_calls, _test_calls, _captured_config) = + setup_project_with_native_reports( + "work", + ProjectTestSetup { + report_xml: JUNIT_FAILURE_REPORT_FIXTURE, + yax_log: "12:00:00.000 [INF] failed", + enterprise_exit: 17, + build_fail: false, + timeout_seconds: 5, + sleep_seconds: None, + native_reports: NativeReportFixture::Complete, + }, + &[], + ); + + let output = v8_runner_command() + .args([ + "--config", + &config_path.display().to_string(), + "--json-message", + "test", + "yaxunit", + "all", + ]) + .output() + .expect("run"); + + assert!(!output.status.success()); + let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); + assert_eq!(payload["data"]["error_kind"], "test_failures"); + assert_eq!(payload["data"]["execution"]["status"], "failed"); + assert_eq!(payload["data"]["report"]["summary"]["total"], 2); + assert_eq!(payload["data"]["report"]["summary"]["failed"], 1); +} + +#[test] +fn yaxunit_fixture_report_selectors_follow_format_after_reordering() { + // Mutation caught: selecting reports by array position writes each fixture to the wrong path. + let dir = temp_workspace(); + let fixture_workspace = dir.path().join("fixture-workspace"); + let script_path = dir.path().join("1cv8c"); + let config_path = fixture_workspace.join("config.json"); + let captured_config = fixture_workspace.join("captured-config.json"); + let calls_log = fixture_workspace.join("calls.log"); + let junit_path = fixture_workspace.join("junit").join("result.xml"); + let allure_path = fixture_workspace.join("allure"); + let yaxunit_log = fixture_workspace.join("runner.log"); + let platform_log = fixture_workspace.join("platform.log"); + fs::create_dir_all(&fixture_workspace).expect("fixture workspace"); + fs::write( + &config_path, + serde_json::to_vec_pretty(&serde_json::json!({ + "reports": [ + {"format": "allure", "path": allure_path}, + {"format": "jUnit", "path": junit_path} + ], + "logging": {"file": yaxunit_log} + })) + .expect("serialize config"), + ) + .expect("write config"); + write_test_script( + &script_path, + &calls_log, + &captured_config, + NativeReportOutput { + report_xml: JUNIT_SMOKE_REPORT_FIXTURE, + fixture: NativeReportFixture::Complete, + }, + "12:00:00.000 [INF] ok", + 0, + None, + ); + + let output = Command::new(&script_path) + .current_dir(&fixture_workspace) + .args([ + "/C", + &format!("RunUnitTests={}", config_path.display()), + "/Out", + &platform_log.display().to_string(), + ]) + .output() + .expect("run fixture script"); + + assert!( + output.status.success(), + "status={:?}\nstderr={}", + output.status.code(), + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + fs::read_to_string(junit_path).expect("junit report"), + format!("{JUNIT_SMOKE_REPORT_FIXTURE}\n") + ); + assert!(allure_path.join("fixture-result.json").is_file()); +} + +#[test] +fn yaxunit_fixture_report_selectors_fail_closed_before_writing_missing_format() { + // Mutation caught: removing selector failure propagation writes JUnit outside the fixture workspace. + let dir = temp_workspace(); + let fixture_workspace = dir.path().join("fixture-workspace"); + let outside_workspace = dir.path().join("outside-workspace"); + let script_path = dir.path().join("1cv8c"); + let config_path = fixture_workspace.join("config.json"); + let captured_config = fixture_workspace.join("captured-config.json"); + let calls_log = fixture_workspace.join("calls.log"); + let outside_junit = outside_workspace.join("result.xml"); + let yaxunit_log = fixture_workspace.join("runner.log"); + let platform_log = fixture_workspace.join("platform.log"); + fs::create_dir_all(&fixture_workspace).expect("fixture workspace"); + fs::write( + &config_path, + serde_json::to_vec_pretty(&serde_json::json!({ + "reports": [{"format": "jUnit", "path": outside_junit}], + "logging": {"file": yaxunit_log} + })) + .expect("serialize config"), + ) + .expect("write config"); + write_test_script( + &script_path, + &calls_log, + &captured_config, + NativeReportOutput { + report_xml: JUNIT_SMOKE_REPORT_FIXTURE, + fixture: NativeReportFixture::Complete, + }, + "12:00:00.000 [INF] ok", + 0, + None, + ); + + let output = Command::new(&script_path) + .current_dir(&fixture_workspace) + .args([ + "/C", + &format!("RunUnitTests={}", config_path.display()), + "/Out", + &platform_log.display().to_string(), + ]) + .output() + .expect("run fixture script"); + + assert!(!output.status.success()); + assert!( + !outside_workspace.exists(), + "fixture wrote outside its workspace" + ); +} + +#[test] +fn test_yaxunit_missing_or_empty_native_reports_are_invalid_output() { + // Break caught: accepting incomplete native reports turns infrastructure failures into success. + for (name, native_reports, error_kind) in [ + ( + "missing-junit", + NativeReportFixture::MissingJunit, + "junit_not_produced", + ), + ( + "empty-junit", + NativeReportFixture::EmptyJunit, + "junit_empty", + ), + ( + "missing-allure", + NativeReportFixture::MissingAllure, + "allure_not_produced", + ), + ( + "empty-allure", + NativeReportFixture::EmptyAllure, + "allure_empty", + ), + ] { + let (_dir, config_path, _build_calls, _test_calls, _captured_config) = + setup_project_with_native_reports( + name, + ProjectTestSetup { + report_xml: JUNIT_SMOKE_REPORT_FIXTURE, + yax_log: "12:00:00.000 [INF] incomplete", + enterprise_exit: 0, + build_fail: false, + timeout_seconds: 5, + sleep_seconds: None, + native_reports, + }, + &[], + ); + + let output = v8_runner_command() + .args([ + "--config", + &config_path.display().to_string(), + "--json-message", + "test", + "yaxunit", + "all", + ]) + .output() + .expect("run"); + + assert!(!output.status.success(), "{name}"); + let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); + assert_eq!( + payload["data"]["execution"]["status"], "invalid_output", + "{name}" + ); + assert_eq!(payload["data"]["error_kind"], error_kind, "{name}"); + } +} + +#[test] +fn test_yaxunit_repeated_runs_retain_distinct_roots() { + // Break caught: reusing a run directory would overwrite retained diagnostics. + let (_dir, config_path, _build_calls, _test_calls, _captured_config) = setup_project( + "work", + JUNIT_SMOKE_REPORT_FIXTURE, + "12:00:00.000 [INF] ok", + 0, + false, + 5, + None, + ); + + let mut retained_roots = Vec::new(); + for _ in 0..2 { + let output = v8_runner_command() + .args([ + "--config", + &config_path.display().to_string(), + "--json-message", + "test", + "yaxunit", + "all", + ]) + .output() + .expect("run"); + assert!(output.status.success()); + let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); + let root = PathBuf::from( + payload["data"]["retained_paths"]["run_dir"] + .as_str() + .expect("run directory"), + ); + assert!(root.is_dir()); + retained_roots.push(root); + } + + assert_ne!(retained_roots[0], retained_roots[1]); +} + +#[test] +fn test_va_nonzero_exit_with_failed_junit_reports_test_failures() { + // Break caught: Vanessa process exits must not mask report-proven test failures. + let (_dir, config_path, _build_calls, _test_calls, _captured_params) = + setup_va_project_with_native_reports( + JUNIT_FAILURE_REPORT_FIXTURE, + &[], + "work", + 17, + NativeReportFixture::Complete, + ); + + let output = v8_runner_command() + .args([ + "--config", + &config_path.display().to_string(), + "--json-message", + "test", + "va", + ]) + .output() + .expect("run"); + + assert!(!output.status.success()); + let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); + assert_eq!(payload["data"]["error_kind"], "test_failures"); + assert_eq!(payload["data"]["execution"]["status"], "failed"); + assert_eq!(payload["data"]["report"]["summary"]["total"], 2); + assert_eq!(payload["data"]["report"]["summary"]["failed"], 1); +} + +#[test] +fn test_va_missing_or_empty_native_reports_are_invalid_output() { + // Break caught: Vanessa-specific report paths must be validated before success is returned. + for (name, native_reports, error_kind) in [ + ( + "missing-junit", + NativeReportFixture::MissingJunit, + "junit_not_produced", + ), + ( + "empty-junit", + NativeReportFixture::EmptyJunit, + "junit_empty", + ), + ( + "missing-allure", + NativeReportFixture::MissingAllure, + "allure_not_produced", + ), + ( + "empty-allure", + NativeReportFixture::EmptyAllure, + "allure_empty", + ), + ] { + let (_dir, config_path, _build_calls, _test_calls, _captured_params) = + setup_va_project_with_native_reports( + JUNIT_SMOKE_REPORT_FIXTURE, + &[], + name, + 0, + native_reports, + ); + + let output = v8_runner_command() + .args([ + "--config", + &config_path.display().to_string(), + "--json-message", + "test", + "va", + ]) + .output() + .expect("run"); + + assert!(!output.status.success(), "{name}"); + let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); + assert_eq!( + payload["data"]["execution"]["status"], "invalid_output", + "{name}" + ); + assert_eq!(payload["data"]["error_kind"], error_kind, "{name}"); + } } #[test] @@ -1234,6 +1770,25 @@ fn test_module_build_failure_prevents_enterprise_launch() { let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); assert_eq!(payload["ok"], false); assert_eq!(payload["data"]["error_kind"], "build_failed"); + let run_dir = payload["data"]["retained_paths"]["run_dir"] + .as_str() + .expect("retained run dir"); + assert!(Path::new(run_dir).is_dir()); + assert_eq!( + payload["data"]["execution"]["artifacts"]["root_dir"], + run_dir + ); + assert!(payload["data"]["execution"]["artifacts"]["items"] + .as_array() + .expect("artifacts") + .iter() + .any(|artifact| { + artifact["kind"] == "run_directory" + && artifact["role"] == "run_dir" + && artifact["path"] == run_dir + })); + assert_eq!(payload["steps"][0]["name"], "prepare_artifacts"); + assert_eq!(payload["steps"][1]["name"], "build"); } #[test] @@ -1292,7 +1847,10 @@ fn test_module_edt_extension_build_uses_full_load_before_enterprise_launch() { &install_dir.join("bin").join("1cv8c"), &test_calls, &captured_config, - JUNIT_SMOKE_REPORT_FIXTURE, + NativeReportOutput { + report_xml: JUNIT_SMOKE_REPORT_FIXTURE, + fixture: NativeReportFixture::Complete, + }, YAXUNIT_LOG_FIXTURE, 0, None, @@ -1382,7 +1940,10 @@ fn repeated_test_skips_unchanged_source_backed_tool_extension_build() { &install_dir.join("bin").join("1cv8c"), &test_calls, &captured_config, - JUNIT_SMOKE_REPORT_FIXTURE, + NativeReportOutput { + report_xml: JUNIT_SMOKE_REPORT_FIXTURE, + fixture: NativeReportFixture::Complete, + }, YAXUNIT_LOG_FIXTURE, 0, None, @@ -1503,12 +2064,11 @@ stack trace line 2 let retained_config_path = compact_json["data"]["retained_paths"]["config_json"] .as_str() .expect("config path"); - let retained_sentinel = compact_json["data"]["retained_paths"]["sentinel"] - .as_str() - .expect("sentinel"); let retained_config = fs::read_to_string(retained_config_path).expect("retained config"); assert!(retained_config.contains("\"modules\": [")); - assert!(Path::new(retained_sentinel).exists()); + assert!(compact_json["data"]["retained_paths"] + .as_object() + .is_some_and(|paths| !paths.contains_key("sentinel"))); assert!(fs::read_to_string(captured_config) .expect("captured config") .contains("Foo")); @@ -1566,8 +2126,21 @@ fn test_timeout_retains_artifacts() { let payload: Value = serde_json::from_slice(&output.stdout).expect("json"); assert_eq!(payload["data"]["execution"]["status"], "timed_out"); - let platform_log = payload["data"]["retained_paths"]["platform_log"] + let run_dir = payload["data"]["retained_paths"]["run_dir"] .as_str() - .expect("platform log"); - assert!(!platform_log.is_empty()); + .expect("retained run directory"); + let expected_platform_log = Path::new(run_dir).join("enterprise.out.log"); + match &payload["data"]["retained_paths"]["platform_log"] { + Value::String(platform_log) => { + assert_eq!(Path::new(platform_log), expected_platform_log); + assert!(expected_platform_log.is_file()); + } + Value::Null => { + assert!( + !expected_platform_log.is_file(), + "an existing platform log must be published in retained_paths" + ); + } + value => panic!("platform_log must be a string or null, got {value}"), + } } diff --git a/tests/mcp_stdio.rs b/tests/mcp_stdio.rs index bf680ea..34f067d 100644 --- a/tests/mcp_stdio.rs +++ b/tests/mcp_stdio.rs @@ -343,7 +343,7 @@ fn setup_designer_suite_project() -> (tempfile::TempDir, PathBuf, PathBuf, PathB write_script(&platform_dir.join("bin").join("1cv8"), &designer_script); let enterprise_script = format!( - "args=\"$*\"\nprintf '%s\\n' \"$args\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncase \"$args\" in\n *\"RunUnitTests=\"*)\n cfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\n cp \"$cfg\" '{}'\n report=$(awk -F '\"' '/reportPath/ {{print $4; exit}}' \"$cfg\")\n ylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\n mkdir -p \"$(dirname \"$report\")\" \"$(dirname \"$ylog\")\"\n cat <<'XML' > \"$report\"\n\nXML\n cat <<'LOG' > \"$ylog\"\n12:00:00.000 [INF] ok\nLOG\n if [ -n \"$out\" ]; then mkdir -p \"$(dirname \"$out\")\" && : > \"$out\"; fi\n exit 0\n ;;\n *)\n sleep 1\n exit 0\n ;;\nesac", + "args=\"$*\"\nprintf '%s\\n' \"$args\" >> '{}'\npayload=\"\"\nout=\"\"\nprev=\"\"\nfor arg in \"$@\"; do\n if [ \"$prev\" = \"/C\" ]; then payload=\"$arg\"; fi\n case \"$arg\" in /C*) payload=\"${{arg#/C}}\" ;; esac\n if [ \"$prev\" = \"/Out\" ]; then out=\"$arg\"; fi\n prev=\"$arg\"\ndone\ncase \"$args\" in\n *\"RunUnitTests=\"*)\n cfg=$(printf '%s' \"$payload\" | sed 's/^\"//; s/\"$//; s/^RunUnitTests=//')\n cp \"$cfg\" '{}'\n report=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'jUnit'))\nPY\n) || exit $?\n[ -n \"$report\" ] || exit 1\nallure_dir=$(python3 - <<'PY' \"$cfg\"\nimport json, sys\nwith open(sys.argv[1], 'r', encoding='utf-8') as fh:\n reports = json.load(fh)['reports']\n print(next(report['path'] for report in reports if report['format'] == 'allure'))\nPY\n) || exit $?\n[ -n \"$allure_dir\" ] || exit 1\n ylog=$(awk -F '\"' '/\"file\"/ {{print $4; exit}}' \"$cfg\")\n[ -n \"$ylog\" ] || exit 1\n mkdir -p \"$(dirname \"$report\")\" \"$allure_dir\" \"$(dirname \"$ylog\")\"\n cat <<'XML' > \"$report\"\n\nXML\n printf '%s\\n' '{{\"uuid\":\"fixture\",\"name\":\"fixture\",\"status\":\"passed\",\"stage\":\"finished\"}}' > \"$allure_dir/fixture-result.json\"\n cat <<'LOG' > \"$ylog\"\n12:00:00.000 [INF] ok\nLOG\n if [ -n \"$out\" ]; then mkdir -p \"$(dirname \"$out\")\" && : > \"$out\"; fi\n exit 0\n ;;\n *)\n sleep 1\n exit 0\n ;;\nesac", enterprise_calls_log.display(), captured_config.display() ); diff --git a/tests/snapshots/cli_test__test_module_compact_json.snap b/tests/snapshots/cli_test__test_module_compact_json.snap index 7e6bf15..bc88357 100644 --- a/tests/snapshots/cli_test__test_module_compact_json.snap +++ b/tests/snapshots/cli_test__test_module_compact_json.snap @@ -1,6 +1,6 @@ --- source: tests/cli_test.rs -assertion_line: 693 +assertion_line: 1685 expression: compact_snapshot --- Object { @@ -14,9 +14,9 @@ Object { "artifacts": Object { "items": Array [ Object { - "kind": String("run_directory"), - "path": String(""), - "role": String("run_dir"), + "kind": String("allure_results"), + "path": String(""), + "role": String("allure_results"), }, Object { "kind": String("config"), @@ -24,14 +24,9 @@ Object { "role": String("config"), }, Object { - "kind": String("report"), + "kind": String("junit_xml"), "path": String(""), - "role": String("report"), - }, - Object { - "kind": String("runner_log"), - "path": String(""), - "role": String("runner_log"), + "role": String("junit_xml"), }, Object { "kind": String("platform_log"), @@ -39,9 +34,14 @@ Object { "role": String("platform_log"), }, Object { - "kind": String("sentinel"), - "path": String(""), - "role": String("sentinel"), + "kind": String("run_directory"), + "path": String(""), + "role": String("run_dir"), + }, + Object { + "kind": String("runner_log"), + "path": String(""), + "role": String("runner_log"), }, ], "root_dir": String(""), @@ -123,11 +123,11 @@ Object { }, }, "retained_paths": Object { + "allure_results": String(""), "config_json": String(""), "junit_xml": String(""), "platform_log": String(""), "run_dir": String(""), - "sentinel": String(""), "yaxunit_log": String(""), }, "target": Object { @@ -146,20 +146,20 @@ Object { "steps": Array [ Object { "duration_ms": String(""), - "kind": String("platform_command"), - "message": String("build completed"), - "name": String("build"), + "kind": String("prepare_workspace"), + "message": String("created "), + "name": String("prepare_artifacts"), "ok": Bool(true), "status": String("succeeded"), + "target": String(""), }, Object { "duration_ms": String(""), - "kind": String("prepare_workspace"), - "message": String("created "), - "name": String("prepare_artifacts"), + "kind": String("platform_command"), + "message": String("build completed"), + "name": String("build"), "ok": Bool(true), "status": String("succeeded"), - "target": String(""), }, Object { "duration_ms": String(""), @@ -182,11 +182,20 @@ Object { Object { "duration_ms": String(""), "kind": String("parse_output"), - "message": String("parsed 2 test cases"), + "message": String("parsed 2 test cases from native JUnit reports"), "name": String("parse_junit"), "ok": Bool(true), "status": String("succeeded"), - "target": String(""), + "target": String(""), + }, + Object { + "duration_ms": String(""), + "kind": String("parse_output"), + "message": String("validated native Allure results"), + "name": String("validate_allure"), + "ok": Bool(true), + "status": String("succeeded"), + "target": String(""), }, Object { "duration_ms": String(""), diff --git a/tests/snapshots/cli_test__test_module_full_json.snap b/tests/snapshots/cli_test__test_module_full_json.snap index b1d23aa..98e6ffa 100644 --- a/tests/snapshots/cli_test__test_module_full_json.snap +++ b/tests/snapshots/cli_test__test_module_full_json.snap @@ -1,6 +1,6 @@ --- source: tests/cli_test.rs -assertion_line: 694 +assertion_line: 1686 expression: full_snapshot --- Object { @@ -14,9 +14,9 @@ Object { "artifacts": Object { "items": Array [ Object { - "kind": String("run_directory"), - "path": String(""), - "role": String("run_dir"), + "kind": String("allure_results"), + "path": String(""), + "role": String("allure_results"), }, Object { "kind": String("config"), @@ -24,14 +24,9 @@ Object { "role": String("config"), }, Object { - "kind": String("report"), + "kind": String("junit_xml"), "path": String(""), - "role": String("report"), - }, - Object { - "kind": String("runner_log"), - "path": String(""), - "role": String("runner_log"), + "role": String("junit_xml"), }, Object { "kind": String("platform_log"), @@ -39,9 +34,14 @@ Object { "role": String("platform_log"), }, Object { - "kind": String("sentinel"), - "path": String(""), - "role": String("sentinel"), + "kind": String("run_directory"), + "path": String(""), + "role": String("run_dir"), + }, + Object { + "kind": String("runner_log"), + "path": String(""), + "role": String("runner_log"), }, ], "root_dir": String(""), @@ -135,11 +135,11 @@ Object { }, }, "retained_paths": Object { + "allure_results": String(""), "config_json": String(""), "junit_xml": String(""), "platform_log": String(""), "run_dir": String(""), - "sentinel": String(""), "yaxunit_log": String(""), }, "target": Object { @@ -158,20 +158,20 @@ Object { "steps": Array [ Object { "duration_ms": String(""), - "kind": String("platform_command"), - "message": String("build completed"), - "name": String("build"), + "kind": String("prepare_workspace"), + "message": String("created "), + "name": String("prepare_artifacts"), "ok": Bool(true), "status": String("succeeded"), + "target": String(""), }, Object { "duration_ms": String(""), - "kind": String("prepare_workspace"), - "message": String("created "), - "name": String("prepare_artifacts"), + "kind": String("platform_command"), + "message": String("build completed"), + "name": String("build"), "ok": Bool(true), "status": String("succeeded"), - "target": String(""), }, Object { "duration_ms": String(""), @@ -194,11 +194,20 @@ Object { Object { "duration_ms": String(""), "kind": String("parse_output"), - "message": String("parsed 2 test cases"), + "message": String("parsed 2 test cases from native JUnit reports"), "name": String("parse_junit"), "ok": Bool(true), "status": String("succeeded"), - "target": String(""), + "target": String(""), + }, + Object { + "duration_ms": String(""), + "kind": String("parse_output"), + "message": String("validated native Allure results"), + "name": String("validate_allure"), + "ok": Bool(true), + "status": String("succeeded"), + "target": String(""), }, Object { "duration_ms": String(""),