diff --git a/.github/actions/run/action.yml b/.github/actions/run/action.yml index 1fbd3ea..b159cb3 100644 --- a/.github/actions/run/action.yml +++ b/.github/actions/run/action.yml @@ -1,5 +1,5 @@ name: 'TWD CLI Run' -description: 'Run TWD browser tests with Puppeteer, including optional contract validation reporting' +description: 'Run TWD browser tests with Puppeteer, optionally as one shard of a parallel matrix, including contract validation reporting' branding: icon: 'check-circle' color: 'green' @@ -13,6 +13,25 @@ inputs: description: 'Post contract validation report as a PR comment (requires pull-requests: write permission)' required: false default: 'false' + shard: + description: >- + Run one shard of the suite, as / (e.g. 2/4). Each shard + discovers the whole suite and takes every nth test, so the test count + never has to be known in advance. Leave empty to run everything in one + job. Join the shards afterwards with `npx twd-cli merge`. + required: false + default: '' + report-dir: + description: 'Where the shard report is written. Only used when `shard` is set.' + required: false + default: '.twd/run' + upload-report: + description: >- + Upload the shard report as an artifact named `twd-run-`, which is + the layout `twd-cli merge` expects after actions/download-artifact. Only + used when `shard` is set. + required: false + default: 'true' runs: using: 'composite' @@ -29,10 +48,48 @@ runs: shell: bash run: npx puppeteer browsers install chrome + - name: Resolve shard + id: shard + shell: bash + env: + SHARD: ${{ inputs.shard }} + run: | + if [ -z "$SHARD" ]; then + echo "index=" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Artifact names cannot contain "/", so the index alone identifies the + # shard. twd-cli itself validates the spec and fails loudly on a bad + # one, so this only has to be good enough to name the artifact. + if ! printf '%s' "$SHARD" | grep -qE '^[0-9]+/[0-9]+$'; then + echo "::error::Invalid shard \"$SHARD\". Expected /, e.g. 2/4." + exit 1 + fi + echo "index=${SHARD%%/*}" >> "$GITHUB_OUTPUT" + - name: Run TWD tests shell: bash working-directory: ${{ inputs.working-directory }} - run: npx twd-cli run + env: + SHARD: ${{ inputs.shard }} + REPORT_DIR: ${{ inputs.report-dir }} + run: | + if [ -n "$SHARD" ]; then + npx twd-cli run --shard "$SHARD" --report-dir "$REPORT_DIR" + else + npx twd-cli run + fi + + - name: Upload shard report + # always(): a red shard must still upload, or `merge` cannot tell "this + # shard failed" from "this shard never ran", and it reports the gap as a + # missing artifact instead of the real failure. + if: always() && inputs.shard != '' && inputs.upload-report == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: twd-run-${{ steps.shard.outputs.index }} + path: ${{ inputs.working-directory }}/${{ inputs.report-dir }} + if-no-files-found: error - name: Post contract report to PR if: inputs.contract-report == 'true' && github.event_name == 'pull_request' @@ -42,7 +99,16 @@ runs: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + SHARD: ${{ inputs.shard }} run: | + # A sharded run deliberately writes no contract markdown: every shard + # would overwrite the others with a fraction of the mocks. `twd-cli + # merge` writes it, so the comment belongs in the merge job. + if [ -n "$SHARD" ]; then + echo "::notice::Skipping the contract PR comment: this is shard $SHARD. Post it from the job that runs \`twd-cli merge\`, which is what writes the report." + exit 0 + fi + REPORT_PATH=$(node -e " const fs = require('fs'); try { diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e155947..c94ce2b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -93,3 +93,124 @@ jobs: echo "" >> "$REPORT_PATH" echo "[View full report →](${RUN_URL})" >> "$REPORT_PATH" gh pr comment "${PR_NUMBER}" --body-file "$REPORT_PATH" + + e2e-sharded: + runs-on: ubuntu-latest + + strategy: + # Without this, the first red shard cancels its siblings and the merge job + # sees gaps it cannot distinguish from a crashed shard. + fail-fast: false + matrix: + # The "2" here must match the shard total below (--shard N/2). Bump both + # together, or shards will disagree on the total and merge will reject them. + shard: [1, 2] + + steps: + - name: Checkout repo + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 + with: + node-version: 24 + cache: npm + + - name: Install CLI dependencies + run: npm ci + + - name: Install test-example-app dependencies + working-directory: test-example-app + run: npm install + + - name: Install mock service worker + working-directory: test-example-app + run: npx twd-js init public --save + + - name: Cache Puppeteer browsers + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: ~/.cache/puppeteer + key: ${{ runner.os }}-puppeteer-${{ hashFiles('package-lock.json') }} + restore-keys: | + ${{ runner.os }}-puppeteer- + + - name: Install Chrome for Puppeteer + run: npx puppeteer browsers install chrome + + - name: Start dev server + working-directory: test-example-app + run: | + nohup npx vite --host > vite.log 2>&1 & + npx wait-on http://localhost:5173 --timeout 30000 + + - name: Run TWD tests for this shard + working-directory: test-example-app + run: node ../bin/twd-cli.js run --shard ${{ matrix.shard }}/2 + + - name: Upload shard report + # Always: a red shard must still upload, or merge cannot tell "this shard + # failed" from "this shard never ran". + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: twd-run-${{ matrix.shard }} + path: test-example-app/.twd/run + if-no-files-found: error + + e2e-merge: + runs-on: ubuntu-latest + needs: [e2e-sharded] + # Runs even though a shard job may have exited 1. Without this the merged + # summary — the point of the exercise — is never printed. + if: ${{ !cancelled() }} + + steps: + - name: Checkout repo + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5 + with: + node-version: 24 + cache: npm + + - name: Install CLI dependencies + run: npm ci + + - name: Download shard reports + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: twd-run-* + path: test-example-app/.twd/shards + + - name: Merge shard reports + working-directory: test-example-app + run: node ../bin/twd-cli.js merge .twd/shards + + - name: Verify the merged report + working-directory: test-example-app + run: | + # Cheap sanity guard, not a real gate: the "Merge shard reports" step + # above has no `if: always()`, so we only reach here after `twd-cli + # merge` exited 0 — and runMerge only writes merged-run.json once it + # has decided to succeed (src/mergeCommand.js). This check protects + # against that assumption changing later, but it cannot fail today. + if [ ! -f .twd/merged-run.json ]; then + echo "ERROR: merged report not generated" + exit 1 + fi + node -e " + const r = require('./.twd/merged-run.json'); + if (r.tests.length !== r.discovery.totalTests) { + console.error('ERROR: ' + r.tests.length + ' merged tests but ' + + r.discovery.totalTests + ' discovered'); + process.exit(1); + } + const executed = r.shards.reduce((sum, s) => sum + s.executed, 0); + if (executed !== r.tests.length) { + console.error('ERROR: shards report ' + executed + ' executed but merged ' + r.tests.length + ' tests'); + process.exit(1); + } + console.log('Merged ' + r.tests.length + ' tests from ' + r.shards.length + ' shards'); + " diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fe713b..a146799 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## 1.5.0 (2026-08-25) + +* feat(shard): `--shard /` runs one slice of the suite so a run can be split across parallel CI jobs. Each shard discovers the whole suite itself and takes every nth test, so the test count never has to be known in advance +* feat(shard): a sharded run writes `run.json` and `coverage.json` to `./.twd/run` (`--report-dir` to change it) — the first machine-readable output twd-cli has had +* feat(merge): `npx twd-cli merge ` joins shard reports into one report covering test results, coverage and contract validation, prints a single summary with a per-shard breakdown, and owns the exit code +* feat(merge): a missing shard report is an error naming the gap, not a silently incomplete report. Shards also fingerprint the test list they discovered, so shards that saw different test sets refuse to merge +* chore(packaging): a `files` allowlist in package.json — the published package is now just `bin/`, `src/`, `README.md`, `CHANGELOG.md` and `LICENSE`. `tests/`, `test-example-app/`, `docs/` and the repo tooling were all being published and no longer are, taking the tarball from ~209 kB to ~33 kB (99 files to 25, ~850 kB to ~101 kB unpacked). Nothing that was importable before has moved +* note: **sharding ships as a beta feature.** It is strictly additive, so a run without `--shard` is unaffected, but which tests land in which shard is not yet a stable contract — a later release is likely to group by top-level `describe` so a suite always stays in one shard +* note: no behavior change without `--shard`. A plain run writes the same files, prints the same output, and exits the same way as 1.4.0 + +Sharding needs three things right in the workflow: `fail-fast: false` on the +matrix, `if: always()` on the shard's artifact upload, and +`if: ${{ !cancelled() }}` on the merge job. Each one breaks the run differently +if left out. See [docs/sharding.md](docs/sharding.md) for a runnable workflow. + +A normal release: `npm install twd-cli` gets it. The *sharding feature* is the +part marked beta — everything else in this version is stable. + ## 1.4.0 (2026-07-28) * feat(record): video recording for twd-cli runs (#13) ([94c21e0](https://github.com/BRIKEV/twd-cli/commit/94c21e0)), closes [#13](https://github.com/BRIKEV/twd-cli/issues/13) diff --git a/README.md b/README.md index 580037e..bea5e1e 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ CI/CD runner for [TWD (Test while developing)](https://brikev.github.io/twd/) - [Recording](#recording): capture a run to video, paced so it is watchable - [Contract Validation](#contract-validation): check your mocks against OpenAPI specs - [CI/CD Integration](#cicd-integration): GitHub Action and custom setups +- [Sharding across CI jobs](#sharding-across-ci-jobs) **(beta)**: split a long run across parallel jobs ([details](docs/sharding.md)) - [How It Works](#how-it-works) - [Requirements](#requirements) @@ -349,6 +350,34 @@ When `contractReportPath` is set and you use the action with `contract-report: ' Failed validations are included in a collapsible details section with a link to the full CI log. +## Sharding across CI jobs + +> **Beta.** Strictly additive: a run without `--shard` behaves exactly as before, +> so turning this on cannot affect your existing pipeline. How tests are assigned +> to shards may still change — see [docs/sharding.md](docs/sharding.md). + +Long suites can be split across parallel CI jobs. Each shard runs one slice of +the suite and writes a report; `twd-cli merge` joins them into a single summary +and owns the exit code. + +```bash +npx twd-cli run --shard 2/4 # "I am job 2 of 4" +npx twd-cli merge .twd/shards # join the reports back together +``` + +The `4` is how many jobs you are running, **not** how many tests exist — each +shard discovers the whole suite itself and keeps every 4th test, so the suite can +grow without a workflow edit. + +**Sharding only pays on long suites.** It trades fixed per-job setup for parallel +execution, so a suite that runs in seconds comes out *slower*. As a rule of +thumb, two shards win once test time is more than twice the merge job's cost. + +See **[docs/sharding.md](docs/sharding.md)** for the full workflow, the three +conditions that are easy to get wrong, the break-even maths with measured +numbers, and the caveats — test independence, `maxFailures` being per shard, and +coverage on a red run. + ## Requirements - Node.js >= 20.19.x diff --git a/bin/twd-cli.js b/bin/twd-cli.js index a5b0d84..5c6a15b 100755 --- a/bin/twd-cli.js +++ b/bin/twd-cli.js @@ -1,14 +1,35 @@ #!/usr/bin/env node -import { runTests } from '../src/index.js'; -import { parseRunArgs } from '../src/parseArgs.js'; +// runTests and runMerge are imported inside their branches, not here. A static +// import of src/index.js pulls in puppeteer, so `twd-cli merge` — which never +// opens a browser — would otherwise load the whole browser-automation graph +// before it even looked at argv. +import { parseRunArgs, parseMergeArgs } from '../src/parseArgs.js'; const command = process.argv[2]; if (command === 'run') { try { - const { testFilters, record } = parseRunArgs(process.argv.slice(3)); - const hasFailures = await runTests({ testFilters, recordOverrides: record }); + const { testFilters, record, shard, reportDir } = parseRunArgs(process.argv.slice(3)); + const { runTests } = await import('../src/index.js'); + const hasFailures = await runTests({ + testFilters, + recordOverrides: record, + shard, + reportDir, + }); + process.exit(hasFailures ? 1 : 0); + } catch (error) { + if (!error?.reported) { + console.error(error?.message ?? String(error)); + } + process.exit(1); + } +} else if (command === 'merge') { + try { + const { dir, out } = parseMergeArgs(process.argv.slice(3)); + const { runMerge } = await import('../src/mergeCommand.js'); + const hasFailures = runMerge({ dir, out }); process.exit(hasFailures ? 1 : 0); } catch (error) { if (!error?.reported) { @@ -26,13 +47,26 @@ Usage: contains (case-insensitive). Repeatable; multiple --test values are OR'd. npx twd-cli run --record Record the run to a video file + npx twd-cli run --shard 2/4 (beta) Run only this shard's slice of the + suite and write a report to ./.twd/run + npx twd-cli merge (beta) Merge shard reports from into + one report, exit 1 if the run failed Examples: npx twd-cli run --test "shows error" npx twd-cli run --test "Login" --test "Signup" + npx twd-cli run --shard 2/4 + npx twd-cli merge .twd/shards Options: --test "" Filter tests by "suite > test" path (repeatable, OR'd) + --shard / (beta) Run slice i of n. Each shard discovers the + whole suite and takes every nth test, so the count + never has to be known in advance. Implies a report. + Which tests land in which shard may change. + --report-dir Where to write the shard report (default ./.twd/run) + --out merge only: where to write the merged report + (default ./.twd/merged-run.json) --record Record the run to a video file (requires ffmpeg) --record-dir Output directory (default ./twd-artifacts) --record-speed Playback speed, e.g. 0.5 for half speed diff --git a/docs/sharding.md b/docs/sharding.md new file mode 100644 index 0000000..0b869a1 --- /dev/null +++ b/docs/sharding.md @@ -0,0 +1,210 @@ +# Sharding across CI jobs + +> **Beta.** Sharding is new and marked beta on purpose. It is strictly additive — +> a run without `--shard` behaves exactly as it did before, writes the same +> files, and exits the same way — so enabling it cannot affect your existing +> pipeline. What may still change is **how tests are assigned to shards**: today +> each shard takes every nth test from the discovered list, and a future release +> is likely to group by top-level `describe` instead, so a suite always stays in +> one shard. Do not build anything that depends on *which* tests land in a given +> shard. Everything else — the flags, the report files, `merge`'s output and exit +> code — is stable. + +A single run walks the whole suite in one browser. Sharding splits it across +parallel CI jobs instead, then joins the results back into one report. + +```bash +npx twd-cli run --shard 2/4 # "I am job 2 of 4" +npx twd-cli merge .twd/shards # join the reports, decide the exit code +``` + +The `4` is how many jobs you are running, **not** how many tests exist. You never +need to know the test count: each shard boots its own browser, discovers the whole +suite exactly as a normal run does, and keeps every 4th test. Add tests and the +same 4 jobs just split more of them. + +Each shard writes `run.json` and `coverage.json` to `./.twd/run` (change it with +`--report-dir`). `merge` reads the downloaded shard directories, combines test +results, coverage and contract validation, prints one summary, and exits non-zero +if anything failed anywhere. + +## A complete workflow + +This runs as-is. The bundled action installs Chrome, runs the shard, and uploads +its report under the name `merge` expects. + +```yaml +name: TWD tests (sharded) + +on: + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + # Without this, the first red shard cancels its siblings and the merge job + # sees gaps it cannot tell apart from a shard that crashed. + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-node@v5 + with: + node-version: 24 + cache: npm + + - run: npm ci + + - name: Start the dev server + run: | + nohup npm run dev > dev.log 2>&1 & + npx wait-on http://localhost:5173 --timeout 60000 + + - name: Run this shard + uses: BRIKEV/twd-cli/.github/actions/run@main + with: + shard: ${{ matrix.shard }}/4 + + merge: + runs-on: ubuntu-latest + needs: [test] + # Runs even though a shard job may have exited 1. Without this a red shard + # short-circuits the workflow and the merged summary never prints. + if: ${{ !cancelled() }} + + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-node@v5 + with: + node-version: 24 + cache: npm + + - run: npm ci + + - uses: actions/download-artifact@v4 + with: + pattern: twd-run-* + path: .twd/shards + + - name: Merge the shard reports + run: npx twd-cli merge .twd/shards +``` + +`merge` owns the final exit code: it fails if any test failed in any shard, if a +contract was violated in `error` mode, or if a shard report is missing entirely. + +### Posting the contract report + +A sharded run deliberately writes no contract markdown per shard — each would +overwrite the others with a fraction of the mocks — so `merge` writes it, and the +PR comment belongs in the merge job: + +```yaml + merge: + permissions: + contents: read + pull-requests: write + steps: + # ...as above, through "Merge the shard reports"... + + - name: Post contract report to PR + if: github.event_name == 'pull_request' && hashFiles('.twd/contract-report.md') != '' + env: + GH_TOKEN: ${{ github.token }} + run: gh pr comment "${{ github.event.pull_request.number }}" --body-file .twd/contract-report.md +``` + +## Without the bundled action + +If you drive the CLI directly, you own the two steps the action was doing for +you — installing Chrome, and uploading the report with `if: always()`: + +```yaml + - run: npx puppeteer browsers install chrome + - run: npx twd-cli run --shard ${{ matrix.shard }}/4 + - uses: actions/upload-artifact@v4 + # A red shard must still upload, or merge cannot tell "this shard failed" + # from "this shard never ran". + if: always() + with: + name: twd-run-${{ matrix.shard }} + path: .twd/run + if-no-files-found: error +``` + +## The three conditions that matter + +Each of these breaks a sharded run in a different way, and all three are easy to +leave out: + +| Condition | Where | What breaks without it | +|---|---|---| +| `fail-fast: false` | the shard matrix | the first red shard cancels its siblings, and `merge` reports their reports as missing | +| `if: always()` | the shard's artifact upload | a red shard uploads nothing, so `merge` cannot distinguish failure from a crash | +| `if: ${{ !cancelled() }}` | the merge job | a red shard short-circuits the workflow and the merged summary never prints | + +## When sharding pays + +Sharding trades fixed per-job setup for parallel test execution, so it only wins +once test time dominates. With per-job overhead `V`, total test time `T`, and a +merge job costing `M`, the wall clock goes from `V + T` to `V + T/N + M`. So `N` +shards help only when: + +``` +M < T (1 - 1/N) → for two shards, roughly T > 2M +``` + +Measured on a real suite of 256 browser tests, where `V` was ~115s and the merge +job ~126s (89s of which was a SonarCloud scan): + +| Shards | Wall clock | Runner time | +|---|---|---| +| 1 | 12.6 min | baseline | +| 2 | ~8.1 min | +15% | +| 4 | 6.5 min | +47% | + +Two things to take from that. Wall clock has a floor of `V + M` no matter how far +you shard, so the returns fall off quickly — past four shards you pay a lot for +seconds. And sharding always costs *more* total compute than it saves in latency, +because every shard repeats `V`. If you are billed for runner minutes, or your +runner concurrency is contended, prefer the smallest `N` that gets you under your +target. + +On a short suite sharding is simply slower: this project's own 71-test suite goes +from 25s in one job to 41s across two plus a merge. + +## Caveats + +- **Coverage.** Each shard writes its own `coverage.json`; `merge` combines them + into `.nyc_output/out.json` — but only when the whole run is green, matching how + a single run behaves. `merge` reports how many shards contributed. +- **Missing shards are an error.** If a shard job dies before uploading, `merge` + refuses and names the gap rather than silently reporting 3 of 4 shards as a + complete green run. +- **Tests must register identically in every job.** Each shard fingerprints the + ordered list of `"suite > test"` paths it discovered and `merge` verifies they + match. Registering tests conditionally — behind a feature flag, a date, + `Math.random()` — makes the fingerprints diverge and `merge` will say so. + (Paths rather than internal test ids: `twd-js` assigns those at registration + time and they differ on every page load, so each shard's browser sees its own.) +- **`maxFailures` is per shard.** Four shards at the default of 10 can reach 40 + failures between them before all four bail. +- **`--test` and `--shard` compose:** filters resolve first, then the filtered list + is sharded. As with any filtered run, coverage is skipped. +- **Recording** produces one clip per shard; they are not concatenated. +- **A missing shard leaves no merged report on disk.** `merge` throws before it + writes `.twd/merged-run.json`, so a CI step that uploads that path with + `if: always()` will find nothing when a shard is missing. The error message on + stderr is the diagnosis in that case. +- **`record.filename` collides under sharding.** Only the *derived* recording + filename is per-shard. If `record.filename` is set explicitly in + `twd.config.json`, every shard writes to the same video path. Use the derived + name, or a per-shard `--record-dir`, when recording a sharded run. +- **Assignment may change.** See the beta note at the top: which tests land in + which shard is not part of the stable contract yet. diff --git a/docs/superpowers/specs/2026-08-19-shardable-run-artifacts-design.md b/docs/superpowers/specs/2026-08-19-shardable-run-artifacts-design.md new file mode 100644 index 0000000..ffa0d40 --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-shardable-run-artifacts-design.md @@ -0,0 +1,473 @@ +# Shardable run artifacts and a merge command - Design + +Date: 2026-08-19 +Status: Ready to plan +Repo: `twd-cli` only. No twd-js changes. + +## Problem + +A twd-cli run is a single Puppeteer process walking the whole suite. On a large +suite that is the slowest step in CI, and there is no way to divide it. + +Running Puppeteer concurrently *inside* one job was tried and abandoned: several +browsers contending for one runner's CPU made runs flaky enough to be useless. +Separate CI jobs avoid that entirely, because each job gets its own runner, its +own dev server, and its own browser. Nothing is shared, so nothing contends. + +What blocks that today is not concurrency. It is that a run has no +machine-readable output. Every structured value `runTests()` builds is printed +and discarded; the function returns a bare boolean (`src/index.js:359`). Four +parallel jobs would produce four consoles and nothing joinable. + +## Scope + +In scope: splitting a run across jobs, writing each job's results to disk, and +merging those results back into one report covering test results, coverage, and +contract validation. + +Out of scope: concatenating per-shard videos (each shard records its own clip; +the report lists them). Timing-aware shard balancing (needs a persisted timing +artifact — revisit if count-based balance proves uneven). Dynamic matrix sizing +and a `twd-cli list` command (a fixed matrix needs neither; see "The shard +model"). Reducing contract-report noise, which is its own tracked concern. + +This spec also does not reopen the AI-friendly-output decision that there is one +console format and no `--reporter` flag +(`docs/superpowers/specs/2026-07-06-ai-friendly-output-design.md`). The report +JSON is an artifact for merging, not an alternative console reporter. Text +output stays the default and is unchanged. + +## What already exists + +The data is already the right shape; only the output layer is missing. + +| Concern | Exists today as | Merge story | +|---|---|---| +| Test results | `testStatus` = `[{id, status, error?, retryAttempt?}]` plus `handlers` (`src/index.js:281`, `:210`) | Concat. Each test runs in exactly one shard, so ids never collide. `handlers` is identical in every shard. | +| Coverage | `.nyc_output/out.json`, raw Istanbul JSON (`src/index.js:338`) | Natively mergeable via `istanbul-lib-coverage`. No new merge logic. | +| Contracts | `validateMocks()` returns `{results, skipped}`, plain objects (`src/contracts.js:109`) | Concat. Occurrence counters are per-process but keyed by `testId`, so no cross-shard collision. | +| Summary | Built by `formatRunComplete`, then discarded | Recomputed from merged test results. | + +Sharding primitives exist too: `orderedTestIds()` in `src/testOrder.js` already +yields a deterministic pre-order id list. + +## The shard model + +`--shard 2/4` means "I am job 2 of 4 parallel jobs". The `4` is the job count, +chosen by the workflow author. **The test count is never needed in advance.** + +Each shard enumerates the suite itself — `src/index.js:125` already reads +`window.__TWD_STATE__.handlers` on every run — then keeps every Nth id: + +```js +// src/shard.js +export function selectShardIds(ids, index, total) { + return ids.filter((_, i) => i % total === index - 1); +} +``` + +``` +120 tests discovered (indices 0..119), identical in all 4 jobs +shard 1/4 -> i % 4 === 0 -> 0, 4, 8, ... (30 tests) +shard 2/4 -> i % 4 === 1 -> 1, 5, 9, ... (30 tests) +shard 3/4 -> i % 4 === 2 -> 2, 6, 10, ... (30 tests) +shard 4/4 -> i % 4 === 3 -> 3, 7, 11, ... (30 tests) +``` + +Round-robin rather than contiguous slicing: it balances better when adjacent +tests have similar cost, at the price of suite locality. Nothing in twd-js +requires a suite to run contiguously, so locality buys nothing here. + +Consequences worth stating, because they are what make a fixed matrix safe: + +- The suite can grow without a YAML edit. 200 tests split across the same 4 + jobs; they just take longer. +- **An empty shard is legal.** 3 tests across 4 shards leaves shard 4 with + nothing. It writes a valid report with `tests: []`. Not an error. + +## Report schema + +The load-bearing decision: **a merged report is shape-identical to a +single-shard report.** `shards` is always an array — one entry for a single run, +N after merging. This makes merge associative, lets every formatter work on +both, and makes a normal run the N=1 case with no second code path. + +```jsonc +{ + "schemaVersion": 2, + "shards": [ + { "index": 2, "total": 4, + "startedAt": "2026-08-19T10:00:00.000Z", + "endedAt": "2026-08-19T10:00:12.345Z", + "durationMs": 12345, + "executed": 30, "notRun": 0, "failed": 0, "stoppedEarly": false, + "coverageFile": "coverage.json", + "recording": { "file": "login.mp4", "bytes": 481920 } } + ], + "discovery": { "totalTests": 120, "fingerprint": "sha256:abc123..." }, + "selection": { "filters": [], "selectedTests": 120 }, + "handlers": [ { "id": "...", "name": "...", "parent": "...", "type": "test" } ], + "tests": [ { "id": "...", "status": "pass", "retryAttempt": 2, + "path": "Login > shows error", "index": 7 } ], + "contracts": { "configured": true, "partial": false, "results": [], "skipped": [] } +} +``` + +`handlers` passes through exactly as enumerated. `tests` is the in-page status +array plus two fields the shard resolves before writing — `path` for display and +`index` for identity (see the correction below) — so `formatRunComplete` and +`generateContractMarkdown` need no data massaging. `contracts` is +`validateMocks()`'s return value verbatim plus two flags. + +Each shard descriptor carries its own `failed` count. Merged `tests` do not +record which shard ran them, so without it the per-shard breakdown line +(`Shards: 1 ✓30 | 2 ✗30 | ...`) could not be rendered — and knowing *which* +shard went red is most of that line's value. + +`selection.filters` holds the `--test` values. Filters and shards compose: +filters resolve first, then the filtered list is sharded. There is no companion +`mode` field — a report only exists under `--shard`, so it would be a constant. + +`selection.selectedTests` is the count the shards actually divided, which is what +`executed + notRun` must sum to. `discovery.totalTests` counts the whole suite, +so comparing against *that* reports a phantom slicing bug on any correct +`--test` + `--shard` run. + +Coverage is **referenced, not embedded** — `coverageFile` names a sibling file. +This keeps `run.json` readable by eye and keeps coverage in stock Istanbul +format so `nyc` tooling works on it untouched. A coverage blob is routinely +several megabytes; embedding it would make every report unreadable. + +Duration is ambiguous once jobs run in parallel, so two figures are reported: +wall-clock span (`max(endedAt) - min(startedAt)`), which is what the developer +waited, and total compute (sum of `durationMs`), which is what was paid for. For +a single shard they are equal. Both are **derived from `shards[]` at render +time**, not stored as fields — the per-shard timestamps are sufficient, and +storing derived totals would let them drift out of agreement under merge. + +### `discovery.fingerprint` is the safety net + +> **Correction (fixed in review, report schema v2).** This section originally +> specified the hash over test **ids**. That could never work: `twd-js` mints +> ids with `Math.random()` at registration time, so every page load — and +> therefore every shard's browser — invents different ids for the same tests. +> The hash was a per-load nonce and `merge` rejected every correct multi-shard +> run. Identity is now split in two: `tests[].path` (the `"suite > test"` +> string, resolved in the shard that ran the test) is what the fingerprint +> hashes and what the summary displays, and `tests[].index` (position in the +> discovered order) is the cross-shard identity key. The same defect made the +> "no test id appears in two reports" check below vacuous; it is keyed on +> `index` now. + +The fingerprint is a hash of `{ orderedPaths: test" path, in order>, filters: }`. + +Round-robin sharding is correct only if every job enumerates an identical test +set. That silently breaks if the app registers tests conditionally — a feature +flag, a date, `Math.random()` — or if two shard jobs somehow build different +code. Without the fingerprint that manifests as tests quietly never running and +a green build. With it, merge refuses and explains why. This is the part of the +design least safe to drop. + +## Hard constraint: no change to non-sharded runs + +A run without `--shard` must behave exactly as 1.4.0 does — same console output, +same files written, same exit code. The feature is strictly additive, and every +behavior change below is gated on sharding being active. This is what makes the +work safe to ship as a beta that existing users can install without reading a +migration note. + +Two changes needed scoping to honor this, and both reduce to one extra term in an +existing conditional: + +| Today (`src/index.js`) | Becomes | +|---|---| +| `config.coverage && !hasFailures && !selectedIds` (`:325`) | `config.coverage && filters.length === 0 && (sharded \|\| !hasFailures)` | +| `!stoppedEarly && config.contracts?.length` (`:296`) | `(sharded \|\| !stoppedEarly) && config.contracts?.length` | + +With `sharded === false` each reduces to today's expression exactly — the +`!selectedIds` guard and `filters.length === 0` are the same predicate, since +`selectedIds` is only set by `--test` on a non-sharded run. + +The remaining additions cannot affect an existing run by construction: new flags +are inert when absent, report writing happens only under `--shard`, `merge` is a +new subcommand, and `formatRunComplete`'s new optional `shards` param changes +output only when more than one shard is present. + +## Coverage: the gate moves up a level + +Today coverage is gated twice (`src/index.js:325`): + +```js +if (config.coverage && !hasFailures && !selectedIds) { +``` + +Both gates change. + +`!selectedIds` exists so a `--test` filter cannot produce a misleading +project-wide number. A shard slice is not a user filter, so the rule becomes: +**collect coverage unless `selection.filters` is non-empty.** A filtered run +still skips, sharded or not. + +`!hasFailures` is the more interesting one, and per the constraint above it is +relaxed **only when sharded**. `hasFailures` is per shard, so +applying it at shard level gives the worst outcome: shards 1, 2 and 4 write +coverage, shard 3 goes red and writes none, and merge emits a report that looks +complete while missing a quarter of the code paths. Silent understatement is +worse than absence. + +The same policy therefore applies one level up: + +- **Shards always write `coverage.json`.** No shard-level failure gate, so a + shard's file is never mysteriously absent. +- **Merge writes `.nyc_output/out.json` only when the merged run is green.** + +Net policy is unchanged — a red run yields no coverage — but under sharding it is +keyed on the true global result rather than on one shard's. + +Where coverage lands depends on whether reporting is active, and the two paths +are mutually exclusive on purpose: + +- **Without `--shard`** (today's normal run): `.nyc_output/out.json`, written + exactly as now, including still being skipped on failure. No change at all. +- **With `--shard`**: `/coverage.json` only. It is + deliberately *not* also written to `.nyc_output/out.json`, because one shard's + partial coverage sitting at the path `nyc` reads by default would masquerade as + the whole run's. Under sharding, `.nyc_output/out.json` is written by `merge` + and by nothing else. + +Since a red run still exits 1, no coverage gate can be fooled by the relaxed +failure gate on the sharded path. + +## maxFailures stays per shard + +Cross-job coordination is impossible without an external store, so each shard +gets the full `maxFailures` budget (default 10) independently. Four shards can +therefore accumulate up to 40 failures before all four bail. + +This is documented, not fixed, and the reason it is acceptable is that the +budget exists to stop CI burning time on a fundamentally broken app — with the +suite already divided N ways, each shard reaches its own limit fast enough that +the extra wasted time is not noticeable. Dividing the budget instead +(`ceil(maxFailures / total)`) was considered and rejected: a shard stopping at 3 +failures is hard to explain from its own log, and it makes the CLI depend on the +shard count to compute a threshold. + +A bailing **shard** no longer skips contract validation. Today `stoppedEarly` +skips it outright (`src/index.js:296`, `:317`); under sharding it instead +validates what it collected and sets `contracts.partial: true`, so merge can +report exactly what is missing rather than silently dropping a quarter of the +mocks. The console report gains a partial banner in place of the skip message. +A non-sharded early-stopped run keeps skipping validation, as today. + +## `twd-cli merge ` + +Discovery globs `/*/run.json`, which is exactly `download-artifact`'s +layout (one directory per artifact name). `/run.json` is also accepted for +the degenerate single-report case. + +Validation runs before anything is combined. All of these are fatal: + +- at least one report found +- all `schemaVersion` equal (otherwise: mismatched twd-cli versions across jobs) +- all `discovery.fingerprint` equal +- all `shards[].total` equal, and `shards[].index` covers `1..total` exactly — + no gaps, no duplicates +- no test *position* (`tests[].index`) appears in two reports — see the + correction above; ids cannot serve as a key + +Combining is then mechanical. `tests`, `contracts.results` and +`contracts.skipped` concat. `shards` concats sorted by index. `handlers` and +`discovery` come from the first report, already proven identical. +`contracts.partial` is the OR across shards. One cross-check: `sum(executed) + +sum(notRun)` must equal `discovery.totalTests`; a mismatch warns, since it +indicates a shard-math bug rather than user error. + +Output goes three places: + +1. Merged report to `--out` (default `.twd/merged-run.json`). +2. Merged coverage to `.nyc_output/out.json`, only when the merged run is green. +3. Markdown to `contractReportPath` when configured — so the existing PR-comment + step at `.github/actions/run/action.yml:37` keeps working unchanged. + +**Merge owns the exit code**: 1 on any test failure, any `error`-mode contract +violation, or any validation failure above. Otherwise 0. + +``` +--- Run complete --- + Passed: 114 | Failed: 6 | Skipped: 0 + Duration: 38.2s wall (2m14s compute across 4 shards) + + Shards: 1 ✓30 | 2 ✗30 | 3 ✓30 | 4 ✓30 + + Failed tests (6): + × Checkout > applies discount code + Expected 90 but got 100 (at http://localhost:5173/cart) +``` + +## Modules + +| Module | Responsibility | +|---|---| +| `src/shard.js` | `selectShardIds(ids, index, total)` | +| `src/runReport.js` | `buildRunReport({...})` -> plain object. No I/O. | +| `src/reportFiles.js` | Write `run.json` + `coverage.json`; discover and read shard dirs | +| `src/mergeReports.js` | `mergeRunReports([reports])` -> same shape, plus the validation above | +| `src/mergeCoverage.js` | `istanbul-lib-coverage` `CoverageMap.merge()` | + +Existing files: `parseArgs.js` learns the new flags; `index.js` slices ids and +calls `buildRunReport` + `writeRunReport` instead of discarding its locals; +`bin/twd-cli.js` gains the `merge` command; `testSummary.js` gains an optional +`shards` param that prints the per-shard breakdown when there is more than one. + +`formatRunComplete`'s loose-argument signature is kept as-is and called with +fields destructured from the report, rather than being rewritten to take a +report object. Less churn, and the formatter stays dumb. + +## CLI flags + +``` +npx twd-cli run --shard 2/4 # a shard; writes a report +npx twd-cli run --shard 1/1 # the non-sharded case: one shard, one report +npx twd-cli run --report-dir

# default .twd/run +npx twd-cli merge

+npx twd-cli merge --out

# default .twd/merged-run.json +``` + +Report writing is driven entirely by `--shard`, so no existing run starts +littering the working tree. There is deliberately no separate `--report` flag: +a shard that writes nothing is useless, so the report is a property of sharding, +and `--shard 1/1` already expresses "one shard, write its report". A dedicated +`--report` would add a second flag, an implication rule to document and test, +and a third code path in `parseArgs` for something no consumer needs yet. It is +a one-line addition later if one turns up. + +Each flag accepts both `--flag value` and `--flag=value`, matching the existing +`readValue` helper in `src/parseArgs.js`. + +### One deliberate divergence: `--shard` validates strictly + +`src/parseArgs.js:28` silently ignores a malformed `--record-speed`. `--shard` +must not follow that precedent. `--shard 5/4`, `--shard 0/4` and `--shard abc` +would each silently run zero tests and exit 0 — a green build that tested +nothing. Invalid shard specs error and exit 1. + +## Failure modes + +| Situation | Behavior | +|---|---| +| Shard crashed before upload, artifact missing | error, names the missing index — never a silent 3-of-4 green | +| Shard uploaded but bailed at `maxFailures` | merged report carries its `notRun`, contracts flagged `partial`, exit driven by the real failures | +| Shards saw different test sets | error citing conditional registration or mismatched code | +| Same test position in two shards | error — shard math bug | +| Fewer tests than shards | valid: empty shard writes `tests: []` | +| `--shard 5/4`, `0/4`, `abc` | parse error, exit 1 | +| A shard's `coverage.json` absent | that shard does not contribute; merge states the contributor count | +| `merge` on empty or missing dir | error, exit 1 | +| `--shard` and `--test` together | compose; filters feed the fingerprint so differently-filtered shards cannot merge; coverage skipped | + +## CI shape + +```yaml +jobs: + test: + strategy: + fail-fast: false # or one red shard cancels the rest + matrix: { shard: [1, 2, 3, 4] } + steps: + - ...checkout, npm ci, chrome, dev server... + - run: npx twd-cli run --shard ${{ matrix.shard }}/4 + - uses: actions/upload-artifact + if: always() # a red shard must still upload + with: + name: twd-run-${{ matrix.shard }} + path: .twd/run + + merge: + needs: [test] + if: ${{ !cancelled() }} # runs even though a shard went red + steps: + - uses: actions/checkout@v5 # merge reads twd.config.json + - uses: actions/setup-node@v5 + with: { node-version: 24, cache: npm } + - run: npm ci # merge needs twd-cli installed + - uses: actions/download-artifact + with: { pattern: twd-run-*, path: .twd/shards } + - run: npx twd-cli merge .twd/shards # owns the final exit code +``` + +Three easy-to-miss details, all load-bearing: + +- `fail-fast: false`, or the first red shard cancels its siblings and merge sees + gaps. +- `if: always()` on upload, or a red shard uploads nothing and merge cannot + distinguish "shard failed" from "shard never ran". +- `if: ${{ !cancelled() }}` on merge, or a red shard short-circuits the workflow + and the merged summary — the entire point — is never printed. + +Action versions are written as `@v5` above for readability; the real workflow +SHA-pins them, matching `.github/workflows/e2e.yml`. + +## Dependency change + +`istanbul-lib-coverage` is currently present only transitively, via the +`@vitest/coverage-v8` devDependency. It gets promoted to a real `dependency`. +Small and battle-tested, but it means running `npm run lock:linux` afterwards so +the wasm32-wasi transitive deps stay correct for Linux CI. + +## Release + +Ships as a prerelease so it can be exercised against a real suite before it +becomes the default install. `package.json` goes to `1.5.0-beta.0`, and the +GitHub Release is marked as a prerelease — `publish.yml:27` already routes +prereleases to the `beta` dist-tag, so `npm install twd-cli` keeps resolving to +1.4.0 and testers opt in with `npm install twd-cli@beta`. No workflow change. + +Per the repo's release process the version bump is its own commit carrying +`package.json`, the lockfile regenerated with `npm run lock:linux`, and a +hand-written CHANGELOG entry. The `conventional-changelog` script is not used. + +## Testing + +Existing constraints hold: no test may require a real browser or a real ffmpeg +binary, and `vi.mock('fs')` auto-mocks `statSync` to `undefined`. + +New files: `tests/shard.test.js`, `tests/runReport.test.js`, +`tests/reportFiles.test.js`, `tests/mergeReports.test.js`, +`tests/mergeCoverage.test.js`. + +Two of the cases are properties rather than examples, and they are what make the +whole scheme trustworthy: + +- **Partition**: the union of all shards equals the input list, with every id + appearing exactly once. +- **Associativity**: `merge([merge([a, b]), c])` deep-equals `merge([a, b, c])`. + +If both hold, sharding cannot silently lose or double-run a test. + +Alongside them: gap detection, fingerprint mismatch, duplicate ids, empty +shards, coverage counts summing across shards, `--shard` parse validation, and +extensions to `tests/runTests.test.js` asserting the shard slice reaches +`runByIds`, the report is written, and **coverage is written despite failures** +(the gate change). + +The constraint above needs its own explicit coverage, not just inference: tests +asserting that with no `--shard` flag, a failing run still writes **no** +coverage file and an early-stopped run still **skips** contract validation. Those +are the two conditionals that were touched, so they are the two that could +silently regress an existing user. + +Unit tests cannot exercise the real Actions plumbing, so +`.github/workflows/e2e.yml` gains a 2-shard-plus-merge run against +`test-example-app`. That is what would catch a missing `if: always()` or a wrong +artifact path. + +## Value + +Wall-clock CI time divides by the shard count without the flakiness that killed +in-process parallelism, because separate jobs share no CPU, no dev server, and +no browser. + +Secondarily, the run report is the machine-readable output twd-cli has never +had. Merging is its first consumer; agent-driven TDD loops reading results +without parsing console text are an obvious second. diff --git a/package-lock.json b/package-lock.json index 1dc1dbf..fbe8787 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "twd-cli", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "twd-cli", - "version": "1.4.0", + "version": "1.5.0", "license": "ISC", "dependencies": { + "istanbul-lib-coverage": "^3.2.2", "openapi-mock-validator": "^0.3.0", "puppeteer": "^25.3.0" }, @@ -137,35 +138,38 @@ } }, "node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", + "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { - "@emnapi/wasi-threads": "1.2.2", + "@emnapi/wasi-threads": "1.2.3", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "tslib": "^2.4.0" } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", + "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", "dev": true, "license": "MIT", "optional": true, + "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -348,9 +352,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -368,9 +369,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -388,9 +386,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -408,9 +403,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -428,9 +420,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -448,9 +437,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -496,6 +482,40 @@ "node": "^20.19.0 || >=22.12.0" } }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@rolldown/binding-win32-arm64-msvc": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", @@ -1156,7 +1176,6 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=8" @@ -1369,9 +1388,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1393,9 +1409,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1417,9 +1430,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1441,9 +1451,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/package.json b/package.json index 440576d..abdf87d 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,19 @@ { "name": "twd-cli", - "version": "1.4.0", + "version": "1.5.0", "description": "CLI tool for running TWD tests with Puppeteer", "type": "module", "main": "src/index.js", "bin": { "twd-cli": "./bin/twd-cli.js" }, + "files": [ + "bin", + "src", + "README.md", + "CHANGELOG.md", + "LICENSE" + ], "scripts": { "test": "vitest", "test:ci": "vitest --run --coverage", @@ -24,6 +31,7 @@ "author": "", "license": "ISC", "dependencies": { + "istanbul-lib-coverage": "^3.2.2", "openapi-mock-validator": "^0.3.0", "puppeteer": "^25.3.0" }, diff --git a/src/index.js b/src/index.js index 3d4f348..db025a5 100644 --- a/src/index.js +++ b/src/index.js @@ -11,6 +11,9 @@ import { selectTestIds } from './filterTests.js'; import { explainError } from './diagnostics.js'; import { orderedTestIds, chunk } from './testOrder.js'; import { resolveRecordFilename } from './recordFilename.js'; +import { selectShardIds } from './shard.js'; +import { buildRunReport } from './runReport.js'; +import { writeRunReport, DEFAULT_REPORT_DIR, COVERAGE_FILE } from './reportFiles.js'; import { assertFfmpegAvailable, applyRecordingFraming, @@ -43,7 +46,8 @@ function recordedFileSize(absPath) { } export async function runTests(options = {}) { - const { testFilters = [], recordOverrides = {} } = options; + const { testFilters = [], recordOverrides = {}, shard = null, reportDir = null } = options; + const sharded = Boolean(shard); let browser; let config; let startedAt = null; @@ -52,6 +56,7 @@ export async function runTests(options = {}) { let recorder = null; let recordOutput = null; let recordOutputPath = null; + let recordingInfo = null; // Stops the screencast at most once. Must always run before browser.close(): // if the browser goes first, ffmpeg is orphaned and the file is truncated. @@ -161,7 +166,24 @@ export async function runTests(options = {}) { } // Resolve the ordered id list to run: the filter result, or all tests. - const baseIds = selectedIds ?? orderedTestIds(registeredHandlers); + // + // allTestIds is the full ordered list, before filtering or slicing. Its + // order is what the fingerprint covers (as paths — the ids themselves are + // random per page load) and its length is discovery.totalTests, so every + // shard agrees on both regardless of which slice it took. filteredIds is + // the list the shards divide, so it is what executed + notRun must total. + const allTestIds = orderedTestIds(registeredHandlers); + const filteredIds = selectedIds ?? allTestIds; + const baseIds = sharded + ? selectShardIds(filteredIds, shard.index, shard.total) + : filteredIds; + + if (sharded) { + console.log( + `Shard ${shard.index}/${shard.total}: running ${baseIds.length} of ${filteredIds.length} test(s).` + ); + } + const chunks = chunk(baseIds, config.chunkSize); // Recording starts here, not earlier: the output path is fixed up front by @@ -274,12 +296,14 @@ export async function runTests(options = {}) { 'Chrome only emits video frames when the page repaints, so a run with no visible changes (or no tests) records nothing.' ); } else { + recordingInfo = { file: recordOutput, bytes: recordedFileSize(recordOutputPath) }; console.log(`Recorded ${executed} test(s) to ${recordOutput}`); } } const testStatus = partialStatus; - const durationMs = Date.now() - startedAt; + const endedAt = Date.now(); + const durationMs = endedAt - startedAt; const notRun = baseIds.length - executed; // Exit with appropriate code @@ -292,9 +316,24 @@ export async function runTests(options = {}) { } } - // Contract validation (skipped on an early stop — the data is partial) - if (!stoppedEarly && config.contracts && config.contracts.length > 0) { - if (collectedMocks.size === 0) { + // Contract validation. A sharded run validates even after an early stop and + // flags the result partial, so merge can say exactly what is missing rather + // than silently dropping a shard's worth of mocks. A non-sharded run keeps + // skipping, exactly as before. + const contractsConfigured = Boolean(config.contracts && config.contracts.length > 0); + let contractsBlock = { + configured: contractsConfigured, + partial: false, + results: [], + skipped: [], + }; + + if (contractsConfigured && (sharded || !stoppedEarly)) { + // Never under sharding. A shard whose slice exercised no mocks — and any + // shard with an empty slice — collects nothing, which is normal, so this + // would advertise a twd-js version problem that does not exist on the + // happy path of every sharded CI run. + if (collectedMocks.size === 0 && !sharded) { console.log('\nNo mocks collected — ensure twd-js supports contract collection'); } const validationOutput = validateMocks(collectedMocks, contractValidators); @@ -303,46 +342,78 @@ export async function runTests(options = {}) { hasFailures = true; } - // Write markdown report for CI/PR integration - if (config.contractReportPath) { + contractsBlock = { + configured: true, + partial: stoppedEarly, + results: validationOutput.results, + skipped: validationOutput.skipped, + }; + + if (stoppedEarly) { + console.log('\n⚠ Contract data is partial — this shard stopped early.'); + } + + // Write markdown report for CI/PR integration. + // + // Only a whole run produces a meaningful markdown report. Under sharding + // each shard would overwrite the others with a quarter of the picture, so + // `merge` writes it instead. + if (config.contractReportPath && !sharded) { const reportPath = path.resolve(workingDir, config.contractReportPath); - const reportDir = path.dirname(reportPath); - if (!fs.existsSync(reportDir)) { - fs.mkdirSync(reportDir, { recursive: true }); + const reportDirPath = path.dirname(reportPath); + if (!fs.existsSync(reportDirPath)) { + fs.mkdirSync(reportDirPath, { recursive: true }); } const markdown = generateContractMarkdown(validationOutput); fs.writeFileSync(reportPath, markdown); console.log(`Contract report written to ${config.contractReportPath}`); } - } else if (stoppedEarly && config.contracts && config.contracts.length > 0) { + } else if (contractsConfigured && stoppedEarly) { console.log('\nSkipping contract validation — run stopped early (partial data).'); } - // Handle code coverage if enabled (skipped when a --test filter is active) + // Handle code coverage if enabled. + // + // The filter gate is unchanged: a --test filter still suppresses coverage, + // because a filtered run's number is a misleading project-wide figure. A + // shard slice is not a filter. + // + // The failure gate is relaxed for sharded runs only. hasFailures is per + // shard, so applying it here would let three green shards write coverage + // while a red fourth writes none — a merged report that looks complete but + // is missing a quarter of the code paths. `merge` applies the gate to the + // true global result instead. if (selectedIds && config.coverage) { console.log('Skipping coverage collection (test filter active).'); } - if (config.coverage && !hasFailures && !selectedIds) { - const coverage = await page.evaluate(() => window.__coverage__); - if (coverage) { - const coverageDir = path.resolve(workingDir, config.coverageDir); - const nycDir = path.resolve(workingDir, config.nycOutputDir); - - if (!fs.existsSync(nycDir)) { - fs.mkdirSync(nycDir, { recursive: true }); - } - if (!fs.existsSync(coverageDir)) { - fs.mkdirSync(coverageDir, { recursive: true }); - } - const coveragePath = path.join(nycDir, 'out.json'); - fs.writeFileSync(coveragePath, JSON.stringify(coverage)); - console.log(`Code coverage data written to ${coveragePath}`); - } else { + let coverageData = null; + if (config.coverage && !selectedIds && (sharded || !hasFailures)) { + coverageData = await page.evaluate(() => window.__coverage__); + if (!coverageData) { console.log('No code coverage data found.'); } } + // A sharded run's coverage goes to the report dir and nowhere else. Writing + // it to .nyc_output/out.json — the path nyc reads by default — would let one + // shard's partial data masquerade as the whole run's. + if (coverageData && !sharded) { + const coverageDir = path.resolve(workingDir, config.coverageDir); + const nycDir = path.resolve(workingDir, config.nycOutputDir); + + if (!fs.existsSync(nycDir)) { + fs.mkdirSync(nycDir, { recursive: true }); + } + if (!fs.existsSync(coverageDir)) { + fs.mkdirSync(coverageDir, { recursive: true }); + } + + const coveragePath = path.join(nycDir, 'out.json'); + fs.writeFileSync(coveragePath, JSON.stringify(coverageData)); + console.log(`Code coverage data written to ${coveragePath}`); + } + await browser.close(); // The run-complete block is always the last output of a completed run @@ -356,6 +427,32 @@ export async function runTests(options = {}) { maxFailures: config.maxFailures, })); + // Written last, and only for a sharded run. A run that threw never gets + // here on purpose: its artifact stays absent, and `merge` reports the gap as + // "a shard job likely failed before uploading", which is the accurate + // diagnosis. A half-written report would be a worse lie. + if (sharded) { + const dir = reportDir ?? DEFAULT_REPORT_DIR; + const report = buildRunReport({ + shard, + startedAt, + endedAt, + allTestIds, + filteredIds, + filters: testFilters, + handlers, + tests: testStatus, + executed, + notRun, + stoppedEarly, + coverageFile: coverageData ? COVERAGE_FILE : null, + recording: recordingInfo, + contracts: contractsBlock, + }); + const { reportPath } = writeRunReport(dir, report, coverageData); + console.log(`Shard report written to ${reportPath}`); + } + return hasFailures; } catch (error) { diff --git a/src/mergeCommand.js b/src/mergeCommand.js new file mode 100644 index 0000000..a929962 --- /dev/null +++ b/src/mergeCommand.js @@ -0,0 +1,142 @@ +import fs from 'fs'; +import path from 'path'; +import { loadConfig } from './config.js'; +import { + readShardReports, + readShardCoverage, + DEFAULT_MERGED_OUT, +} from './reportFiles.js'; +import { + mergeRunReports, + findMissingShards, + reportTimings, + reportTotals, +} from './mergeReports.js'; +import { mergeCoverage } from './mergeCoverage.js'; +import { formatRunComplete } from './testSummary.js'; +import { printContractReport } from './contractReport.js'; +import { generateContractMarkdown } from './contractMarkdown.js'; + +/** + * Joins per-shard reports into one and reports on the whole run. + * + * This function owns the run's exit code. Shard jobs each exit 1 on their own + * failures, so the workflow only reaches here with `if: !cancelled()`, and the + * merged verdict is the one that counts. + */ +export function runMerge({ dir, out = null } = {}) { + if (!dir) { + throw new Error('Usage: twd-cli merge

[--out ]'); + } + + const config = loadConfig(); + const workingDir = process.cwd(); + + const found = readShardReports(dir); + if (found.length === 0) { + throw new Error( + `No shard reports found in ${dir}. ` + + 'Expected /*/run.json (the actions/download-artifact layout) or /run.json.' + ); + } + + // readShardReports returns readdir order, so twd-run-10 sorts before + // twd-run-2. mergeRunReports sorts shards[] by index but concatenates tests + // in argument order, so without this the merged report's tests and its shard + // list disagree about ordering. + found.sort((a, b) => (a.report.shards[0]?.index ?? 0) - (b.report.shards[0]?.index ?? 0)); + + const merged = mergeRunReports(found.map((f) => f.report)); + + // Completeness is enforced here rather than inside mergeRunReports, which must + // stay associative. A gap is never a warning: a silent 3-of-4 merge reads as a + // complete green run. + const missing = findMissingShards(merged); + if (missing.length > 0) { + const total = merged.shards[0].total; + throw new Error( + `Missing shard report(s): ${missing.map((i) => `${i}/${total}`).join(', ')}. ` + + 'A shard job likely failed before uploading its artifact — check that the ' + + 'upload step runs with `if: always()`.' + ); + } + + const outPath = path.resolve(workingDir, out ?? DEFAULT_MERGED_OUT); + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + fs.writeFileSync(outPath, `${JSON.stringify(merged, null, 2)}\n`); + console.log(`Merged report written to ${outPath}`); + + let hasFailures = merged.tests.some((test) => test.status === 'fail'); + + if (merged.contracts.configured) { + const validationOutput = { + results: merged.contracts.results, + skipped: merged.contracts.skipped, + }; + if (printContractReport(validationOutput)) { + hasFailures = true; + } + if (merged.contracts.partial) { + console.warn( + 'Warning: contract data is partial — at least one shard stopped early, so ' + + 'some mocks were never collected.' + ); + } + if (config.contractReportPath) { + const reportPath = path.resolve(workingDir, config.contractReportPath); + fs.mkdirSync(path.dirname(reportPath), { recursive: true }); + fs.writeFileSync(reportPath, generateContractMarkdown(validationOutput)); + console.log(`Contract report written to ${config.contractReportPath}`); + } + } + + // A red run yields no coverage — the same policy a single run has always had, + // but keyed on the whole merged result instead of one shard's. + if (config.coverage) { + const coverages = found.map((f) => + readShardCoverage(f.dir, f.report.shards[0]?.coverageFile ?? null) + ); + const contributors = coverages.filter(Boolean).length; + + if (contributors === 0) { + console.log('No coverage data found in any shard.'); + } else if (hasFailures) { + console.log( + `Skipping merged coverage — the run has failures ` + + `(${contributors}/${found.length} shard(s) had data).` + ); + } else { + const nycDir = path.resolve(workingDir, config.nycOutputDir); + fs.mkdirSync(nycDir, { recursive: true }); + fs.writeFileSync(path.join(nycDir, 'out.json'), JSON.stringify(mergeCoverage(coverages))); + console.log( + `Coverage merged from ${contributors}/${found.length} shards to ` + + `${config.nycOutputDir}/out.json` + ); + } + } + + const totals = reportTotals(merged); + if (!totals.consistent) { + console.warn( + `Warning: shard totals do not add up — ${totals.executed} executed + ` + + `${totals.notRun} not run != ${totals.expected} selected. ` + + 'This points at a shard-slicing bug, not at your tests.' + ); + } + + const timings = reportTimings(merged); + console.log(''); + console.log(formatRunComplete({ + testStatus: merged.tests, + handlers: merged.handlers, + durationMs: timings.wallMs, + computeMs: timings.computeMs, + notRun: totals.notRun, + shards: merged.shards, + stoppedEarly: merged.shards.some((s) => s.stoppedEarly), + maxFailures: config.maxFailures, + })); + + return hasFailures; +} diff --git a/src/mergeCoverage.js b/src/mergeCoverage.js new file mode 100644 index 0000000..f7172e7 --- /dev/null +++ b/src/mergeCoverage.js @@ -0,0 +1,26 @@ +import libCoverage from 'istanbul-lib-coverage'; + +/** + * Combines per-shard Istanbul coverage into one map. + * + * This is the one artifact that needed no bespoke merge logic: summing hit + * counts across runs of the same code is exactly what CoverageMap.merge does, + * and it is the same operation `nyc merge` performs. + * + * Nulls are skipped rather than rejected — a shard that collected no coverage + * (a filtered run, or one that never loaded instrumented code) reads back as + * null and simply does not contribute. + * + * Inputs are cloned before merging. `istanbul-lib-coverage`'s `FileCoverage` + * wraps a plain coverage object by reference instead of copying it, so the + * first shard's raw object would otherwise become the map's live storage and + * get mutated in place (`this.data.s = ...`) once a later shard's counts for + * the same file are merged in. + */ +export function mergeCoverage(coverageObjects) { + const map = libCoverage.createCoverageMap({}); + for (const coverage of coverageObjects) { + if (coverage) map.merge(structuredClone(coverage)); + } + return map.toJSON(); +} diff --git a/src/mergeReports.js b/src/mergeReports.js new file mode 100644 index 0000000..a0e2e7d --- /dev/null +++ b/src/mergeReports.js @@ -0,0 +1,163 @@ +import { REPORT_SCHEMA_VERSION } from './runReport.js'; + +/** + * Combines shard reports into one report of the same shape. + * + * Only *consistency* is validated here — the things that stay true under a + * partial merge. Completeness (is 1..total all present?) is checked by + * findMissingShards, called from the merge command, because a 2-of-3 merge is a + * legal intermediate value: rejecting it here would make + * mergeRunReports([mergeRunReports([a, b]), c]) throw and destroy + * associativity, which is the property that proves no test is lost or doubled. + */ +export function mergeRunReports(reports) { + if (!reports.length) { + throw new Error( + 'No shard reports to merge. Expected /*/run.json or /run.json.' + ); + } + + const versions = [...new Set(reports.map((r) => r.schemaVersion))]; + if (versions.length > 1) { + throw new Error( + `Shard reports disagree on schemaVersion (${versions.sort().join(', ')}). ` + + 'Every shard job must run the same twd-cli version.' + ); + } + + // Agreement is not enough. Reports from a newer twd-cli agree with each other + // and would be merged by an older binary against a schema it does not + // understand — the exact silent mis-merge the field exists to prevent. + if (versions[0] !== REPORT_SCHEMA_VERSION) { + throw new Error( + `Shard reports use report schema v${versions[0]}, but this twd-cli reads ` + + `v${REPORT_SCHEMA_VERSION}. Every shard job and the merge job must run the ` + + 'same twd-cli version.' + ); + } + + const fingerprints = new Set(reports.map((r) => r.discovery.fingerprint)); + if (fingerprints.size > 1) { + throw new Error( + 'Shard reports discovered different test sets, so they cannot be merged. ' + + 'Either tests are registered conditionally (a feature flag, a date, ' + + 'Math.random), or the shard jobs did not build the same code.' + ); + } + + const shards = reports.flatMap((r) => r.shards); + + const totals = [...new Set(shards.map((s) => s.total))]; + if (totals.length > 1) { + throw new Error( + `Shard reports disagree on shard total (${totals.sort((a, b) => a - b).join(', ')}). ` + + 'Every shard job must pass the same --shard total.' + ); + } + + const byIndex = new Set(); + for (const shard of shards) { + if (byIndex.has(shard.index)) { + throw new Error(`Shard ${shard.index}/${shard.total} appears more than once.`); + } + byIndex.add(shard.index); + } + + // Overlap is detected on tests[].index — the test's position in the discovered + // order — not on tests[].id. twd-js ids are Math.random() per page load, so + // every shard invents its own and an id-keyed check can never fire: it looked + // like a guard while proving nothing. Position is deterministic, so it is a + // real one. The path cannot be the key either: duplicate test names share a + // path and may legally land in different shards. + const tests = []; + const positions = new Set(); + for (const report of reports) { + for (const test of report.tests) { + if (test.index != null) { + if (positions.has(test.index)) { + throw new Error( + `Test "${test.path ?? test.id}" (position ${test.index}) appears in more ` + + 'than one shard — the shard slices overlap.' + ); + } + positions.add(test.index); + } + tests.push(test); + } + } + + const first = reports[0]; + + return { + schemaVersion: first.schemaVersion, + // Copy before sorting: sort mutates, and the input reports are the caller's. + shards: [...shards].sort((a, b) => a.index - b.index), + discovery: first.discovery, + selection: first.selection, + // Only the first shard's map, and only its own ids resolve in it — twd-js + // ids are per-page-load random. That is why every test carries its own + // resolved `path` and renderers prefer it; this stays for the ids it can + // still explain and for diagnostics. Taking the first is the documented + // contract, pinned by a test. + handlers: first.handlers, + tests, + contracts: { + configured: first.contracts.configured, + partial: reports.some((r) => r.contracts.partial), + results: reports.flatMap((r) => r.contracts.results), + skipped: reports.flatMap((r) => r.contracts.skipped), + }, + }; +} + +/** + * Shard indices in 1..total that no report accounted for. + * + * A gap almost always means a shard job died before uploading its artifact. It + * must be loud: a silent 3-of-4 merge reads as a complete green run. + */ +export function findMissingShards(report) { + const total = report.shards[0]?.total ?? 0; + const present = new Set(report.shards.map((s) => s.index)); + const missing = []; + for (let i = 1; i <= total; i++) { + if (!present.has(i)) missing.push(i); + } + return missing; +} + +/** + * Wall clock (what the developer waited) and compute (what was paid for). + * + * Derived rather than stored, so the two can never drift out of agreement with + * the per-shard timestamps they come from. + */ +export function reportTimings(report) { + const starts = report.shards.map((s) => Date.parse(s.startedAt)); + const ends = report.shards.map((s) => Date.parse(s.endedAt)); + return { + wallMs: Math.max(...ends) - Math.min(...starts), + computeMs: report.shards.reduce((sum, s) => sum + s.durationMs, 0), + }; +} + +/** + * Executed and not-run totals, plus whether they account for every discovered + * test. An inconsistent result points at a shard-math bug, not user error. + */ +export function reportTotals(report) { + const executed = report.shards.reduce((sum, s) => sum + s.executed, 0); + const notRun = report.shards.reduce((sum, s) => sum + s.notRun, 0); + // Against the count the shards divided, not everything discovered. With + // --test active those differ by every excluded test, and comparing to + // discovery.totalTests reported a slicing bug on a correct run. The fallback + // is unreachable for a v2 report; it only keeps a hand-built one from + // comparing against undefined and always warning. + const expected = report.selection?.selectedTests ?? report.discovery.totalTests; + return { + executed, + notRun, + expected, + consistent: executed + notRun === expected, + }; +} diff --git a/src/parseArgs.js b/src/parseArgs.js index 75f9965..873897f 100644 --- a/src/parseArgs.js +++ b/src/parseArgs.js @@ -1,36 +1,52 @@ +import { parseShardSpec } from './shard.js'; + +// Reads a flag's value in either `--flag value` or `--flag=value` form, and +// reports how many tokens it consumed. Shared by both parsers. +function readValue(argv, token, prefix, index) { + if (token === prefix) { + return { value: argv[index + 1], consumed: argv[index + 1] !== undefined ? 2 : 1 }; + } + return { value: token.slice(prefix.length + 1), consumed: 1 }; +} + export function parseRunArgs(argv) { const testFilters = []; const record = {}; - - const readValue = (token, prefix, index) => { - if (token === prefix) { - return { value: argv[index + 1], consumed: argv[index + 1] !== undefined ? 2 : 1 }; - } - return { value: token.slice(prefix.length + 1), consumed: 1 }; - }; + let shard = null; + let reportDir = null; for (let i = 0; i < argv.length; i++) { const token = argv[i]; if (token === '--test' || token.startsWith('--test=')) { - const { value, consumed } = readValue(token, '--test', i); + const { value, consumed } = readValue(argv, token, '--test', i); if (value !== undefined) testFilters.push(value); i += consumed - 1; + } else if (token === '--shard' || token.startsWith('--shard=')) { + const { value, consumed } = readValue(argv, token, '--shard', i); + // Throws on a malformed spec. A silently-ignored --shard would run zero + // tests and exit 0. + shard = parseShardSpec(value); + i += consumed - 1; + } else if (token === '--report-dir' || token.startsWith('--report-dir=')) { + const { value, consumed } = readValue(argv, token, '--report-dir', i); + if (value !== undefined) reportDir = value; + i += consumed - 1; } else if (token === '--record') { record.enabled = true; } else if (token === '--record-dir' || token.startsWith('--record-dir=')) { - const { value, consumed } = readValue(token, '--record-dir', i); + const { value, consumed } = readValue(argv, token, '--record-dir', i); if (value !== undefined) record.dir = value; i += consumed - 1; } else if (token === '--record-speed' || token.startsWith('--record-speed=')) { - const { value, consumed } = readValue(token, '--record-speed', i); + const { value, consumed } = readValue(argv, token, '--record-speed', i); const parsed = Number(value); if (value !== undefined && Number.isFinite(parsed) && parsed > 0) { record.speed = parsed; } i += consumed - 1; } else if (token === '--record-pace' || token.startsWith('--record-pace=')) { - const { value, consumed } = readValue(token, '--record-pace', i); + const { value, consumed } = readValue(argv, token, '--record-pace', i); const parsed = Number(value); if (value !== undefined && Number.isFinite(parsed) && parsed > 0) { record.pace = parsed; @@ -39,5 +55,26 @@ export function parseRunArgs(argv) { } } - return { testFilters, record }; + return { testFilters, record, shard, reportDir }; +} + +// `twd-cli merge [--out ]`. The directory is the first positional +// token; anything after the first is ignored. +export function parseMergeArgs(argv) { + let dir = null; + let out = null; + + for (let i = 0; i < argv.length; i++) { + const token = argv[i]; + + if (token === '--out' || token.startsWith('--out=')) { + const { value, consumed } = readValue(argv, token, '--out', i); + if (value !== undefined) out = value; + i += consumed - 1; + } else if (!token.startsWith('--') && dir === null) { + dir = token; + } + } + + return { dir, out }; } diff --git a/src/reportFiles.js b/src/reportFiles.js new file mode 100644 index 0000000..747eb20 --- /dev/null +++ b/src/reportFiles.js @@ -0,0 +1,87 @@ +import fs from 'fs'; +import path from 'path'; + +export const DEFAULT_REPORT_DIR = './.twd/run'; +export const DEFAULT_MERGED_OUT = './.twd/merged-run.json'; +export const RUN_REPORT_FILE = 'run.json'; +export const COVERAGE_FILE = 'coverage.json'; + +/** + * Writes one shard's report, and its coverage when it collected any. + * + * The report is pretty-printed because a human reads it when a merge complains. + * Coverage is not: it is machine input for nyc, routinely several megabytes, and + * indenting it would roughly double the artifact size for no benefit. + */ +export function writeRunReport(dir, report, coverage = null) { + fs.mkdirSync(dir, { recursive: true }); + + const reportPath = path.join(dir, RUN_REPORT_FILE); + fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + + let coveragePath = null; + if (coverage) { + coveragePath = path.join(dir, COVERAGE_FILE); + fs.writeFileSync(coveragePath, JSON.stringify(coverage)); + } + + return { reportPath, coveragePath }; +} + +function readJson(file, label) { + const raw = fs.readFileSync(file, 'utf-8'); + try { + return JSON.parse(raw); + } catch (err) { + throw new Error(`Could not parse ${label} at ${file}: ${err.message}`); + } +} + +/** + * Finds every shard report under `dir`. + * + * actions/download-artifact lays each artifact out as its own directory, so the + * normal shape is `//run.json`. A bare `/run.json` is + * also accepted, which is what a local single-shard run produces. + */ +export function readShardReports(dir) { + const found = []; + + const direct = path.join(dir, RUN_REPORT_FILE); + if (fs.existsSync(direct)) { + found.push({ dir, report: readJson(direct, RUN_REPORT_FILE) }); + } + + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + // Missing or unreadable directory: the caller reports "no reports found", + // which is a better message than an ENOENT stack. + return found; + } + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const shardDir = path.join(dir, entry.name); + const file = path.join(shardDir, RUN_REPORT_FILE); + if (fs.existsSync(file)) { + found.push({ dir: shardDir, report: readJson(file, RUN_REPORT_FILE) }); + } + } + + return found; +} + +/** + * Reads a shard's coverage, or null when it collected none. + * + * Absence is a normal outcome, not an error: a filtered run skips coverage + * entirely. Merge reports how many shards contributed. + */ +export function readShardCoverage(shardDir, coverageFile) { + if (!coverageFile) return null; + const file = path.join(shardDir, coverageFile); + if (!fs.existsSync(file)) return null; + return readJson(file, COVERAGE_FILE); +} diff --git a/src/runReport.js b/src/runReport.js new file mode 100644 index 0000000..2ae297f --- /dev/null +++ b/src/runReport.js @@ -0,0 +1,119 @@ +import crypto from 'node:crypto'; +import { buildTestPath } from './buildTestPath.js'; + +export const REPORT_SCHEMA_VERSION = 2; + +/** + * Hash of the full ordered list of test *paths* plus any active --test filters. + * + * Round-robin sharding is correct only if every job enumerates an identical + * test set in an identical order. That breaks silently if the app registers + * tests conditionally — a feature flag, a date, Math.random — or if two shard + * jobs did not build the same code: tests quietly never run and the build stays + * green. Shards compare fingerprints at merge time so it becomes an error. + * + * Paths, not ids. twd-js mints test ids with `Math.random()` at registration + * time, so the same test carries a different id on every page load — and every + * shard boots its own browser. Hashing ids made the fingerprint a per-load + * nonce that could never match, so `merge` rejected every correct multi-shard + * run. A `"suite > test"` path is derived from names, so it is stable across + * loads, and it is the same string `--test` already filters on. The check is + * also strictly stronger this way: a conditionally-registered test still + * changes the hash, because its path drops out of the ordered list. + * + * Filters are OR'd, so their order carries no meaning and is normalized away. + */ +export function fingerprintTests(orderedPaths, filters = []) { + const payload = JSON.stringify({ + orderedPaths, + filters: [...filters].sort(), + }); + const digest = crypto.createHash('sha256').update(payload).digest('hex'); + return `sha256:${digest}`; +} + +/** + * Assembles the on-disk run report. Pure: no I/O, no clock reads. + * + * `shards` is an array even for a single run, because a merged report has the + * same shape as a single-shard one. That is what makes merging associative and + * lets one set of formatters render both. + * + * `startedAt` and `endedAt` are epoch milliseconds; the report stores ISO + * strings plus the derived duration. + * + * `allTestIds` is the whole discovered suite in order; `filteredIds` is what + * survived `--test` and therefore what the shards actually divide between them. + * They are the same list when no filter is active. + */ +export function buildRunReport({ + shard, + startedAt, + endedAt, + allTestIds, + filteredIds = null, + filters = [], + handlers, + tests, + executed, + notRun, + stoppedEarly, + coverageFile = null, + recording = null, + contracts = null, +}) { + // Resolved here, in the shard that ran the tests, because this is the only + // place the handler map is valid: ids are per-page-load random, so shard 2's + // ids mean nothing in shard 1's handler map — and shard 1's map is the one a + // merged report keeps. + const orderedPaths = allTestIds.map((id) => buildTestPath(id, handlers)); + const positions = new Map(allTestIds.map((id, i) => [id, i])); + const selectedIds = filteredIds ?? allTestIds; + + return { + schemaVersion: REPORT_SCHEMA_VERSION, + shards: [ + { + index: shard.index, + total: shard.total, + startedAt: new Date(startedAt).toISOString(), + endedAt: new Date(endedAt).toISOString(), + durationMs: endedAt - startedAt, + executed, + notRun, + // Merged reports do not record which shard ran a test, so the per-shard + // breakdown line could not be rendered without this count. + failed: tests.filter((t) => t.status === 'fail').length, + stoppedEarly, + coverageFile, + recording, + }, + ], + discovery: { + totalTests: allTestIds.length, + fingerprint: fingerprintTests(orderedPaths, filters), + }, + // selectedTests is the count the shards divided, which is what + // executed + notRun must add up to. Comparing against totalTests instead + // reports a slicing bug on any correct `--test` + `--shard` run. + selection: { filters: [...filters], selectedTests: selectedIds.length }, + handlers, + tests: tests.map((test) => ({ + ...test, + // For display. May be null if the handler somehow went missing, so every + // renderer has to tolerate that. + path: buildTestPath(test.id, handlers), + // Identity. Position in the shard-independent ordered list is stable + // across shards because registration order is deterministic, unlike the + // random id. The path cannot serve as the key: two tests may share one + // (duplicate names) and can legally land in different shards. + index: positions.has(test.id) ? positions.get(test.id) : null, + })), + contracts: contracts ?? { + configured: false, + partial: false, + results: [], + skipped: [], + }, + }; +} diff --git a/src/shard.js b/src/shard.js new file mode 100644 index 0000000..81447ac --- /dev/null +++ b/src/shard.js @@ -0,0 +1,38 @@ +// Parses a "/" shard spec, e.g. "2/4" for job 2 of 4. +// +// This throws where src/parseArgs.js silently ignores a malformed +// --record-speed, and the divergence is deliberate: "--shard 5/4" would select +// no tests and exit 0, which reads as a green build that tested nothing. +export function parseShardSpec(value) { + const raw = typeof value === 'string' ? value.trim() : value; + const match = /^(\d+)\/(\d+)$/.exec(String(raw ?? '')); + if (!match) { + throw new Error( + `Invalid --shard "${value}". Expected /, e.g. --shard 2/4.` + ); + } + + const index = Number(match[1]); + const total = Number(match[2]); + + if (total < 1) { + throw new Error(`Invalid --shard "${value}". Total must be at least 1.`); + } + if (index < 1 || index > total) { + throw new Error( + `Invalid --shard "${value}". Index must be between 1 and ${total}.` + ); + } + + return { index, total }; +} + +// Round-robin slice of an ordered id list. `index` is 1-based. +// +// Round-robin rather than contiguous: it balances better when adjacent tests +// have similar cost, and nothing in twd-js requires a suite to run +// contiguously, so locality buys nothing. An empty result is a legal outcome — +// 3 tests across 4 shards leaves the fourth with nothing to run. +export function selectShardIds(ids, index, total) { + return ids.filter((_, i) => i % total === index - 1); +} diff --git a/src/testSummary.js b/src/testSummary.js index 7497005..f2eb971 100644 --- a/src/testSummary.js +++ b/src/testSummary.js @@ -1,5 +1,20 @@ import { buildTestPath } from './buildTestPath.js'; +/** + * Display name for one test result. + * + * `entry.path` is preferred because it was resolved inside the shard that ran + * the test, where the handler map was valid. A merged report keeps only the + * first shard's handlers, and twd-js ids are random per page load, so + * buildTestPath cannot resolve anything from shards 2..n — every failure would + * print a bare random id in the one place the merge exists to produce. The + * buildTestPath call stays for a live, non-sharded run, whose entries carry no + * path. + */ +function resolvePath(entry, handlers) { + return entry.path ?? buildTestPath(entry.id, handlers) ?? entry.id; +} + export function formatRunComplete({ testStatus, handlers, @@ -7,6 +22,8 @@ export function formatRunComplete({ notRun = 0, stoppedEarly = false, maxFailures, + shards = null, + computeMs = null, }) { const passed = testStatus.filter((t) => t.status === 'pass').length; const failed = testStatus.filter((t) => t.status === 'fail').length; @@ -18,13 +35,25 @@ export function formatRunComplete({ ` Passed: ${passed} | Failed: ${failed} | Skipped: ${skipped}`, ]; if (notRun > 0) lines.push(` Not run: ${notRun}`); - lines.push(` Duration: ${duration}s`); + + // A merged run has two meaningful durations: the span the developer waited, + // and the compute it consumed. A single run has only one, and its line must + // stay byte-identical to what it has always printed. + const merged = Array.isArray(shards) && shards.length > 1; + if (merged) { + const compute = (computeMs / 1000).toFixed(1); + lines.push(` Duration: ${duration}s wall | ${compute}s across ${shards.length} shards`); + const cells = shards.map((s) => `${s.index} ${s.failed > 0 ? '✗' : '✓'}${s.executed}`); + lines.push(` Shards: ${cells.join(' | ')}`); + } else { + lines.push(` Duration: ${duration}s`); + } const failures = testStatus.filter((t) => t.status === 'fail'); if (failures.length > 0) { lines.push('', ` Failed tests (${failures.length}):`); for (const failure of failures) { - const testPath = buildTestPath(failure.id, handlers) ?? failure.id; + const testPath = resolvePath(failure, handlers); lines.push(` × ${testPath}`); if (failure.error) { lines.push(` ${String(failure.error).replace(/\n/g, '\n ')}`); @@ -36,7 +65,7 @@ export function formatRunComplete({ if (retried.length > 0) { lines.push('', ` Retried (${retried.length}):`); for (const t of retried) { - const testPath = buildTestPath(t.id, handlers) ?? t.id; + const testPath = resolvePath(t, handlers); lines.push(` ✓ ${testPath} (passed on attempt ${t.retryAttempt})`); } } diff --git a/tests/mergeCommand.test.js b/tests/mergeCommand.test.js new file mode 100644 index 0000000..e7f663a --- /dev/null +++ b/tests/mergeCommand.test.js @@ -0,0 +1,343 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +vi.mock('fs'); +vi.mock('../src/config.js', () => ({ loadConfig: vi.fn() })); +vi.mock('../src/contractReport.js', () => ({ printContractReport: vi.fn() })); +vi.mock('../src/reportFiles.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readShardReports: vi.fn(), readShardCoverage: vi.fn() }; +}); + +import fs from 'fs'; +import { loadConfig } from '../src/config.js'; +import { printContractReport } from '../src/contractReport.js'; +import { readShardReports, readShardCoverage } from '../src/reportFiles.js'; +import { runMerge } from '../src/mergeCommand.js'; +import { REPORT_SCHEMA_VERSION } from '../src/runReport.js'; + +const HANDLERS = [ + { id: 's1', name: 'Login', parent: null, type: 'suite' }, + { id: 't1', name: 'a', parent: 's1', type: 'test' }, + { id: 't2', name: 'b', parent: 's1', type: 'test' }, +]; + +// Only shard 1's ids exist in HANDLERS, matching reality: twd-js mints ids with +// Math.random() at registration, so each shard's browser invents its own. Every +// entry carries the path its own shard resolved, plus its position in the +// discovered order as the cross-shard identity. +function shardReport(index, overrides = {}) { + const { + total = 2, + tests = [{ id: index === 1 ? 't1' : `r${index}-x`, path: `Login > ${'ab'[index - 1]}`, index: index - 1, status: 'pass' }], + failed = 0, + selectedTests = 2, + } = overrides; + return { + schemaVersion: REPORT_SCHEMA_VERSION, + shards: [{ + index, total, + startedAt: `2026-08-19T10:00:0${index}.000Z`, + endedAt: `2026-08-19T10:00:1${index}.000Z`, + durationMs: 10_000, + executed: 1, notRun: 0, failed, + stoppedEarly: false, coverageFile: 'coverage.json', recording: null, + }], + discovery: { totalTests: 2, fingerprint: 'sha256:same' }, + selection: { filters: [], selectedTests }, + handlers: HANDLERS, + tests, + contracts: { configured: false, partial: false, results: [], skipped: [] }, + }; +} + +const baseConfig = { + coverage: true, + nycOutputDir: './.nyc_output', + maxFailures: 10, +}; + +function writtenFiles() { + return vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f)); +} + +describe('runMerge', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadConfig).mockReturnValue({ ...baseConfig }); + vi.mocked(readShardCoverage).mockReturnValue(null); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('requires a directory', () => { + expect(() => runMerge({})).toThrow(/Usage: twd-cli merge/); + }); + + it('errors when no shard reports were found', () => { + vi.mocked(readShardReports).mockReturnValue([]); + expect(() => runMerge({ dir: '.twd/shards' })).toThrow(/No shard reports found/); + }); + + // The failure that must never be silent: three green shards and one that + // never uploaded would otherwise read as a complete green run. + it('errors and names the gap when a shard is missing', () => { + vi.mocked(readShardReports).mockReturnValue([{ dir: 'a', report: shardReport(1) }]); + expect(() => runMerge({ dir: '.twd/shards' })).toThrow(/Missing shard report\(s\): 2\/2/); + }); + + it('mentions if: always() in the missing-shard message', () => { + vi.mocked(readShardReports).mockReturnValue([{ dir: 'a', report: shardReport(2) }]); + expect(() => runMerge({ dir: '.twd/shards' })).toThrow(/if: always\(\)/); + }); + + it('writes the merged report to the default path', () => { + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2) }, + ]); + + expect(runMerge({ dir: '.twd/shards' })).toBe(false); + + const merged = writtenFiles().find((f) => f.endsWith('merged-run.json')); + expect(merged).toBeDefined(); + }); + + it('honors --out', () => { + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2) }, + ]); + + runMerge({ dir: '.twd/shards', out: 'custom.json' }); + + expect(writtenFiles().some((f) => f.endsWith('custom.json'))).toBe(true); + }); + + // readShardReports returns readdir order, which is lexicographic, so + // twd-run-10 comes back before twd-run-2. mergeRunReports sorts shards[] by + // index but concatenates tests in argument order, so without a sort here the + // merged artifact's test order and its shard list would disagree. + it('merges shards in index order whatever order they were found in', () => { + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'twd-run-2', report: shardReport(2) }, + { dir: 'twd-run-1', report: shardReport(1) }, + ]); + + runMerge({ dir: '.twd/shards' }); + + const call = vi.mocked(fs.writeFileSync).mock.calls + .find(([f]) => String(f).endsWith('merged-run.json')); + const merged = JSON.parse(String(call[1])); + expect(merged.tests.map((t) => t.index)).toEqual([0, 1]); + expect(merged.shards.map((s) => s.index)).toEqual([1, 2]); + }); + + it('returns true when any shard had a failing test', () => { + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2, { tests: [{ id: 't2', status: 'fail', error: 'boom' }], failed: 1 }) }, + ]); + + expect(runMerge({ dir: '.twd/shards' })).toBe(true); + }); + + it('merges coverage when the run is green', () => { + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2) }, + ]); + vi.mocked(readShardCoverage).mockReturnValue({}); + + runMerge({ dir: '.twd/shards' }); + + expect(writtenFiles().some((f) => f.includes('.nyc_output'))).toBe(true); + }); + + // The user's rule, applied to the true global result rather than one shard's. + it('skips merged coverage when the run is red', () => { + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2, { tests: [{ id: 't2', status: 'fail', error: 'boom' }], failed: 1 }) }, + ]); + vi.mocked(readShardCoverage).mockReturnValue({}); + + runMerge({ dir: '.twd/shards' }); + + expect(writtenFiles().some((f) => f.includes('.nyc_output'))).toBe(false); + }); + + it('reports how many shards contributed coverage', () => { + const log = vi.spyOn(console, 'log'); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2) }, + ]); + vi.mocked(readShardCoverage).mockReturnValueOnce({}).mockReturnValueOnce(null); + + runMerge({ dir: '.twd/shards' }); + + expect(log.mock.calls.flat().join('\n')).toMatch(/Coverage merged from 1\/2 shards/); + }); + + it('says so when no shard had coverage', () => { + const log = vi.spyOn(console, 'log'); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2) }, + ]); + + runMerge({ dir: '.twd/shards' }); + + expect(log.mock.calls.flat().join('\n')).toMatch(/No coverage data found/); + }); + + it('returns true when contracts report an error-mode violation', () => { + const withContracts = (i) => ({ + ...shardReport(i), + contracts: { configured: true, partial: false, results: [{ alias: 'a' }], skipped: [] }, + }); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: withContracts(1) }, + { dir: 'b', report: withContracts(2) }, + ]); + vi.mocked(printContractReport).mockReturnValue(true); + + expect(runMerge({ dir: '.twd/shards' })).toBe(true); + }); + + it('warns when contract data is partial', () => { + const warn = vi.spyOn(console, 'warn'); + const partial = (i, isPartial) => ({ + ...shardReport(i), + contracts: { configured: true, partial: isPartial, results: [], skipped: [] }, + }); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: partial(1, false) }, + { dir: 'b', report: partial(2, true) }, + ]); + vi.mocked(printContractReport).mockReturnValue(false); + + runMerge({ dir: '.twd/shards' }); + + expect(warn.mock.calls.flat().join('\n')).toMatch(/contract data is partial/i); + }); + + // A sharded run skips the markdown report on purpose — each shard would + // overwrite the others with a quarter of the picture. Merge is where the whole + // picture exists, so this is the only place it can be written. + it('writes the contract markdown report that sharded runs skip', () => { + vi.mocked(loadConfig).mockReturnValue({ ...baseConfig, contractReportPath: './contract-report.md' }); + const configured = (i) => ({ + ...shardReport(i), + contracts: { configured: true, partial: false, results: [], skipped: [] }, + }); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: configured(1) }, + { dir: 'b', report: configured(2) }, + ]); + vi.mocked(printContractReport).mockReturnValue(false); + + runMerge({ dir: '.twd/shards' }); + + expect(writtenFiles().some((f) => f.endsWith('contract-report.md'))).toBe(true); + }); + + // Executed + not-run has to account for every selected test. When it does + // not, the shard math dropped tests on the floor and nothing else would say so. + it('warns when the shard totals do not account for every selected test', () => { + const warn = vi.spyOn(console, 'warn'); + const wrongTotal = (i) => shardReport(i, { selectedTests: 3 }); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: wrongTotal(1) }, + { dir: 'b', report: wrongTotal(2) }, + ]); + + runMerge({ dir: '.twd/shards' }); + + expect(warn.mock.calls.flat().join('\n')).toMatch(/shard totals do not add up/); + }); + + // The false alarm this replaces: a --test filter narrows what the shards + // divide, so a correct filtered run has executed + notRun below + // discovery.totalTests and used to be reported as a shard-slicing bug. + it('does not warn when a filter narrowed what the shards divided', () => { + const warn = vi.spyOn(console, 'warn'); + const filtered = (i) => { + const report = shardReport(i); + report.discovery = { totalTests: 40, fingerprint: 'sha256:same' }; + report.selection = { filters: ['Login'], selectedTests: 2 }; + return report; + }; + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: filtered(1) }, + { dir: 'b', report: filtered(2) }, + ]); + + runMerge({ dir: '.twd/shards' }); + + expect(warn.mock.calls.flat().join('\n')).not.toMatch(/do not add up/); + }); + + it('prints the merged run-complete block with a shard breakdown', () => { + const log = vi.spyOn(console, 'log'); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { dir: 'b', report: shardReport(2) }, + ]); + + runMerge({ dir: '.twd/shards' }); + + const output = log.mock.calls.flat().join('\n'); + expect(output).toContain('--- Run complete ---'); + expect(output).toContain('Shards: 1 ✓1 | 2 ✓1'); + }); + + // The merged summary is the one output the merge exists to produce, and a + // failure from any shard but the first used to render as a raw random id: the + // merged report keeps only shard 1's handlers, which cannot resolve shard 2's + // ids. Each entry now carries the path its own shard resolved. + it('names failed tests from later shards instead of printing their raw ids', () => { + const log = vi.spyOn(console, 'log'); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { + dir: 'b', + report: shardReport(2, { + failed: 1, + tests: [{ id: 'k3j2h1g9d', path: 'Login > b', index: 1, status: 'fail', error: 'boom' }], + }), + }, + ]); + + const hasFailures = runMerge({ dir: '.twd/shards' }); + + const output = log.mock.calls.flat().join('\n'); + expect(hasFailures).toBe(true); + expect(output).toContain('× Login > b'); + expect(output).not.toContain('k3j2h1g9d'); + }); + + // Retried tests render through the same path resolution. + it('names retried tests from later shards', () => { + const log = vi.spyOn(console, 'log'); + vi.mocked(readShardReports).mockReturnValue([ + { dir: 'a', report: shardReport(1) }, + { + dir: 'b', + report: shardReport(2, { + tests: [{ id: 'zzzflaky', path: 'Login > b', index: 1, status: 'pass', retryAttempt: 2 }], + }), + }, + ]); + + runMerge({ dir: '.twd/shards' }); + + const output = log.mock.calls.flat().join('\n'); + expect(output).toContain('✓ Login > b (passed on attempt 2)'); + expect(output).not.toContain('zzzflaky'); + }); +}); diff --git a/tests/mergeCoverage.test.js b/tests/mergeCoverage.test.js new file mode 100644 index 0000000..97bfdb7 --- /dev/null +++ b/tests/mergeCoverage.test.js @@ -0,0 +1,70 @@ +import { describe, it, expect } from 'vitest'; +import { mergeCoverage } from '../src/mergeCoverage.js'; + +// Minimal but structurally valid Istanbul file coverage. istanbul-lib-coverage +// validates the shape, so the maps cannot be omitted. +function fileCoverage(path, statementHits, fnHits = 0) { + return { + path, + statementMap: { 0: { start: { line: 1, column: 0 }, end: { line: 1, column: 10 } } }, + fnMap: { + 0: { + name: 'f', + decl: { start: { line: 1, column: 0 }, end: { line: 1, column: 1 } }, + loc: { start: { line: 1, column: 0 }, end: { line: 3, column: 1 } }, + }, + }, + branchMap: {}, + s: { 0: statementHits }, + f: { 0: fnHits }, + b: {}, + }; +} + +describe('mergeCoverage', () => { + it('sums statement hits for the same file across shards', () => { + const merged = mergeCoverage([ + { '/app/src/a.js': fileCoverage('/app/src/a.js', 1) }, + { '/app/src/a.js': fileCoverage('/app/src/a.js', 2) }, + ]); + expect(merged['/app/src/a.js'].s[0]).toBe(3); + }); + + it('sums function hits for the same file across shards', () => { + const merged = mergeCoverage([ + { '/app/src/a.js': fileCoverage('/app/src/a.js', 1, 4) }, + { '/app/src/a.js': fileCoverage('/app/src/a.js', 1, 5) }, + ]); + expect(merged['/app/src/a.js'].f[0]).toBe(9); + }); + + it('unions files that only one shard touched', () => { + const merged = mergeCoverage([ + { '/app/src/a.js': fileCoverage('/app/src/a.js', 1) }, + { '/app/src/b.js': fileCoverage('/app/src/b.js', 7) }, + ]); + expect(Object.keys(merged).sort()).toEqual(['/app/src/a.js', '/app/src/b.js']); + expect(merged['/app/src/b.js'].s[0]).toBe(7); + }); + + // A shard with no coverage file reads back as null and must not break the merge. + it('skips null and undefined entries', () => { + const merged = mergeCoverage([ + null, + { '/app/src/a.js': fileCoverage('/app/src/a.js', 2) }, + undefined, + ]); + expect(merged['/app/src/a.js'].s[0]).toBe(2); + }); + + it('returns an empty map for no input', () => { + expect(mergeCoverage([])).toEqual({}); + expect(mergeCoverage([null])).toEqual({}); + }); + + it('does not mutate its inputs', () => { + const first = { '/app/src/a.js': fileCoverage('/app/src/a.js', 1) }; + mergeCoverage([first, { '/app/src/a.js': fileCoverage('/app/src/a.js', 5) }]); + expect(first['/app/src/a.js'].s[0]).toBe(1); + }); +}); diff --git a/tests/mergeReports.test.js b/tests/mergeReports.test.js new file mode 100644 index 0000000..33f3c89 --- /dev/null +++ b/tests/mergeReports.test.js @@ -0,0 +1,251 @@ +import { describe, it, expect } from 'vitest'; +import { + mergeRunReports, + findMissingShards, + reportTimings, + reportTotals, +} from '../src/mergeReports.js'; +import { REPORT_SCHEMA_VERSION } from '../src/runReport.js'; + +const HANDLERS = [ + { id: 's1', name: 'Login', parent: null, type: 'suite' }, + { id: 't1', name: 'a', parent: 's1', type: 'test' }, + { id: 't2', name: 'b', parent: 's1', type: 'test' }, + { id: 't3', name: 'c', parent: 's1', type: 'test' }, +]; + +const FINGERPRINT = 'sha256:deadbeef'; + +// Ids are per-page-load random in reality, so shard n's report carries ids +// nothing else can resolve. index (position in the discovered order) and path +// are what stay stable, and they are what the merge relies on. +function makeReport(index, overrides = {}) { + const { + total = 3, + tests = [{ id: `r${index}-t`, path: `Login > ${'abc'[index - 1]}`, index: index - 1, status: 'pass' }], + startedAt = `2026-08-19T10:00:0${index}.000Z`, + endedAt = `2026-08-19T10:00:1${index}.000Z`, + durationMs = 10_000, + executed = 1, + notRun = 0, + failed = 0, + stoppedEarly = false, + coverageFile = 'coverage.json', + contracts = { configured: true, partial: false, results: [], skipped: [] }, + fingerprint = FINGERPRINT, + schemaVersion = REPORT_SCHEMA_VERSION, + totalTests = 3, + selectedTests = 3, + } = overrides; + + return { + schemaVersion, + shards: [{ + index, total, startedAt, endedAt, durationMs, + executed, notRun, failed, stoppedEarly, coverageFile, recording: null, + }], + discovery: { totalTests, fingerprint }, + selection: { filters: [], selectedTests }, + handlers: HANDLERS, + tests, + contracts, + }; +} + +describe('mergeRunReports', () => { + it('concatenates tests across shards', () => { + const merged = mergeRunReports([makeReport(1), makeReport(2), makeReport(3)]); + expect(merged.tests.map((t) => t.index)).toEqual([0, 1, 2]); + expect(merged.tests.map((t) => t.path)).toEqual(['Login > a', 'Login > b', 'Login > c']); + }); + + it('sorts shard descriptors by index regardless of input order', () => { + const merged = mergeRunReports([makeReport(3), makeReport(1), makeReport(2)]); + expect(merged.shards.map((s) => s.index)).toEqual([1, 2, 3]); + }); + + it('keeps the single-report shape', () => { + const merged = mergeRunReports([makeReport(1), makeReport(2), makeReport(3)]); + expect(Object.keys(merged).sort()).toEqual( + ['contracts', 'discovery', 'handlers', 'schemaVersion', 'selection', 'shards', 'tests'], + ); + expect(merged.handlers).toEqual(HANDLERS); + expect(merged.discovery).toEqual({ totalTests: 3, fingerprint: FINGERPRINT }); + }); + + // Locks the documented first-wins semantics. These fields are invariant across + // valid shards by construction, so nothing upstream distinguishes first from + // last — only this test does. + it('takes discovery, selection, handlers and contracts.configured from the first report', () => { + const first = makeReport(1, { totalTests: 3 }); + const second = makeReport(2, { totalTests: 3 }); + second.discovery = { ...second.discovery, totalTests: 99 }; + second.selection = { filters: ['not-the-first'] }; + second.handlers = [{ id: 'other', name: 'Other', parent: null, type: 'suite' }]; + second.contracts = { ...second.contracts, configured: false }; + + const merged = mergeRunReports([first, second]); + + expect(merged.discovery.totalTests).toBe(3); + expect(merged.selection).toEqual({ filters: [], selectedTests: 3 }); + expect(merged.handlers).toEqual(HANDLERS); + expect(merged.contracts.configured).toBe(true); + }); + + // The property that proves nothing is lost or doubled. It only holds because + // completeness is checked outside this function. + it('is associative', () => { + const a = makeReport(1); + const b = makeReport(2); + const c = makeReport(3); + expect(mergeRunReports([mergeRunReports([a, b]), c])) + .toEqual(mergeRunReports([a, b, c])); + expect(mergeRunReports([a, mergeRunReports([b, c])])) + .toEqual(mergeRunReports([a, b, c])); + }); + + it('accepts a partial merge without complaining about gaps', () => { + const merged = mergeRunReports([makeReport(1), makeReport(2)]); + expect(merged.shards.map((s) => s.index)).toEqual([1, 2]); + }); + + it('concatenates contract results and skipped entries', () => { + const merged = mergeRunReports([ + makeReport(1, { contracts: { configured: true, partial: false, results: [{ alias: 'a' }], skipped: [{ alias: 'x' }] } }), + makeReport(2, { contracts: { configured: true, partial: false, results: [{ alias: 'b' }], skipped: [] } }), + ]); + expect(merged.contracts.results).toEqual([{ alias: 'a' }, { alias: 'b' }]); + expect(merged.contracts.skipped).toEqual([{ alias: 'x' }]); + }); + + it('ORs the contracts partial flag', () => { + const partial = makeReport(2, { contracts: { configured: true, partial: true, results: [], skipped: [] } }); + expect(mergeRunReports([makeReport(1), partial]).contracts.partial).toBe(true); + expect(mergeRunReports([makeReport(1), makeReport(2)]).contracts.partial).toBe(false); + }); + + it('throws on an empty input', () => { + expect(() => mergeRunReports([])).toThrow(/No shard reports/); + }); + + it('throws when schema versions disagree', () => { + expect(() => mergeRunReports([ + makeReport(1), + makeReport(2, { schemaVersion: REPORT_SCHEMA_VERSION + 1 }), + ])).toThrow(/schemaVersion/); + }); + + // Agreeing with each other is not enough. Reports from a newer twd-cli agree, + // and merging them against a schema this binary does not know is the silent + // mis-merge the field exists to prevent. + it('throws when every shard agrees on a version this build does not read', () => { + const future = REPORT_SCHEMA_VERSION + 1; + expect(() => mergeRunReports([ + makeReport(1, { schemaVersion: future }), + makeReport(2, { schemaVersion: future }), + ])).toThrow(new RegExp(`schema v${future}, but this twd-cli reads v${REPORT_SCHEMA_VERSION}`)); + }); + + // The safety net: shards that saw different test sets must never be combined. + it('throws when fingerprints disagree', () => { + expect(() => mergeRunReports([makeReport(1), makeReport(2, { fingerprint: 'sha256:other' })])) + .toThrow(/different test sets/); + }); + + it('throws when shard totals disagree', () => { + expect(() => mergeRunReports([makeReport(1), makeReport(2, { total: 4 })])) + .toThrow(/shard total/); + }); + + it('throws when the same shard index appears twice', () => { + expect(() => mergeRunReports([makeReport(1), makeReport(1)])) + .toThrow(/more than once/); + }); + + // Overlap is keyed on position, not id: shards never agree on ids, so the old + // id check could not fire at all. + it('throws when the same position appears in two shards', () => { + expect(() => mergeRunReports([ + makeReport(1, { tests: [{ id: 'aaa', path: 'Login > a', index: 0, status: 'pass' }] }), + makeReport(2, { tests: [{ id: 'zzz', path: 'Login > a', index: 0, status: 'pass' }] }), + ])).toThrow(/"Login > a" \(position 0\) appears in more than one shard/); + }); + + // Two tests may legitimately share a "suite > test" path and land in + // different shards. Keying identity on the path would fail a correct run. + it('accepts duplicate paths in different shards when the positions differ', () => { + const merged = mergeRunReports([ + makeReport(1, { tests: [{ id: 'aaa', path: 'Login > a', index: 0, status: 'pass' }] }), + makeReport(2, { tests: [{ id: 'zzz', path: 'Login > a', index: 1, status: 'pass' }] }), + ]); + expect(merged.tests).toHaveLength(2); + }); + + it('does not mutate the input reports', () => { + const a = makeReport(1); + const b = makeReport(2); + mergeRunReports([b, a]); + expect(a.shards).toHaveLength(1); + expect(b.shards[0].index).toBe(2); + }); +}); + +describe('findMissingShards', () => { + it('returns an empty array when every shard is present', () => { + expect(findMissingShards(mergeRunReports([makeReport(1), makeReport(2), makeReport(3)]))) + .toEqual([]); + }); + + it('names the gaps', () => { + expect(findMissingShards(mergeRunReports([makeReport(1), makeReport(3)]))).toEqual([2]); + expect(findMissingShards(mergeRunReports([makeReport(2)]))).toEqual([1, 3]); + }); +}); + +describe('reportTimings', () => { + // Wall clock is what the developer waited; compute is what was paid for. + it('reports wall clock as the span and compute as the sum', () => { + const merged = mergeRunReports([makeReport(1), makeReport(2), makeReport(3)]); + // starts 10:00:01..03, ends 10:00:11..13 -> span 12s; 3 x 10s compute + expect(reportTimings(merged)).toEqual({ wallMs: 12_000, computeMs: 30_000 }); + }); + + it('makes wall and compute equal for a single shard', () => { + const single = mergeRunReports([makeReport(1, { total: 1 })]); + const { wallMs, computeMs } = reportTimings(single); + expect(wallMs).toBe(computeMs); + }); +}); + +describe('reportTotals', () => { + it('sums executed and notRun and confirms they account for the selection', () => { + const merged = mergeRunReports([makeReport(1), makeReport(2), makeReport(3)]); + expect(reportTotals(merged)) + .toEqual({ executed: 3, notRun: 0, expected: 3, consistent: true }); + }); + + it('flags totals that do not add up to the selected count', () => { + const merged = mergeRunReports([makeReport(1), makeReport(2)]); + expect(reportTotals(merged).consistent).toBe(false); + }); + + it('counts a bailed shard\'s notRun', () => { + const merged = mergeRunReports([ + makeReport(1), + makeReport(2, { executed: 1, notRun: 1, stoppedEarly: true, failed: 1 }), + makeReport(3), + ]); + expect(reportTotals(merged)) + .toEqual({ executed: 3, notRun: 1, expected: 3, consistent: false }); + }); + + // The regression this fixes: --test narrows what the shards divide, so + // comparing against every discovered test called a correct run a slicing bug. + it('measures a filtered run against the filtered count, not the whole suite', () => { + const merged = mergeRunReports([ + makeReport(1, { totalTests: 40, selectedTests: 2, tests: [] }), + makeReport(2, { totalTests: 40, selectedTests: 2, tests: [] }), + ]); + expect(reportTotals(merged)).toEqual({ executed: 2, notRun: 0, expected: 2, consistent: true }); + }); +}); diff --git a/tests/parseArgs.test.js b/tests/parseArgs.test.js index b2eb066..d9103a4 100644 --- a/tests/parseArgs.test.js +++ b/tests/parseArgs.test.js @@ -1,15 +1,17 @@ import { describe, it, expect } from "vitest"; -import { parseRunArgs } from "../src/parseArgs.js"; +import { parseRunArgs, parseMergeArgs } from "../src/parseArgs.js"; describe("parseRunArgs", () => { it("returns empty filters when no args", () => { - expect(parseRunArgs([])).toEqual({ testFilters: [], record: {} }); + expect(parseRunArgs([])).toEqual({ testFilters: [], record: {}, shard: null, reportDir: null }); }); it("parses a single --test ", () => { expect(parseRunArgs(['--test', 'shows error'])).toEqual({ testFilters: ['shows error'], record: {}, + shard: null, + reportDir: null, }); }); @@ -17,6 +19,8 @@ describe("parseRunArgs", () => { expect(parseRunArgs(['--test', 'Login', '--test', 'Signup'])).toEqual({ testFilters: ['Login', 'Signup'], record: {}, + shard: null, + reportDir: null, }); }); @@ -24,17 +28,21 @@ describe("parseRunArgs", () => { expect(parseRunArgs(['--test=Login'])).toEqual({ testFilters: ['Login'], record: {}, + shard: null, + reportDir: null, }); }); it("ignores a trailing --test with no value", () => { - expect(parseRunArgs(['--test'])).toEqual({ testFilters: [], record: {} }); + expect(parseRunArgs(['--test'])).toEqual({ testFilters: [], record: {}, shard: null, reportDir: null }); }); it("ignores unknown tokens", () => { expect(parseRunArgs(['--verbose', '--test', 'Login'])).toEqual({ testFilters: ['Login'], record: {}, + shard: null, + reportDir: null, }); }); @@ -71,6 +79,8 @@ describe("parseRunArgs", () => { expect(parseRunArgs(['--record', '--test', 'checkout', '--record-speed=0.5'])).toEqual({ testFilters: ['checkout'], record: { enabled: true, speed: 0.5 }, + shard: null, + reportDir: null, }); }); @@ -93,7 +103,65 @@ describe("parseRunArgs", () => { expect(parseRunArgs(['--record', '--test', 'checkout', '--record-pace=500'])).toEqual({ testFilters: ['checkout'], record: { enabled: true, pace: 500 }, + shard: null, + reportDir: null, }); }); }); + +describe('parseRunArgs shard and report flags', () => { + it('parses --shard in both forms', () => { + expect(parseRunArgs(['--shard', '2/4']).shard).toEqual({ index: 2, total: 4 }); + expect(parseRunArgs(['--shard=3/4']).shard).toEqual({ index: 3, total: 4 }); + }); + + it('throws on an invalid --shard instead of ignoring it', () => { + expect(() => parseRunArgs(['--shard', '5/4'])).toThrow(/Invalid --shard/); + expect(() => parseRunArgs(['--shard=abc'])).toThrow(/Invalid --shard/); + }); + + it('throws on a trailing --shard with no value', () => { + expect(() => parseRunArgs(['--shard'])).toThrow(/Invalid --shard/); + }); + + it('parses --report-dir in both forms', () => { + expect(parseRunArgs(['--report-dir', './out']).reportDir).toBe('./out'); + expect(parseRunArgs(['--report-dir=./out']).reportDir).toBe('./out'); + }); + + it('ignores a trailing --report-dir with no value', () => { + expect(parseRunArgs(['--report-dir']).reportDir).toBeNull(); + }); + + it('combines --shard with --test filters and record flags', () => { + expect(parseRunArgs(['--shard', '2/4', '--test', 'Login', '--record'])).toEqual({ + testFilters: ['Login'], + record: { enabled: true }, + shard: { index: 2, total: 4 }, + reportDir: null, + }); + }); +}); + +describe('parseMergeArgs', () => { + it('reads the directory as the first positional', () => { + expect(parseMergeArgs(['.twd/shards'])).toEqual({ dir: '.twd/shards', out: null }); + }); + + it('parses --out in both forms', () => { + expect(parseMergeArgs(['.twd/shards', '--out', 'merged.json'])) + .toEqual({ dir: '.twd/shards', out: 'merged.json' }); + expect(parseMergeArgs(['.twd/shards', '--out=merged.json'])) + .toEqual({ dir: '.twd/shards', out: 'merged.json' }); + }); + + it('returns a null dir when none is given', () => { + expect(parseMergeArgs([])).toEqual({ dir: null, out: null }); + expect(parseMergeArgs(['--out=merged.json'])).toEqual({ dir: null, out: 'merged.json' }); + }); + + it('takes only the first positional as the directory', () => { + expect(parseMergeArgs(['a', 'b']).dir).toBe('a'); + }); +}); diff --git a/tests/reportFiles.test.js b/tests/reportFiles.test.js new file mode 100644 index 0000000..da203bd --- /dev/null +++ b/tests/reportFiles.test.js @@ -0,0 +1,134 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +vi.mock('fs'); + +import fs from 'fs'; +import { + writeRunReport, + readShardReports, + readShardCoverage, + RUN_REPORT_FILE, + COVERAGE_FILE, + DEFAULT_REPORT_DIR, +} from '../src/reportFiles.js'; + +const report = { schemaVersion: 1, tests: [] }; + +describe('writeRunReport', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('creates the directory recursively', () => { + writeRunReport('.twd/run', report, null); + expect(fs.mkdirSync).toHaveBeenCalledWith('.twd/run', { recursive: true }); + }); + + it('writes pretty-printed JSON so the report is readable by eye', () => { + writeRunReport('.twd/run', report, null); + const [file, body] = vi.mocked(fs.writeFileSync).mock.calls[0]; + expect(file).toBe(`.twd/run/${RUN_REPORT_FILE}`); + expect(body).toBe(`${JSON.stringify(report, null, 2)}\n`); + }); + + it('does not write a coverage file when there is no coverage', () => { + writeRunReport('.twd/run', report, null); + expect(fs.writeFileSync).toHaveBeenCalledTimes(1); + expect(writeRunReport('.twd/run', report, null).coveragePath).toBeNull(); + }); + + // Coverage stays raw and unformatted: it is machine input for nyc, routinely + // several megabytes, and pretty-printing it would double the artifact size. + it('writes coverage compactly alongside the report', () => { + const coverage = { '/a.js': { s: { 0: 1 } } }; + const result = writeRunReport('.twd/run', report, coverage); + expect(fs.writeFileSync).toHaveBeenCalledWith( + `.twd/run/${COVERAGE_FILE}`, + JSON.stringify(coverage), + ); + expect(result.coveragePath).toBe(`.twd/run/${COVERAGE_FILE}`); + }); +}); + +describe('readShardReports', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + // This is download-artifact's layout: one directory per artifact name. + it('reads run.json from each child directory', () => { + vi.mocked(fs.existsSync).mockImplementation((p) => String(p).endsWith(RUN_REPORT_FILE)); + vi.mocked(fs.readdirSync).mockReturnValue([ + { name: 'twd-run-1', isDirectory: () => true }, + { name: 'twd-run-2', isDirectory: () => true }, + { name: 'notes.txt', isDirectory: () => false }, + ]); + vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(report)); + + const found = readShardReports('.twd/shards'); + + expect(found.map((f) => f.dir)).toEqual([ + '.twd/shards', + '.twd/shards/twd-run-1', + '.twd/shards/twd-run-2', + ]); + expect(found[0].report).toEqual(report); + }); + + it('skips child directories with no run.json', () => { + vi.mocked(fs.existsSync).mockImplementation((p) => String(p).includes('twd-run-1')); + vi.mocked(fs.readdirSync).mockReturnValue([ + { name: 'twd-run-1', isDirectory: () => true }, + { name: 'empty', isDirectory: () => true }, + ]); + vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify(report)); + + expect(readShardReports('.twd/shards').map((f) => f.dir)) + .toEqual(['.twd/shards/twd-run-1']); + }); + + it('returns an empty array when the directory does not exist', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + vi.mocked(fs.readdirSync).mockImplementation(() => { + throw Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + }); + expect(readShardReports('.twd/nope')).toEqual([]); + }); + + it('explains which file failed to parse', () => { + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readdirSync).mockReturnValue([]); + vi.mocked(fs.readFileSync).mockReturnValue('{ not json'); + expect(() => readShardReports('.twd/shards')).toThrow(/run\.json/); + }); +}); + +describe('readShardCoverage', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('reads the named coverage file from the shard directory', () => { + vi.mocked(fs.existsSync).mockReturnValue(true); + vi.mocked(fs.readFileSync).mockReturnValue('{"/a.js":{}}'); + expect(readShardCoverage('.twd/shards/twd-run-1', COVERAGE_FILE)).toEqual({ '/a.js': {} }); + }); + + it('returns null when the shard recorded no coverage file', () => { + expect(readShardCoverage('.twd/shards/twd-run-1', null)).toBeNull(); + expect(fs.existsSync).not.toHaveBeenCalled(); + }); + + // A shard whose coverage file is absent simply does not contribute; merge + // reports the contributor count rather than failing. + it('returns null when the file is missing on disk', () => { + vi.mocked(fs.existsSync).mockReturnValue(false); + expect(readShardCoverage('.twd/shards/twd-run-1', COVERAGE_FILE)).toBeNull(); + }); +}); + +describe('defaults', () => { + it('defaults the report dir to .twd/run', () => { + expect(DEFAULT_REPORT_DIR).toBe('./.twd/run'); + }); +}); diff --git a/tests/runReport.test.js b/tests/runReport.test.js new file mode 100644 index 0000000..0a68ae0 --- /dev/null +++ b/tests/runReport.test.js @@ -0,0 +1,184 @@ +import { describe, it, expect } from 'vitest'; +import { buildRunReport, fingerprintTests, REPORT_SCHEMA_VERSION } from '../src/runReport.js'; + +const handlers = [ + { id: 's1', name: 'Login', parent: null, type: 'suite' }, + { id: 't1', name: 'works', parent: 's1', type: 'test' }, + { id: 't2', name: 'also works', parent: 's1', type: 'test' }, + { id: 't3', name: 'still works', parent: 's1', type: 'test' }, +]; + +const PATHS = ['Login > works', 'Login > also works', 'Login > still works']; + +function build(overrides = {}) { + return buildRunReport({ + shard: { index: 2, total: 4 }, + startedAt: 1_000, + endedAt: 4_500, + allTestIds: ['t1', 't2', 't3'], + filters: [], + handlers, + tests: [{ id: 't1', status: 'pass' }], + executed: 1, + notRun: 0, + stoppedEarly: false, + ...overrides, + }); +} + +describe('fingerprintTests', () => { + it('is stable for the same input', () => { + expect(fingerprintTests(['a', 'b'])).toBe(fingerprintTests(['a', 'b'])); + }); + + it('is prefixed with the algorithm', () => { + expect(fingerprintTests(['a'])).toMatch(/^sha256:[0-9a-f]{64}$/); + }); + + // Order matters: round-robin slicing is only correct if every shard sees the + // same list in the same order. + it('changes when the path order changes', () => { + expect(fingerprintTests(['a', 'b'])).not.toBe(fingerprintTests(['b', 'a'])); + }); + + it('changes when the path set changes', () => { + expect(fingerprintTests(['a', 'b'])).not.toBe(fingerprintTests(['a', 'b', 'c'])); + }); + + it('changes when the filters differ', () => { + expect(fingerprintTests(['a'], ['Login'])).not.toBe(fingerprintTests(['a'], ['Cart'])); + expect(fingerprintTests(['a'], [])).not.toBe(fingerprintTests(['a'], ['Login'])); + }); + + // Filters are OR'd, so their order is not meaningful and must not split + // otherwise-identical shards. + it('ignores the order the filters were given in', () => { + expect(fingerprintTests(['a'], ['Login', 'Cart'])) + .toBe(fingerprintTests(['a'], ['Cart', 'Login'])); + }); +}); + +describe('buildRunReport', () => { + // Pins the literal on purpose. Every other assertion in the suite derives + // from REPORT_SCHEMA_VERSION, so editing the constant would otherwise leave + // all 438 tests green — and the version's only job is to be a tripwire for + // shards produced by mismatched twd-cli builds. + it('is at schema version 2', () => { + expect(REPORT_SCHEMA_VERSION).toBe(2); + }); + + it('stamps the schema version', () => { + expect(build().schemaVersion).toBe(REPORT_SCHEMA_VERSION); + }); + + it('wraps a single shard descriptor in an array', () => { + const report = build(); + expect(report.shards).toHaveLength(1); + expect(report.shards[0]).toMatchObject({ + index: 2, total: 4, executed: 1, notRun: 0, failed: 0, stoppedEarly: false, + }); + }); + + it('derives durationMs and ISO timestamps from epoch millis', () => { + const shard = build().shards[0]; + expect(shard.durationMs).toBe(3500); + expect(shard.startedAt).toBe(new Date(1_000).toISOString()); + expect(shard.endedAt).toBe(new Date(4_500).toISOString()); + }); + + it('counts this shard\'s failures', () => { + const report = build({ + tests: [ + { id: 't1', status: 'pass' }, + { id: 't2', status: 'fail', error: 'boom' }, + { id: 't3', status: 'skip' }, + ], + }); + expect(report.shards[0].failed).toBe(1); + }); + + it('records total discovered tests and the fingerprint', () => { + const report = build(); + expect(report.discovery.totalTests).toBe(3); + expect(report.discovery.fingerprint).toBe(fingerprintTests(PATHS, [])); + }); + + // The whole point of the path-based fingerprint: twd-js ids are Math.random() + // per page load, so two shards of the same suite never agree on ids. If the + // fingerprint were keyed on them, merge would reject every correct run. + it('fingerprints the same suite identically when the ids differ', () => { + const shardTwoHandlers = handlers.map((h) => ({ + ...h, + id: `x${h.id}`, + parent: h.parent ? `x${h.parent}` : h.parent, + })); + const shardTwo = build({ + allTestIds: ['xt1', 'xt2', 'xt3'], + handlers: shardTwoHandlers, + tests: [{ id: 'xt1', status: 'pass' }], + }); + expect(shardTwo.discovery.fingerprint).toBe(build().discovery.fingerprint); + }); + + it('carries handlers through untouched', () => { + expect(build().handlers).toEqual(handlers); + }); + + // path is for display, index is for identity. See buildRunReport. + it('stamps each test with its resolved path and its position in the order', () => { + expect(build({ tests: [{ id: 't3', status: 'fail', error: 'boom' }] }).tests).toEqual([ + { id: 't3', status: 'fail', error: 'boom', path: 'Login > still works', index: 2 }, + ]); + }); + + // Positions are shard-independent, so a merged report can detect a genuine + // overlap with them where random ids proved nothing. + it('numbers positions from the full ordered list, not the shard slice', () => { + const report = build({ + tests: [{ id: 't2', status: 'pass' }, { id: 't3', status: 'pass' }], + }); + expect(report.tests.map((t) => t.index)).toEqual([1, 2]); + }); + + // Renderers fall back on a null path; inventing one would be worse. + it('leaves path null and index null when the handler is missing', () => { + const report = build({ tests: [{ id: 'ghost', status: 'pass' }] }); + expect(report.tests[0]).toEqual({ id: 'ghost', status: 'pass', path: null, index: null }); + }); + + it('copies the filters rather than aliasing them', () => { + const filters = ['Login']; + const report = build({ filters }); + filters.push('Cart'); + expect(report.selection.filters).toEqual(['Login']); + }); + + // What executed + notRun has to add up to. With --test active this is smaller + // than discovery.totalTests, and comparing against the latter reported a + // shard-slicing bug on every correct filtered run. + it('records the filtered count the shards divided', () => { + expect(build({ filteredIds: ['t2', 't3'], filters: ['works'] }).selection.selectedTests) + .toBe(2); + }); + + it('falls back to the whole suite when no filter is active', () => { + expect(build().selection.selectedTests).toBe(3); + }); + + it('defaults contracts to an unconfigured empty block', () => { + expect(build().contracts).toEqual({ + configured: false, partial: false, results: [], skipped: [], + }); + }); + + it('passes a contracts block through when given', () => { + const contracts = { configured: true, partial: true, results: [{ alias: 'a' }], skipped: [] }; + expect(build({ contracts }).contracts).toEqual(contracts); + }); + + it('defaults coverageFile and recording to null', () => { + const shard = build().shards[0]; + expect(shard.coverageFile).toBeNull(); + expect(shard.recording).toBeNull(); + }); +}); diff --git a/tests/runTests.test.js b/tests/runTests.test.js index fbf5da3..547af64 100644 --- a/tests/runTests.test.js +++ b/tests/runTests.test.js @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { runTests } from "../src/index.js"; +import { REPORT_SCHEMA_VERSION } from "../src/runReport.js"; vi.mock('fs'); vi.mock('puppeteer'); @@ -1183,3 +1184,374 @@ describe("runTests pacing", () => { }); }); + +describe('runTests sharding', () => { + function fourTests() { + return { + handlers: [ + { id: '1', name: 'a', type: 'test' }, + { id: '2', name: 'b', type: 'test' }, + { id: '3', name: 'c', type: 'test' }, + { id: '4', name: 'd', type: 'test' }, + ], + testStatus: [{ id: '2', status: 'pass' }, { id: '4', status: 'pass' }], + }; + } + + function runJson() { + const call = vi.mocked(fs.writeFileSync).mock.calls + .find(([file]) => String(file).endsWith('run.json')); + return call ? JSON.parse(call[1]) : null; + } + + // runJson() and the not.toHaveBeenCalled() assertions read fs call history, so + // this block cannot inherit another describe's. loadConfig has to be re-stubbed + // too: clearAllMocks keeps implementations, so without this the previous block's + // recording-enabled config (with a pace) leaks in. + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig }); + vi.mocked(assertFfmpegAvailable).mockReset(); + vi.mocked(fs.statSync).mockReset(); + vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // Round-robin: shard 2 of 2 takes the odd indices, i.e. the 2nd and 4th ids. + it('runs only its round-robin slice', async () => { + const { handlers, testStatus } = fourTests(); + const page = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + await runTests({ shard: { index: 2, total: 2 } }); + + expect(page.evaluate).toHaveBeenCalledWith(expect.any(Function), 2, ['2', '4']); + }); + + it('writes a run report to the default report dir', async () => { + const { handlers, testStatus } = fourTests(); + const page = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + await runTests({ shard: { index: 2, total: 2 } }); + + expect(fs.mkdirSync).toHaveBeenCalledWith('./.twd/run', { recursive: true }); + const report = runJson(); + expect(report.schemaVersion).toBe(REPORT_SCHEMA_VERSION); + expect(report.shards[0]).toMatchObject({ index: 2, total: 2, executed: 2 }); + expect(report.discovery.totalTests).toBe(4); + expect(report.tests.map((t) => t.id)).toEqual(['2', '4']); + }); + + // path is resolved here, in the shard that ran the test, because twd-js ids + // are random per page load: shard 2's ids do not exist in the handler map a + // merged report keeps, so the merged summary could not name these tests + // otherwise. index is the cross-shard identity — the same number in every + // shard, unlike the id. + it('stamps each result with its resolved path and its position in the order', async () => { + const { handlers, testStatus } = fourTests(); + const page = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + await runTests({ shard: { index: 2, total: 2 } }); + + expect(runJson().tests).toEqual([ + { id: '2', status: 'pass', path: 'b', index: 1 }, + { id: '4', status: 'pass', path: 'd', index: 3 }, + ]); + }); + + // The fingerprint is over paths, not ids, so two shards of the same suite + // agree on it even though their browsers minted entirely different ids. With + // ids it could never match and merge rejected every correct sharded run. + it('fingerprints identically across shards whose ids differ', async () => { + const { handlers, testStatus } = fourTests(); + const first = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(first)); + await runTests({ shard: { index: 1, total: 2 } }); + const shardOne = runJson().discovery.fingerprint; + + vi.clearAllMocks(); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig }); + // Same suite, same order, all-new ids — exactly what a second browser does. + const relabeled = handlers.map((h) => ({ ...h, id: `x${h.id}` })); + const second = createMockPage({ handlers: relabeled, testStatus: [] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(second)); + await runTests({ shard: { index: 2, total: 2 } }); + + expect(runJson().discovery.fingerprint).toBe(shardOne); + }); + + it('honors --report-dir', async () => { + const { handlers, testStatus } = fourTests(); + const page = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + await runTests({ shard: { index: 1, total: 1 }, reportDir: './out' }); + + expect(fs.mkdirSync).toHaveBeenCalledWith('./out', { recursive: true }); + }); + + it('writes no report when not sharded', async () => { + const { handlers, testStatus } = fourTests(); + const page = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + await runTests(); + + expect(runJson()).toBeNull(); + }); + + // 3 tests across 4 shards leaves the fourth with nothing to run. It must + // still write a valid report, or merge sees a gap it cannot explain. + it('writes a valid empty report when its slice is empty', async () => { + const handlers = [ + { id: '1', name: 'a', type: 'test' }, + { id: '2', name: 'b', type: 'test' }, + { id: '3', name: 'c', type: 'test' }, + ]; + const page = createMockPage({ handlers, testStatus: [] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + const hasFailures = await runTests({ shard: { index: 4, total: 4 } }); + + expect(hasFailures).toBe(false); + const report = runJson(); + expect(report.tests).toEqual([]); + expect(report.shards[0]).toMatchObject({ index: 4, total: 4, executed: 0, failed: 0 }); + // The fingerprint still covers the whole suite, so an empty shard merges + // cleanly with the three that ran something. + expect(report.discovery.totalTests).toBe(3); + }); + + // Filters resolve first, then the filtered list is sharded. A filtered run's + // coverage is a misleading project-wide number, sharded or not. + it('skips coverage when a filter is combined with a shard', async () => { + const { handlers, testStatus } = fourTests(); + const page = createMockPage({ handlers, testStatus }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig, coverage: true }); + + await runTests({ shard: { index: 1, total: 2 }, testFilters: ['a'] }); + + const files = vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f)); + expect(files.some((f) => f.endsWith('coverage.json'))).toBe(false); + expect(runJson().shards[0].coverageFile).toBeNull(); + expect(runJson().selection.filters).toEqual(['a']); + }); + + // Filters resolve first, so the shards divide the filtered list, and that is + // the count executed + notRun has to add up to. Recording the unfiltered + // total instead made merge print a shard-slicing warning on a correct run. + it('records the filtered count as the selection the shards divided', async () => { + const { handlers } = fourTests(); + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'pass' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + + await runTests({ shard: { index: 1, total: 2 }, testFilters: ['a'] }); + + const report = runJson(); + expect(report.discovery.totalTests).toBe(4); + expect(report.selection.selectedTests).toBe(1); + expect(report.shards[0].executed + report.shards[0].notRun) + .toBe(report.selection.selectedTests); + }); +}); + +describe('runTests non-regression: non-sharded behavior is unchanged', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig }); + vi.mocked(assertFfmpegAvailable).mockReset(); + vi.mocked(fs.statSync).mockReset(); + vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + // The !hasFailures coverage gate is relaxed only for sharded runs. A plain + // failing run must still write nothing, exactly as in 1.4.0. + it('writes no coverage when a non-sharded run fails', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'fail', error: 'boom' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig, coverage: true }); + + await runTests(); + + expect(fs.writeFileSync).not.toHaveBeenCalled(); + }); + + // The successful coverage write. Splitting its if/else apart is the most + // delicate edit in the sharding change and nothing else exercised this block, + // so pin the destination: nyc reads .nyc_output/out.json by default. + it('still writes coverage to .nyc_output/out.json when a non-sharded run passes', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'pass' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig, coverage: true }); + + await runTests(); + + const files = vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f)); + expect(files).toHaveLength(1); + expect(files[0].includes('.nyc_output')).toBe(true); + expect(files[0].endsWith('out.json')).toBe(true); + }); + + // Likewise the early-stop contract skip. + it('still skips contract validation when a non-sharded run stops early', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'fail', error: 'boom' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ + ...defaultMockConfig, maxFailures: 1, contracts: [{ source: 'api.json' }], + }); + vi.mocked(loadContracts).mockResolvedValue([]); + + await runTests(); + + expect(validateMocks).not.toHaveBeenCalled(); + }); + + // The twd-js version hint gained a `&& !sharded` gate. A plain run with + // contracts configured and nothing collected really may be running a twd-js + // that cannot collect, so it must still say so. + it('still hints at twd-js when a non-sharded run collects no mocks', async () => { + const log = vi.spyOn(console, 'log'); + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'pass' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ + ...defaultMockConfig, contracts: [{ source: 'api.json' }], + }); + vi.mocked(loadContracts).mockResolvedValue([]); + vi.mocked(validateMocks).mockReturnValue({ results: [], skipped: [] }); + vi.mocked(printContractReport).mockReturnValue(false); + + await runTests(); + + expect(log.mock.calls.flat().join('\n')).toContain('No mocks collected'); + }); + + // The markdown report gained a `&& !sharded` gate. A plain run must still + // write it, and this is the only test that executes that block at all. + it('still writes the contract markdown report on a non-sharded run', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'pass' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ + ...defaultMockConfig, + contracts: [{ source: 'api.json' }], + contractReportPath: './contract-report.md', + }); + vi.mocked(loadContracts).mockResolvedValue([]); + vi.mocked(validateMocks).mockReturnValue({ results: [], skipped: [] }); + vi.mocked(printContractReport).mockReturnValue(false); + + await runTests(); + + const files = vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f)); + expect(files.some((f) => f.endsWith('contract-report.md'))).toBe(true); + }); +}); + +describe('runTests sharded behavior changes', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig }); + vi.mocked(assertFfmpegAvailable).mockReset(); + vi.mocked(fs.statSync).mockReset(); + vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('writes coverage for a sharded run that fails', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'fail', error: 'boom' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig, coverage: true }); + + await runTests({ shard: { index: 1, total: 1 } }); + + const files = vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f)); + expect(files.some((f) => f.endsWith('coverage.json'))).toBe(true); + // Never the path nyc reads by default: a shard's partial coverage there + // would masquerade as the whole run's. + expect(files.some((f) => f.includes('.nyc_output'))).toBe(false); + }); + + // Every shard would overwrite the others with a fraction of the picture, so + // the markdown report is merge's job. The run.json still gets written. + it('writes no contract markdown report when sharded', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'pass' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ + ...defaultMockConfig, + contracts: [{ source: 'api.json' }], + contractReportPath: './contract-report.md', + }); + vi.mocked(loadContracts).mockResolvedValue([]); + vi.mocked(validateMocks).mockReturnValue({ results: [], skipped: [] }); + vi.mocked(printContractReport).mockReturnValue(false); + + await runTests({ shard: { index: 1, total: 1 } }); + + const files = vi.mocked(fs.writeFileSync).mock.calls.map(([f]) => String(f)); + expect(files.some((f) => f.endsWith('contract-report.md'))).toBe(false); + expect(files.some((f) => f.endsWith('run.json'))).toBe(true); + }); + + // A shard whose slice exercised no mocks — and any shard with an empty slice — + // collects nothing, which is normal. Printing the twd-js version hint there + // advertises a problem that does not exist, on the happy path of every + // sharded CI run. + it('does not hint at twd-js when a sharded run collects no mocks', async () => { + const log = vi.spyOn(console, 'log'); + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'pass' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ + ...defaultMockConfig, contracts: [{ source: 'api.json' }], + }); + vi.mocked(loadContracts).mockResolvedValue([]); + vi.mocked(validateMocks).mockReturnValue({ results: [], skipped: [] }); + vi.mocked(printContractReport).mockReturnValue(false); + + await runTests({ shard: { index: 1, total: 2 } }); + + expect(log.mock.calls.flat().join('\n')).not.toContain('No mocks collected'); + // Still validated what it did collect — only the hint is suppressed. + expect(validateMocks).toHaveBeenCalled(); + }); + + it('validates contracts on a sharded early stop and marks them partial', async () => { + const handlers = [{ id: '1', name: 'a', type: 'test' }]; + const page = createMockPage({ handlers, testStatus: [{ id: '1', status: 'fail', error: 'boom' }] }); + vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page)); + vi.mocked(loadConfig).mockReturnValue({ + ...defaultMockConfig, maxFailures: 1, contracts: [{ source: 'api.json' }], + }); + vi.mocked(loadContracts).mockResolvedValue([]); + vi.mocked(validateMocks).mockReturnValue({ results: [{ alias: 'a' }], skipped: [] }); + vi.mocked(printContractReport).mockReturnValue(false); + + await runTests({ shard: { index: 1, total: 1 } }); + + expect(validateMocks).toHaveBeenCalled(); + const call = vi.mocked(fs.writeFileSync).mock.calls + .find(([file]) => String(file).endsWith('run.json')); + const report = JSON.parse(call[1]); + expect(report.contracts).toMatchObject({ configured: true, partial: true }); + expect(report.contracts.results).toEqual([{ alias: 'a' }]); + }); +}); diff --git a/tests/shard.test.js b/tests/shard.test.js new file mode 100644 index 0000000..b3e5bd4 --- /dev/null +++ b/tests/shard.test.js @@ -0,0 +1,73 @@ +import { describe, it, expect } from 'vitest'; +import { parseShardSpec, selectShardIds } from '../src/shard.js'; + +describe('parseShardSpec', () => { + it('parses /', () => { + expect(parseShardSpec('2/4')).toEqual({ index: 2, total: 4 }); + expect(parseShardSpec('1/1')).toEqual({ index: 1, total: 1 }); + expect(parseShardSpec(' 3/4 ')).toEqual({ index: 3, total: 4 }); + }); + + // Unlike --record-speed, a bad --shard must never be silently ignored: it + // would run zero tests and exit 0, reading as a green build that tested + // nothing. + it('throws when the index exceeds the total', () => { + expect(() => parseShardSpec('5/4')).toThrow(/between 1 and 4/); + }); + + it('throws on a zero or negative index', () => { + expect(() => parseShardSpec('0/4')).toThrow(/between 1 and 4/); + expect(() => parseShardSpec('-1/4')).toThrow(/Expected \//); + }); + + it('throws on a zero total', () => { + expect(() => parseShardSpec('2/0')).toThrow(/at least 1/); + }); + + it('throws on unparseable input', () => { + for (const bad of ['abc', '', '2', '2/', '/4', '2.5/4', '2/4/6', undefined, null]) { + expect(() => parseShardSpec(bad)).toThrow(/Expected \//); + } + }); + + it('names the offending value in the message', () => { + expect(() => parseShardSpec('9/2')).toThrow(/"9\/2"/); + }); +}); + +describe('selectShardIds', () => { + const ids = Array.from({ length: 12 }, (_, i) => `t${i}`); + + it('takes every Nth id at its own offset', () => { + expect(selectShardIds(ids, 1, 4)).toEqual(['t0', 't4', 't8']); + expect(selectShardIds(ids, 2, 4)).toEqual(['t1', 't5', 't9']); + expect(selectShardIds(ids, 4, 4)).toEqual(['t3', 't7', 't11']); + }); + + it('returns everything when total is 1', () => { + expect(selectShardIds(ids, 1, 1)).toEqual(ids); + }); + + // 3 tests across 4 shards leaves the fourth with nothing. Legal, not an error. + it('returns an empty slice when there are fewer ids than shards', () => { + expect(selectShardIds(['a', 'b', 'c'], 4, 4)).toEqual([]); + expect(selectShardIds([], 1, 4)).toEqual([]); + }); + + // The property that makes sharding trustworthy: nothing lost, nothing doubled. + it('partitions the input — every id lands in exactly one shard', () => { + const many = Array.from({ length: 37 }, (_, i) => `t${i}`); + const total = 5; + const slices = Array.from({ length: total }, (_, i) => selectShardIds(many, i + 1, total)); + const counts = new Map(); + for (const id of slices.flat()) counts.set(id, (counts.get(id) ?? 0) + 1); + expect(counts.size).toBe(many.length); + expect([...counts.values()]).toEqual(many.map(() => 1)); + }); + + it('does not mutate its input', () => { + const input = ['a', 'b', 'c']; + selectShardIds(input, 1, 2); + expect(input).toEqual(['a', 'b', 'c']); + }); +}); diff --git a/tests/testSummary.test.js b/tests/testSummary.test.js index 4950d75..b8a45a2 100644 --- a/tests/testSummary.test.js +++ b/tests/testSummary.test.js @@ -25,6 +25,41 @@ describe('formatRunComplete', () => { ); }); + // A merged report keeps only the first shard's handlers, and twd-js ids are + // random per page load, so a later shard's failure cannot be resolved from + // them — it used to print as a raw id in the merged summary. Each entry now + // carries the path its own shard resolved. + it('prefers the path the shard resolved over its own handler lookup', () => { + const block = formatRunComplete({ + testStatus: [ + { id: 'k3j2h1g9d', path: 'Cart > removes an item', status: 'fail', error: 'boom' }, + { id: 'z9y8x7w6v', path: 'Cart > applies a coupon', status: 'pass', retryAttempt: 2 }, + ], + handlers, + durationMs: 1000, + }); + expect(block).toContain('× Cart > removes an item'); + expect(block).toContain('✓ Cart > applies a coupon (passed on attempt 2)'); + expect(block).not.toContain('k3j2h1g9d'); + expect(block).not.toContain('z9y8x7w6v'); + }); + + // A live non-sharded run carries no path, and a null path is a legal value. + it('falls back to the handler lookup, then the raw id', () => { + const block = formatRunComplete({ + testStatus: [ + { id: 't1', status: 'fail', error: 'a' }, + { id: 't2', path: null, status: 'fail', error: 'b' }, + { id: 'ghost', status: 'fail', error: 'c' }, + ], + handlers, + durationMs: 1000, + }); + expect(block).toContain('× Login > shows error on wrong password'); + expect(block).toContain('× Login > redirects on success'); + expect(block).toContain('× ghost'); + }); + it('counts skipped tests', () => { const block = formatRunComplete({ testStatus: [ @@ -178,3 +213,47 @@ describe('formatRunComplete', () => { expect(block).not.toContain('0 test(s) were not run'); }); }); + +describe('formatRunComplete with shards', () => { + const handlers = [ + { id: 's1', name: 'Login', parent: null, type: 'suite' }, + { id: 't1', name: 'works', parent: 's1', type: 'test' }, + ]; + const testStatus = [{ id: 't1', status: 'pass' }]; + + function shard(index, overrides = {}) { + return { index, total: 4, executed: 30, failed: 0, notRun: 0, stoppedEarly: false, ...overrides }; + } + + it('adds a shard breakdown line when more than one shard merged', () => { + const output = formatRunComplete({ + testStatus, handlers, durationMs: 38_200, computeMs: 134_200, + shards: [shard(1), shard(2, { failed: 3 }), shard(3), shard(4)], + }); + expect(output).toContain('Shards: 1 ✓30 | 2 ✗30 | 3 ✓30 | 4 ✓30'); + }); + + it('reports wall clock and compute separately for a merged run', () => { + const output = formatRunComplete({ + testStatus, handlers, durationMs: 38_200, computeMs: 134_200, + shards: [shard(1), shard(2)], + }); + expect(output).toContain('Duration: 38.2s wall | 134.2s across 2 shards'); + }); + + // The existing single-run format must not shift. + it('keeps the plain duration line when there are no shards', () => { + const output = formatRunComplete({ testStatus, handlers, durationMs: 4_200 }); + expect(output).toContain('Duration: 4.2s'); + expect(output).not.toContain('wall'); + expect(output).not.toContain('Shards:'); + }); + + it('keeps the plain duration line for a single shard', () => { + const output = formatRunComplete({ + testStatus, handlers, durationMs: 4_200, computeMs: 4_200, shards: [shard(1, { total: 1 })], + }); + expect(output).toContain('Duration: 4.2s'); + expect(output).not.toContain('Shards:'); + }); +});