diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml new file mode 100644 index 0000000..dc2ec34 --- /dev/null +++ b/.github/workflows/selftest.yml @@ -0,0 +1,308 @@ +# Self-test: exercise the action end-to-end on a real GitHub runner against the +# dev backend. The V2 project has frontend-only tests, which the V2 execution +# path skips — so it deterministically drives the partial-run guard (no browser +# execution, no credits). +name: selftest + +# Runs on push to main / dev / ci/** (not every feature branch), so an unrelated +# commit doesn't fire dev-backend runs (dev-uptime/rate-limit noise). The +# credit-burning V3 jobs are further gated to workflow_dispatch only. +on: + push: + branches: [main, dev, "ci/**"] + workflow_dispatch: + +jobs: + # allow-partial=false (default): FE tests are skipped on V2, so the action + # must FAIL the job. We run it with continue-on-error and assert it failed — + # proving the guard fires on a real runner while keeping this job green. + v2-guard-blocks-partial: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "false" + - name: Assert the guard fired (action should have failed ON SKIPPED TESTS) + run: | + echo "action outcome: ${{ steps.run.outcome }} skipped: ${{ steps.run.outputs.skipped }}" + if [ "${{ steps.run.outcome }}" != "failure" ]; then + echo "::error::Expected the action to FAIL (FE skipped, allow-partial=false), but it did not." + exit 1 + fi + # Distinguish "the partial-run guard fired" from "the action broke for an + # UNRELATED reason (CLI crash, bad key, dev down)" — outcome==failure + # alone can't. The guard only fires when tests were actually skipped. + if [ -z "${{ steps.run.outputs.skipped }}" ] || [ "${{ steps.run.outputs.skipped }}" = "0" ]; then + echo "::error::Action failed but skipped=${{ steps.run.outputs.skipped }} — it did NOT fail via the partial-run guard (something else broke)." + exit 1 + fi + echo "OK — partial-run guard fired (skipped=${{ steps.run.outputs.skipped }})." + + # allow-partial=true: same skip, but the job stays green (warning only). + v2-allow-partial: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "true" + - name: Assert the V2-skip premise held (no browser execution / no credits) + run: | + echo "skipped=${{ steps.run.outputs.skipped }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + echo "junit=${{ steps.run.outputs.junit-file }}" + # This scenario relies on the V2 account SKIPPING FE tests (no execution, + # no credits). skipped=0 means the account likely no longer runs on V2 + # (FE now executes + costs credits) — fail LOUDLY instead of silently + # burning credits on every push. Doubles as an account-drift tripwire. + if [ -z "${{ steps.run.outputs.skipped }}" ] || [ "${{ steps.run.outputs.skipped }}" = "0" ]; then + echo "::error::Expected FE tests SKIPPED on the V2 account (skipped>0), got skipped=${{ steps.run.outputs.skipped }}. The test account may have been flipped to V3 (tests now execute + cost credits) — check featureFlags.v3." + exit 1 + fi + echo "OK — FE skipped on V2 as expected; no execution, no credits." + + # allow-partial=true must NOT green a CRASHED run. A broken run (auth / network + # / CLI crash) exits non-zero with an empty summary — passed=0, failed=0 — which + # the allow-partial branch used to `exit 0` on (green with only a ::warning). A + # bogus (well-formed but invalid) key forces exactly that shape: the CLI passes + # its client-side format check, reaches the server, and 401s. The action must + # RED. Skips never change the CLI exit code, so this can only be a crash. + allow-partial-crash-reds: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: "sk-user-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "true" + - name: Assert allow-partial did NOT swallow the crash (action should FAIL) + env: + O: ${{ steps.run.outcome }} + SKIPPED: ${{ steps.run.outputs.skipped }} + PASSED: ${{ steps.run.outputs.passed }} + FAILED: ${{ steps.run.outputs.failed }} + TOTAL: ${{ steps.run.outputs.total }} + run: | + echo "outcome=$O skipped=$SKIPPED passed=$PASSED failed=$FAILED total=$TOTAL" + if [ "$O" != "failure" ]; then + echo "::error::allow-partial greened a crashed run (bogus key → CLI non-zero exit, empty summary). The crash-swallow gate did not fire." + exit 1 + fi + # Prove the failure came from the CRASH gate, not from a real failed>0 + # run (which would also make outcome=failure). A crash has the + # empty-summary shape: no verdict counted at all. skipped stays 0 too + # (a crash is not a skip). + if [ "${PASSED:-0}" != "0" ] || [ "${FAILED:-0}" != "0" ] || [ "${TOTAL:-0}" != "0" ]; then + echo "::error::Expected the empty-summary crash shape (passed=0 failed=0 total=0), got passed=$PASSED failed=$FAILED total=$TOTAL — the action failed via a real verdict, not the crash gate." + exit 1 + fi + if [ -n "$SKIPPED" ] && [ "$SKIPPED" != "0" ]; then + echo "::error::Expected a crash (skipped=0), got skipped=$SKIPPED — failed via the wrong path." + exit 1 + fi + echo "OK — allow-partial reds a crashed run (empty-summary shape) instead of greening it." + + # Single-test mode must RED a crash too (symmetric to the batch gate above), and + # this is what proves the v3-single-test assertions can actually fail: a bogus + # key fails at auth (non-zero exit, no --summary-file written), so total stays 0 + # and error-code surfaces. If total==1 / error-code were still phantom, this job + # could not go red. + single-test-crash-reds: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: "sk-user-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + test-id: "77b9daa0-f3ed-4883-8e26-66055a66d595" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + - name: Assert single-test crash REDS with load-bearing outputs + env: + O: ${{ steps.run.outcome }} + EC: ${{ steps.run.outputs.error-code }} + TOTAL: ${{ steps.run.outputs.total }} + run: | + echo "outcome=$O error-code='$EC' total=$TOTAL" + if [ "$O" != "failure" ]; then + echo "::error::single-test crash greened (bogus key). The status+exit gate did not fire." + exit 1 + fi + # A crash writes no summary → total=0 (so v3-single-test's total==1 is + # load-bearing), and the API error surfaces on error-code (so that output + # is real, not the phantom the previous version read). + [ "${TOTAL:-0}" = "0" ] || { echo "::error::expected total=0 on a crash (no summary written), got $TOTAL"; exit 1; } + [ -n "$EC" ] || { echo "::error::expected error-code set on an auth failure, got empty"; exit 1; } + echo "OK — single-test crash reds; total=0 + error-code='$EC' prove the outputs are load-bearing." + + # Real-user shape: reference the action REMOTELY by ref (what a consumer's + # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just + # `uses` + api-key + project. This exercises GitHub's action-resolution path + # (fetch the repo at the ref, load its action.yml, run the composite). + # Runs on `main` only: a remote `uses` resolves the ref from the DEFAULT + # branch's published action, so on a feature branch it would test stale code. + # After this lands on main it validates the real consumer shape against main. + # `endpoint-url` / `allow-partial` are here only because the dev test project + # is frontend-only on V2; a real V3/prod project with passing tests needs + # neither. + real-user-remote-ref: + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + steps: + - uses: TestSprite/testsprite-action@main + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "true" + + # V3 real execution: a V3 project runs frontend tests natively, so this + # actually executes a browser test (filtered to one) and produces a JUnit + # report with real test cases — exercising upload-artifact, per-test + # annotations, and the job summary. continue-on-error because the browser test + # may pass or fail; we verify the plumbing (report has cases + is uploaded), + # not the verdict. Manual-only to control credit usage. + v3-real-execution: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} + project: "f01fd7ee-91ba-465e-bce6-2cb1dc14346d" + filter: "Sign in from the login page" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + report-file: "results/junit.xml" + - name: Verify a JUnit report with real test cases was produced + run: | + echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + test -s results/junit.xml || { echo "::error::no JUnit report written"; exit 1; } + grep -q ""; exit 1; } + echo "OK — JUnit report with test cases:"; cat results/junit.xml + + # Single-test-by-id mode on a real runner with the PUBLISHED CLI (no Gap A). + # A single test runs → verdict from the run envelope → red/green + counts. + # (Annotation appears once a CLI with single-test CI output is published.) + v3-single-test: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Plant a stale report at the default path (as if a prior --all run wrote + # it). Single-test mode must NOT upload it or advertise it as its output. + - run: echo '' > testsprite-junit.xml + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} + # A stable test in the dedicated FE-fixtures project (a simple + # "home page" assertion), not the cal demo — more reliable per run. + test-id: "77b9daa0-f3ed-4883-8e26-66055a66d595" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + - name: Verify single-test verdict + outputs + env: + O: ${{ steps.run.outcome }} + EC: ${{ steps.run.outputs.error-code }} + PASSED: ${{ steps.run.outputs.passed }} + FAILED: ${{ steps.run.outputs.failed }} + TOTAL: ${{ steps.run.outputs.total }} + JUNIT: ${{ steps.run.outputs.junit-file }} + run: | + echo "outcome=$O error-code='$EC' passed=$PASSED failed=$FAILED total=$TOTAL junit-file='$JUNIT'" + # Tolerate a transient backend blip (auth/throttle) — error-code is a real + # action output now, so this branch actually fires when it should. + if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then + echo "::warning::transient backend error ($EC) — skipping single-test validation this run." + exit 0 + fi + # The run must have DISPATCHED (no API-layer error). + [ -z "$EC" ] || { echo "::error::single test did not run (error: $EC)"; exit 1; } + # total==1 means a real run wrote a --summary-file (a crash / auth / network + # failure never does — total stays 0). But total also counts a `timeout` + # status, so it alone doesn't prove a VERDICT was reached... + [ "$TOTAL" = "1" ] || { echo "::error::expected total=1 (a real single-test run wrote a summary), got $TOTAL"; exit 1; } + # ...so passed+failed==1 is the load-bearing one: it proves the run reached + # a real pass/fail verdict, not a timeout/unknown that still counts in total. + [ "$((PASSED + FAILED))" = "1" ] || { echo "::error::expected exactly one verdict (passed+failed==1), got passed=$PASSED failed=$FAILED — a timeout/unknown status, not a real result."; exit 1; } + # And the action's JOB OUTCOME must be coherent with that verdict — the + # single-test path's whole contract is verdict → job status. 77b9daa0 is a + # stable passing fixture, so a pass MUST green; if the browser flaked to a + # real FAIL, that's tolerated but the action MUST then have red-ed. + if [ "${PASSED:-0}" = "1" ]; then + [ "$O" = "success" ] || { echo "::error::test passed but the action outcome was '$O' (expected success)"; exit 1; } + else + [ "$O" = "failure" ] || { echo "::error::test failed (failed=$FAILED) but the action outcome was '$O' (expected failure)"; exit 1; } + echo "::warning::single-test fixture flaked to a failing verdict this run — the action correctly red-ed." + fi + # junit-file output must be EMPTY in single-test mode even though a + # stale testsprite-junit.xml is present (Codex finding). + [ -z "$JUNIT" ] || { echo "::error::junit-file should be empty in single-test mode, got '$JUNIT'"; exit 1; } + echo "OK — single test dispatched, reached a verdict coherent with outcome=$O (passed=$PASSED failed=$FAILED), no stale JUnit advertised." + + # Regression job for the timeout gate (the fix that was silently wrong for two + # rounds). A gate nobody has seen go red is not yet a gate — this forces it. + # `timeout: 1` + a filter narrowed to ONE V3 test: the run dispatches, the poll + # gives up at 1s while the browser test is still running, the CLI exits 7 with + # timedOut=1, and the action MUST red. The assertion is timed-out==1 (not just + # outcome==failure), so it proves the red came from THIS gate and not a real + # failed>0 or a crash. workflow_dispatch-only: it costs one dispatched run's + # credits (hence the narrowed filter, same as v3-real-execution). + timeout-gate-reds: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} + project: "f01fd7ee-91ba-465e-bce6-2cb1dc14346d" + filter: "Sign in from the login page" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "true" + timeout: "1" + - name: Verify the timeout gate red-ed the batch + env: + O: ${{ steps.run.outcome }} + EC: ${{ steps.run.outputs.error-code }} + TO: ${{ steps.run.outputs.timed-out }} + FAILED: ${{ steps.run.outputs.failed }} + run: | + echo "outcome=$O error-code='$EC' timed-out=$TO failed=$FAILED" + # A transient backend error means the run never dispatched, so no timeout + # could occur — the gate can't be exercised. Skip rather than false-fail. + if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then + echo "::warning::transient backend error ($EC) — the run never dispatched, skipping the timeout-gate assertion this run." + exit 0 + fi + # The action MUST have red-ed (allow-partial is true, so ONLY the timeout + # gate can red this — a passing/deferred/conflict batch would be green). + [ "$O" = "failure" ] || { echo "::error::expected the action to RED on a timed-out batch under allow-partial, got outcome=$O"; exit 1; } + # ...and the red MUST come from the timeout gate specifically. + [ "$TO" = "1" ] || { echo "::error::expected timed-out=1 (the timeout gate fired), got timed-out=$TO — the red came from a different path"; exit 1; } + # A real failing verdict reds via failed>0, not this gate; assert it wasn't that. + [ "${FAILED:-0}" = "0" ] || { echo "::error::expected failed=0 (a real failure would red via a different path), got failed=$FAILED"; exit 1; } + echo "OK — allow-partial red-ed a timed-out batch via the timeout gate (timed-out=1, failed=0)." diff --git a/README.md b/README.md index b0d9bac..ef875c0 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,9 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | ----------------- | -------------------- | --------------------------------------------------------------------------- | | `api-key` | — (required) | TestSprite API key. Pass a secret. | | `project` | `""` | Project id. If empty, `TESTSPRITE_PROJECT_ID` must be set. | -| `filter` | `""` | Only run tests whose name contains this substring. | -| `target-url` | `""` | Target URL override (V2 path only; ignored on V3). | +| `test-id` | `""` | Run a single test by id (whole-project run otherwise). `project` ignored; `filter` must NOT be set (mutually exclusive — fails fast if both given); no JUnit (batch-only). | +| `filter` | `""` | Only run tests whose name contains this substring. Full-project run only; mutually exclusive with `test-id`. | +| `target-url` | `""` | Target URL override. **Single-test (`test-id`) runs only** — the CLI rejects it on a full-project run, so setting it without `test-id` fails fast. | | `report-file` | `testsprite-junit.xml` | JUnit XML path. | | `cli-version` | `latest` | npm version/dist-tag of `@testsprite/testsprite-cli`. | | `allow-partial` | `false` | If false, fail the job when tests are skipped (see below). | @@ -52,15 +53,19 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | `endpoint-url` | `""` | API base URL override. | | `node-version` | `lts/*` | Node version used to install/run the CLI. | | `upload-artifact` | `true` | Upload the JUnit report as a workflow artifact. | +| `artifact-name` | `testsprite-junit` | Name of the uploaded JUnit artifact. Give each a **unique** name when the action runs more than once (matrix / multiple projects) in one workflow — `upload-artifact@v4` fails on duplicate names. | ## Outputs -| Output | Description | -| ------------ | ---------------------------------- | -| `junit-file` | Path to the JUnit report. | -| `passed` | Number of tests that passed. | -| `failed` | Number of tests that did not pass. | -| `total` | Total tests in the summary. | +| Output | Description | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `junit-file` | Path to the JUnit report (empty in single-test mode — JUnit is batch-only). | +| `passed` | Number of tests that passed. | +| `failed` | Number of tests that ran to a **failing verdict** (`failed`/`blocked`) — not deferred, conflicted, or skipped. | +| `total` | Total tests in the summary. | +| `skipped` | Tests skipped and not run (full-project runs, e.g. frontend tests on the V2 path; `0` in single-test mode). | +| `timed-out` | Tests that ran but did **not** reach a verdict within `timeout` (status `timeout`). Counted separately from `failed`; under `allow-partial: true` these still fail the job. | +| `error-code` | The CLI's API-layer error code when the run failed **before** a verdict (e.g. `AUTH_INVALID`, `NOT_FOUND`, `UNAVAILABLE`); empty otherwise. | ## CI-native output diff --git a/action.yml b/action.yml index 11ef5a0..20a7ffa 100644 --- a/action.yml +++ b/action.yml @@ -12,18 +12,29 @@ branding: inputs: api-key: - description: "TestSprite API key. Pass a secret, e.g. ${{ secrets.TESTSPRITE_API_KEY }}." + description: "TestSprite API key. Store it as an encrypted repository secret and pass it here." required: true project: description: "Project id to run. If empty, TESTSPRITE_PROJECT_ID must be set in the env." required: false default: "" + test-id: + description: > + Run a single test by id instead of the whole project. When set, `project` + is ignored and `filter` must NOT be set (test-id and filter are mutually + exclusive — the action fails fast if both are given). No JUnit report is + produced (the CLI's JUnit report is batch-only); the job still goes + red/green on the verdict and emits a summary + annotation. + required: false + default: "" filter: - description: "Only run tests whose name contains this substring (case-insensitive)." + description: > + Only run tests whose name contains this substring (case-insensitive). + Applies to the full-project run only; mutually exclusive with test-id. required: false default: "" target-url: - description: "Optional target URL override (applied on the V2 path; ignored on V3)." + description: "Optional target URL override. Single-test (test-id) runs only — the CLI rejects it on a full-project run, so setting it without test-id fails fast." required: false default: "" report-file: @@ -57,20 +68,33 @@ inputs: description: "Upload the JUnit report as a workflow artifact." required: false default: "true" + artifact-name: + description: "Name of the uploaded JUnit artifact. actions/upload-artifact@v4 fails on a duplicate name within one workflow run, so give each a UNIQUE name when this action runs more than once (a matrix, or several projects) in the same workflow." + required: false + default: "testsprite-junit" outputs: junit-file: - description: "Path to the JUnit report." - value: ${{ inputs.report-file }} + description: "Path to the JUnit report (empty in single-test mode — JUnit is batch-only)." + value: ${{ inputs.test-id == '' && inputs.report-file || '' }} passed: description: "Number of tests that passed." value: ${{ steps.run.outputs.passed }} failed: - description: "Number of tests that did not pass." + description: "Number of tests that failed to a verdict (failed/blocked) — not deferred/skipped." value: ${{ steps.run.outputs.failed }} total: description: "Total tests in the summary." value: ${{ steps.run.outputs.total }} + skipped: + description: "Tests skipped and not run (full-project runs; 0 in single-test mode)." + value: ${{ steps.run.outputs.skipped }} + timed-out: + description: "Tests that ran but did not reach a verdict within `timeout` (status 'timeout'). Counted separately from `failed`; under allow-partial these still fail the job." + value: ${{ steps.run.outputs.timed-out }} + error-code: + description: "The CLI's API-layer error code when the run failed BEFORE a verdict (e.g. AUTH_INVALID, NOT_FOUND, VALIDATION_ERROR, UNAVAILABLE, RATE_LIMITED); empty on a dispatched run. Lets a caller tell a transient/infra failure from a real test failure." + value: ${{ steps.run.outputs.error-code }} runs: using: "composite" @@ -82,7 +106,10 @@ runs: - name: Install testsprite CLI shell: bash - run: npm install -g @testsprite/testsprite-cli@${{ inputs.cli-version }} + env: + # Via env, not interpolated into the command, so the value can't inject shell. + INPUT_CLI_VERSION: ${{ inputs.cli-version }} + run: npm install -g "@testsprite/testsprite-cli@${INPUT_CLI_VERSION}" - name: Run TestSprite tests id: run @@ -94,50 +121,127 @@ runs: TESTSPRITE_API_URL: ${{ inputs.endpoint-url }} TESTSPRITE_PROJECT_ID: ${{ inputs.project }} INPUT_REPORT_FILE: ${{ inputs.report-file }} + INPUT_TEST_ID: ${{ inputs.test-id }} INPUT_FILTER: ${{ inputs.filter }} INPUT_TARGET_URL: ${{ inputs.target-url }} INPUT_TIMEOUT: ${{ inputs.timeout }} INPUT_ALLOW_PARTIAL: ${{ inputs.allow-partial }} run: | set -uo pipefail - report="$INPUT_REPORT_FILE" + + # Emit zero-valued outputs on an early validation exit so a consumer reading + # e.g. steps.run.outputs.total gets `0`, not an empty string. + emit_zero_outputs() { + { + echo "passed=0"; echo "failed=0"; echo "total=0"; echo "skipped=0" + echo "timed-out=0" + # error-code is the CLI's API-layer error code, not this action's own + # input rejections — leave it empty here so a consumer can't mistake a + # local validation refusal for a CLI/API error. + echo "error-code=" + } >> "$GITHUB_OUTPUT" + } + + # target-url is single-test only: the published CLI rejects `--target-url` + # together with `--all` (VALIDATION_ERROR / exit 5), so on the full-project + # path it would ALWAYS fail with a CLI-internal message about a flag the + # consumer never typed. Reject it here with a clear one instead. + if [ -n "$INPUT_TARGET_URL" ] && [ -z "$INPUT_TEST_ID" ]; then + emit_zero_outputs + echo "::error title=TestSprite::target-url applies to a single-test run only (the CLI rejects --target-url with --all). Set test-id, or remove target-url." + exit 1 + fi + if [ -n "$INPUT_TEST_ID" ] && [ -n "$INPUT_FILTER" ]; then + emit_zero_outputs + echo "::error title=TestSprite::test-id and filter are mutually exclusive (filter only applies to a full-project run)." + exit 1 + fi + summary="$(mktemp)" envjson="$(mktemp)" + errfile="$(mktemp)" + passed=0; failed=0; total=0; skipped=0; timed_out=0; error_code="" + + # --output json puts the machine run envelope on stdout (captured here); CLI + # annotations AND the API-error envelope go to stderr. We need stderr BOTH + # live (so a cancelled/killed step still shows what streamed) AND captured to + # a file (to parse the API-error envelope). A background `tee` reading a fifo + # gives both: the CLI's stderr → fifo → tee → $errfile + the terminal, in + # real time. `wait "$tee_pid"` after the run makes the file read deterministic + # (bash does not reap a process substitution, so a plain `2> >(tee …)` would + # race the parse below). Only one of the two branches runs, so one tee suffices. + # mktemp -d (not -u): create the dir, then name the fifo inside it — avoids the + # name-then-create TOCTOU a bare `mktemp -u` has. + tmpd="$(mktemp -d)"; runlog="$tmpd/stderr"; mkfifo "$runlog" + tee "$errfile" < "$runlog" >&2 & + tee_pid=$! + + if [ -n "$INPUT_TEST_ID" ]; then + # Single test by id. Write a --summary-file too (the CLI supports it for a + # single `test run --wait`): the counts then come from a real + # artifact, so a crash / auth / network failure that writes no summary + # leaves total=0 — distinguishable from a real run's total=1. JUnit stays + # batch-only, so no --report here. + args=(test run "$INPUT_TEST_ID" --wait --output json --summary-file "$summary" --timeout "$INPUT_TIMEOUT") + if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi + set +e; testsprite "${args[@]}" > "$envjson" 2> "$runlog"; code=$?; set -e + status=$(jq -r '.status // "unknown"' "$envjson" 2>/dev/null || echo unknown) + else + # Whole project (optionally name-filtered), with a JUnit report. + # --gh-output auto-enables under GITHUB_ACTIONS=true, so ::error:: + # annotations + the $GITHUB_STEP_SUMMARY table are emitted for free. + report="$INPUT_REPORT_FILE" + # The CLI requires the report's parent dir to exist; create it so a + # nested report-file path (e.g. results/junit.xml) doesn't fail. + mkdir -p "$(dirname "$report")" + args=(test run --all --wait --output json + --report junit --report-file "$report" + --summary-file "$summary" + --timeout "$INPUT_TIMEOUT") + if [ -n "$INPUT_FILTER" ]; then args+=(--filter "$INPUT_FILTER"); fi + set +e; testsprite "${args[@]}" > "$envjson" 2> "$runlog"; code=$?; set -e + # Skipped (frontend-on-V2 etc.) live on the raw envelope, not the summary. + skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) + fi - # --gh-output auto-enables under GITHUB_ACTIONS=true, so ::error:: - # annotations + the $GITHUB_STEP_SUMMARY table are emitted for free. - # --output json puts the machine envelope on stdout (captured here) and - # routes annotations to stderr (shown in the log + parsed by Actions). - args=(test run --all --wait --output json - --report junit --report-file "$report" - --summary-file "$summary" - --timeout "$INPUT_TIMEOUT") - if [ -n "$INPUT_FILTER" ]; then args+=(--filter "$INPUT_FILTER"); fi - if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi - - set +e - testsprite "${args[@]}" > "$envjson" - code=$? - set -e - - # Counts from the machine summary (best-effort). - passed=0; failed=0; total=0 + # Close the fifo write end (both branches are done) and reap tee, so $errfile + # is fully flushed before it is parsed below. `|| true` because the composite + # shell runs with `-e` (GitHub's default `bash -eo pipefail`): a non-zero tee + # must not kill the step BEFORE $GITHUB_OUTPUT is written and a verdict reached. + wait "$tee_pid" || true; rm -rf "$tmpd" + + # Counts from the summary artifact (both paths). `passed`/`total` are the + # CLI's; `failed` = FAILING verdicts only (failed/blocked from runs[].status) + # — NOT the summary's `failed`, which folds in deferred/conflict/not-found + # (partials, not failures). No summary written ⇒ counts stay 0 (a crash). if [ -s "$summary" ]; then passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) - failed=$(jq -r '.failed // 0' "$summary" 2>/dev/null || echo 0) - total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) + total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) + failed=$(jq -r '[.runs[]? | select(.status == "failed" or .status == "blocked")] | length' "$summary" 2>/dev/null || echo 0) + # Timed-out runs carry status "timeout" (NOT counted in `failed` above) and + # come back on CLI exit 7 — which allow-partial otherwise treats as an + # accepted partial. Surface the count so the verdict below can red it. + timed_out=$(jq -r '.timedOut // 0' "$summary" 2>/dev/null || echo 0) fi + # error-code = the CLI's API-layer error code when the run failed BEFORE a + # verdict. Its JSON envelope is on stderr under --output json (stdout carries + # the run envelope, empty on such a failure), preceded by the idempotency-key + # line — so slice from the first line-leading `{` before parsing. + error_code=$(sed -n '/^{/,$p' "$errfile" | jq -r '.error.code // ""' 2>/dev/null || echo "") + { echo "passed=$passed" echo "failed=$failed" echo "total=$total" + echo "skipped=$skipped" + echo "timed-out=$timed_out" + echo "error-code=$error_code" } >> "$GITHUB_OUTPUT" - # Partial-run guard: skipped tests (e.g. frontend tests on the V2 path) - # do NOT affect the CLI exit code, so a partial batch can exit 0. Detect - # them from the raw envelope and fail unless allow-partial is set. - skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) - if [ "${skipped:-0}" -gt 0 ]; then + # Partial-run guard (full-project runs only): skipped tests are invisible + # to a pass/fail count, so a batch that skipped everything can look clean. + # Fail unless allow-partial is set. + if [ -z "$INPUT_TEST_ID" ] && [ "${skipped:-0}" -gt 0 ]; then echo "::warning title=TestSprite::${skipped} test(s) were skipped and did not run (e.g. frontend tests on the V2 execution path)." if [ "$INPUT_ALLOW_PARTIAL" != "true" ]; then echo "::error title=TestSprite::${skipped} test(s) skipped — failing the job. Set allow-partial: true to accept partial runs, or use a V3 project / test list for full coverage." @@ -145,12 +249,77 @@ runs: fi fi + # Single-test verdict — green iff the test passed AND the CLI exited 0. + # Requiring BOTH signals is the same rule as the allow-partial branch + # below: a `passed` status with a non-zero exit is a post-verdict failure + # (cleanup / upload / network), not a pass — never green a broken run. + if [ -n "$INPUT_TEST_ID" ]; then + if [ "$status" = "passed" ] && [ "$code" -eq 0 ]; then exit 0; fi + echo "::error title=TestSprite::single test ${INPUT_TEST_ID} did not pass (status: ${status}, exit: ${code})." + exit 1 + fi + + # Zero-scope guard (BOTH strict and allow-partial). A clean exit that ran + # NOTHING — a zero-match --filter, or a project with no tests — is a false + # green in a CI gate, and allow-partial is about accepting a PARTIAL run, not + # an EMPTY one: red it either way so a typo'd filter or a mis-scoped run isn't + # a silent pass. (skipped>0 is a different case, handled by the guard above; + # a non-zero exit means something was attempted, so it's not zero-scope.) + if [ "$code" -eq 0 ] && [ "${total:-0}" -eq 0 ] && [ "${skipped:-0}" -eq 0 ]; then + echo "::error title=TestSprite::no tests ran (a --filter that matched nothing, or a project with no tests). Failing so a mis-scoped run isn't a silent green." + exit 1 + fi + + # Full-project verdict under allow-partial: green a PARTIAL run (skips / + # rate-deferred / all-conflict), red a real failure or a crash. + if [ "$INPUT_ALLOW_PARTIAL" = "true" ]; then + if [ "${failed:-0}" -gt 0 ]; then exit 1; fi + # A timed-out run is NOT a partial dispatch — the case ran and never + # reached a verdict, so its result is unknown, not deferred. allow-partial + # must red it (otherwise, since a timeout exits 7 and is excluded from the + # crash gate below, ANY batch with a timed-out run would go green). + if [ "${timed_out:-0}" -gt 0 ]; then + echo "::error title=TestSprite::${timed_out} run(s) did not reach a verdict within ${INPUT_TIMEOUT}s. Raise timeout, or poll later with test wait ." + exit 1 + fi + # The timed_out check above can only red a timeout the summary RECORDED. + # exit 7 means "rate-deferred OR timed out"; the summary is how we tell them + # apart. If the CLI exited 7 but wrote NO summary (emitCiArtifacts warns and + # continues if --summary-file can't be written — timed_out would then read 0), + # the outcome is indeterminate: fail closed rather than green a possibly + # timed-out batch. (Scoped to 7 only — exit 6 is all-conflict, which throws + # before any run reaches a timeout, so it has nothing to hide.) + if [ "$code" -eq 7 ] && [ ! -s "$summary" ]; then + echo "::error title=TestSprite::allow-partial: CLI exited 7 (rate-deferred or timed out) but wrote no summary — cannot confirm no run timed out. Failing." + exit 7 + fi + # exit 6 (nothing dispatched — every case already in flight) and 7 + # (rate-deferred / timed out) are PARTIAL dispatch, which is exactly what + # allow-partial accepts. Any OTHER non-zero exit is a crash / auth / + # network / validation error — never green that. (A genuine timeout also + # exits 7 but was already red-ed above by the timed_out check.) + if [ "$code" -ne 0 ] && [ "$code" -ne 6 ] && [ "$code" -ne 7 ]; then + echo "::error title=TestSprite::allow-partial: CLI exited ${code} with no failing verdict — a crash/auth/network/validation error, not a partial run. Failing." + exit "$code" + fi + if [ "$((passed + failed))" -eq 0 ]; then + echo "::warning title=TestSprite::allow-partial: no tests executed to a verdict (all skipped/deferred/conflict) — the job is green but nothing was verified." + fi + exit 0 + fi + + # Default (strict) path. The zero-scope guard already ran above; here a + # non-zero exit (a real failure verdict, crash, auth, or timeout→7) reds the + # job, and a clean exit that ran something is a genuine pass. exit "$code" - name: Upload JUnit report - if: ${{ always() && inputs.upload-artifact == 'true' && hashFiles(inputs.report-file) != '' }} + # Only the full-project run produces a JUnit report. Gate on an empty + # test-id so a single-test run never uploads a stale/pre-existing file at + # the default report path as if it were this run's result. + if: ${{ always() && inputs.test-id == '' && inputs.upload-artifact == 'true' && hashFiles(inputs.report-file) != '' }} uses: actions/upload-artifact@v4 with: - name: testsprite-junit + name: ${{ inputs.artifact-name }} path: ${{ inputs.report-file }} if-no-files-found: ignore