From 7bd0842d186f90233e53ed364f15ec5fad378cfa Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 18:08:45 -0700 Subject: [PATCH 01/20] ci: add selftest workflow (V2 partial-run guard on a real runner) --- .github/workflows/selftest.yml | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/selftest.yml diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml new file mode 100644 index 0000000..08f2e67 --- /dev/null +++ b/.github/workflows/selftest.yml @@ -0,0 +1,51 @@ +# Self-test: exercise the action end-to-end on a real GitHub runner against the +# dev backend. The V2 project has frontend-only tests, which the V2 execution +# path skips — so it deterministically drives the partial-run guard (no browser +# execution, no credits). +name: selftest + +on: + push: + workflow_dispatch: + +jobs: + # allow-partial=false (default): FE tests are skipped on V2, so the action + # must FAIL the job. We run it with continue-on-error and assert it failed — + # proving the guard fires on a real runner while keeping this job green. + v2-guard-blocks-partial: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} + project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" + endpoint-url: "https://api.testsprite.com:11443" + allow-partial: "false" + - name: Assert the guard fired (action should have failed) + run: | + echo "action outcome: ${{ steps.run.outcome }}" + if [ "${{ steps.run.outcome }}" != "failure" ]; then + echo "::error::Expected the action to FAIL (FE skipped, allow-partial=false), but it did not." + exit 1 + fi + echo "OK — partial-run guard fired as expected." + + # allow-partial=true: same skip, but the job stays green (warning only). + v2-allow-partial: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} + project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" + endpoint-url: "https://api.testsprite.com:11443" + allow-partial: "true" + - name: Show outputs + run: | + echo "passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + echo "junit=${{ steps.run.outputs.junit-file }}" From 736b57ebca2ffafdf375051482d94d70f0635b01 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 18:10:54 -0700 Subject: [PATCH 02/20] fix(action): drop ${{ }} example from api-key description (invalid in action manifest) --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 11ef5a0..e0e9098 100644 --- a/action.yml +++ b/action.yml @@ -12,7 +12,7 @@ branding: inputs: api-key: - description: "TestSprite API key. Pass a secret, e.g. ${{ secrets.TESTSPRITE_API_KEY }}." + description: "TestSprite API key. Store it as an encrypted repository secret and pass it here." required: true project: description: "Project id to run. If empty, TESTSPRITE_PROJECT_ID must be set in the env." From 0f314469521f1b6cacb44bd6e3f7262d613bd744 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 18:16:27 -0700 Subject: [PATCH 03/20] ci: add real-user-shape job (remote uses: ref, no scaffolding) --- .github/workflows/selftest.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 08f2e67..2e7c378 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -49,3 +49,20 @@ jobs: run: | echo "passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" echo "junit=${{ steps.run.outputs.junit-file }}" + + # Real-user shape: reference the action REMOTELY by ref (what a consumer's + # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just + # `uses` + api-key + project. This exercises GitHub's action-resolution path + # (fetch the repo at the pinned ref, load its action.yml, run the composite). + # `endpoint-url` / `allow-partial` are here only because the dev test project + # is frontend-only on V2; a real V3/prod project with passing tests needs + # neither. + real-user-remote-ref: + runs-on: ubuntu-latest + steps: + - uses: TestSprite/testsprite-action@736b57ebca2ffafdf375051482d94d70f0635b01 + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} + project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" + endpoint-url: "https://api.testsprite.com:11443" + allow-partial: "true" From a592d3141536d82b30668bbc8150504749bed794 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 18:23:27 -0700 Subject: [PATCH 04/20] ci: add v3-real-execution job (real browser run, junit + annotations) --- .github/workflows/selftest.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 2e7c378..2d2f85d 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -66,3 +66,30 @@ jobs: project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" endpoint-url: "https://api.testsprite.com:11443" allow-partial: "true" + + # V3 real execution: a V3 project runs frontend tests natively, so this + # actually executes a browser test (filtered to one) and produces a JUnit + # report with real test cases — exercising upload-artifact, per-test + # annotations, and the job summary. continue-on-error because the browser test + # may pass or fail; we verify the plumbing (report has cases + is uploaded), + # not the verdict. Manual-only to control credit usage. + v3-real-execution: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} + project: "90048e9c-4de3-4266-bee5-5d722df101b4" + filter: "price filtering" + endpoint-url: "https://api.testsprite.com:11443" + report-file: "results/junit.xml" + - name: Verify a JUnit report with real test cases was produced + run: | + echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + test -s results/junit.xml || { echo "::error::no JUnit report written"; exit 1; } + grep -q ""; exit 1; } + echo "OK — JUnit report with test cases:"; cat results/junit.xml From 3b8939629bd97a13c172cc338316401fc3260ac1 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 18:32:28 -0700 Subject: [PATCH 05/20] ci: gate scenario jobs by event so dispatch runs v3 in isolation --- .github/workflows/selftest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 2d2f85d..de7d053 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -13,6 +13,7 @@ jobs: # must FAIL the job. We run it with continue-on-error and assert it failed — # proving the guard fires on a real runner while keeping this job green. v2-guard-blocks-partial: + if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -35,6 +36,7 @@ jobs: # allow-partial=true: same skip, but the job stays green (warning only). v2-allow-partial: + if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -58,6 +60,7 @@ jobs: # is frontend-only on V2; a real V3/prod project with passing tests needs # neither. real-user-remote-ref: + if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest steps: - uses: TestSprite/testsprite-action@736b57ebca2ffafdf375051482d94d70f0635b01 From 6c0a7680be35b776af07e319e21b67d43aa31efe Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 18:35:02 -0700 Subject: [PATCH 06/20] fix(action): create report-file parent dir before running (nested paths failed validation) --- action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/action.yml b/action.yml index e0e9098..0015db3 100644 --- a/action.yml +++ b/action.yml @@ -101,6 +101,9 @@ runs: run: | set -uo pipefail report="$INPUT_REPORT_FILE" + # The CLI requires the report's parent directory to exist; create it so a + # nested report-file path (e.g. results/junit.xml) doesn't fail validation. + mkdir -p "$(dirname "$report")" summary="$(mktemp)" envjson="$(mktemp)" From fa8133dbc4ad9f29bd00027fb09675254698efc9 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 10 Aug 2026 19:05:52 -0700 Subject: [PATCH 07/20] feat(action): add test-id input (run a single test by id) --- .github/workflows/selftest.yml | 21 ++++++++ README.md | 3 +- action.yml | 98 ++++++++++++++++++++++------------ 3 files changed, 86 insertions(+), 36 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index de7d053..fcb30ef 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -96,3 +96,24 @@ jobs: test -s results/junit.xml || { echo "::error::no JUnit report written"; exit 1; } grep -q ""; exit 1; } echo "OK — JUnit report with test cases:"; cat results/junit.xml + + # Single-test-by-id mode on a real runner with the PUBLISHED CLI (no Gap A). + # A single test runs → verdict from the run envelope → red/green + counts. + # (Annotation appears once a CLI with single-test CI output is published.) + v3-single-test: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} + test-id: "46872b89-f74f-4443-8fe6-36a93cdb0dc2" + endpoint-url: "https://api.testsprite.com:11443" + - name: Verify single-test verdict + outputs + run: | + echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + test "${{ steps.run.outputs.total }}" = "1" || { echo "::error::expected total=1 for a single test"; exit 1; } + echo "OK — single test produced total=1 and a pass/fail verdict." diff --git a/README.md b/README.md index b0d9bac..aa2b790 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,8 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | ----------------- | -------------------- | --------------------------------------------------------------------------- | | `api-key` | — (required) | TestSprite API key. Pass a secret. | | `project` | `""` | Project id. If empty, `TESTSPRITE_PROJECT_ID` must be set. | -| `filter` | `""` | Only run tests whose name contains this substring. | +| `test-id` | `""` | Run a single test by id (whole-project run otherwise). `project`/`filter` ignored; no JUnit (batch-only). | +| `filter` | `""` | Only run tests whose name contains this substring. Full-project run only; mutually exclusive with `test-id`. | | `target-url` | `""` | Target URL override (V2 path only; ignored on V3). | | `report-file` | `testsprite-junit.xml` | JUnit XML path. | | `cli-version` | `latest` | npm version/dist-tag of `@testsprite/testsprite-cli`. | diff --git a/action.yml b/action.yml index 0015db3..a8ec4b1 100644 --- a/action.yml +++ b/action.yml @@ -18,8 +18,18 @@ inputs: description: "Project id to run. If empty, TESTSPRITE_PROJECT_ID must be set in the env." required: false default: "" + test-id: + description: > + Run a single test by id instead of the whole project. When set, `project` + and `filter` are ignored and no JUnit report is produced (the CLI's JUnit + report is batch-only); the job still goes red/green on the verdict and + emits a summary + annotation. + required: false + default: "" filter: - description: "Only run tests whose name contains this substring (case-insensitive)." + description: > + Only run tests whose name contains this substring (case-insensitive). + Applies to the full-project run only; mutually exclusive with test-id. required: false default: "" target-url: @@ -94,57 +104,75 @@ runs: TESTSPRITE_API_URL: ${{ inputs.endpoint-url }} TESTSPRITE_PROJECT_ID: ${{ inputs.project }} INPUT_REPORT_FILE: ${{ inputs.report-file }} + INPUT_TEST_ID: ${{ inputs.test-id }} INPUT_FILTER: ${{ inputs.filter }} INPUT_TARGET_URL: ${{ inputs.target-url }} INPUT_TIMEOUT: ${{ inputs.timeout }} INPUT_ALLOW_PARTIAL: ${{ inputs.allow-partial }} run: | set -uo pipefail - report="$INPUT_REPORT_FILE" - # The CLI requires the report's parent directory to exist; create it so a - # nested report-file path (e.g. results/junit.xml) doesn't fail validation. - mkdir -p "$(dirname "$report")" + if [ -n "$INPUT_TEST_ID" ] && [ -n "$INPUT_FILTER" ]; then + echo "::error title=TestSprite::test-id and filter are mutually exclusive (filter only applies to a full-project run)." + exit 1 + fi summary="$(mktemp)" envjson="$(mktemp)" + passed=0; failed=0; total=0 - # --gh-output auto-enables under GITHUB_ACTIONS=true, so ::error:: - # annotations + the $GITHUB_STEP_SUMMARY table are emitted for free. # --output json puts the machine envelope on stdout (captured here) and - # routes annotations to stderr (shown in the log + parsed by Actions). - args=(test run --all --wait --output json - --report junit --report-file "$report" - --summary-file "$summary" - --timeout "$INPUT_TIMEOUT") - if [ -n "$INPUT_FILTER" ]; then args+=(--filter "$INPUT_FILTER"); fi - if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi - - set +e - testsprite "${args[@]}" > "$envjson" - code=$? - set -e - - # Counts from the machine summary (best-effort). - passed=0; failed=0; total=0 - if [ -s "$summary" ]; then - passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) - failed=$(jq -r '.failed // 0' "$summary" 2>/dev/null || echo 0) - total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) + # routes CLI annotations to stderr (shown in the log + parsed by Actions). + if [ -n "$INPUT_TEST_ID" ]; then + # Single test by id. The CLI's JUnit report is batch-only, so there is + # no report here; the verdict comes from the run envelope's status and + # the non-zero exit reds the job. The CLI emits the ::error annotation + # itself under GITHUB_ACTIONS on a CLI version with single-test CI + # output (DEV-772); older versions just red/green without an annotation. + args=(test run "$INPUT_TEST_ID" --wait --output json --timeout "$INPUT_TIMEOUT") + if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi + set +e; testsprite "${args[@]}" > "$envjson"; code=$?; set -e + status=$(jq -r '.status // "unknown"' "$envjson" 2>/dev/null || echo unknown) + total=1 + if [ "$status" = "passed" ]; then passed=1; else failed=1; fi + else + # Whole project (optionally name-filtered), with a JUnit report. + # --gh-output auto-enables under GITHUB_ACTIONS=true, so ::error:: + # annotations + the $GITHUB_STEP_SUMMARY table are emitted for free. + report="$INPUT_REPORT_FILE" + # The CLI requires the report's parent dir to exist; create it so a + # nested report-file path (e.g. results/junit.xml) doesn't fail. + mkdir -p "$(dirname "$report")" + args=(test run --all --wait --output json + --report junit --report-file "$report" + --summary-file "$summary" + --timeout "$INPUT_TIMEOUT") + if [ -n "$INPUT_FILTER" ]; then args+=(--filter "$INPUT_FILTER"); fi + if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi + set +e; testsprite "${args[@]}" > "$envjson"; code=$?; set -e + if [ -s "$summary" ]; then + passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) + failed=$(jq -r '.failed // 0' "$summary" 2>/dev/null || echo 0) + total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) + fi fi + { echo "passed=$passed" echo "failed=$failed" echo "total=$total" } >> "$GITHUB_OUTPUT" - # Partial-run guard: skipped tests (e.g. frontend tests on the V2 path) - # do NOT affect the CLI exit code, so a partial batch can exit 0. Detect - # them from the raw envelope and fail unless allow-partial is set. - skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) - if [ "${skipped:-0}" -gt 0 ]; then - echo "::warning title=TestSprite::${skipped} test(s) were skipped and did not run (e.g. frontend tests on the V2 execution path)." - if [ "$INPUT_ALLOW_PARTIAL" != "true" ]; then - echo "::error title=TestSprite::${skipped} test(s) skipped — failing the job. Set allow-partial: true to accept partial runs, or use a V3 project / test list for full coverage." - exit 1 + # Partial-run guard (full-project runs only): skipped tests (e.g. frontend + # tests on the V2 path) do NOT affect the CLI exit code, so a partial + # batch can exit 0. Detect them from the envelope and fail unless + # allow-partial is set. + if [ -z "$INPUT_TEST_ID" ]; then + skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) + if [ "${skipped:-0}" -gt 0 ]; then + echo "::warning title=TestSprite::${skipped} test(s) were skipped and did not run (e.g. frontend tests on the V2 execution path)." + if [ "$INPUT_ALLOW_PARTIAL" != "true" ]; then + echo "::error title=TestSprite::${skipped} test(s) skipped — failing the job. Set allow-partial: true to accept partial runs, or use a V3 project / test list for full coverage." + exit 1 + fi fi fi From dc14c9d6b55104958b406edb2ac5eacde694afe0 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 11 Aug 2026 14:25:18 -0700 Subject: [PATCH 08/20] fix(action): don't upload/advertise a JUnit report in single-test mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-test runs don't produce a JUnit report, but the upload step and junit-file output still keyed on the report path — so a stale testsprite-junit.xml (e.g. from a prior --all run in the same job) could be uploaded as this run's result and misattribute old data. Gate the upload on an empty test-id and emit an empty junit-file output in single-test mode. selftest v3-single-test now plants a stale report and asserts it is neither advertised nor uploaded. --- .github/workflows/selftest.yml | 10 ++++++++-- action.yml | 9 ++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index fcb30ef..a11c6df 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -105,6 +105,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + # Plant a stale report at the default path (as if a prior --all run wrote + # it). Single-test mode must NOT upload it or advertise it as its output. + - run: echo '' > testsprite-junit.xml - id: run continue-on-error: true uses: ./ @@ -114,6 +117,9 @@ jobs: endpoint-url: "https://api.testsprite.com:11443" - name: Verify single-test verdict + outputs run: | - echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }} junit-file='${{ steps.run.outputs.junit-file }}'" test "${{ steps.run.outputs.total }}" = "1" || { echo "::error::expected total=1 for a single test"; exit 1; } - echo "OK — single test produced total=1 and a pass/fail verdict." + # junit-file output must be EMPTY in single-test mode even though a + # stale testsprite-junit.xml is present (Codex finding). + test -z "${{ steps.run.outputs.junit-file }}" || { echo "::error::junit-file should be empty in single-test mode, got '${{ steps.run.outputs.junit-file }}'"; exit 1; } + echo "OK — single test: total=1, no JUnit advertised despite a stale file." diff --git a/action.yml b/action.yml index a8ec4b1..6a19b57 100644 --- a/action.yml +++ b/action.yml @@ -70,8 +70,8 @@ inputs: outputs: junit-file: - description: "Path to the JUnit report." - value: ${{ inputs.report-file }} + description: "Path to the JUnit report (empty in single-test mode — JUnit is batch-only)." + value: ${{ inputs.test-id == '' && inputs.report-file || '' }} passed: description: "Number of tests that passed." value: ${{ steps.run.outputs.passed }} @@ -179,7 +179,10 @@ runs: exit "$code" - name: Upload JUnit report - if: ${{ always() && inputs.upload-artifact == 'true' && hashFiles(inputs.report-file) != '' }} + # Only the full-project run produces a JUnit report. Gate on an empty + # test-id so a single-test run never uploads a stale/pre-existing file at + # the default report path as if it were this run's result. + if: ${{ always() && inputs.test-id == '' && inputs.upload-artifact == 'true' && hashFiles(inputs.report-file) != '' }} uses: actions/upload-artifact@v4 with: name: testsprite-junit From 302981b6a167d9127e52cf45bb08c93d794e1cd9 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 18 Aug 2026 15:41:58 -0700 Subject: [PATCH 09/20] fix(action): allow-partial must green a fully-skipped run; self-test real-user-ref on main only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The self-test caught a real defect: with allow-partial=true, a frontend-only V2 project dispatches nothing and the CLI exits 7 (incomplete), which the action's final `exit "$code"` propagated — reddening the job despite allow-partial. Under allow-partial (full-project runs) accept a skipped/deferred-only batch as GREEN, reddening only on a real test failure (failed>0). Also point the real-user-remote-ref self-test job at @main and gate it to main-only: a remote `uses:` resolves from the default branch, so pinning a stale feature-branch SHA tested old code and always failed. It now validates the real consumer shape against main after the change lands. --- .github/workflows/selftest.yml | 9 ++++++--- action.yml | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index a11c6df..be39e33 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -55,15 +55,18 @@ jobs: # Real-user shape: reference the action REMOTELY by ref (what a consumer's # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just # `uses` + api-key + project. This exercises GitHub's action-resolution path - # (fetch the repo at the pinned ref, load its action.yml, run the composite). + # (fetch the repo at the ref, load its action.yml, run the composite). + # Runs on `main` only: a remote `uses` resolves the ref from the DEFAULT + # branch's published action, so on a feature branch it would test stale code. + # After this lands on main it validates the real consumer shape against main. # `endpoint-url` / `allow-partial` are here only because the dev test project # is frontend-only on V2; a real V3/prod project with passing tests needs # neither. real-user-remote-ref: - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} runs-on: ubuntu-latest steps: - - uses: TestSprite/testsprite-action@736b57ebca2ffafdf375051482d94d70f0635b01 + - uses: TestSprite/testsprite-action@main with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" diff --git a/action.yml b/action.yml index 6a19b57..3261aef 100644 --- a/action.yml +++ b/action.yml @@ -162,9 +162,9 @@ runs: } >> "$GITHUB_OUTPUT" # Partial-run guard (full-project runs only): skipped tests (e.g. frontend - # tests on the V2 path) do NOT affect the CLI exit code, so a partial - # batch can exit 0. Detect them from the envelope and fail unless - # allow-partial is set. + # tests on the V2 path) are invisible to a pass/fail count, so a batch + # that skipped everything can look clean. Detect them from the envelope + # and fail unless allow-partial is set. if [ -z "$INPUT_TEST_ID" ]; then skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) if [ "${skipped:-0}" -gt 0 ]; then @@ -176,6 +176,16 @@ runs: fi fi + # Final verdict. Under allow-partial (full-project runs), a batch that only + # skipped/deferred tests still exits NON-ZERO from the CLI — a fully + # frontend-only V2 project dispatches nothing and the CLI exits 7 + # (incomplete). Accept that as GREEN, reddening only on a real test + # failure; otherwise allow-partial can't actually make a skipped run pass. + if [ "$INPUT_ALLOW_PARTIAL" = "true" ] && [ -z "$INPUT_TEST_ID" ]; then + [ "${failed:-0}" -gt 0 ] && exit 1 + exit 0 + fi + exit "$code" - name: Upload JUnit report From 830a99d552a0d041388f0e78ed2d16b328c8d315 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 18 Aug 2026 16:03:46 -0700 Subject: [PATCH 10/20] chore(selftest): parameterize dev host + project/test ids via repo secrets Move the hardcoded dev endpoint (api.testsprite.com:11443) and the dev project/test ids out of the workflow file and into repo secrets (TESTSPRITE_ENDPOINT_URL / TESTSPRITE_V2_PROJECT / TESTSPRITE_V3_PROJECT / TESTSPRITE_V3_TEST_ID). Secrets are masked in Actions logs and absent from the YAML, so nothing dev-specific leaks if this repo is ever made public. The API keys were already secrets. --- .github/workflows/selftest.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index be39e33..aec771c 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -22,8 +22,8 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} - project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" - endpoint-url: "https://api.testsprite.com:11443" + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} allow-partial: "false" - name: Assert the guard fired (action should have failed) run: | @@ -44,8 +44,8 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} - project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" - endpoint-url: "https://api.testsprite.com:11443" + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} allow-partial: "true" - name: Show outputs run: | @@ -69,8 +69,8 @@ jobs: - uses: TestSprite/testsprite-action@main with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }} - project: "19475ce4-44c9-41e2-b1f6-5dc73d58cf36" - endpoint-url: "https://api.testsprite.com:11443" + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} allow-partial: "true" # V3 real execution: a V3 project runs frontend tests natively, so this @@ -89,9 +89,9 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} - project: "90048e9c-4de3-4266-bee5-5d722df101b4" + project: ${{ secrets.TESTSPRITE_V3_PROJECT }} filter: "price filtering" - endpoint-url: "https://api.testsprite.com:11443" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} report-file: "results/junit.xml" - name: Verify a JUnit report with real test cases was produced run: | @@ -116,8 +116,8 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} - test-id: "46872b89-f74f-4443-8fe6-36a93cdb0dc2" - endpoint-url: "https://api.testsprite.com:11443" + test-id: ${{ secrets.TESTSPRITE_V3_TEST_ID }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} - name: Verify single-test verdict + outputs run: | echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }} junit-file='${{ steps.run.outputs.junit-file }}'" From 60de3366a6426948fdb70b8be0d6cff8c71d6076 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 18 Aug 2026 16:21:04 -0700 Subject: [PATCH 11/20] fix(action): count real failures from runs[].status; harden self-test assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial-review follow-ups: - allow-partial now reds only on a REAL test failure. `failed` was read from the CLI summary's `.failed` = total-passed-timedOut, which folds deferred / conflict / not-found (partials) into the count — so allow-partial would red a batch that merely couldn't dispatch everything. Now `failed` = runs[].status in {failed,blocked}, and allow-partial reds only on those. - allow-partial that executed NOTHING (all skipped/deferred) now emits a loud ::warning instead of a silent green. - Single-test verdict derives red/green from the run status (authoritative), not the CLI exit code alone, so a blocked/timeout single test can't leave the job green while the `failed` output is 1. - New `skipped` action output. - Self-test: v2-guard asserts skipped>0 (proves the guard fired, not that the action broke for an unrelated reason); v2-allow-partial asserts skipped>0 as a V2-skip-premise + account-drift/credit tripwire (skipped=0 => account may be V3, executing + burning credits). Real-backend jobs restricted to main + ci/**. - Dropped an internal ticket ref from a comment. --- .github/workflows/selftest.yml | 29 +++++++++++--- action.yml | 70 +++++++++++++++++++++------------- 2 files changed, 68 insertions(+), 31 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index aec771c..14c71ff 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -4,8 +4,11 @@ # execution, no credits). name: selftest +# Real-backend jobs run only on main + ci/** (not every feature branch), so an +# unrelated commit doesn't fire dev-backend runs (dev-uptime/rate-limit noise). on: push: + branches: [main, "ci/**"] workflow_dispatch: jobs: @@ -25,14 +28,21 @@ jobs: project: ${{ secrets.TESTSPRITE_V2_PROJECT }} endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} allow-partial: "false" - - name: Assert the guard fired (action should have failed) + - name: Assert the guard fired (action should have failed ON SKIPPED TESTS) run: | - echo "action outcome: ${{ steps.run.outcome }}" + echo "action outcome: ${{ steps.run.outcome }} skipped: ${{ steps.run.outputs.skipped }}" if [ "${{ steps.run.outcome }}" != "failure" ]; then echo "::error::Expected the action to FAIL (FE skipped, allow-partial=false), but it did not." exit 1 fi - echo "OK — partial-run guard fired as expected." + # Distinguish "the partial-run guard fired" from "the action broke for an + # UNRELATED reason (CLI crash, bad key, dev down)" — outcome==failure + # alone can't. The guard only fires when tests were actually skipped. + if [ -z "${{ steps.run.outputs.skipped }}" ] || [ "${{ steps.run.outputs.skipped }}" = "0" ]; then + echo "::error::Action failed but skipped=${{ steps.run.outputs.skipped }} — it did NOT fail via the partial-run guard (something else broke)." + exit 1 + fi + echo "OK — partial-run guard fired (skipped=${{ steps.run.outputs.skipped }})." # allow-partial=true: same skip, but the job stays green (warning only). v2-allow-partial: @@ -47,10 +57,19 @@ jobs: project: ${{ secrets.TESTSPRITE_V2_PROJECT }} endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} allow-partial: "true" - - name: Show outputs + - name: Assert the V2-skip premise held (no browser execution / no credits) run: | - echo "passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" + echo "skipped=${{ steps.run.outputs.skipped }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}" echo "junit=${{ steps.run.outputs.junit-file }}" + # This scenario relies on the V2 account SKIPPING FE tests (no execution, + # no credits). skipped=0 means the account likely no longer runs on V2 + # (FE now executes + costs credits) — fail LOUDLY instead of silently + # burning credits on every push. Doubles as an account-drift tripwire. + if [ -z "${{ steps.run.outputs.skipped }}" ] || [ "${{ steps.run.outputs.skipped }}" = "0" ]; then + echo "::error::Expected FE tests SKIPPED on the V2 account (skipped>0), got skipped=${{ steps.run.outputs.skipped }}. The test account may have been flipped to V3 (tests now execute + cost credits) — check featureFlags.v3." + exit 1 + fi + echo "OK — FE skipped on V2 as expected; no execution, no credits." # Real-user shape: reference the action REMOTELY by ref (what a consumer's # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just diff --git a/action.yml b/action.yml index 3261aef..ffbc8c4 100644 --- a/action.yml +++ b/action.yml @@ -76,11 +76,14 @@ outputs: description: "Number of tests that passed." value: ${{ steps.run.outputs.passed }} failed: - description: "Number of tests that did not pass." + description: "Number of tests that failed to a verdict (failed/blocked) — not deferred/skipped." value: ${{ steps.run.outputs.failed }} total: description: "Total tests in the summary." value: ${{ steps.run.outputs.total }} + skipped: + description: "Tests skipped and not run (full-project runs; 0 in single-test mode)." + value: ${{ steps.run.outputs.skipped }} runs: using: "composite" @@ -117,16 +120,15 @@ runs: fi summary="$(mktemp)" envjson="$(mktemp)" - passed=0; failed=0; total=0 + passed=0; failed=0; total=0; skipped=0 # --output json puts the machine envelope on stdout (captured here) and # routes CLI annotations to stderr (shown in the log + parsed by Actions). if [ -n "$INPUT_TEST_ID" ]; then # Single test by id. The CLI's JUnit report is batch-only, so there is - # no report here; the verdict comes from the run envelope's status and - # the non-zero exit reds the job. The CLI emits the ::error annotation - # itself under GITHUB_ACTIONS on a CLI version with single-test CI - # output (DEV-772); older versions just red/green without an annotation. + # no report here; the verdict comes from the run envelope's status. The + # CLI emits the ::error annotation itself under GITHUB_ACTIONS on a CLI + # version with single-test CI output; older versions just red/green. args=(test run "$INPUT_TEST_ID" --wait --output json --timeout "$INPUT_TIMEOUT") if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi set +e; testsprite "${args[@]}" > "$envjson"; code=$?; set -e @@ -150,39 +152,55 @@ runs: set +e; testsprite "${args[@]}" > "$envjson"; code=$?; set -e if [ -s "$summary" ]; then passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) - failed=$(jq -r '.failed // 0' "$summary" 2>/dev/null || echo 0) total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) + # `failed` = tests that ran to a FAILING verdict (failed/blocked), + # counted from runs[].status — NOT the summary's `failed`, which folds + # in deferred / conflict / not-found (partials, not failures) and would + # red an allow-partial run that merely couldn't dispatch everything. + failed=$(jq -r '[.runs[]? | select(.status == "failed" or .status == "blocked")] | length' "$summary" 2>/dev/null || echo 0) fi + # Skipped (frontend-on-V2 etc.) live on the raw envelope, not the summary. + skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) fi { echo "passed=$passed" echo "failed=$failed" echo "total=$total" + echo "skipped=$skipped" } >> "$GITHUB_OUTPUT" - # Partial-run guard (full-project runs only): skipped tests (e.g. frontend - # tests on the V2 path) are invisible to a pass/fail count, so a batch - # that skipped everything can look clean. Detect them from the envelope - # and fail unless allow-partial is set. - if [ -z "$INPUT_TEST_ID" ]; then - skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) - if [ "${skipped:-0}" -gt 0 ]; then - echo "::warning title=TestSprite::${skipped} test(s) were skipped and did not run (e.g. frontend tests on the V2 execution path)." - if [ "$INPUT_ALLOW_PARTIAL" != "true" ]; then - echo "::error title=TestSprite::${skipped} test(s) skipped — failing the job. Set allow-partial: true to accept partial runs, or use a V3 project / test list for full coverage." - exit 1 - fi + # Partial-run guard (full-project runs only): skipped tests are invisible + # to a pass/fail count, so a batch that skipped everything can look clean. + # Fail unless allow-partial is set. + if [ -z "$INPUT_TEST_ID" ] && [ "${skipped:-0}" -gt 0 ]; then + echo "::warning title=TestSprite::${skipped} test(s) were skipped and did not run (e.g. frontend tests on the V2 execution path)." + if [ "$INPUT_ALLOW_PARTIAL" != "true" ]; then + echo "::error title=TestSprite::${skipped} test(s) skipped — failing the job. Set allow-partial: true to accept partial runs, or use a V3 project / test list for full coverage." + exit 1 fi fi - # Final verdict. Under allow-partial (full-project runs), a batch that only - # skipped/deferred tests still exits NON-ZERO from the CLI — a fully - # frontend-only V2 project dispatches nothing and the CLI exits 7 - # (incomplete). Accept that as GREEN, reddening only on a real test - # failure; otherwise allow-partial can't actually make a skipped run pass. - if [ "$INPUT_ALLOW_PARTIAL" = "true" ] && [ -z "$INPUT_TEST_ID" ]; then - [ "${failed:-0}" -gt 0 ] && exit 1 + # Single-test verdict — green iff the test passed. Derived from the status + # (authoritative, matches the outputs above), not the CLI exit code alone, + # so a blocked/timeout single test can't leave the job green while the + # `failed` output is 1. + if [ -n "$INPUT_TEST_ID" ]; then + [ "$status" = "passed" ] && exit 0 + echo "::error title=TestSprite::single test ${INPUT_TEST_ID} did not pass (status: ${status})." + exit 1 + fi + + # Full-project verdict under allow-partial: red ONLY on a real test failure + # (failed/blocked, from runs[].status above). A skipped/deferred-only batch + # that reached no failing verdict is accepted as GREEN — but if NOTHING ran + # to a verdict (all skipped/deferred), say so loudly so a mis-scoped run + # isn't a silent green. + if [ "$INPUT_ALLOW_PARTIAL" = "true" ]; then + if [ "${failed:-0}" -gt 0 ]; then exit 1; fi + if [ "$((passed + failed))" -eq 0 ]; then + echo "::warning title=TestSprite::allow-partial: no tests executed to a verdict (all skipped/deferred) — the job is green but nothing was verified." + fi exit 0 fi From 96d967fade60b19f8101613f60bac86a70bd9865 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 18 Aug 2026 16:38:38 -0700 Subject: [PATCH 12/20] chore(selftest): also run on dev pushes (dev-first flow) --- .github/workflows/selftest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 14c71ff..7d465ad 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -8,7 +8,7 @@ name: selftest # unrelated commit doesn't fire dev-backend runs (dev-uptime/rate-limit noise). on: push: - branches: [main, "ci/**"] + branches: [main, dev, "ci/**"] workflow_dispatch: jobs: From e796eaf59f18e8506c418ec50e600b3ba1b63ea0 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 18 Aug 2026 19:51:26 -0700 Subject: [PATCH 13/20] test(selftest): point v3-real filter at a real project test name The hardcoded 'price filtering' matched no test in the configured V3 project, so the run produced an empty JUnit and failed the report check. Use a test name that exists in the project so v3-real actually executes one browser test. --- .github/workflows/selftest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 7d465ad..3c966ff 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -109,7 +109,7 @@ jobs: with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} project: ${{ secrets.TESTSPRITE_V3_PROJECT }} - filter: "price filtering" + filter: "Sign in from the login page" endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} report-file: "results/junit.xml" - name: Verify a JUnit report with real test cases was produced From c67b495ecececc08742293953c32c6b22f49104f Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 18 Aug 2026 20:00:52 -0700 Subject: [PATCH 14/20] test(selftest): inline the V3 project/test ids instead of secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They are dev fixture ids, not sensitive. As secrets, GitHub masked them to *** everywhere — including inside the dashboard deep-link in the job summary, so the rendered link pointed at /test-cases/*** and 404'd. Inlining them keeps the summary link clickable with real ids. --- .github/workflows/selftest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 3c966ff..8247576 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -108,7 +108,7 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} - project: ${{ secrets.TESTSPRITE_V3_PROJECT }} + project: "f01fd7ee-91ba-465e-bce6-2cb1dc14346d" filter: "Sign in from the login page" endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} report-file: "results/junit.xml" @@ -135,7 +135,7 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} - test-id: ${{ secrets.TESTSPRITE_V3_TEST_ID }} + test-id: "dd0b3be0-ba8b-4e0f-9dd9-61ad38a45b3a" endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} - name: Verify single-test verdict + outputs run: | From 3d7ab7bc6caf2c384bc137165f200f0ca4e14376 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 24 Aug 2026 18:16:49 -0700 Subject: [PATCH 15/20] fix(action): allow-partial must not green a crashed run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A non-zero CLI exit with no failing verdict (auth/network/crash → empty summary, passed=0/failed=0) previously hit the allow-partial `exit 0` and greened the job with only a ::warning — a CI gate reporting success on a broken pipeline. Skips never change the CLI exit code, so `code != 0` in the allow-partial branch can only be a crash: respect it and exit non-zero. A genuine all-skip (code=0) still stays green. Self-test: a bogus-key job asserts the action REDS under allow-partial instead of greening. --- .github/workflows/selftest.yml | 34 ++++++++++++++++++++++++++++++++++ action.yml | 7 +++++++ 2 files changed, 41 insertions(+) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 8247576..e4d8087 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -71,6 +71,40 @@ jobs: fi echo "OK — FE skipped on V2 as expected; no execution, no credits." + # allow-partial=true must NOT green a CRASHED run. A broken run (auth / network + # / CLI crash) exits non-zero with an empty summary — passed=0, failed=0 — which + # the allow-partial branch used to `exit 0` on (green with only a ::warning). A + # bogus (well-formed but invalid) key forces exactly that shape: the CLI passes + # its client-side format check, reaches the server, and 401s. The action must + # RED. Skips never change the CLI exit code, so this can only be a crash. + allow-partial-crash-reds: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: "sk-user-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + project: ${{ secrets.TESTSPRITE_V2_PROJECT }} + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "true" + - name: Assert allow-partial did NOT swallow the crash (action should FAIL) + run: | + echo "outcome=${{ steps.run.outcome }} skipped=${{ steps.run.outputs.skipped }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }}" + if [ "${{ steps.run.outcome }}" != "failure" ]; then + echo "::error::allow-partial greened a crashed run (bogus key → CLI non-zero exit, empty summary). The crash-swallow gate did not fire." + exit 1 + fi + # A crash is not a skip — the failure must be the crash gate, not the + # partial-run guard (which only fires when tests were actually skipped). + if [ -n "${{ steps.run.outputs.skipped }}" ] && [ "${{ steps.run.outputs.skipped }}" != "0" ]; then + echo "::error::Expected a crash (skipped=0), got skipped=${{ steps.run.outputs.skipped }} — failed via the wrong path." + exit 1 + fi + echo "OK — allow-partial reds a crashed run instead of greening it." + # Real-user shape: reference the action REMOTELY by ref (what a consumer's # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just # `uses` + api-key + project. This exercises GitHub's action-resolution path diff --git a/action.yml b/action.yml index ffbc8c4..257b10c 100644 --- a/action.yml +++ b/action.yml @@ -198,6 +198,13 @@ runs: # isn't a silent green. if [ "$INPUT_ALLOW_PARTIAL" = "true" ]; then if [ "${failed:-0}" -gt 0 ]; then exit 1; fi + # A non-zero CLI exit with no failing verdict is a crash / auth / network + # error, NOT a skip — skips never change the CLI exit code, so `code != 0` + # here can only mean the run broke. Never green a broken pipeline. + if [ "$code" -ne 0 ]; then + echo "::error title=TestSprite::allow-partial: CLI exited ${code} and no test failed to a verdict — a crash/auth/network error, not a skip. Failing." + exit "$code" + fi if [ "$((passed + failed))" -eq 0 ]; then echo "::warning title=TestSprite::allow-partial: no tests executed to a verdict (all skipped/deferred) — the job is green but nothing was verified." fi From d8a3cdbb774012d12578a350607135be29c872da Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Mon, 24 Aug 2026 18:44:43 -0700 Subject: [PATCH 16/20] fix(action): single-test must not green a non-zero CLI exit; tighten self-test gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symmetric to the allow-partial fix: single-test mode greened on `status==passed` alone, ignoring the CLI exit code — a post-verdict cleanup/upload/network failure (status=passed, code!=0) went green. Now requires status=passed AND code==0. Self-test tightening: - allow-partial-crash-reds: also assert the empty-summary shape (passed=0 failed=0 total=0), so it proves the failure came from the CRASH gate, not a real failed>0 run (which would also make outcome=failure). - v3-single-test: point at a stable FE-fixtures test (not the cal demo), assert the run DISPATCHED (error-code empty) and reached a coherent verdict (passed+failed==1) rather than only total=1 (written unconditionally). Not a hard passed=1 — a real browser run can flake and a flaky-red self-test is worse. --- .github/workflows/selftest.yml | 62 +++++++++++++++++++++++++++------- action.yml | 12 +++---- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index e4d8087..4cd981d 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -91,19 +91,31 @@ jobs: endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} allow-partial: "true" - name: Assert allow-partial did NOT swallow the crash (action should FAIL) + env: + O: ${{ steps.run.outcome }} + SKIPPED: ${{ steps.run.outputs.skipped }} + PASSED: ${{ steps.run.outputs.passed }} + FAILED: ${{ steps.run.outputs.failed }} + TOTAL: ${{ steps.run.outputs.total }} run: | - echo "outcome=${{ steps.run.outcome }} skipped=${{ steps.run.outputs.skipped }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }}" - if [ "${{ steps.run.outcome }}" != "failure" ]; then + echo "outcome=$O skipped=$SKIPPED passed=$PASSED failed=$FAILED total=$TOTAL" + if [ "$O" != "failure" ]; then echo "::error::allow-partial greened a crashed run (bogus key → CLI non-zero exit, empty summary). The crash-swallow gate did not fire." exit 1 fi - # A crash is not a skip — the failure must be the crash gate, not the - # partial-run guard (which only fires when tests were actually skipped). - if [ -n "${{ steps.run.outputs.skipped }}" ] && [ "${{ steps.run.outputs.skipped }}" != "0" ]; then - echo "::error::Expected a crash (skipped=0), got skipped=${{ steps.run.outputs.skipped }} — failed via the wrong path." + # Prove the failure came from the CRASH gate, not from a real failed>0 + # run (which would also make outcome=failure). A crash has the + # empty-summary shape: no verdict counted at all. skipped stays 0 too + # (a crash is not a skip). + if [ "${PASSED:-0}" != "0" ] || [ "${FAILED:-0}" != "0" ] || [ "${TOTAL:-0}" != "0" ]; then + echo "::error::Expected the empty-summary crash shape (passed=0 failed=0 total=0), got passed=$PASSED failed=$FAILED total=$TOTAL — the action failed via a real verdict, not the crash gate." + exit 1 + fi + if [ -n "$SKIPPED" ] && [ "$SKIPPED" != "0" ]; then + echo "::error::Expected a crash (skipped=0), got skipped=$SKIPPED — failed via the wrong path." exit 1 fi - echo "OK — allow-partial reds a crashed run instead of greening it." + echo "OK — allow-partial reds a crashed run (empty-summary shape) instead of greening it." # Real-user shape: reference the action REMOTELY by ref (what a consumer's # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just @@ -169,13 +181,39 @@ jobs: uses: ./ with: api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} - test-id: "dd0b3be0-ba8b-4e0f-9dd9-61ad38a45b3a" + # A stable test in the dedicated FE-fixtures project (a simple + # "home page" assertion), not the cal demo — more reliable per run. + test-id: "77b9daa0-f3ed-4883-8e26-66055a66d595" endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} - name: Verify single-test verdict + outputs + env: + O: ${{ steps.run.outcome }} + EC: ${{ steps.run.outputs.error-code }} + PASSED: ${{ steps.run.outputs.passed }} + FAILED: ${{ steps.run.outputs.failed }} + TOTAL: ${{ steps.run.outputs.total }} + JUNIT: ${{ steps.run.outputs.junit-file }} run: | - echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }} junit-file='${{ steps.run.outputs.junit-file }}'" - test "${{ steps.run.outputs.total }}" = "1" || { echo "::error::expected total=1 for a single test"; exit 1; } + echo "outcome=$O error-code='$EC' passed=$PASSED failed=$FAILED total=$TOTAL junit-file='$JUNIT'" + # Tolerate a transient backend blip (auth/throttle), same as v3-real. + if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then + echo "::warning::transient backend error ($EC) — skipping single-test validation this run." + exit 0 + fi + # The run must have DISPATCHED (no API-layer error) — else total=1 is the + # hardcoded single-test default, not evidence the test actually ran. + [ -z "$EC" ] || { echo "::error::single test did not run (error: $EC)"; exit 1; } + [ "$TOTAL" = "1" ] || { echo "::error::expected total=1 for a single test, got $TOTAL"; exit 1; } + # And reached a REAL verdict — passed+failed==1. total=1 alone is written + # unconditionally, so this is what distinguishes a run that produced a + # verdict (pass OR fail) from an unknown/broken run the job would else green. + # NOT a hard passed=1: a real browser run can flake, and a flaky RED + # self-test is worse than one that proves the path reached a verdict. + if [ "$(( ${PASSED:-0} + ${FAILED:-0} ))" != "1" ]; then + echo "::error::single test reached no coherent verdict (passed=$PASSED failed=$FAILED) — unknown/broken run, not a real pass/fail." + exit 1 + fi # junit-file output must be EMPTY in single-test mode even though a # stale testsprite-junit.xml is present (Codex finding). - test -z "${{ steps.run.outputs.junit-file }}" || { echo "::error::junit-file should be empty in single-test mode, got '${{ steps.run.outputs.junit-file }}'"; exit 1; } - echo "OK — single test: total=1, no JUnit advertised despite a stale file." + [ -z "$JUNIT" ] || { echo "::error::junit-file should be empty in single-test mode, got '$JUNIT'"; exit 1; } + echo "OK — single test dispatched, reached a coherent verdict (passed=$PASSED failed=$FAILED), no stale JUnit advertised." diff --git a/action.yml b/action.yml index 257b10c..0b3266c 100644 --- a/action.yml +++ b/action.yml @@ -181,13 +181,13 @@ runs: fi fi - # Single-test verdict — green iff the test passed. Derived from the status - # (authoritative, matches the outputs above), not the CLI exit code alone, - # so a blocked/timeout single test can't leave the job green while the - # `failed` output is 1. + # Single-test verdict — green iff the test passed AND the CLI exited 0. + # Requiring BOTH signals is the same rule as the allow-partial branch + # below: a `passed` status with a non-zero exit is a post-verdict failure + # (cleanup / upload / network), not a pass — never green a broken run. if [ -n "$INPUT_TEST_ID" ]; then - [ "$status" = "passed" ] && exit 0 - echo "::error title=TestSprite::single test ${INPUT_TEST_ID} did not pass (status: ${status})." + if [ "$status" = "passed" ] && [ "$code" -eq 0 ]; then exit 0; fi + echo "::error title=TestSprite::single test ${INPUT_TEST_ID} did not pass (status: ${status}, exit: ${code})." exit 1 fi From 5d15815da8902b705e77a64aeb18b5299a423427 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 25 Aug 2026 13:57:07 -0700 Subject: [PATCH 17/20] fix(action): real single-test counts + error-code output; partial vs crash; strict-path false-green; target-url guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Release-gate review follow-ups: - error-code is now a REAL output (was referenced but never declared). Captured from the CLI's API-error envelope on stderr (--output json puts it there), sliced from the first line-leading brace past the idempotency-key line. - single-test counts come from --summary-file (the CLI writes one for a single `test run --wait` too), so a crash/auth failure that writes no summary leaves total=0 — the self-test's total==1 assertion is now load-bearing, not the old unconditional total=1. Added single-test-crash-reds to prove it reds. - allow-partial no longer mislabels a PARTIAL dispatch as a crash: exit 6 (all-conflict) and 7 (rate-deferred/timeout) are accepted; only other non-zero exits red. Fixes deferred-batch being failed with a crash message. - strict (default) path now reds a run that executed NOTHING (zero-match filter or empty project, code=0/total=0/skipped=0) instead of a silent green. - target-url rejected on the full-project path (the CLI errors on --target-url with --all) with a clear message; docs corrected to single-test-only. - nits: cli-version via env (no shell interpolation); early-exit paths emit zero-valued outputs; stale on-push header comment fixed. - README Outputs table updated (failed semantics, skipped + error-code rows, single-test junit caveat). --- .github/workflows/selftest.yml | 63 ++++++++++++++----- README.md | 16 ++--- action.yml | 112 ++++++++++++++++++++++----------- 3 files changed, 134 insertions(+), 57 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 4cd981d..75334e9 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -4,8 +4,9 @@ # execution, no credits). name: selftest -# Real-backend jobs run only on main + ci/** (not every feature branch), so an -# unrelated commit doesn't fire dev-backend runs (dev-uptime/rate-limit noise). +# Runs on push to main / dev / ci/** (not every feature branch), so an unrelated +# commit doesn't fire dev-backend runs (dev-uptime/rate-limit noise). The +# credit-burning V3 jobs are further gated to workflow_dispatch only. on: push: branches: [main, dev, "ci/**"] @@ -117,6 +118,41 @@ jobs: fi echo "OK — allow-partial reds a crashed run (empty-summary shape) instead of greening it." + # Single-test mode must RED a crash too (symmetric to the batch gate above), and + # this is what proves the v3-single-test assertions can actually fail: a bogus + # key fails at auth (non-zero exit, no --summary-file written), so total stays 0 + # and error-code surfaces. If total==1 / error-code were still phantom, this job + # could not go red. + single-test-crash-reds: + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: "sk-user-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + test-id: "77b9daa0-f3ed-4883-8e26-66055a66d595" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + - name: Assert single-test crash REDS with load-bearing outputs + env: + O: ${{ steps.run.outcome }} + EC: ${{ steps.run.outputs.error-code }} + TOTAL: ${{ steps.run.outputs.total }} + run: | + echo "outcome=$O error-code='$EC' total=$TOTAL" + if [ "$O" != "failure" ]; then + echo "::error::single-test crash greened (bogus key). The status+exit gate did not fire." + exit 1 + fi + # A crash writes no summary → total=0 (so v3-single-test's total==1 is + # load-bearing), and the API error surfaces on error-code (so that output + # is real, not the phantom the previous version read). + [ "${TOTAL:-0}" = "0" ] || { echo "::error::expected total=0 on a crash (no summary written), got $TOTAL"; exit 1; } + [ -n "$EC" ] || { echo "::error::expected error-code set on an auth failure, got empty"; exit 1; } + echo "OK — single-test crash reds; total=0 + error-code='$EC' prove the outputs are load-bearing." + # Real-user shape: reference the action REMOTELY by ref (what a consumer's # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just # `uses` + api-key + project. This exercises GitHub's action-resolution path @@ -195,24 +231,21 @@ jobs: JUNIT: ${{ steps.run.outputs.junit-file }} run: | echo "outcome=$O error-code='$EC' passed=$PASSED failed=$FAILED total=$TOTAL junit-file='$JUNIT'" - # Tolerate a transient backend blip (auth/throttle), same as v3-real. + # Tolerate a transient backend blip (auth/throttle) — error-code is a real + # action output now, so this branch actually fires when it should. if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then echo "::warning::transient backend error ($EC) — skipping single-test validation this run." exit 0 fi - # The run must have DISPATCHED (no API-layer error) — else total=1 is the - # hardcoded single-test default, not evidence the test actually ran. + # The run must have DISPATCHED (no API-layer error). [ -z "$EC" ] || { echo "::error::single test did not run (error: $EC)"; exit 1; } - [ "$TOTAL" = "1" ] || { echo "::error::expected total=1 for a single test, got $TOTAL"; exit 1; } - # And reached a REAL verdict — passed+failed==1. total=1 alone is written - # unconditionally, so this is what distinguishes a run that produced a - # verdict (pass OR fail) from an unknown/broken run the job would else green. - # NOT a hard passed=1: a real browser run can flake, and a flaky RED - # self-test is worse than one that proves the path reached a verdict. - if [ "$(( ${PASSED:-0} + ${FAILED:-0} ))" != "1" ]; then - echo "::error::single test reached no coherent verdict (passed=$PASSED failed=$FAILED) — unknown/broken run, not a real pass/fail." - exit 1 - fi + # total==1 is now load-bearing: single-test counts come from the CLI's + # --summary-file, which a crash / auth / network failure never writes + # (total stays 0). So total==1 means a real run produced a result — NOT + # the old unconditional total=1. (Deliberately not a hard passed==1: a + # real browser run can flake, and a flaky-RED self-test is worse than one + # that proves the single-test path produced a verdict.) + [ "$TOTAL" = "1" ] || { echo "::error::expected total=1 (a real single-test run wrote a summary), got $TOTAL"; exit 1; } # junit-file output must be EMPTY in single-test mode even though a # stale testsprite-junit.xml is present (Codex finding). [ -z "$JUNIT" ] || { echo "::error::junit-file should be empty in single-test mode, got '$JUNIT'"; exit 1; } diff --git a/README.md b/README.md index aa2b790..15f6396 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | `project` | `""` | Project id. If empty, `TESTSPRITE_PROJECT_ID` must be set. | | `test-id` | `""` | Run a single test by id (whole-project run otherwise). `project`/`filter` ignored; no JUnit (batch-only). | | `filter` | `""` | Only run tests whose name contains this substring. Full-project run only; mutually exclusive with `test-id`. | -| `target-url` | `""` | Target URL override (V2 path only; ignored on V3). | +| `target-url` | `""` | Target URL override. **Single-test (`test-id`) runs only** — the CLI rejects it on a full-project run, so setting it without `test-id` fails fast. | | `report-file` | `testsprite-junit.xml` | JUnit XML path. | | `cli-version` | `latest` | npm version/dist-tag of `@testsprite/testsprite-cli`. | | `allow-partial` | `false` | If false, fail the job when tests are skipped (see below). | @@ -56,12 +56,14 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). ## Outputs -| Output | Description | -| ------------ | ---------------------------------- | -| `junit-file` | Path to the JUnit report. | -| `passed` | Number of tests that passed. | -| `failed` | Number of tests that did not pass. | -| `total` | Total tests in the summary. | +| Output | Description | +| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `junit-file` | Path to the JUnit report (empty in single-test mode — JUnit is batch-only). | +| `passed` | Number of tests that passed. | +| `failed` | Number of tests that ran to a **failing verdict** (`failed`/`blocked`) — not deferred, conflicted, or skipped. | +| `total` | Total tests in the summary. | +| `skipped` | Tests skipped and not run (full-project runs, e.g. frontend tests on the V2 path; `0` in single-test mode). | +| `error-code` | The CLI's API-layer error code when the run failed **before** a verdict (e.g. `AUTH_INVALID`, `NOT_FOUND`, `UNAVAILABLE`); empty otherwise. | ## CI-native output diff --git a/action.yml b/action.yml index 0b3266c..4f71ff5 100644 --- a/action.yml +++ b/action.yml @@ -33,7 +33,7 @@ inputs: required: false default: "" target-url: - description: "Optional target URL override (applied on the V2 path; ignored on V3)." + description: "Optional target URL override. Single-test (test-id) runs only — the CLI rejects it on a full-project run, so setting it without test-id fails fast." required: false default: "" report-file: @@ -84,6 +84,9 @@ outputs: skipped: description: "Tests skipped and not run (full-project runs; 0 in single-test mode)." value: ${{ steps.run.outputs.skipped }} + error-code: + description: "The CLI's API-layer error code when the run failed BEFORE a verdict (e.g. AUTH_INVALID, NOT_FOUND, VALIDATION_ERROR, UNAVAILABLE, RATE_LIMITED); empty on a dispatched run. Lets a caller tell a transient/infra failure from a real test failure." + value: ${{ steps.run.outputs.error-code }} runs: using: "composite" @@ -95,7 +98,10 @@ runs: - name: Install testsprite CLI shell: bash - run: npm install -g @testsprite/testsprite-cli@${{ inputs.cli-version }} + env: + # Via env, not interpolated into the command, so the value can't inject shell. + INPUT_CLI_VERSION: ${{ inputs.cli-version }} + run: npm install -g "@testsprite/testsprite-cli@${INPUT_CLI_VERSION}" - name: Run TestSprite tests id: run @@ -114,27 +120,50 @@ runs: INPUT_ALLOW_PARTIAL: ${{ inputs.allow-partial }} run: | set -uo pipefail + + # Emit zero-valued outputs on an early validation exit so a consumer reading + # e.g. steps.run.outputs.total gets `0`, not an empty string. + emit_zero_outputs() { + { + echo "passed=0"; echo "failed=0"; echo "total=0"; echo "skipped=0" + echo "error-code=VALIDATION_ERROR" + } >> "$GITHUB_OUTPUT" + } + + # target-url is single-test only: the published CLI rejects `--target-url` + # together with `--all` (VALIDATION_ERROR / exit 5), so on the full-project + # path it would ALWAYS fail with a CLI-internal message about a flag the + # consumer never typed. Reject it here with a clear one instead. + if [ -n "$INPUT_TARGET_URL" ] && [ -z "$INPUT_TEST_ID" ]; then + emit_zero_outputs + echo "::error title=TestSprite::target-url applies to a single-test run only (the CLI rejects --target-url with --all). Set test-id, or remove target-url." + exit 1 + fi if [ -n "$INPUT_TEST_ID" ] && [ -n "$INPUT_FILTER" ]; then + emit_zero_outputs echo "::error title=TestSprite::test-id and filter are mutually exclusive (filter only applies to a full-project run)." exit 1 fi + summary="$(mktemp)" envjson="$(mktemp)" - passed=0; failed=0; total=0; skipped=0 + errfile="$(mktemp)" + passed=0; failed=0; total=0; skipped=0; error_code="" - # --output json puts the machine envelope on stdout (captured here) and - # routes CLI annotations to stderr (shown in the log + parsed by Actions). + # --output json puts the machine run envelope on stdout (captured here); CLI + # annotations AND the API-error envelope go to stderr — captured to a file + # for parsing, then re-surfaced (`cat >&2`) so Actions still renders them. if [ -n "$INPUT_TEST_ID" ]; then - # Single test by id. The CLI's JUnit report is batch-only, so there is - # no report here; the verdict comes from the run envelope's status. The - # CLI emits the ::error annotation itself under GITHUB_ACTIONS on a CLI - # version with single-test CI output; older versions just red/green. - args=(test run "$INPUT_TEST_ID" --wait --output json --timeout "$INPUT_TIMEOUT") + # Single test by id. Write a --summary-file too (the CLI supports it for a + # single `test run --wait`): the counts then come from a real + # artifact, so a crash / auth / network failure that writes no summary + # leaves total=0 — distinguishable from a real run's total=1. JUnit stays + # batch-only, so no --report here. + args=(test run "$INPUT_TEST_ID" --wait --output json --summary-file "$summary" --timeout "$INPUT_TIMEOUT") if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi - set +e; testsprite "${args[@]}" > "$envjson"; code=$?; set -e + set +e; testsprite "${args[@]}" > "$envjson" 2> "$errfile"; code=$?; set -e + cat "$errfile" >&2 status=$(jq -r '.status // "unknown"' "$envjson" 2>/dev/null || echo unknown) - total=1 - if [ "$status" = "passed" ]; then passed=1; else failed=1; fi else # Whole project (optionally name-filtered), with a JUnit report. # --gh-output auto-enables under GITHUB_ACTIONS=true, so ::error:: @@ -148,26 +177,33 @@ runs: --summary-file "$summary" --timeout "$INPUT_TIMEOUT") if [ -n "$INPUT_FILTER" ]; then args+=(--filter "$INPUT_FILTER"); fi - if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi - set +e; testsprite "${args[@]}" > "$envjson"; code=$?; set -e - if [ -s "$summary" ]; then - passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) - total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) - # `failed` = tests that ran to a FAILING verdict (failed/blocked), - # counted from runs[].status — NOT the summary's `failed`, which folds - # in deferred / conflict / not-found (partials, not failures) and would - # red an allow-partial run that merely couldn't dispatch everything. - failed=$(jq -r '[.runs[]? | select(.status == "failed" or .status == "blocked")] | length' "$summary" 2>/dev/null || echo 0) - fi + set +e; testsprite "${args[@]}" > "$envjson" 2> "$errfile"; code=$?; set -e + cat "$errfile" >&2 # Skipped (frontend-on-V2 etc.) live on the raw envelope, not the summary. skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) fi + # Counts from the summary artifact (both paths). `passed`/`total` are the + # CLI's; `failed` = FAILING verdicts only (failed/blocked from runs[].status) + # — NOT the summary's `failed`, which folds in deferred/conflict/not-found + # (partials, not failures). No summary written ⇒ counts stay 0 (a crash). + if [ -s "$summary" ]; then + passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) + total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) + failed=$(jq -r '[.runs[]? | select(.status == "failed" or .status == "blocked")] | length' "$summary" 2>/dev/null || echo 0) + fi + # error-code = the CLI's API-layer error code when the run failed BEFORE a + # verdict. Its JSON envelope is on stderr under --output json (stdout carries + # the run envelope, empty on such a failure), preceded by the idempotency-key + # line — so slice from the first line-leading `{` before parsing. + error_code=$(sed -n '/^{/,$p' "$errfile" | jq -r '.error.code // ""' 2>/dev/null || echo "") + { echo "passed=$passed" echo "failed=$failed" echo "total=$total" echo "skipped=$skipped" + echo "error-code=$error_code" } >> "$GITHUB_OUTPUT" # Partial-run guard (full-project runs only): skipped tests are invisible @@ -191,26 +227,32 @@ runs: exit 1 fi - # Full-project verdict under allow-partial: red ONLY on a real test failure - # (failed/blocked, from runs[].status above). A skipped/deferred-only batch - # that reached no failing verdict is accepted as GREEN — but if NOTHING ran - # to a verdict (all skipped/deferred), say so loudly so a mis-scoped run - # isn't a silent green. + # Full-project verdict under allow-partial: green a PARTIAL run (skips / + # rate-deferred / all-conflict), red a real failure or a crash. if [ "$INPUT_ALLOW_PARTIAL" = "true" ]; then if [ "${failed:-0}" -gt 0 ]; then exit 1; fi - # A non-zero CLI exit with no failing verdict is a crash / auth / network - # error, NOT a skip — skips never change the CLI exit code, so `code != 0` - # here can only mean the run broke. Never green a broken pipeline. - if [ "$code" -ne 0 ]; then - echo "::error title=TestSprite::allow-partial: CLI exited ${code} and no test failed to a verdict — a crash/auth/network error, not a skip. Failing." + # exit 6 (nothing dispatched — every case already in flight) and 7 + # (rate-deferred / timed out) are PARTIAL dispatch, which is exactly what + # allow-partial accepts. Any OTHER non-zero exit is a crash / auth / + # network / validation error — never green that. + if [ "$code" -ne 0 ] && [ "$code" -ne 6 ] && [ "$code" -ne 7 ]; then + echo "::error title=TestSprite::allow-partial: CLI exited ${code} with no failing verdict — a crash/auth/network/validation error, not a partial run. Failing." exit "$code" fi if [ "$((passed + failed))" -eq 0 ]; then - echo "::warning title=TestSprite::allow-partial: no tests executed to a verdict (all skipped/deferred) — the job is green but nothing was verified." + echo "::warning title=TestSprite::allow-partial: no tests executed to a verdict (all skipped/deferred/conflict) — the job is green but nothing was verified." fi exit 0 fi + # Default (strict) path. A clean exit that ran NOTHING — a zero-match + # --filter, or a project with no tests — is a false green in a CI gate: + # red it so a typo'd filter or a mis-scoped run isn't a silent pass. + if [ "$code" -eq 0 ] && [ "${total:-0}" -eq 0 ] && [ "${skipped:-0}" -eq 0 ]; then + echo "::error title=TestSprite::no tests ran (a --filter that matched nothing, or a project with no tests). Failing so a mis-scoped run isn't a silent green." + exit 1 + fi + exit "$code" - name: Upload JUnit report From 675213109ef0bf7c9365422ccc41a76ec42b3355 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 25 Aug 2026 14:01:33 -0700 Subject: [PATCH 18/20] fix(action): configurable artifact-name to avoid upload-artifact@v4 duplicate-name failure actions/upload-artifact@v4 hard-fails on a duplicate artifact name within one workflow run, so the fixed 'testsprite-junit' name broke any matrix or multi-project workflow that invoked the action twice. Expose an artifact-name input (default unchanged) so each invocation can use a unique name. --- README.md | 1 + action.yml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 15f6396..6e61ea3 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | `endpoint-url` | `""` | API base URL override. | | `node-version` | `lts/*` | Node version used to install/run the CLI. | | `upload-artifact` | `true` | Upload the JUnit report as a workflow artifact. | +| `artifact-name` | `testsprite-junit` | Name of the uploaded JUnit artifact. Give each a **unique** name when the action runs more than once (matrix / multiple projects) in one workflow — `upload-artifact@v4` fails on duplicate names. | ## Outputs diff --git a/action.yml b/action.yml index 4f71ff5..6ff2c50 100644 --- a/action.yml +++ b/action.yml @@ -67,6 +67,10 @@ inputs: description: "Upload the JUnit report as a workflow artifact." required: false default: "true" + artifact-name: + description: "Name of the uploaded JUnit artifact. actions/upload-artifact@v4 fails on a duplicate name within one workflow run, so give each a UNIQUE name when this action runs more than once (a matrix, or several projects) in the same workflow." + required: false + default: "testsprite-junit" outputs: junit-file: @@ -262,6 +266,6 @@ runs: if: ${{ always() && inputs.test-id == '' && inputs.upload-artifact == 'true' && hashFiles(inputs.report-file) != '' }} uses: actions/upload-artifact@v4 with: - name: testsprite-junit + name: ${{ inputs.artifact-name }} path: ${{ inputs.report-file }} if-no-files-found: ignore From 88a20acfa7a855af7a7672b677dac75bb867e8cc Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 25 Aug 2026 15:39:33 -0700 Subject: [PATCH 19/20] fix(action): red timed-out runs under allow-partial; harden single-test self-test - allow-partial no longer greens a batch containing a timed-out run: a timeout (status 'timeout', CLI exit 7) is not a partial dispatch. Derive timedOut from the summary, add a timed-out output, red the job when >0. - hoist the zero-scope guard (clean exit, ran nothing) above the allow-partial branch so an empty-scope run reds regardless of allow-partial. - capture CLI stderr live via a fifo+tee (wait-reaped) so a cancelled step still shows what streamed, and the error-code parse is deterministic. - emit_zero_outputs leaves error-code empty (a local input rejection is not a CLI/API error code). - v3-single-test: restore passed+failed==1 and assert the job outcome is coherent with the verdict (pass greens, fail reds). - docs: test-id + filter are mutually exclusive (hard error), not 'ignored'. --- .github/workflows/selftest.yml | 24 ++++++++--- README.md | 3 +- action.yml | 79 +++++++++++++++++++++++++--------- 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 75334e9..ae2a12a 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -239,14 +239,24 @@ jobs: fi # The run must have DISPATCHED (no API-layer error). [ -z "$EC" ] || { echo "::error::single test did not run (error: $EC)"; exit 1; } - # total==1 is now load-bearing: single-test counts come from the CLI's - # --summary-file, which a crash / auth / network failure never writes - # (total stays 0). So total==1 means a real run produced a result — NOT - # the old unconditional total=1. (Deliberately not a hard passed==1: a - # real browser run can flake, and a flaky-RED self-test is worse than one - # that proves the single-test path produced a verdict.) + # total==1 means a real run wrote a --summary-file (a crash / auth / network + # failure never does — total stays 0). But total also counts a `timeout` + # status, so it alone doesn't prove a VERDICT was reached... [ "$TOTAL" = "1" ] || { echo "::error::expected total=1 (a real single-test run wrote a summary), got $TOTAL"; exit 1; } + # ...so passed+failed==1 is the load-bearing one: it proves the run reached + # a real pass/fail verdict, not a timeout/unknown that still counts in total. + [ "$((PASSED + FAILED))" = "1" ] || { echo "::error::expected exactly one verdict (passed+failed==1), got passed=$PASSED failed=$FAILED — a timeout/unknown status, not a real result."; exit 1; } + # And the action's JOB OUTCOME must be coherent with that verdict — the + # single-test path's whole contract is verdict → job status. 77b9daa0 is a + # stable passing fixture, so a pass MUST green; if the browser flaked to a + # real FAIL, that's tolerated but the action MUST then have red-ed. + if [ "${PASSED:-0}" = "1" ]; then + [ "$O" = "success" ] || { echo "::error::test passed but the action outcome was '$O' (expected success)"; exit 1; } + else + [ "$O" = "failure" ] || { echo "::error::test failed (failed=$FAILED) but the action outcome was '$O' (expected failure)"; exit 1; } + echo "::warning::single-test fixture flaked to a failing verdict this run — the action correctly red-ed." + fi # junit-file output must be EMPTY in single-test mode even though a # stale testsprite-junit.xml is present (Codex finding). [ -z "$JUNIT" ] || { echo "::error::junit-file should be empty in single-test mode, got '$JUNIT'"; exit 1; } - echo "OK — single test dispatched, reached a coherent verdict (passed=$PASSED failed=$FAILED), no stale JUnit advertised." + echo "OK — single test dispatched, reached a verdict coherent with outcome=$O (passed=$PASSED failed=$FAILED), no stale JUnit advertised." diff --git a/README.md b/README.md index 6e61ea3..ef875c0 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | ----------------- | -------------------- | --------------------------------------------------------------------------- | | `api-key` | — (required) | TestSprite API key. Pass a secret. | | `project` | `""` | Project id. If empty, `TESTSPRITE_PROJECT_ID` must be set. | -| `test-id` | `""` | Run a single test by id (whole-project run otherwise). `project`/`filter` ignored; no JUnit (batch-only). | +| `test-id` | `""` | Run a single test by id (whole-project run otherwise). `project` ignored; `filter` must NOT be set (mutually exclusive — fails fast if both given); no JUnit (batch-only). | | `filter` | `""` | Only run tests whose name contains this substring. Full-project run only; mutually exclusive with `test-id`. | | `target-url` | `""` | Target URL override. **Single-test (`test-id`) runs only** — the CLI rejects it on a full-project run, so setting it without `test-id` fails fast. | | `report-file` | `testsprite-junit.xml` | JUnit XML path. | @@ -64,6 +64,7 @@ pass — or if tests were skipped (see [`allow-partial`](#partial-runs)). | `failed` | Number of tests that ran to a **failing verdict** (`failed`/`blocked`) — not deferred, conflicted, or skipped. | | `total` | Total tests in the summary. | | `skipped` | Tests skipped and not run (full-project runs, e.g. frontend tests on the V2 path; `0` in single-test mode). | +| `timed-out` | Tests that ran but did **not** reach a verdict within `timeout` (status `timeout`). Counted separately from `failed`; under `allow-partial: true` these still fail the job. | | `error-code` | The CLI's API-layer error code when the run failed **before** a verdict (e.g. `AUTH_INVALID`, `NOT_FOUND`, `UNAVAILABLE`); empty otherwise. | ## CI-native output diff --git a/action.yml b/action.yml index 6ff2c50..79f5bce 100644 --- a/action.yml +++ b/action.yml @@ -21,9 +21,10 @@ inputs: test-id: description: > Run a single test by id instead of the whole project. When set, `project` - and `filter` are ignored and no JUnit report is produced (the CLI's JUnit - report is batch-only); the job still goes red/green on the verdict and - emits a summary + annotation. + is ignored and `filter` must NOT be set (test-id and filter are mutually + exclusive — the action fails fast if both are given). No JUnit report is + produced (the CLI's JUnit report is batch-only); the job still goes + red/green on the verdict and emits a summary + annotation. required: false default: "" filter: @@ -88,6 +89,9 @@ outputs: skipped: description: "Tests skipped and not run (full-project runs; 0 in single-test mode)." value: ${{ steps.run.outputs.skipped }} + timed-out: + description: "Tests that ran but did not reach a verdict within `timeout` (status 'timeout'). Counted separately from `failed`; under allow-partial these still fail the job." + value: ${{ steps.run.outputs.timed-out }} error-code: description: "The CLI's API-layer error code when the run failed BEFORE a verdict (e.g. AUTH_INVALID, NOT_FOUND, VALIDATION_ERROR, UNAVAILABLE, RATE_LIMITED); empty on a dispatched run. Lets a caller tell a transient/infra failure from a real test failure." value: ${{ steps.run.outputs.error-code }} @@ -130,7 +134,11 @@ runs: emit_zero_outputs() { { echo "passed=0"; echo "failed=0"; echo "total=0"; echo "skipped=0" - echo "error-code=VALIDATION_ERROR" + echo "timed-out=0" + # error-code is the CLI's API-layer error code, not this action's own + # input rejections — leave it empty here so a consumer can't mistake a + # local validation refusal for a CLI/API error. + echo "error-code=" } >> "$GITHUB_OUTPUT" } @@ -152,11 +160,20 @@ runs: summary="$(mktemp)" envjson="$(mktemp)" errfile="$(mktemp)" - passed=0; failed=0; total=0; skipped=0; error_code="" + passed=0; failed=0; total=0; skipped=0; timed_out=0; error_code="" # --output json puts the machine run envelope on stdout (captured here); CLI - # annotations AND the API-error envelope go to stderr — captured to a file - # for parsing, then re-surfaced (`cat >&2`) so Actions still renders them. + # annotations AND the API-error envelope go to stderr. We need stderr BOTH + # live (so a cancelled/killed step still shows what streamed) AND captured to + # a file (to parse the API-error envelope). A background `tee` reading a fifo + # gives both: the CLI's stderr → fifo → tee → $errfile + the terminal, in + # real time. `wait "$tee_pid"` after the run makes the file read deterministic + # (bash does not reap a process substitution, so a plain `2> >(tee …)` would + # race the parse below). Only one of the two branches runs, so one tee suffices. + runlog="$(mktemp -u)"; mkfifo "$runlog" + tee "$errfile" < "$runlog" >&2 & + tee_pid=$! + if [ -n "$INPUT_TEST_ID" ]; then # Single test by id. Write a --summary-file too (the CLI supports it for a # single `test run --wait`): the counts then come from a real @@ -165,8 +182,7 @@ runs: # batch-only, so no --report here. args=(test run "$INPUT_TEST_ID" --wait --output json --summary-file "$summary" --timeout "$INPUT_TIMEOUT") if [ -n "$INPUT_TARGET_URL" ]; then args+=(--target-url "$INPUT_TARGET_URL"); fi - set +e; testsprite "${args[@]}" > "$envjson" 2> "$errfile"; code=$?; set -e - cat "$errfile" >&2 + set +e; testsprite "${args[@]}" > "$envjson" 2> "$runlog"; code=$?; set -e status=$(jq -r '.status // "unknown"' "$envjson" 2>/dev/null || echo unknown) else # Whole project (optionally name-filtered), with a JUnit report. @@ -181,12 +197,15 @@ runs: --summary-file "$summary" --timeout "$INPUT_TIMEOUT") if [ -n "$INPUT_FILTER" ]; then args+=(--filter "$INPUT_FILTER"); fi - set +e; testsprite "${args[@]}" > "$envjson" 2> "$errfile"; code=$?; set -e - cat "$errfile" >&2 + set +e; testsprite "${args[@]}" > "$envjson" 2> "$runlog"; code=$?; set -e # Skipped (frontend-on-V2 etc.) live on the raw envelope, not the summary. skipped=$(jq -r '((.skippedFrontend // []) + (.skippedIntegration // [])) | length' "$envjson" 2>/dev/null || echo 0) fi + # Close the fifo write end (both branches are done) and reap tee, so $errfile + # is fully flushed before it is parsed below. + wait "$tee_pid"; rm -f "$runlog" + # Counts from the summary artifact (both paths). `passed`/`total` are the # CLI's; `failed` = FAILING verdicts only (failed/blocked from runs[].status) # — NOT the summary's `failed`, which folds in deferred/conflict/not-found @@ -195,6 +214,10 @@ runs: passed=$(jq -r '.passed // 0' "$summary" 2>/dev/null || echo 0) total=$(jq -r '.total // 0' "$summary" 2>/dev/null || echo 0) failed=$(jq -r '[.runs[]? | select(.status == "failed" or .status == "blocked")] | length' "$summary" 2>/dev/null || echo 0) + # Timed-out runs carry status "timeout" (NOT counted in `failed` above) and + # come back on CLI exit 7 — which allow-partial otherwise treats as an + # accepted partial. Surface the count so the verdict below can red it. + timed_out=$(jq -r '.timedOut // 0' "$summary" 2>/dev/null || echo 0) fi # error-code = the CLI's API-layer error code when the run failed BEFORE a # verdict. Its JSON envelope is on stderr under --output json (stdout carries @@ -207,6 +230,7 @@ runs: echo "failed=$failed" echo "total=$total" echo "skipped=$skipped" + echo "timed-out=$timed_out" echo "error-code=$error_code" } >> "$GITHUB_OUTPUT" @@ -231,14 +255,34 @@ runs: exit 1 fi + # Zero-scope guard (BOTH strict and allow-partial). A clean exit that ran + # NOTHING — a zero-match --filter, or a project with no tests — is a false + # green in a CI gate, and allow-partial is about accepting a PARTIAL run, not + # an EMPTY one: red it either way so a typo'd filter or a mis-scoped run isn't + # a silent pass. (skipped>0 is a different case, handled by the guard above; + # a non-zero exit means something was attempted, so it's not zero-scope.) + if [ "$code" -eq 0 ] && [ "${total:-0}" -eq 0 ] && [ "${skipped:-0}" -eq 0 ]; then + echo "::error title=TestSprite::no tests ran (a --filter that matched nothing, or a project with no tests). Failing so a mis-scoped run isn't a silent green." + exit 1 + fi + # Full-project verdict under allow-partial: green a PARTIAL run (skips / # rate-deferred / all-conflict), red a real failure or a crash. if [ "$INPUT_ALLOW_PARTIAL" = "true" ]; then if [ "${failed:-0}" -gt 0 ]; then exit 1; fi + # A timed-out run is NOT a partial dispatch — the case ran and never + # reached a verdict, so its result is unknown, not deferred. allow-partial + # must red it (otherwise, since a timeout exits 7 and is excluded from the + # crash gate below, ANY batch with a timed-out run would go green). + if [ "${timed_out:-0}" -gt 0 ]; then + echo "::error title=TestSprite::${timed_out} run(s) did not reach a verdict within ${INPUT_TIMEOUT}s. Raise timeout, or poll later with test wait ." + exit 1 + fi # exit 6 (nothing dispatched — every case already in flight) and 7 # (rate-deferred / timed out) are PARTIAL dispatch, which is exactly what # allow-partial accepts. Any OTHER non-zero exit is a crash / auth / - # network / validation error — never green that. + # network / validation error — never green that. (A genuine timeout also + # exits 7 but was already red-ed above by the timed_out check.) if [ "$code" -ne 0 ] && [ "$code" -ne 6 ] && [ "$code" -ne 7 ]; then echo "::error title=TestSprite::allow-partial: CLI exited ${code} with no failing verdict — a crash/auth/network/validation error, not a partial run. Failing." exit "$code" @@ -249,14 +293,9 @@ runs: exit 0 fi - # Default (strict) path. A clean exit that ran NOTHING — a zero-match - # --filter, or a project with no tests — is a false green in a CI gate: - # red it so a typo'd filter or a mis-scoped run isn't a silent pass. - if [ "$code" -eq 0 ] && [ "${total:-0}" -eq 0 ] && [ "${skipped:-0}" -eq 0 ]; then - echo "::error title=TestSprite::no tests ran (a --filter that matched nothing, or a project with no tests). Failing so a mis-scoped run isn't a silent green." - exit 1 - fi - + # Default (strict) path. The zero-scope guard already ran above; here a + # non-zero exit (a real failure verdict, crash, auth, or timeout→7) reds the + # job, and a clean exit that ran something is a genuine pass. exit "$code" - name: Upload JUnit report From 522c40a8a08ac6aa1c1c16b637a7d9b4ffb57704 Mon Sep 17 00:00:00 2001 From: Fangyuan_Zhang Date: Tue, 25 Aug 2026 16:23:42 -0700 Subject: [PATCH 20/20] test(action): regression job for the timeout gate; harden fifo + 6/7 exclusion - add timeout-gate-reds self-test (workflow_dispatch): allow-partial + a one-test filter + timeout:1 forces a timed-out batch; asserts the action reds AND that timed-out==1 (the red came from this gate, not a real failed>0). - wait "$tee_pid" || true: the composite shell runs under -e, so a non-zero tee must not kill the step before outputs/verdict are written. - fail closed when the CLI exits 7 but wrote no summary (indeterminate deferred-vs-timeout); scoped to 7 only (exit 6 all-conflict has no timeout). - mktemp -d + fifo inside it, avoiding the name-then-create TOCTOU. --- .github/workflows/selftest.yml | 46 ++++++++++++++++++++++++++++++++++ action.yml | 21 +++++++++++++--- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index ae2a12a..dc2ec34 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -260,3 +260,49 @@ jobs: # stale testsprite-junit.xml is present (Codex finding). [ -z "$JUNIT" ] || { echo "::error::junit-file should be empty in single-test mode, got '$JUNIT'"; exit 1; } echo "OK — single test dispatched, reached a verdict coherent with outcome=$O (passed=$PASSED failed=$FAILED), no stale JUnit advertised." + + # Regression job for the timeout gate (the fix that was silently wrong for two + # rounds). A gate nobody has seen go red is not yet a gate — this forces it. + # `timeout: 1` + a filter narrowed to ONE V3 test: the run dispatches, the poll + # gives up at 1s while the browser test is still running, the CLI exits 7 with + # timedOut=1, and the action MUST red. The assertion is timed-out==1 (not just + # outcome==failure), so it proves the red came from THIS gate and not a real + # failed>0 or a crash. workflow_dispatch-only: it costs one dispatched run's + # credits (hence the narrowed filter, same as v3-real-execution). + timeout-gate-reds: + if: ${{ github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - id: run + continue-on-error: true + uses: ./ + with: + api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }} + project: "f01fd7ee-91ba-465e-bce6-2cb1dc14346d" + filter: "Sign in from the login page" + endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }} + allow-partial: "true" + timeout: "1" + - name: Verify the timeout gate red-ed the batch + env: + O: ${{ steps.run.outcome }} + EC: ${{ steps.run.outputs.error-code }} + TO: ${{ steps.run.outputs.timed-out }} + FAILED: ${{ steps.run.outputs.failed }} + run: | + echo "outcome=$O error-code='$EC' timed-out=$TO failed=$FAILED" + # A transient backend error means the run never dispatched, so no timeout + # could occur — the gate can't be exercised. Skip rather than false-fail. + if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then + echo "::warning::transient backend error ($EC) — the run never dispatched, skipping the timeout-gate assertion this run." + exit 0 + fi + # The action MUST have red-ed (allow-partial is true, so ONLY the timeout + # gate can red this — a passing/deferred/conflict batch would be green). + [ "$O" = "failure" ] || { echo "::error::expected the action to RED on a timed-out batch under allow-partial, got outcome=$O"; exit 1; } + # ...and the red MUST come from the timeout gate specifically. + [ "$TO" = "1" ] || { echo "::error::expected timed-out=1 (the timeout gate fired), got timed-out=$TO — the red came from a different path"; exit 1; } + # A real failing verdict reds via failed>0, not this gate; assert it wasn't that. + [ "${FAILED:-0}" = "0" ] || { echo "::error::expected failed=0 (a real failure would red via a different path), got failed=$FAILED"; exit 1; } + echo "OK — allow-partial red-ed a timed-out batch via the timeout gate (timed-out=1, failed=0)." diff --git a/action.yml b/action.yml index 79f5bce..20a7ffa 100644 --- a/action.yml +++ b/action.yml @@ -170,7 +170,9 @@ runs: # real time. `wait "$tee_pid"` after the run makes the file read deterministic # (bash does not reap a process substitution, so a plain `2> >(tee …)` would # race the parse below). Only one of the two branches runs, so one tee suffices. - runlog="$(mktemp -u)"; mkfifo "$runlog" + # mktemp -d (not -u): create the dir, then name the fifo inside it — avoids the + # name-then-create TOCTOU a bare `mktemp -u` has. + tmpd="$(mktemp -d)"; runlog="$tmpd/stderr"; mkfifo "$runlog" tee "$errfile" < "$runlog" >&2 & tee_pid=$! @@ -203,8 +205,10 @@ runs: fi # Close the fifo write end (both branches are done) and reap tee, so $errfile - # is fully flushed before it is parsed below. - wait "$tee_pid"; rm -f "$runlog" + # is fully flushed before it is parsed below. `|| true` because the composite + # shell runs with `-e` (GitHub's default `bash -eo pipefail`): a non-zero tee + # must not kill the step BEFORE $GITHUB_OUTPUT is written and a verdict reached. + wait "$tee_pid" || true; rm -rf "$tmpd" # Counts from the summary artifact (both paths). `passed`/`total` are the # CLI's; `failed` = FAILING verdicts only (failed/blocked from runs[].status) @@ -278,6 +282,17 @@ runs: echo "::error title=TestSprite::${timed_out} run(s) did not reach a verdict within ${INPUT_TIMEOUT}s. Raise timeout, or poll later with test wait ." exit 1 fi + # The timed_out check above can only red a timeout the summary RECORDED. + # exit 7 means "rate-deferred OR timed out"; the summary is how we tell them + # apart. If the CLI exited 7 but wrote NO summary (emitCiArtifacts warns and + # continues if --summary-file can't be written — timed_out would then read 0), + # the outcome is indeterminate: fail closed rather than green a possibly + # timed-out batch. (Scoped to 7 only — exit 6 is all-conflict, which throws + # before any run reaches a timeout, so it has nothing to hide.) + if [ "$code" -eq 7 ] && [ ! -s "$summary" ]; then + echo "::error title=TestSprite::allow-partial: CLI exited 7 (rate-deferred or timed out) but wrote no summary — cannot confirm no run timed out. Failing." + exit 7 + fi # exit 6 (nothing dispatched — every case already in flight) and 7 # (rate-deferred / timed out) are PARTIAL dispatch, which is exactly what # allow-partial accepts. Any OTHER non-zero exit is a crash / auth /