Beta promotion: dev → main (composite action + self-test)

.github/workflows/selftest.yml

@@ -0,0 +1,308 @@
+# Self-test: exercise the action end-to-end on a real GitHub runner against the
+# dev backend. The V2 project has frontend-only tests, which the V2 execution
+# path skips — so it deterministically drives the partial-run guard (no browser
+# execution, no credits).
+name: selftest
+
+# Runs on push to main / dev / ci/** (not every feature branch), so an unrelated
+# commit doesn't fire dev-backend runs (dev-uptime/rate-limit noise). The
+# credit-burning V3 jobs are further gated to workflow_dispatch only.
+on:
+  push:
+    branches: [main, dev, "ci/**"]
+  workflow_dispatch:
+
+jobs:
+  # allow-partial=false (default): FE tests are skipped on V2, so the action
+  # must FAIL the job. We run it with continue-on-error and assert it failed —
+  # proving the guard fires on a real runner while keeping this job green.
+  v2-guard-blocks-partial:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: run
+        continue-on-error: true
+        uses: ./
+        with:
+          api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }}
+          project: ${{ secrets.TESTSPRITE_V2_PROJECT }}
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+          allow-partial: "false"
+      - name: Assert the guard fired (action should have failed ON SKIPPED TESTS)
+        run: |
+          echo "action outcome: ${{ steps.run.outcome }}  skipped: ${{ steps.run.outputs.skipped }}"
+          if [ "${{ steps.run.outcome }}" != "failure" ]; then
+            echo "::error::Expected the action to FAIL (FE skipped, allow-partial=false), but it did not."
+            exit 1
+          fi
+          # Distinguish "the partial-run guard fired" from "the action broke for an
+          # UNRELATED reason (CLI crash, bad key, dev down)" — outcome==failure
+          # alone can't. The guard only fires when tests were actually skipped.
+          if [ -z "${{ steps.run.outputs.skipped }}" ] || [ "${{ steps.run.outputs.skipped }}" = "0" ]; then
+            echo "::error::Action failed but skipped=${{ steps.run.outputs.skipped }} — it did NOT fail via the partial-run guard (something else broke)."
+            exit 1
+          fi
+          echo "OK — partial-run guard fired (skipped=${{ steps.run.outputs.skipped }})."
+
+  # allow-partial=true: same skip, but the job stays green (warning only).
+  v2-allow-partial:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: run
+        uses: ./
+        with:
+          api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }}
+          project: ${{ secrets.TESTSPRITE_V2_PROJECT }}
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+          allow-partial: "true"
+      - name: Assert the V2-skip premise held (no browser execution / no credits)
+        run: |
+          echo "skipped=${{ steps.run.outputs.skipped }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}"
+          echo "junit=${{ steps.run.outputs.junit-file }}"
+          # This scenario relies on the V2 account SKIPPING FE tests (no execution,
+          # no credits). skipped=0 means the account likely no longer runs on V2
+          # (FE now executes + costs credits) — fail LOUDLY instead of silently
+          # burning credits on every push. Doubles as an account-drift tripwire.
+          if [ -z "${{ steps.run.outputs.skipped }}" ] || [ "${{ steps.run.outputs.skipped }}" = "0" ]; then
+            echo "::error::Expected FE tests SKIPPED on the V2 account (skipped>0), got skipped=${{ steps.run.outputs.skipped }}. The test account may have been flipped to V3 (tests now execute + cost credits) — check featureFlags.v3."
+            exit 1
+          fi
+          echo "OK — FE skipped on V2 as expected; no execution, no credits."
+
+  # allow-partial=true must NOT green a CRASHED run. A broken run (auth / network
+  # / CLI crash) exits non-zero with an empty summary — passed=0, failed=0 — which
+  # the allow-partial branch used to `exit 0` on (green with only a ::warning). A
+  # bogus (well-formed but invalid) key forces exactly that shape: the CLI passes
+  # its client-side format check, reaches the server, and 401s. The action must
+  # RED. Skips never change the CLI exit code, so this can only be a crash.
+  allow-partial-crash-reds:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: run
+        continue-on-error: true
+        uses: ./
+        with:
+          api-key: "sk-user-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+          project: ${{ secrets.TESTSPRITE_V2_PROJECT }}
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+          allow-partial: "true"
+      - name: Assert allow-partial did NOT swallow the crash (action should FAIL)
+        env:
+          O: ${{ steps.run.outcome }}
+          SKIPPED: ${{ steps.run.outputs.skipped }}
+          PASSED: ${{ steps.run.outputs.passed }}
+          FAILED: ${{ steps.run.outputs.failed }}
+          TOTAL: ${{ steps.run.outputs.total }}
+        run: |
+          echo "outcome=$O skipped=$SKIPPED passed=$PASSED failed=$FAILED total=$TOTAL"
+          if [ "$O" != "failure" ]; then
+            echo "::error::allow-partial greened a crashed run (bogus key → CLI non-zero exit, empty summary). The crash-swallow gate did not fire."
+            exit 1
+          fi
+          # Prove the failure came from the CRASH gate, not from a real failed>0
+          # run (which would also make outcome=failure). A crash has the
+          # empty-summary shape: no verdict counted at all. skipped stays 0 too
+          # (a crash is not a skip).
+          if [ "${PASSED:-0}" != "0" ] || [ "${FAILED:-0}" != "0" ] || [ "${TOTAL:-0}" != "0" ]; then
+            echo "::error::Expected the empty-summary crash shape (passed=0 failed=0 total=0), got passed=$PASSED failed=$FAILED total=$TOTAL — the action failed via a real verdict, not the crash gate."
+            exit 1
+          fi
+          if [ -n "$SKIPPED" ] && [ "$SKIPPED" != "0" ]; then
+            echo "::error::Expected a crash (skipped=0), got skipped=$SKIPPED — failed via the wrong path."
+            exit 1
+          fi
+          echo "OK — allow-partial reds a crashed run (empty-summary shape) instead of greening it."
+
+  # Single-test mode must RED a crash too (symmetric to the batch gate above), and
+  # this is what proves the v3-single-test assertions can actually fail: a bogus
+  # key fails at auth (non-zero exit, no --summary-file written), so total stays 0
+  # and error-code surfaces. If total==1 / error-code were still phantom, this job
+  # could not go red.
+  single-test-crash-reds:
+    if: ${{ github.event_name == 'push' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: run
+        continue-on-error: true
+        uses: ./
+        with:
+          api-key: "sk-user-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+          test-id: "77b9daa0-f3ed-4883-8e26-66055a66d595"
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+      - name: Assert single-test crash REDS with load-bearing outputs
+        env:
+          O: ${{ steps.run.outcome }}
+          EC: ${{ steps.run.outputs.error-code }}
+          TOTAL: ${{ steps.run.outputs.total }}
+        run: |
+          echo "outcome=$O error-code='$EC' total=$TOTAL"
+          if [ "$O" != "failure" ]; then
+            echo "::error::single-test crash greened (bogus key). The status+exit gate did not fire."
+            exit 1
+          fi
+          # A crash writes no summary → total=0 (so v3-single-test's total==1 is
+          # load-bearing), and the API error surfaces on error-code (so that output
+          # is real, not the phantom the previous version read).
+          [ "${TOTAL:-0}" = "0" ] || { echo "::error::expected total=0 on a crash (no summary written), got $TOTAL"; exit 1; }
+          [ -n "$EC" ] || { echo "::error::expected error-code set on an auth failure, got empty"; exit 1; }
+          echo "OK — single-test crash reds; total=0 + error-code='$EC' prove the outputs are load-bearing."
+
+  # Real-user shape: reference the action REMOTELY by ref (what a consumer's
+  # workflow does), NOT `./`. No checkout, no continue-on-error, no assert — just
+  # `uses` + api-key + project. This exercises GitHub's action-resolution path
+  # (fetch the repo at the ref, load its action.yml, run the composite).
+  # Runs on `main` only: a remote `uses` resolves the ref from the DEFAULT
+  # branch's published action, so on a feature branch it would test stale code.
+  # After this lands on main it validates the real consumer shape against main.
+  # `endpoint-url` / `allow-partial` are here only because the dev test project
+  # is frontend-only on V2; a real V3/prod project with passing tests needs
+  # neither.
+  real-user-remote-ref:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: TestSprite/testsprite-action@main
+        with:
+          api-key: ${{ secrets.TESTSPRITE_API_KEY_V2 }}
+          project: ${{ secrets.TESTSPRITE_V2_PROJECT }}
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+          allow-partial: "true"
+
+  # V3 real execution: a V3 project runs frontend tests natively, so this
+  # actually executes a browser test (filtered to one) and produces a JUnit
+  # report with real test cases — exercising upload-artifact, per-test
+  # annotations, and the job summary. continue-on-error because the browser test
+  # may pass or fail; we verify the plumbing (report has cases + is uploaded),
+  # not the verdict. Manual-only to control credit usage.
+  v3-real-execution:
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: run
+        continue-on-error: true
+        uses: ./
+        with:
+          api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }}
+          project: "f01fd7ee-91ba-465e-bce6-2cb1dc14346d"
+          filter: "Sign in from the login page"
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+          report-file: "results/junit.xml"
+      - name: Verify a JUnit report with real test cases was produced
+        run: |
+          echo "outcome=${{ steps.run.outcome }} passed=${{ steps.run.outputs.passed }} failed=${{ steps.run.outputs.failed }} total=${{ steps.run.outputs.total }}"
+          test -s results/junit.xml || { echo "::error::no JUnit report written"; exit 1; }
+          grep -q "<testcase" results/junit.xml || { echo "::error::JUnit report has no <testcase>"; exit 1; }
+          echo "OK — JUnit report with test cases:"; cat results/junit.xml
+
+  # Single-test-by-id mode on a real runner with the PUBLISHED CLI (no Gap A).
+  # A single test runs → verdict from the run envelope → red/green + counts.
+  # (Annotation appears once a CLI with single-test CI output is published.)
+  v3-single-test:
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # Plant a stale report at the default path (as if a prior --all run wrote
+      # it). Single-test mode must NOT upload it or advertise it as its output.
+      - run: echo '<testsuites><testsuite tests="1"/></testsuites>' > testsprite-junit.xml
+      - id: run
+        continue-on-error: true
+        uses: ./
+        with:
+          api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }}
+          # A stable test in the dedicated FE-fixtures project (a simple
+          # "home page" assertion), not the cal demo — more reliable per run.
+          test-id: "77b9daa0-f3ed-4883-8e26-66055a66d595"
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+      - name: Verify single-test verdict + outputs
+        env:
+          O: ${{ steps.run.outcome }}
+          EC: ${{ steps.run.outputs.error-code }}
+          PASSED: ${{ steps.run.outputs.passed }}
+          FAILED: ${{ steps.run.outputs.failed }}
+          TOTAL: ${{ steps.run.outputs.total }}
+          JUNIT: ${{ steps.run.outputs.junit-file }}
+        run: |
+          echo "outcome=$O error-code='$EC' passed=$PASSED failed=$FAILED total=$TOTAL junit-file='$JUNIT'"
+          # Tolerate a transient backend blip (auth/throttle) — error-code is a real
+          # action output now, so this branch actually fires when it should.
+          if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then
+            echo "::warning::transient backend error ($EC) — skipping single-test validation this run."
+            exit 0
+          fi
+          # The run must have DISPATCHED (no API-layer error).
+          [ -z "$EC" ] || { echo "::error::single test did not run (error: $EC)"; exit 1; }
+          # total==1 means a real run wrote a --summary-file (a crash / auth / network
+          # failure never does — total stays 0). But total also counts a `timeout`
+          # status, so it alone doesn't prove a VERDICT was reached...
+          [ "$TOTAL" = "1" ] || { echo "::error::expected total=1 (a real single-test run wrote a summary), got $TOTAL"; exit 1; }
+          # ...so passed+failed==1 is the load-bearing one: it proves the run reached
+          # a real pass/fail verdict, not a timeout/unknown that still counts in total.
+          [ "$((PASSED + FAILED))" = "1" ] || { echo "::error::expected exactly one verdict (passed+failed==1), got passed=$PASSED failed=$FAILED — a timeout/unknown status, not a real result."; exit 1; }
+          # And the action's JOB OUTCOME must be coherent with that verdict — the
+          # single-test path's whole contract is verdict → job status. 77b9daa0 is a
+          # stable passing fixture, so a pass MUST green; if the browser flaked to a
+          # real FAIL, that's tolerated but the action MUST then have red-ed.
+          if [ "${PASSED:-0}" = "1" ]; then
+            [ "$O" = "success" ] || { echo "::error::test passed but the action outcome was '$O' (expected success)"; exit 1; }
+          else
+            [ "$O" = "failure" ] || { echo "::error::test failed (failed=$FAILED) but the action outcome was '$O' (expected failure)"; exit 1; }
+            echo "::warning::single-test fixture flaked to a failing verdict this run — the action correctly red-ed."
+          fi
+          # junit-file output must be EMPTY in single-test mode even though a
+          # stale testsprite-junit.xml is present (Codex finding).
+          [ -z "$JUNIT" ] || { echo "::error::junit-file should be empty in single-test mode, got '$JUNIT'"; exit 1; }
+          echo "OK — single test dispatched, reached a verdict coherent with outcome=$O (passed=$PASSED failed=$FAILED), no stale JUnit advertised."
+
+  # Regression job for the timeout gate (the fix that was silently wrong for two
+  # rounds). A gate nobody has seen go red is not yet a gate — this forces it.
+  # `timeout: 1` + a filter narrowed to ONE V3 test: the run dispatches, the poll
+  # gives up at 1s while the browser test is still running, the CLI exits 7 with
+  # timedOut=1, and the action MUST red. The assertion is timed-out==1 (not just
+  # outcome==failure), so it proves the red came from THIS gate and not a real
+  # failed>0 or a crash. workflow_dispatch-only: it costs one dispatched run's
+  # credits (hence the narrowed filter, same as v3-real-execution).
+  timeout-gate-reds:
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: run
+        continue-on-error: true
+        uses: ./
+        with:
+          api-key: ${{ secrets.TESTSPRITE_API_KEY_V3 }}
+          project: "f01fd7ee-91ba-465e-bce6-2cb1dc14346d"
+          filter: "Sign in from the login page"
+          endpoint-url: ${{ secrets.TESTSPRITE_ENDPOINT_URL }}
+          allow-partial: "true"
+          timeout: "1"
+      - name: Verify the timeout gate red-ed the batch
+        env:
+          O: ${{ steps.run.outcome }}
+          EC: ${{ steps.run.outputs.error-code }}
+          TO: ${{ steps.run.outputs.timed-out }}
+          FAILED: ${{ steps.run.outputs.failed }}
+        run: |
+          echo "outcome=$O error-code='$EC' timed-out=$TO failed=$FAILED"
+          # A transient backend error means the run never dispatched, so no timeout
+          # could occur — the gate can't be exercised. Skip rather than false-fail.
+          if [ "$EC" = "UNAVAILABLE" ] || [ "$EC" = "RATE_LIMITED" ]; then
+            echo "::warning::transient backend error ($EC) — the run never dispatched, skipping the timeout-gate assertion this run."
+            exit 0
+          fi
+          # The action MUST have red-ed (allow-partial is true, so ONLY the timeout
+          # gate can red this — a passing/deferred/conflict batch would be green).
+          [ "$O" = "failure" ] || { echo "::error::expected the action to RED on a timed-out batch under allow-partial, got outcome=$O"; exit 1; }
+          # ...and the red MUST come from the timeout gate specifically.
+          [ "$TO" = "1" ] || { echo "::error::expected timed-out=1 (the timeout gate fired), got timed-out=$TO — the red came from a different path"; exit 1; }
+          # A real failing verdict reds via failed>0, not this gate; assert it wasn't that.
+          [ "${FAILED:-0}" = "0" ] || { echo "::error::expected failed=0 (a real failure would red via a different path), got failed=$FAILED"; exit 1; }
+          echo "OK — allow-partial red-ed a timed-out batch via the timeout gate (timed-out=1, failed=0)."