diff --git a/.github/actions/check-test-outcome/action.yml b/.github/actions/check-test-outcome/action.yml index d87754d..c912345 100644 --- a/.github/actions/check-test-outcome/action.yml +++ b/.github/actions/check-test-outcome/action.yml @@ -9,8 +9,8 @@ description: >- The rules themselves are tools/check-test-outcome.py in this repository, the one runnable definition this action and a developer both call, so the CI - answer and the by-hand answer cannot differ. It reads surefire and pytest - summaries and sums every module's line. + answer and the by-hand answer cannot differ. It reads the surefire, pytest, + Go, VSTest and Catch2 summaries, and sums every module's line. inputs: log: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 70b0180..aa58e4a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 293 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 300 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the @@ -178,6 +178,34 @@ jobs: echo "::error::the check passed a VSTest run over an assembly holding no test" exit 1 fi + # The Catch2 dialect, which the DuckDB test runner writes and a + # MobilityDuck suite tees. The passing form's test-case count + # EXCLUDES the skipped cases, so the floor is read off the sum. + printf 'All tests passed (2695 assertions in 102 test cases)\n' \ + > "$RUNNER_TEMP/fx-catch-ok.log" + if ! tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-ok.log" --min-tests 102; then + echo "::error::the check refused a Catch2 log that satisfies both rules" + exit 1 + fi + if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-ok.log" --min-tests 103; then + echo "::error::the check passed a Catch2 suite below its floor" + exit 1 + fi + printf 'All tests passed (3 skip''ped tests, 2695 assertions in 99 test cases)\n' \ + > "$RUNNER_TEMP/fx-catch-skip.log" + if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-skip.log" --min-tests 102; then + echo "::error::the check passed a Catch2 log reporting skipped tests" + exit 1 + fi + # The failure table carries `101 passed`, which the pytest pattern + # matches too; reading it as pytest would report 101 as the total. + printf 'test cases: 102 | 101 passed | 1 failed\n' \ + > "$RUNNER_TEMP/fx-catch-table.log" + if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-table.log" --min-tests 102 \ + | grep -q 'pytest'; then + echo "::error::the check read a Catch2 failure table as pytest" + exit 1 + fi echo "the check refuses a skip, a shrunken suite, and a log with no summary, in every dialect it reads" - name: Upload meos-idl.json as artefact diff --git a/tests/test_test_outcome.py b/tests/test_test_outcome.py index dc128c2..10f6b40 100644 --- a/tests/test_test_outcome.py +++ b/tests/test_test_outcome.py @@ -76,6 +76,57 @@ def test_vstest_sums_every_assembly(self): self.assertEqual(2, len(lines)) +class Catch2Tests(unittest.TestCase): + """The Catch2 console reporter, which DuckDB's test runner writes.""" + + def test_catch2_reads_the_line_a_mobilityduck_run_writes(self): + # Verbatim from a MobilityDuck CI run of its 102 sqllogictest files. + total, skipped, dialect, _ = outcome.read_summaries( + "All tests passed (2695 assertions in 102 test cases)\n") + self.assertEqual(("catch2", 102, 0), (dialect, total, skipped)) + + def test_catch2_adds_the_skipped_cases_the_passing_count_excludes(self): + # The reporter prints `testCases.passed - skippedTests`, so 99 printed + # beside 3 skipped is a suite of 102. + total, skipped, dialect, _ = outcome.read_summaries( + "All tests passed (3 skipped tests, 2695 assertions in " + "99 test cases)\n") + self.assertEqual(("catch2", 102, 3), (dialect, total, skipped)) + + def test_catch2_reads_the_singular_forms(self): + total, skipped, dialect, _ = outcome.read_summaries( + "All tests passed (1 skipped test, 1 assertion in 1 test case)\n") + self.assertEqual(("catch2", 2, 1), (dialect, total, skipped)) + + def test_catch2_reads_the_failure_table(self): + log = ( + "test cases: 102 | 101 passed | 1 failed\n" + "assertions: 2695 | 2694 passed | 1 failed\n" + ) + total, skipped, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("catch2", 102, 0), (dialect, total, skipped)) + + def test_catch2_reads_the_skipped_column_when_the_table_carries_one(self): + # A column whose count is zero is omitted, so the skipped column is + # present only when it is not zero, and the leading total includes it. + log = "test cases: 102 | 99 passed | 1 failed | 2 skipped\n" + total, skipped, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("catch2", 102, 2), (dialect, total, skipped)) + + def test_catch2_reads_a_wholly_skipped_run(self): + total, skipped, dialect, _ = outcome.read_summaries( + "All tests were skipped (total skipped 7)\n") + self.assertEqual(("catch2", 7, 7), (dialect, total, skipped)) + + def test_catch2_is_read_before_pytest(self): + # THE ORDER IS THE TEST: `101 passed` in the failure table satisfies the + # pytest pattern too, and pytest-first would report 101 as the total, + # missing the failure and any skip beside it. + log = "test cases: 102 | 101 passed | 1 failed\n" + total, _, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("catch2", 102), (dialect, total)) + + class NoSummaryTests(unittest.TestCase): """A log carrying no summary at all names no dialect, which is a failure.""" diff --git a/tools/check-test-outcome.py b/tools/check-test-outcome.py index 0ceb2a7..1a64e5a 100755 --- a/tools/check-test-outcome.py +++ b/tools/check-test-outcome.py @@ -28,6 +28,15 @@ # reading only the last understates the total (measured: MobilityKafka prints 7 # and 4, so its total is 11 rather than the 4 a tail would report). # +# The Catch2 dialect reads the runner DuckDB builds, under which a +# MobilityDuck sqllogictest FILE is one test case. ITS SKIP COUNT CANNOT SEE A +# `mode skip`: that directive silences the statements after it while the file +# still reports as a passing test case, so a suite can retire assertions +# wholesale and this dialect reads `0 skipped` over it. What the dialect does +# hold for such a consumer is the FLOOR — the total moves when a whole file +# leaves the suite. A sqllogictest consumer needs a census of `mode skip` in the +# test SOURCES beside this, and a green answer here is not evidence about them. +# # The Go dialect prints no counts at all: `go test -v` writes one result line # per test and per subtest, and the totals are their tally. A run without -v # writes only `ok 0.42s`, which carries neither a total nor a skip count, @@ -69,6 +78,28 @@ r"(?:Passed|Failed)!\s+-\s+Failed:\s*(\d+),\s*Passed:\s*(\d+)," r"\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)") +# `All tests passed (2695 assertions in 102 test cases)` — what the Catch2 +# console reporter writes when nothing failed, carrying a leading +# `3 skipped tests, ` when any were. The test-case count it prints EXCLUDES the +# skipped ones, so the total is the two added: the reporter writes +# `testCases.passed - skippedTests` there. +CATCH_PASSED = re.compile( + r"All tests passed\s*\(\s*(?:(\d+)\s+skipped tests?,\s*)?" + r"(\d+)\s+assertions?\s+in\s+(\d+)\s+test cases?\s*\)") + +# `test cases: 102 | 101 passed | 1 failed` — the table the same reporter writes +# when anything failed. Here the leading number IS the whole total, skips +# included, and a column whose count is zero is omitted rather than printed, so +# the skipped column is read out of the tail when it is there at all. That +# column sums expected failures with skips; both are tests that did not assert, +# and the guard treats them alike. +CATCH_TOTALS = re.compile(r"\btest cases:\s*(\d+)((?:\s*\|\s*\d+\s+\w+)*)") + +# `All tests were skipped (total skipped 7)` — the one form that reports no +# passing count at all, written when every test skipped. +CATCH_ALL_SKIPPED = re.compile( + r"All tests were skipped\s*\(\s*total skipped\s*(\d+)\s*\)") + def read_summaries(text: str): """Return (total, skipped, dialect, lines) summed over every summary found.""" @@ -104,6 +135,35 @@ def read_summaries(text: str): if lines: return total, skipped, "vstest", lines + # Catch2 is read BEFORE pytest, and the order is load-bearing: the failure + # table's `101 passed` satisfies the pytest pattern, so a failing Catch2 run + # read pytest-first reports the passing count as the total and misses both + # the failures and the skips. + for raw in text.splitlines(): + line = raw.rstrip() + m = CATCH_ALL_SKIPPED.search(line) + if m: + every = int(m.group(1)) + total += every + skipped += every + lines.append(line.strip()) + continue + m = CATCH_PASSED.search(line) + if m: + some = int(m.group(1) or 0) + total += int(m.group(3)) + some + skipped += some + lines.append(line.strip()) + continue + m = CATCH_TOTALS.search(line) + if m: + total += int(m.group(1)) + column = re.search(r"(\d+)\s+skipped", m.group(2)) + skipped += int(column.group(1)) if column else 0 + lines.append(line.strip()) + if lines: + return total, skipped, "catch2", lines + for raw in text.splitlines(): m = PYTEST.search(raw) if m: