From 2b85dcfe12f90f48c46f1b02d4498e933f93d8a7 Mon Sep 17 00:00:00 2001 From: Esteban Zimanyi Date: Thu, 3 Sep 2026 02:32:47 +0200 Subject: [PATCH] Read the Catch2 dialect, so the guard covers a DuckDB consumer WITNESS: a MobilityDuck CI run writes its whole test summary as one line of the Catch2 console reporter DuckDB's test runner builds, reading ... (2695 assertions in 102 test cases) The rules read surefire, pytest, Go and VSTest, so that log names no dialect and reaches the no-summary failure: a sqllogictest consumer cannot use the action at all. MEASURED over that job log: the rules read `catch2`, total 102, 0 skipped, and a floor of 102 is accepted while 103 is refused. The passing form's test-case count EXCLUDES the skipped cases -- the reporter writes `testCases.passed - skippedTests` -- so `3 skipped tests, ... in 99 test cases` is a suite of 102 with 3 skipped, and the two are added rather than read off the printed count. The failure table and the wholly-skipped form are read too. tests/test_test_outcome.py goes 9 cases to 16, and the suite floor 293 to 300. The order is load-bearing and one of those cases holds it: the failure table's `101 passed` satisfies the pytest pattern, so Catch2 is read BEFORE pytest -- pytest-first reports 101 as the total and misses the failure and every skip beside it. WHY, and what this dialect does NOT do: under this runner a sqllogictest FILE is one test case, and a `mode skip` inside a file silences the statements after it while the file still reports as a passing test case. The skip count therefore cannot see a `mode skip`, and only the floor holds -- it moves when a whole file leaves the suite. The module says so where a reader meets the dialect, because a guard whose green answer is silence about the very construct a consumer uses is worse than none. A sqllogictest consumer needs a census of `mode skip` in the test sources beside this. --- .github/actions/check-test-outcome/action.yml | 4 +- .github/workflows/pytest.yml | 30 +++++++++- tests/test_test_outcome.py | 51 ++++++++++++++++ tools/check-test-outcome.py | 60 +++++++++++++++++++ 4 files changed, 142 insertions(+), 3 deletions(-) diff --git a/.github/actions/check-test-outcome/action.yml b/.github/actions/check-test-outcome/action.yml index d87754d..c912345 100644 --- a/.github/actions/check-test-outcome/action.yml +++ b/.github/actions/check-test-outcome/action.yml @@ -9,8 +9,8 @@ description: >- The rules themselves are tools/check-test-outcome.py in this repository, the one runnable definition this action and a developer both call, so the CI - answer and the by-hand answer cannot differ. It reads surefire and pytest - summaries and sums every module's line. + answer and the by-hand answer cannot differ. It reads the surefire, pytest, + Go, VSTest and Catch2 summaries, and sums every module's line. inputs: log: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 70b0180..aa58e4a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -96,7 +96,7 @@ jobs: # carries, or a change to them is not exercised until after it merges. # Consumers use the action; this repository owns the rules. - name: Refuse a skip, and a suite that shrank - run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 293 + run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 300 # The rules earn their place by refusing a log that carries what they # name. Both fixtures are written here rather than tracked, and the @@ -178,6 +178,34 @@ jobs: echo "::error::the check passed a VSTest run over an assembly holding no test" exit 1 fi + # The Catch2 dialect, which the DuckDB test runner writes and a + # MobilityDuck suite tees. The passing form's test-case count + # EXCLUDES the skipped cases, so the floor is read off the sum. + printf 'All tests passed (2695 assertions in 102 test cases)\n' \ + > "$RUNNER_TEMP/fx-catch-ok.log" + if ! tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-ok.log" --min-tests 102; then + echo "::error::the check refused a Catch2 log that satisfies both rules" + exit 1 + fi + if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-ok.log" --min-tests 103; then + echo "::error::the check passed a Catch2 suite below its floor" + exit 1 + fi + printf 'All tests passed (3 skip''ped tests, 2695 assertions in 99 test cases)\n' \ + > "$RUNNER_TEMP/fx-catch-skip.log" + if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-skip.log" --min-tests 102; then + echo "::error::the check passed a Catch2 log reporting skipped tests" + exit 1 + fi + # The failure table carries `101 passed`, which the pytest pattern + # matches too; reading it as pytest would report 101 as the total. + printf 'test cases: 102 | 101 passed | 1 failed\n' \ + > "$RUNNER_TEMP/fx-catch-table.log" + if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-table.log" --min-tests 102 \ + | grep -q 'pytest'; then + echo "::error::the check read a Catch2 failure table as pytest" + exit 1 + fi echo "the check refuses a skip, a shrunken suite, and a log with no summary, in every dialect it reads" - name: Upload meos-idl.json as artefact diff --git a/tests/test_test_outcome.py b/tests/test_test_outcome.py index dc128c2..10f6b40 100644 --- a/tests/test_test_outcome.py +++ b/tests/test_test_outcome.py @@ -76,6 +76,57 @@ def test_vstest_sums_every_assembly(self): self.assertEqual(2, len(lines)) +class Catch2Tests(unittest.TestCase): + """The Catch2 console reporter, which DuckDB's test runner writes.""" + + def test_catch2_reads_the_line_a_mobilityduck_run_writes(self): + # Verbatim from a MobilityDuck CI run of its 102 sqllogictest files. + total, skipped, dialect, _ = outcome.read_summaries( + "All tests passed (2695 assertions in 102 test cases)\n") + self.assertEqual(("catch2", 102, 0), (dialect, total, skipped)) + + def test_catch2_adds_the_skipped_cases_the_passing_count_excludes(self): + # The reporter prints `testCases.passed - skippedTests`, so 99 printed + # beside 3 skipped is a suite of 102. + total, skipped, dialect, _ = outcome.read_summaries( + "All tests passed (3 skipped tests, 2695 assertions in " + "99 test cases)\n") + self.assertEqual(("catch2", 102, 3), (dialect, total, skipped)) + + def test_catch2_reads_the_singular_forms(self): + total, skipped, dialect, _ = outcome.read_summaries( + "All tests passed (1 skipped test, 1 assertion in 1 test case)\n") + self.assertEqual(("catch2", 2, 1), (dialect, total, skipped)) + + def test_catch2_reads_the_failure_table(self): + log = ( + "test cases: 102 | 101 passed | 1 failed\n" + "assertions: 2695 | 2694 passed | 1 failed\n" + ) + total, skipped, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("catch2", 102, 0), (dialect, total, skipped)) + + def test_catch2_reads_the_skipped_column_when_the_table_carries_one(self): + # A column whose count is zero is omitted, so the skipped column is + # present only when it is not zero, and the leading total includes it. + log = "test cases: 102 | 99 passed | 1 failed | 2 skipped\n" + total, skipped, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("catch2", 102, 2), (dialect, total, skipped)) + + def test_catch2_reads_a_wholly_skipped_run(self): + total, skipped, dialect, _ = outcome.read_summaries( + "All tests were skipped (total skipped 7)\n") + self.assertEqual(("catch2", 7, 7), (dialect, total, skipped)) + + def test_catch2_is_read_before_pytest(self): + # THE ORDER IS THE TEST: `101 passed` in the failure table satisfies the + # pytest pattern too, and pytest-first would report 101 as the total, + # missing the failure and any skip beside it. + log = "test cases: 102 | 101 passed | 1 failed\n" + total, _, dialect, _ = outcome.read_summaries(log) + self.assertEqual(("catch2", 102), (dialect, total)) + + class NoSummaryTests(unittest.TestCase): """A log carrying no summary at all names no dialect, which is a failure.""" diff --git a/tools/check-test-outcome.py b/tools/check-test-outcome.py index 0ceb2a7..1a64e5a 100755 --- a/tools/check-test-outcome.py +++ b/tools/check-test-outcome.py @@ -28,6 +28,15 @@ # reading only the last understates the total (measured: MobilityKafka prints 7 # and 4, so its total is 11 rather than the 4 a tail would report). # +# The Catch2 dialect reads the runner DuckDB builds, under which a +# MobilityDuck sqllogictest FILE is one test case. ITS SKIP COUNT CANNOT SEE A +# `mode skip`: that directive silences the statements after it while the file +# still reports as a passing test case, so a suite can retire assertions +# wholesale and this dialect reads `0 skipped` over it. What the dialect does +# hold for such a consumer is the FLOOR — the total moves when a whole file +# leaves the suite. A sqllogictest consumer needs a census of `mode skip` in the +# test SOURCES beside this, and a green answer here is not evidence about them. +# # The Go dialect prints no counts at all: `go test -v` writes one result line # per test and per subtest, and the totals are their tally. A run without -v # writes only `ok 0.42s`, which carries neither a total nor a skip count, @@ -69,6 +78,28 @@ r"(?:Passed|Failed)!\s+-\s+Failed:\s*(\d+),\s*Passed:\s*(\d+)," r"\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)") +# `All tests passed (2695 assertions in 102 test cases)` — what the Catch2 +# console reporter writes when nothing failed, carrying a leading +# `3 skipped tests, ` when any were. The test-case count it prints EXCLUDES the +# skipped ones, so the total is the two added: the reporter writes +# `testCases.passed - skippedTests` there. +CATCH_PASSED = re.compile( + r"All tests passed\s*\(\s*(?:(\d+)\s+skipped tests?,\s*)?" + r"(\d+)\s+assertions?\s+in\s+(\d+)\s+test cases?\s*\)") + +# `test cases: 102 | 101 passed | 1 failed` — the table the same reporter writes +# when anything failed. Here the leading number IS the whole total, skips +# included, and a column whose count is zero is omitted rather than printed, so +# the skipped column is read out of the tail when it is there at all. That +# column sums expected failures with skips; both are tests that did not assert, +# and the guard treats them alike. +CATCH_TOTALS = re.compile(r"\btest cases:\s*(\d+)((?:\s*\|\s*\d+\s+\w+)*)") + +# `All tests were skipped (total skipped 7)` — the one form that reports no +# passing count at all, written when every test skipped. +CATCH_ALL_SKIPPED = re.compile( + r"All tests were skipped\s*\(\s*total skipped\s*(\d+)\s*\)") + def read_summaries(text: str): """Return (total, skipped, dialect, lines) summed over every summary found.""" @@ -104,6 +135,35 @@ def read_summaries(text: str): if lines: return total, skipped, "vstest", lines + # Catch2 is read BEFORE pytest, and the order is load-bearing: the failure + # table's `101 passed` satisfies the pytest pattern, so a failing Catch2 run + # read pytest-first reports the passing count as the total and misses both + # the failures and the skips. + for raw in text.splitlines(): + line = raw.rstrip() + m = CATCH_ALL_SKIPPED.search(line) + if m: + every = int(m.group(1)) + total += every + skipped += every + lines.append(line.strip()) + continue + m = CATCH_PASSED.search(line) + if m: + some = int(m.group(1) or 0) + total += int(m.group(3)) + some + skipped += some + lines.append(line.strip()) + continue + m = CATCH_TOTALS.search(line) + if m: + total += int(m.group(1)) + column = re.search(r"(\d+)\s+skipped", m.group(2)) + skipped += int(column.group(1)) if column else 0 + lines.append(line.strip()) + if lines: + return total, skipped, "catch2", lines + for raw in text.splitlines(): m = PYTEST.search(raw) if m: