feat(diagnostics): surface twd-js failure diagnostics in the run summary

src/failureDiagnostics.js

@@ -0,0 +1,49 @@
+/**
+ * Renders the diagnostics snapshot twd-js hangs off a failed test.
+ *
+ * The snapshot is plain data — `{ location, mockRules }` — so it serialises
+ * straight out of `page.evaluate()` and is rendered here, in Node. That split
+ * is the whole point: the in-page `onFail` is a stringified function with no
+ * module scope, so a formatter shared with twd-js could only reach it through
+ * new public surface on the library (a `window.__twdFormatDiagnostics`).
+ * Carrying the data out instead needs nothing from twd-js, and puts the
+ * rendering where the rest of this package's output already lives.
+ *
+ * Only the mock-rule signal is rendered. twd-js's own block also carries a
+ * `location` row, but every failure message here already ends in
+ * `(at <href>)`, and the full href is strictly more informative than the
+ * snapshot's pathname + search + hash. Printing both would say the same thing
+ * twice.
+ */
+
+// A page with fifteen mocks must not produce fifteen lines. The rules that did
+// fire are only interesting as a count; it is the misses that name the bug.
+const LIST_CAP = 5;
+
+// Aligns a continuation row under the value column of `mock rules  `.
+const CONTINUATION = ' '.repeat(12);
+
+export function formatFailureDiagnostics(diagnostics) {
+  const mockRules = diagnostics?.mockRules;
+  // Omitted by twd-js whenever the test registered no rules, and absent
+  // entirely on a twd-js that predates diagnostics. Never render `0/0`:
+  // "this test mocks nothing" is not a diagnostic, it is the default.
+  if (!mockRules) return [];
+
+  const { registered, triggered } = mockRules;
+  const untriggered = mockRules.untriggered ?? [];
+  const summary = `${triggered}/${registered} triggered`;
+
+  if (untriggered.length === 0) return [`mock rules  ${summary}`];
+  if (untriggered.length === 1) {
+    return [`mock rules  ${summary} — ${untriggered[0]} never requested`];
+  }
+
+  const lines = [`mock rules  ${summary} — ${untriggered.length} never requested`];
+  for (const alias of untriggered.slice(0, LIST_CAP)) {
+    lines.push(`${CONTINUATION}✗ ${alias}`);
+  }
+  const rest = untriggered.length - LIST_CAP;
+  if (rest > 0) lines.push(`${CONTINUATION}+${rest} more`);
+  return lines;
+}

src/index.js

@@ -251,7 +251,17 @@ export async function runTests(options = {}) {
           },
           onFail: (test, err) => {
             test.status = "done";
-            testStatus.push({ id: test.id, status: "fail", error: `${err.message} (at ${window.location.href})` });
+            // The raw snapshot travels out; src/failureDiagnostics.js renders
+            // it in Node. This callback is serialised into the page and has no
+            // module scope, so it cannot reach a formatter, and duplicating one
+            // here would be a copy that drifts. `undefined` on a twd-js without
+            // diagnostics support, and dropped by serialisation.
+            testStatus.push({
+              id: test.id,
+              status: "fail",
+              diagnostics: test.diagnostics,
+              error: `${err.message} (at ${window.location.href})`,
+            });
           },
           onSkip: (test) => {
             test.status = "done";

src/testSummary.js

@@ -1,4 +1,5 @@
 import { buildTestPath } from './buildTestPath.js';
+import { formatFailureDiagnostics } from './failureDiagnostics.js';
 
 /**
  * Display name for one test result.
@@ -55,6 +56,11 @@ export function formatRunComplete({
     for (const failure of failures) {
       const testPath = resolvePath(failure, handlers);
       lines.push(`    × ${testPath}`);
+      // Above the error, not below: a twd-js failure message can carry a full
+      // accessible-roles dump, which would bury the block underneath it.
+      for (const row of formatFailureDiagnostics(failure.diagnostics)) {
+        lines.push(`      ${row}`);
+      }
       if (failure.error) {
         lines.push(`      ${String(failure.error).replace(/\n/g, '\n      ')}`);
       }

tests/failureDiagnostics.test.js

@@ -0,0 +1,92 @@
+import { describe, it, expect } from 'vitest';
+import { formatFailureDiagnostics } from '../src/failureDiagnostics.js';
+
+describe('formatFailureDiagnostics', () => {
+  // The snapshot is absent on a passing test, on a test that failed one attempt
+  // then passed on retry, and on every test run against a twd-js that predates
+  // diagnostics. All three have to render nothing rather than throw.
+  it('renders nothing when there is no snapshot', () => {
+    expect(formatFailureDiagnostics(undefined)).toEqual([]);
+    expect(formatFailureDiagnostics(null)).toEqual([]);
+  });
+
+  // "This test mocks nothing" is the default, not a diagnostic. twd-js omits
+  // mockRules in that case and the block must not invent a `0/0` row.
+  it('renders nothing when the test registered no mock rules', () => {
+    expect(formatFailureDiagnostics({ location: '/checkout' })).toEqual([]);
+  });
+
+  it('renders a bare count when every rule was triggered', () => {
+    expect(formatFailureDiagnostics({
+      location: '/checkout',
+      mockRules: { registered: 6, triggered: 6, untriggered: [] },
+    })).toEqual(['mock rules  6/6 triggered']);
+  });
+
+  // One miss names itself. That single alias is usually the whole answer, so it
+  // is worth the inline room.
+  it('names the alias when exactly one rule was never requested', () => {
+    expect(formatFailureDiagnostics({
+      location: '/cg-1/settings/catalog',
+      mockRules: { registered: 7, triggered: 6, untriggered: ['catalog'] },
+    })).toEqual(['mock rules  6/7 triggered — catalog never requested']);
+  });
+
+  it('lists the aliases under a count when several were never requested', () => {
+    expect(formatFailureDiagnostics({
+      location: '/cg-1',
+      mockRules: { registered: 7, triggered: 4, untriggered: ['catalog', 'profile', 'advisor'] },
+    })).toEqual([
+      'mock rules  4/7 triggered — 3 never requested',
+      '            ✗ catalog',
+      '            ✗ profile',
+      '            ✗ advisor',
+    ]);
+  });
+
+  // A page with fifteen mocks must not produce fifteen lines.
+  it('caps the list at five and counts the remainder', () => {
+    const untriggered = ['a', 'b', 'c', 'd', 'e', 'f', 'g'];
+    expect(formatFailureDiagnostics({
+      location: '/wide',
+      mockRules: { registered: 9, triggered: 2, untriggered },
+    })).toEqual([
+      'mock rules  2/9 triggered — 7 never requested',
+      '            ✗ a',
+      '            ✗ b',
+      '            ✗ c',
+      '            ✗ d',
+      '            ✗ e',
+      '            +2 more',
+    ]);
+  });
+
+  it('shows no remainder line when the list lands exactly on the cap', () => {
+    const lines = formatFailureDiagnostics({
+      location: '/exact',
+      mockRules: { registered: 5, triggered: 0, untriggered: ['a', 'b', 'c', 'd', 'e'] },
+    });
+    expect(lines).toHaveLength(6);
+    expect(lines.some((l) => l.includes('more'))).toBe(false);
+  });
+
+  // A snapshot from a future twd-js that stops sending the array must not throw
+  // in the middle of reporting a failure.
+  it('tolerates a missing untriggered array', () => {
+    expect(formatFailureDiagnostics({
+      location: '/partial',
+      mockRules: { registered: 3, triggered: 3 },
+    })).toEqual(['mock rules  3/3 triggered']);
+  });
+
+  // The location row is deliberately not rendered: the failure message already
+  // ends in `(at <href>)`, which is strictly more informative.
+  it('never renders a location row', () => {
+    const lines = formatFailureDiagnostics({
+      location: '/cg-1/settings/catalog',
+      mockRules: { registered: 2, triggered: 1, untriggered: ['catalog'] },
+    });
+    expect(lines.join('\n')).not.toContain('/cg-1/settings/catalog');
+    expect(lines.join('\n')).not.toContain('location');
+  });
+});

tests/mergeReports.test.js

@@ -249,3 +249,53 @@ describe('reportTotals', () => {
     expect(reportTotals(merged)).toEqual({ executed: 2, notRun: 0, expected: 2, consistent: true });
   });
 });
+
+// A failure's diagnostics snapshot is written by the shard that ran the test
+// and rendered by whoever prints the merged summary, so it has to survive the
+// merge and a JSON round-trip (embedded newlines in `error` included).
+describe('mergeRunReports diagnostics', () => {
+  const diagnostics = {
+    location: '/cg-1/settings/catalog',
+    mockRules: { registered: 7, triggered: 6, untriggered: ['catalog'] },
+  };
+
+  it('carries the snapshot through the merge on the failing shard', () => {
+    const merged = mergeRunReports([
+      makeReport(1, {
+        tests: [{ id: 'r1-t', path: 'Login > a', index: 0, status: 'pass' }],
+      }),
+      makeReport(2, {
+        failed: 1,
+        tests: [{
+          id: 'r2-t',
+          path: 'Login > b',
+          index: 1,
+          status: 'fail',
+          diagnostics,
+          error: 'AssertionError: expected 0 rows\n  at Object.<anonymous>',
+        }],
+      }),
+    ]);
+
+    const failed = merged.tests.find((t) => t.status === 'fail');
+    expect(failed.diagnostics).toEqual(diagnostics);
+
+    const roundTripped = JSON.parse(JSON.stringify(merged));
+    const after = roundTripped.tests.find((t) => t.status === 'fail');
+    expect(after.diagnostics).toEqual(diagnostics);
+    expect(after.error).toContain('\n');
+  });
+
+  // Shards produced by a twd-js without diagnostics merge unchanged.
+  it('merges reports that carry no snapshot at all', () => {
+    const merged = mergeRunReports([
+      makeReport(1, {
+        failed: 1,
+        tests: [{ id: 'r1-t', path: 'Login > a', index: 0, status: 'fail', error: 'boom' }],
+      }),
+      makeReport(2),
+    ]);
+
+    expect(merged.tests.find((t) => t.status === 'fail').diagnostics).toBeUndefined();
+  });
+});

tests/runTests.test.js

@@ -1555,3 +1555,69 @@ describe('runTests sharded behavior changes', () => {
     expect(report.contracts.results).toEqual([{ alias: 'a' }]);
   });
 });
+
+// The in-page onFail is serialised into the browser, so the suite never runs it
+// by mocking page.evaluate. These drive the real callback directly against a
+// stub runner: it is the only place the diagnostics hand-off is observable.
+describe("in-page onFail diagnostics hand-off", () => {
+  let savedWindow;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(loadConfig).mockReturnValue({ ...defaultMockConfig });
+    vi.spyOn(console, 'log').mockImplementation(() => {});
+    savedWindow = global.window;
+  });
+
+  afterEach(() => {
+    global.window = savedWindow;
+    vi.restoreAllMocks();
+  });
+
+  // Runs the function twd-cli hands to page.evaluate, with a stub
+  // window.__testRunner that fails one test.
+  async function runInPageFn(failingTest) {
+    const handlers = [{ id: 't1', name: 'test1', type: 'test' }];
+    const page = createMockPage({ handlers, testStatus: [] });
+    vi.mocked(puppeteer.launch).mockResolvedValue(createMockBrowser(page));
+    await runTests();
+
+    // call 0 is the enumeration pass; call 1 is the first chunk run.
+    const [inPageFn, retryCount, chunkIds] = page.evaluate.mock.calls[1];
+
+    global.window = {
+      location: { href: 'http://localhost:5173/cg-1/settings/catalog' },
+      __testRunner: class {
+        constructor(callbacks) { this.callbacks = callbacks; }
+        async runByIds() { this.callbacks.onFail(failingTest, new Error('boom')); }
+      },
+    };
+
+    return inPageFn(retryCount, chunkIds);
+  }
+
+  it("carries the raw snapshot out on the failure entry", async () => {
+    const diagnostics = {
+      location: '/cg-1/settings/catalog',
+      mockRules: { registered: 7, triggered: 6, untriggered: ['catalog'] },
+    };
+
+    const result = await runInPageFn({ id: 't1', diagnostics });
+
+    expect(result).toEqual([{
+      id: 't1',
+      status: 'fail',
+      diagnostics,
+      error: 'boom (at http://localhost:5173/cg-1/settings/catalog)',
+    }]);
+  });
+
+  // twd-js 1.9.0 and earlier hang nothing off the handler. The entry must still
+  // be well-formed, and the error text unchanged.
+  it("leaves the entry intact when twd-js sends no snapshot", async () => {
+    const result = await runInPageFn({ id: 't1' });
+
+    expect(result[0].diagnostics).toBeUndefined();
+    expect(result[0].error).toBe('boom (at http://localhost:5173/cg-1/settings/catalog)');
+  });
+});

tests/testSummary.test.js

@@ -257,3 +257,72 @@ describe('formatRunComplete with shards', () => {
     expect(output).not.toContain('Shards:');
   });
 });
+
+// The diagnostics snapshot travels out of the page as raw data on the failure
+// entry (src/index.js), and is rendered here rather than in twd-js. See
+// src/failureDiagnostics.js for why the split sits where it does.
+describe('formatRunComplete diagnostics block', () => {
+  const failing = (diagnostics) => ({
+    id: 't1',
+    status: 'fail',
+    diagnostics,
+    error: 'AssertionError: expected 0 rows (at http://localhost:5173/cg-1/settings/catalog)',
+  });
+
+  it('prints the mock-rule row above the error message', () => {
+    const output = formatRunComplete({
+      testStatus: [failing({
+        location: '/cg-1/settings/catalog',
+        mockRules: { registered: 7, triggered: 6, untriggered: ['catalog'] },
+      })],
+      handlers,
+      durationMs: 1000,
+    });
+    expect(output).toContain(
+      '    × Login > shows error on wrong password\n' +
+      '      mock rules  6/7 triggered — catalog never requested\n' +
+      '      AssertionError: expected 0 rows (at http://localhost:5173/cg-1/settings/catalog)'
+    );
+  });
+
+  it('indents every row of a multi-alias block to the error column', () => {
+    const output = formatRunComplete({
+      testStatus: [failing({
+        location: '/cg-1',
+        mockRules: { registered: 4, triggered: 1, untriggered: ['catalog', 'profile'] },
+      })],
+      handlers,
+      durationMs: 1000,
+    });
+    expect(output).toContain(
+      '      mock rules  1/4 triggered — 2 never requested\n' +
+      '                  ✗ catalog\n' +
+      '                  ✗ profile\n'
+    );
+  });
+
+  // twd-js 1.9.0 and earlier send no snapshot at all. The failure must print
+  // exactly as it always has.
+  it('is byte-identical to the old output when no snapshot is present', () => {
+    const args = { handlers, durationMs: 1000 };
+    const withField = formatRunComplete({ testStatus: [failing(undefined)], ...args });
+    const withoutField = formatRunComplete({
+      testStatus: [{ id: 't1', status: 'fail', error: failing().error }],
+      ...args,
+    });
+    expect(withField).toBe(withoutField);
+    expect(withField).not.toContain('mock rules');
+  });
+
+  // A test that failed an attempt then passed carries no snapshot on the pass
+  // entry, so a retried-then-green run stays clean.
+  it('prints no block for a test that passed on retry', () => {
+    const output = formatRunComplete({
+      testStatus: [{ id: 't1', status: 'pass', retryAttempt: 2 }],
+      handlers,
+      durationMs: 1000,
+    });
+    expect(output).not.toContain('mock rules');
+    expect(output).toContain('Retried (1):');
+  });
+});