diff --git a/mcp/src/__tests__/staktrak/playwright-replay.test.ts b/mcp/src/__tests__/staktrak/playwright-replay.test.ts index 056669cc9..083934f50 100644 --- a/mcp/src/__tests__/staktrak/playwright-replay.test.ts +++ b/mcp/src/__tests__/staktrak/playwright-replay.test.ts @@ -345,7 +345,7 @@ test.describe('Playwright Replay Integration', () => { }); }); - test('should continue replay on action errors', async ({ page }) => { + test('should halt replay on action errors (does not run later steps)', async ({ page }) => { const html = createTestPage({ includeStaktrak: true }); await page.setContent(html); @@ -355,7 +355,7 @@ test.describe('Playwright Replay Integration', () => { }); }); - // Test code with invalid selector that will fail + // First step fails; the second step must NOT run because replay halts. const testCode = ` test('test', async ({ page }) => { await page.click('[data-testid="nonexistent-button"]'); @@ -369,16 +369,19 @@ test.describe('Playwright Replay Integration', () => { } }, testCode); - const completed = await waitForCondition( - () => messages.some(m => m.type === 'staktrak-playwright-replay-completed'), + const errored = await waitForCondition( + () => messages.some(m => m.type === 'staktrak-playwright-replay-error'), 5000 ); + expect(errored).toBe(true); - // Should complete despite error - expect(completed).toBe(true); - + // The failing step is index 0, so replay must stop there and never + // reach — or complete — the second step. const errorMsgs = messages.filter(m => m.type === 'staktrak-playwright-replay-error'); - expect(errorMsgs.length).toBeGreaterThan(0); + expect(errorMsgs.length).toBe(1); + expect(errorMsgs[0].fatal).toBe(true); + expect(errorMsgs[0].actionIndex).toBe(0); + expect(messages.some(m => m.type === 'staktrak-playwright-replay-completed')).toBe(false); }); test('should report error details in error messages', async ({ page }) => { @@ -416,7 +419,7 @@ test.describe('Playwright Replay Integration', () => { expect(typeof errorMsg.error).toBe('string'); }); - test('should continue capturing screenshots after action errors', async ({ page }) => { + test('should capture a screenshot of the failure frame when a step errors', async ({ page }) => { const html = createTestPage({ includeStaktrak: true, includeConfig: true }); await page.setContent(html); @@ -429,7 +432,7 @@ test.describe('Playwright Replay Integration', () => { const testCode = ` test('test', async ({ page }) => { await page.click('[data-testid="nonexistent"]'); - await page.waitForURL('http://localhost:3000'); + await page.click('[data-testid="test-button"]'); }); `; @@ -440,16 +443,20 @@ test.describe('Playwright Replay Integration', () => { }, testCode); await waitForCondition( - () => messages.some(m => m.type === 'staktrak-playwright-replay-completed'), + () => messages.some(m => m.type === 'staktrak-playwright-replay-error'), 8000 ); - const screenshotMsgs = extractScreenshotMessages(messages.map(m => ({ data: m }))); const errorMsgs = messages.filter(m => m.type === 'staktrak-playwright-replay-error'); + expect(errorMsgs.length).toBe(1); + expect(errorMsgs[0].fatal).toBe(true); - // Should have error and still capture screenshot - expect(errorMsgs.length).toBeGreaterThan(0); - // Screenshot may or may not be captured depending on implementation + // A failure screenshot should have been captured and referenced by id on + // the error message. + const screenshotMsgs = extractScreenshotMessages(messages.map(m => ({ data: m }))); + expect(screenshotMsgs.length).toBeGreaterThan(0); + expect(errorMsgs[0].screenshotId).toBeTruthy(); + expect(messages.some(m => m.type === 'staktrak-playwright-replay-completed')).toBe(false); }); }); diff --git a/mcp/tests/hooks.js b/mcp/tests/hooks.js index fefce074b..369118d4f 100644 --- a/mcp/tests/hooks.js +++ b/mcp/tests/hooks.js @@ -674,12 +674,26 @@ export function usePlaywrightReplay(iframeRef) { }, ]); - // Don't stop replay on error, just log it console.warn("Playwright replay error:", errorMsg); + + // A fatal error halts the replay (issue #756): tear down the replaying + // state just like a normal stop so the UI doesn't look stuck "playing". + if (data.fatal) { + setIsPlaywrightReplaying(false); + setIsPlaywrightPaused(false); + setPlaywrightStatus("error"); + setCurrentAction(null); + + const errContainer = document.querySelector(".iframe-container"); + if (errContainer) { + errContainer.classList.remove("playwright-replaying"); + } + } + if (data.actionIndex !== undefined) { showPopup( - `Error at action ${data.actionIndex + 1}: ${errorMsg}`, - "warning" + `Replay stopped at action ${data.actionIndex + 1}: ${errorMsg}`, + "error" ); } break; diff --git a/mcp/tests/staktrak/dist/staktrak.js b/mcp/tests/staktrak/dist/staktrak.js index ad8932e8c..20295077b 100644 --- a/mcp/tests/staktrak/dist/staktrak.js +++ b/mcp/tests/staktrak/dist/staktrak.js @@ -3573,8 +3573,10 @@ var userBehaviour = (() => { }, getParentOrigin() ); + return id; } catch (error) { console.error(`[Screenshot] Error capturing for actionIndex=${actionIndex}:`, error); + return null; } } function beginReplay(actions, testCode) { @@ -3649,20 +3651,26 @@ var userBehaviour = (() => { executeNextPlaywrightAction(); }, 500); } catch (error) { - state.errors.push( - `Action ${state.currentActionIndex + 1}: ${error instanceof Error ? error.message : "Unknown error"}` + const message = error instanceof Error ? error.message : "Unknown error"; + state.errors.push(`Action ${state.currentActionIndex + 1}: ${message}`); + state.status = "error" /* ERROR */; + state.timeouts.forEach((id) => clearTimeout(id)); + state.timeouts = []; + const screenshotId = await captureScreenshot( + state.currentActionIndex, + window.location.href ); - state.currentActionIndex++; window.parent.postMessage( { type: "staktrak-playwright-replay-error", - error: error instanceof Error ? error.message : "Unknown error", - actionIndex: state.currentActionIndex - 1, - action + error: message, + actionIndex: state.currentActionIndex, + action, + fatal: true, + screenshotId }, getParentOrigin() ); - executeNextPlaywrightAction(); } } function pausePlaywrightReplay() { diff --git a/mcp/tests/staktrak/src/playwright-replay/index.ts b/mcp/tests/staktrak/src/playwright-replay/index.ts index ce0f832da..afcd3bee5 100644 --- a/mcp/tests/staktrak/src/playwright-replay/index.ts +++ b/mcp/tests/staktrak/src/playwright-replay/index.ts @@ -45,7 +45,7 @@ function getParentOrigin(): string { /** * Capture screenshot and send to parent window */ -async function captureScreenshot(actionIndex: number, url: string): Promise { +async function captureScreenshot(actionIndex: number, url: string): Promise { try { // Get screenshot config from STAKTRAK_CONFIG or use defaults // Note: In cross-origin iframes, STAKTRAK_CONFIG may not be accessible @@ -80,8 +80,11 @@ async function captureScreenshot(actionIndex: number, url: string): Promise { executeNextPlaywrightAction(); }, 500); } catch (error) { - state.errors.push( - `Action ${state.currentActionIndex + 1}: ${error instanceof Error ? error.message : "Unknown error"}` - ); + const message = error instanceof Error ? error.message : "Unknown error"; + state.errors.push(`Action ${state.currentActionIndex + 1}: ${message}`); - state.currentActionIndex++; + // Halt the replay: a failed step means everything after it runs against an + // unexpected page state, so continuing produces misleading results. Stop here + // and let the host surface the failure. (See stakgraph issue #756.) + state.status = ReplayStatus.ERROR; + state.timeouts.forEach((id) => clearTimeout(id as any)); + state.timeouts = []; + + // Capture the screen at the point of failure so the host can show *where* it + // broke — not just the error text. + const screenshotId = await captureScreenshot( + state.currentActionIndex, + window.location.href + ); window.parent.postMessage( { type: "staktrak-playwright-replay-error", - error: error instanceof Error ? error.message : "Unknown error", - actionIndex: state.currentActionIndex - 1, + error: message, + actionIndex: state.currentActionIndex, action: action, + fatal: true, + screenshotId, }, getParentOrigin() ); - - executeNextPlaywrightAction(); } } diff --git a/mcp/tests/staktrak/src/types.ts b/mcp/tests/staktrak/src/types.ts index bd3abd209..155491972 100644 --- a/mcp/tests/staktrak/src/types.ts +++ b/mcp/tests/staktrak/src/types.ts @@ -147,6 +147,7 @@ export enum ReplayStatus { PLAYING = "playing", PAUSED = "paused", COMPLETED = "completed", + ERROR = "error", } export interface ReplayAction {