From 362d75e55cf37db0f8b1ba71e6cfef16e4308b1c Mon Sep 17 00:00:00 2001 From: Dmitry Volovod Date: Mon, 7 Sep 2026 23:39:13 +0300 Subject: [PATCH 1/2] fix(playwright): recognize modern screenshot comparison errors --- lib/adapters/test-result/playwright.ts | 23 +++++- .../lib/adapters/test-result/playwright.ts | 73 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/lib/adapters/test-result/playwright.ts b/lib/adapters/test-result/playwright.ts index 328cc82df..fa0b39678 100644 --- a/lib/adapters/test-result/playwright.ts +++ b/lib/adapters/test-result/playwright.ts @@ -212,7 +212,7 @@ export class PlaywrightTestResultAdapter implements ReporterTestResult { if (/snapshot .*doesn't exist/.test(message) && message.includes('.png')) { result.name = ErrorName.NO_REF_IMAGE; - } else if (message.includes('Screenshot comparison failed')) { + } else if (this._testResult.errors.every(error => this._isScreenshotComparisonError(error))) { result.name = ErrorName.IMAGE_DIFF; } @@ -368,6 +368,27 @@ export class PlaywrightTestResultAdapter implements ReporterTestResult { return _.groupBy(imageAttachments, a => a.name.replace(ANY_IMAGE_ENDING_REGEXP, '')); } + private _isScreenshotComparisonError(error: PlaywrightTestResult['errors'][number]): boolean { + const message = stripAnsi(error.message || ''); + const header = message.split('\n')[0]; + if (header.includes('Screenshot comparison failed')) { + return true; + } + + if (!/^(?:Error: )?expect\((?:page|locator|Buffer)\)\.(?:toHaveScreenshot|toMatchSnapshot)\(expected\)(?: failed)?$/.test(header)) { + return false; + } + + // Modern Playwright uses the same matcher header for diffs and capture errors. + const snapshotName = message.match(/^\s*Snapshot: (.+)\.png\s*$/m)?.[1]; + const states = Object.entries(this._attachmentsByState).filter(([state]) => + snapshotName ? state === snapshotName : this._testResult.errors.length === 1); + + return states.some(([, attachments]) => + [ImageTitleEnding.Expected, ImageTitleEnding.Actual, ImageTitleEnding.Diff].every(ending => + attachments.some(attachment => attachment.name.endsWith(ending)))); + } + get duration(): number { return this._testResult.duration; } diff --git a/test/unit/lib/adapters/test-result/playwright.ts b/test/unit/lib/adapters/test-result/playwright.ts index d26c993cb..ea9fcd783 100644 --- a/test/unit/lib/adapters/test-result/playwright.ts +++ b/test/unit/lib/adapters/test-result/playwright.ts @@ -115,6 +115,79 @@ describe('PlaywrightTestResultAdapter', () => { assert.strictEqual(error?.stack, errorStack); }); + ['locator', 'page', 'Buffer'].forEach(receiver => { + it(`should recognize modern ${receiver} screenshot diffs`, () => { + const matcher = receiver === 'Buffer' ? 'toMatchSnapshot' : 'toHaveScreenshot'; + const suffix = receiver === 'Buffer' ? '' : ' failed'; + const errors = [{message: `Error: expect(${receiver}).${matcher}(expected)${suffix}\n\n Snapshot: state1.png`}]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + assert.equal(adapter.status, FAIL); + }); + }); + + ['Screenshot comparison failed', 'Error: expect(page).toHaveScreenshot(expected) failed'].forEach(message => { + it(`should preserve other failures alongside "${message}"`, () => { + const errors = [{message}, {message: 'Error: expect(received).toBe(expected)'}]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + assert.equal(adapter.status, ERROR); + }); + }); + + it('should not treat screenshot capture failures as acceptable diffs', () => { + const errors = [{message: 'Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: state1.png'}]; + const attachments = [createAttachment('state1-actual.png'), createAttachment('state1-previous.png')]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + assert.equal(adapter.status, ERROR); + }); + + it('should match diff attachments to the failing snapshot', () => { + const errors = [{message: 'Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: state2.png'}]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + }); + + it('should recognize an unnamed modern screenshot diff', () => { + const errors = [{message: 'Error: expect(page).toHaveScreenshot(expected) failed'}]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + }); + + it('should recognize modern screenshot diffs with ANSI formatting', () => { + const errors = [{message: 'Error: \u001b[31mexpect(page).toHaveScreenshot(expected)\u001b[39m failed\n\n Snapshot: state1.png'}]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + }); + + it('should recognize multiple named soft screenshot diffs', () => { + const errors = ['state1', 'state2'].map(state => ({ + message: `Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: ${state}.png` + })); + const attachments = ['state1', 'state2'].flatMap(state => + [ImageTitleEnding.Expected, ImageTitleEnding.Actual, ImageTitleEnding.Diff].map(ending => createAttachment(state + ending))); + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + }); + + it('should not associate an unnamed capture error with another soft assertion diff', () => { + const errors = [ + {message: 'Error: expect(page).toHaveScreenshot(expected) failed\n\n Snapshot: state1.png'}, + {message: 'Error: expect(locator).toHaveScreenshot(expected) failed'} + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + }); + it('should convert multiple errors to a single JSON string', () => { const errors = [ {message: 'First error', stack: 'Error: First error at some-file.ts:5:10'}, From edb83a56693fb4753c2b744a80ecac2a84db2cb7 Mon Sep 17 00:00:00 2001 From: Dmitry Volovod Date: Tue, 8 Sep 2026 00:05:50 +0300 Subject: [PATCH 2/2] test(playwright): make screenshot attachments explicit --- .../lib/adapters/test-result/playwright.ts | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/test/unit/lib/adapters/test-result/playwright.ts b/test/unit/lib/adapters/test-result/playwright.ts index ea9fcd783..81cd7263b 100644 --- a/test/unit/lib/adapters/test-result/playwright.ts +++ b/test/unit/lib/adapters/test-result/playwright.ts @@ -120,7 +120,12 @@ describe('PlaywrightTestResultAdapter', () => { const matcher = receiver === 'Buffer' ? 'toMatchSnapshot' : 'toHaveScreenshot'; const suffix = receiver === 'Buffer' ? '' : ' failed'; const errors = [{message: `Error: expect(${receiver}).${matcher}(expected)${suffix}\n\n Snapshot: state1.png`}]; - const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); assert.equal(adapter.status, FAIL); @@ -130,7 +135,12 @@ describe('PlaywrightTestResultAdapter', () => { ['Screenshot comparison failed', 'Error: expect(page).toHaveScreenshot(expected) failed'].forEach(message => { it(`should preserve other failures alongside "${message}"`, () => { const errors = [{message}, {message: 'Error: expect(received).toBe(expected)'}]; - const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); assert.equal(adapter.status, ERROR); @@ -148,21 +158,36 @@ describe('PlaywrightTestResultAdapter', () => { it('should match diff attachments to the failing snapshot', () => { const errors = [{message: 'Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: state2.png'}]; - const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); }); it('should recognize an unnamed modern screenshot diff', () => { const errors = [{message: 'Error: expect(page).toHaveScreenshot(expected) failed'}]; - const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); }); it('should recognize modern screenshot diffs with ANSI formatting', () => { const errors = [{message: 'Error: \u001b[31mexpect(page).toHaveScreenshot(expected)\u001b[39m failed\n\n Snapshot: state1.png'}]; - const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); }); @@ -183,7 +208,12 @@ describe('PlaywrightTestResultAdapter', () => { {message: 'Error: expect(page).toHaveScreenshot(expected) failed\n\n Snapshot: state1.png'}, {message: 'Error: expect(locator).toHaveScreenshot(expected) failed'} ]; - const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors}), UNKNOWN_ATTEMPT); + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); });