diff --git a/lib/adapters/test-result/playwright.ts b/lib/adapters/test-result/playwright.ts index 328cc82df..fa0b39678 100644 --- a/lib/adapters/test-result/playwright.ts +++ b/lib/adapters/test-result/playwright.ts @@ -212,7 +212,7 @@ export class PlaywrightTestResultAdapter implements ReporterTestResult { if (/snapshot .*doesn't exist/.test(message) && message.includes('.png')) { result.name = ErrorName.NO_REF_IMAGE; - } else if (message.includes('Screenshot comparison failed')) { + } else if (this._testResult.errors.every(error => this._isScreenshotComparisonError(error))) { result.name = ErrorName.IMAGE_DIFF; } @@ -368,6 +368,27 @@ export class PlaywrightTestResultAdapter implements ReporterTestResult { return _.groupBy(imageAttachments, a => a.name.replace(ANY_IMAGE_ENDING_REGEXP, '')); } + private _isScreenshotComparisonError(error: PlaywrightTestResult['errors'][number]): boolean { + const message = stripAnsi(error.message || ''); + const header = message.split('\n')[0]; + if (header.includes('Screenshot comparison failed')) { + return true; + } + + if (!/^(?:Error: )?expect\((?:page|locator|Buffer)\)\.(?:toHaveScreenshot|toMatchSnapshot)\(expected\)(?: failed)?$/.test(header)) { + return false; + } + + // Modern Playwright uses the same matcher header for diffs and capture errors. + const snapshotName = message.match(/^\s*Snapshot: (.+)\.png\s*$/m)?.[1]; + const states = Object.entries(this._attachmentsByState).filter(([state]) => + snapshotName ? state === snapshotName : this._testResult.errors.length === 1); + + return states.some(([, attachments]) => + [ImageTitleEnding.Expected, ImageTitleEnding.Actual, ImageTitleEnding.Diff].every(ending => + attachments.some(attachment => attachment.name.endsWith(ending)))); + } + get duration(): number { return this._testResult.duration; } diff --git a/test/unit/lib/adapters/test-result/playwright.ts b/test/unit/lib/adapters/test-result/playwright.ts index d26c993cb..81cd7263b 100644 --- a/test/unit/lib/adapters/test-result/playwright.ts +++ b/test/unit/lib/adapters/test-result/playwright.ts @@ -115,6 +115,109 @@ describe('PlaywrightTestResultAdapter', () => { assert.strictEqual(error?.stack, errorStack); }); + ['locator', 'page', 'Buffer'].forEach(receiver => { + it(`should recognize modern ${receiver} screenshot diffs`, () => { + const matcher = receiver === 'Buffer' ? 'toMatchSnapshot' : 'toHaveScreenshot'; + const suffix = receiver === 'Buffer' ? '' : ' failed'; + const errors = [{message: `Error: expect(${receiver}).${matcher}(expected)${suffix}\n\n Snapshot: state1.png`}]; + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + assert.equal(adapter.status, FAIL); + }); + }); + + ['Screenshot comparison failed', 'Error: expect(page).toHaveScreenshot(expected) failed'].forEach(message => { + it(`should preserve other failures alongside "${message}"`, () => { + const errors = [{message}, {message: 'Error: expect(received).toBe(expected)'}]; + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + assert.equal(adapter.status, ERROR); + }); + }); + + it('should not treat screenshot capture failures as acceptable diffs', () => { + const errors = [{message: 'Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: state1.png'}]; + const attachments = [createAttachment('state1-actual.png'), createAttachment('state1-previous.png')]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + assert.equal(adapter.status, ERROR); + }); + + it('should match diff attachments to the failing snapshot', () => { + const errors = [{message: 'Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: state2.png'}]; + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + }); + + it('should recognize an unnamed modern screenshot diff', () => { + const errors = [{message: 'Error: expect(page).toHaveScreenshot(expected) failed'}]; + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + }); + + it('should recognize modern screenshot diffs with ANSI formatting', () => { + const errors = [{message: 'Error: \u001b[31mexpect(page).toHaveScreenshot(expected)\u001b[39m failed\n\n Snapshot: state1.png'}]; + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + }); + + it('should recognize multiple named soft screenshot diffs', () => { + const errors = ['state1', 'state2'].map(state => ({ + message: `Error: expect(locator).toHaveScreenshot(expected) failed\n\n Snapshot: ${state}.png` + })); + const attachments = ['state1', 'state2'].flatMap(state => + [ImageTitleEnding.Expected, ImageTitleEnding.Actual, ImageTitleEnding.Diff].map(ending => createAttachment(state + ending))); + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.IMAGE_DIFF); + }); + + it('should not associate an unnamed capture error with another soft assertion diff', () => { + const errors = [ + {message: 'Error: expect(page).toHaveScreenshot(expected) failed\n\n Snapshot: state1.png'}, + {message: 'Error: expect(locator).toHaveScreenshot(expected) failed'} + ]; + const attachments = [ + createAttachment('state1-expected.png'), + createAttachment('state1-diff.png'), + createAttachment('state1-actual.png') + ]; + const adapter = new PlaywrightTestResultAdapter(mkTestCase(), mkTestResult({errors, attachments}), UNKNOWN_ATTEMPT); + + assert.equal(adapter.error?.name, ErrorName.GENERAL_ERROR); + }); + it('should convert multiple errors to a single JSON string', () => { const errors = [ {message: 'First error', stack: 'Error: First error at some-file.ts:5:10'},