From a5c09a0f78c5091c3b277fccc11990a8d383b0ab Mon Sep 17 00:00:00 2001 From: Robert DeLuca Date: Tue, 12 May 2026 00:09:48 -0500 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20Harden=20Swift=20SDK=20package?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split XCTest helpers into their own product so native app targets can depend on the core client without test APIs. Align Swift screenshot payloads with the server comparison contract and document the SPM integration path. --- clients/swift/Example/ExampleUITests.swift | 5 +- clients/swift/INTEGRATION.md | 12 +- clients/swift/Package.swift | 8 +- clients/swift/QUICKSTART.md | 5 +- clients/swift/README.md | 48 +++- .../swift/Sources/Vizzly/VizzlyClient.swift | 260 ++++++++++++------ .../XCTestExtensions.swift | 37 ++- .../Tests/VizzlyTests/VizzlyClientTests.swift | 61 +++- 8 files changed, 304 insertions(+), 132 deletions(-) rename clients/swift/Sources/{Vizzly => VizzlyXCTest}/XCTestExtensions.swift (78%) diff --git a/clients/swift/Example/ExampleUITests.swift b/clients/swift/Example/ExampleUITests.swift index dc166039..9fa8472e 100644 --- a/clients/swift/Example/ExampleUITests.swift +++ b/clients/swift/Example/ExampleUITests.swift @@ -1,5 +1,6 @@ import XCTest import Vizzly +import VizzlyXCTest /// Example UI tests demonstrating Vizzly integration /// @@ -133,7 +134,6 @@ final class ExampleUITests: XCTestCase { func testResponsiveLayout() throws { // Test different device orientations XCUIDevice.shared.orientation = .portrait - sleep(1) // Wait for orientation change app.vizzlyScreenshot( name: "home-portrait", @@ -141,7 +141,6 @@ final class ExampleUITests: XCTestCase { ) XCUIDevice.shared.orientation = .landscapeLeft - sleep(1) app.vizzlyScreenshot( name: "home-landscape", @@ -187,7 +186,7 @@ final class ExampleUITests: XCTestCase { // MARK: - Custom Threshold Example func testWithCustomThreshold() throws { - // Allow up to 5% pixel difference + // Allow a higher comparison threshold for animated content app.vizzlyScreenshot( name: "animation-test", threshold: 5, diff --git a/clients/swift/INTEGRATION.md b/clients/swift/INTEGRATION.md index 0ef06c00..309785c1 100644 --- a/clients/swift/INTEGRATION.md +++ b/clients/swift/INTEGRATION.md @@ -21,7 +21,10 @@ In Xcode: 1. **File → Add Package Dependencies** 2. Enter URL: `https://github.com/vizzly-testing/cli` 3. Select version/branch -4. **Important**: Add the package to your **UI Test target** (not the main app target) +4. Add the `VizzlyXCTest` product to your **UI Test target** + +Use the core `Vizzly` product directly only when you need to send PNG data from +app or test-support code without the XCTest convenience extensions. #### Option B: Local Package @@ -78,6 +81,7 @@ Create or update your UI test file: ```swift import XCTest import Vizzly +import VizzlyXCTest final class MyAppUITests: XCTestCase { @@ -265,13 +269,13 @@ For views with animations or timing-sensitive content: func testAnimatedView() { app.launch() - // Wait for animation to complete - sleep(1) // Or use expectations + let finishedState = app.otherElements["AnimatedBannerReady"] + XCTAssertTrue(finishedState.waitForExistence(timeout: 5)) // Use threshold for slight variations app.vizzlyScreenshot( name: "animated-banner", - threshold: 5 // Allow 5% difference + threshold: 5 ) } ``` diff --git a/clients/swift/Package.swift b/clients/swift/Package.swift index 1d11521b..ba60bc95 100644 --- a/clients/swift/Package.swift +++ b/clients/swift/Package.swift @@ -13,13 +13,19 @@ let package = Package( .library( name: "Vizzly", targets: ["Vizzly"]), + .library( + name: "VizzlyXCTest", + targets: ["VizzlyXCTest"]), ], targets: [ .target( name: "Vizzly", dependencies: []), + .target( + name: "VizzlyXCTest", + dependencies: ["Vizzly"]), .testTarget( name: "VizzlyTests", - dependencies: ["Vizzly"]), + dependencies: ["Vizzly", "VizzlyXCTest"]), ] ) diff --git a/clients/swift/QUICKSTART.md b/clients/swift/QUICKSTART.md index d398e207..83cbe311 100644 --- a/clients/swift/QUICKSTART.md +++ b/clients/swift/QUICKSTART.md @@ -13,7 +13,7 @@ npm install -g @vizzly-testing/cli 1. Open your iOS project in Xcode 2. **File → Add Package Dependencies** 3. Paste: `https://github.com/vizzly-testing/cli` -4. Add to your **UI Test target** (not main app) +4. Add the `VizzlyXCTest` product to your **UI Test target** ## 3. Start TDD Server @@ -28,6 +28,7 @@ vizzly tdd start ```swift import XCTest import Vizzly +import VizzlyXCTest class MyAppUITests: XCTestCase { let app = XCUIApplication() @@ -96,7 +97,7 @@ button.vizzlyScreenshot(name: "submit-button") ### Custom Threshold ```swift -// Allow 5% pixel difference (useful for animations) +// Allow a higher comparison threshold for animated content app.vizzlyScreenshot( name: "animated-view", threshold: 5 diff --git a/clients/swift/README.md b/clients/swift/README.md index c604481a..9be912a1 100644 --- a/clients/swift/README.md +++ b/clients/swift/README.md @@ -7,7 +7,7 @@ Unlike tools that render components in isolation, Vizzly captures screenshots di ## Features - **Zero Configuration** - Auto-discovers Vizzly TDD server -- **Native XCTest Integration** - Simple extensions for `XCUIApplication` and `XCUIElement` +- **Native XCTest Integration** - Simple extensions for `XCUIApplication` and `XCUIElement` via the `VizzlyXCTest` helper product - **iOS & macOS Support** - Works on both platforms - **Automatic Metadata** - Captures device, screen size, and platform info - **TDD Mode** - Local visual testing with instant feedback @@ -22,21 +22,29 @@ Add Vizzly to your test target using Xcode: 1. File → Add Package Dependencies 2. Enter repository URL: `https://github.com/vizzly-testing/cli` -3. Select version and add to your UI test target +3. Select version and add the `VizzlyXCTest` product to your UI test target + +The core `Vizzly` product has no XCTest dependency and can also be used from +native app or test-support code when you want to send PNG data directly. Or add to your `Package.swift`: ```swift dependencies: [ .package(url: "https://github.com/vizzly-testing/cli", from: "1.0.0") +], +targets: [ + .testTarget( + name: "MyAppUITests", + dependencies: [ + .product(name: "VizzlyXCTest", package: "cli") + ] + ) ] ``` -### CocoaPods - -```ruby -pod 'Vizzly', :git => 'https://github.com/vizzly-testing/cli', :branch => 'main' -``` +Vizzly does not currently ship a CocoaPods podspec. Use Swift Package Manager +for native app integration. ## Quick Start @@ -54,6 +62,7 @@ This starts a local server at `http://localhost:47392` that receives screenshots ```swift import XCTest import Vizzly +import VizzlyXCTest class MyUITests: XCTestCase { let app = XCUIApplication() @@ -128,7 +137,7 @@ func testNavigationBar() { ```swift func testAnimatedContent() { - // Allow up to 5% pixel difference (useful for animations) + // Allow a higher comparison threshold for animated content app.vizzlyScreenshot( name: "animated-banner", threshold: 5 @@ -136,6 +145,9 @@ func testAnimatedContent() { } ``` +If `threshold` or `minClusterSize` is omitted, the server's configured +comparison settings are used. + ### Multiple Device Orientations ```swift @@ -187,7 +199,8 @@ extension XCUIApplication { func vizzlyScreenshot( name: String, properties: [String: Any]? = nil, - threshold: Int = 0, + threshold: Double? = nil, + minClusterSize: Int? = nil, fullPage: Bool = false ) -> [String: Any]? } @@ -200,7 +213,8 @@ extension XCUIElement { func vizzlyScreenshot( name: String, properties: [String: Any]? = nil, - threshold: Int = 0 + threshold: Double? = nil, + minClusterSize: Int? = nil ) -> [String: Any]? } ``` @@ -213,7 +227,8 @@ extension XCTestCase { name: String, app: XCUIApplication, properties: [String: Any]? = nil, - threshold: Int = 0, + threshold: Double? = nil, + minClusterSize: Int? = nil, fullPage: Bool = false ) -> [String: Any]? @@ -221,7 +236,8 @@ extension XCTestCase { name: String, element: XCUIElement, properties: [String: Any]? = nil, - threshold: Int = 0 + threshold: Double? = nil, + minClusterSize: Int? = nil ) -> [String: Any]? } ``` @@ -236,7 +252,8 @@ class VizzlyClient { name: String, image: Data, properties: [String: Any]? = nil, - threshold: Int = 0, + threshold: Double? = nil, + minClusterSize: Int? = nil, fullPage: Bool = false ) -> [String: Any]? @@ -254,8 +271,9 @@ class VizzlyClient { The SDK automatically discovers a running Vizzly TDD server using this priority order: 1. **VIZZLY_SERVER_URL environment variable** - Explicitly set server URL -2. **Global server file** - `~/.vizzly/server.json` written by CLI -3. **Default port health check** - Tests `http://localhost:47392/health` +2. **Project server file** - `.vizzly/server.json` in the current directory +3. **Global server file** - `~/.vizzly/server.json` written by CLI +4. **Default port health check** - Tests `http://localhost:47392/health` When you run `vizzly tdd start`, the CLI automatically writes server info to `~/.vizzly/server.json` in your home directory, enabling zero-config discovery from iOS tests. diff --git a/clients/swift/Sources/Vizzly/VizzlyClient.swift b/clients/swift/Sources/Vizzly/VizzlyClient.swift index 8483a39b..a1d01750 100644 --- a/clients/swift/Sources/Vizzly/VizzlyClient.swift +++ b/clients/swift/Sources/Vizzly/VizzlyClient.swift @@ -12,10 +12,22 @@ import AppKit import WatchKit #endif -/// Vizzly visual regression testing client for Swift/iOS +private enum VizzlyClientError: LocalizedError { + case requestTimedOut + + var errorDescription: String? { + switch self { + case .requestTimedOut: + return "request timed out" + } + } +} + +/// Vizzly visual regression testing client for Swift/iOS. /// -/// A lightweight client SDK for capturing screenshots and sending them to Vizzly for visual -/// regression testing. Works with both local TDD mode and cloud builds. +/// A lightweight client SDK for capturing screenshots and sending them to +/// Vizzly for visual regression testing. Works with both local TDD mode and +/// cloud builds. /// /// ## Usage /// @@ -27,6 +39,45 @@ import WatchKit /// client.screenshot(name: "login-screen", image: screenshot.pngRepresentation) /// ``` public final class VizzlyClient { + private struct ScreenshotResponse { + var data: Data? + var response: URLResponse? + var error: Error? + } + + private final class ScreenshotRequestState { + private let lock = NSLock() + private var didTimeOut = false + private var response = ScreenshotResponse() + + func complete(data: Data?, response: URLResponse?, error: Error?) { + lock.lock() + defer { lock.unlock() } + + guard !didTimeOut else { return } + + self.response = ScreenshotResponse( + data: data, + response: response, + error: error + ) + } + + func timeOut() { + lock.lock() + defer { lock.unlock() } + + didTimeOut = true + response.error = VizzlyClientError.requestTimedOut + } + + var currentResponse: ScreenshotResponse { + lock.lock() + defer { lock.unlock() } + + return response + } + } /// Shared singleton instance public static let shared = VizzlyClient() @@ -43,8 +94,10 @@ public final class VizzlyClient { /// Initialize a new Vizzly client /// /// - Parameter serverUrl: Optional server URL. If not provided, auto-discovery will be used. - public init(serverUrl: String? = nil) { - self.serverUrl = serverUrl ?? discoverServerUrl() + /// - Parameter autoDiscover: Whether to discover a local Vizzly server when + /// `serverUrl` is nil. + public init(serverUrl: String? = nil, autoDiscover: Bool = true) { + self.serverUrl = serverUrl ?? (autoDiscover ? discoverServerUrl() : nil) } /// Take a screenshot for visual regression testing @@ -53,7 +106,10 @@ public final class VizzlyClient { /// - name: Unique name for the screenshot /// - image: PNG image data /// - properties: Additional properties to attach (browser, viewport, etc.) - /// - threshold: Pixel difference threshold (0-100) + /// - threshold: Optional CIEDE2000 Delta E threshold. When nil, the + /// Vizzly server configuration is used. + /// - minClusterSize: Optional minimum changed-pixel cluster size to count + /// as a real difference. When nil, the server configuration is used. /// - fullPage: Whether this is a full page screenshot /// - Returns: Response data if successful, nil otherwise @discardableResult @@ -61,7 +117,8 @@ public final class VizzlyClient { name: String, image: Data, properties: [String: Any]? = nil, - threshold: Int = 0, + threshold: Double? = nil, + minClusterSize: Int? = nil, fullPage: Bool = false ) -> [String: Any]? { guard !isDisabled else { return nil } @@ -72,39 +129,16 @@ public final class VizzlyClient { return nil } - // Encode image to base64 - let imageBase64 = image.base64EncodedString() - - // Build payload - var payload: [String: Any] = [ - "name": name, - "image": imageBase64, - "type": "base64", - "threshold": threshold, - "fullPage": fullPage - ] - - // Build properties - merge user properties with auto-detected device info - var mergedProperties: [String: Any] = properties ?? [:] - - // Add device info (user properties take precedence) - let deviceInfo = getDeviceInfo() - for (key, value) in deviceInfo { - if mergedProperties[key] == nil { - mergedProperties[key] = value - } - } - - if !mergedProperties.isEmpty { - payload["properties"] = mergedProperties - } - - // Try to get buildId from multiple sources (priority order): - // 1. Environment variable (for Node.js and other runtimes that support it) - // 2. Server info file (for iOS/Swift where env vars don't propagate to simulator) - if let buildId = getBuildId() { - payload["buildId"] = buildId - } + let payload = Self.makeScreenshotPayload( + name: name, + image: image, + properties: properties, + threshold: threshold, + minClusterSize: minClusterSize, + fullPage: fullPage, + buildId: getBuildId(), + deviceInfo: getDeviceInfo() + ) // Send HTTP request guard let url = URL(string: "\(serverUrl)/screenshot") else { @@ -125,54 +159,35 @@ public final class VizzlyClient { return nil } - // Use semaphore for synchronous request (needed in test context) - let semaphore = DispatchSemaphore(value: 0) - var result: [String: Any]? - - let task = URLSession.shared.dataTask(with: request) { data, response, error in - defer { semaphore.signal() } - - if let error = error { - self.handleError(name: name, error: error) - return - } + let response = send(request) - guard let httpResponse = response as? HTTPURLResponse else { - print("❌ Invalid response for screenshot: \(name)") - self.disable(reason: "failure") - return - } - - guard let data = data else { - print("❌ No data received for screenshot: \(name)") - self.disable(reason: "failure") - return - } + if let error = response.error { + handleError(name: name, error: error) + return nil + } - // Handle non-success responses - if httpResponse.statusCode != 200 { - self.handleNonSuccessResponse( - statusCode: httpResponse.statusCode, - data: data, - name: name - ) - return - } + guard let httpResponse = response.response as? HTTPURLResponse else { + print("❌ Invalid response for screenshot: \(name)") + disable(reason: "failure") + return nil + } - // Parse successful response - do { - if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { - result = json - } - } catch { - print("⚠️ Failed to parse response for \(name): \(error)") - } + guard let data = response.data else { + print("❌ No data received for screenshot: \(name)") + disable(reason: "failure") + return nil } - task.resume() - semaphore.wait() + if httpResponse.statusCode != 200 { + handleNonSuccessResponse( + statusCode: httpResponse.statusCode, + data: data, + name: name + ) + return nil + } - return result + return parseScreenshotResponse(data, name: name) } /// Flush any pending screenshots (no-op for simple client) @@ -217,6 +232,53 @@ public final class VizzlyClient { // MARK: - Private Methods + internal static func makeScreenshotPayload( + name: String, + image: Data, + properties: [String: Any]? = nil, + threshold: Double? = nil, + minClusterSize: Int? = nil, + fullPage: Bool = false, + buildId: String? = nil, + deviceInfo: [String: Any] = [:] + ) -> [String: Any] { + var mergedProperties = properties ?? [:] + + for (key, value) in deviceInfo { + if mergedProperties[key] == nil { + mergedProperties[key] = value + } + } + + if let threshold = threshold { + mergedProperties["threshold"] = threshold + } + + if let minClusterSize = minClusterSize { + mergedProperties["minClusterSize"] = minClusterSize + } + + if fullPage { + mergedProperties["fullPage"] = true + } + + var payload: [String: Any] = [ + "name": name, + "image": image.base64EncodedString(), + "type": "base64" + ] + + if !mergedProperties.isEmpty { + payload["properties"] = mergedProperties + } + + if let buildId = buildId { + payload["buildId"] = buildId + } + + return payload + } + private func getDeviceInfo() -> [String: Any] { var info: [String: Any] = [:] @@ -249,6 +311,37 @@ public final class VizzlyClient { hasWarned = true } + private func send(_ request: URLRequest) -> ScreenshotResponse { + let semaphore = DispatchSemaphore(value: 0) + let state = ScreenshotRequestState() + + let task = URLSession.shared.dataTask(with: request) { data, response, error in + defer { semaphore.signal() } + state.complete( + data: data, + response: response, + error: error + ) + } + + task.resume() + if semaphore.wait(timeout: .now() + request.timeoutInterval + 5) == .timedOut { + state.timeOut() + task.cancel() + } + + return state.currentResponse + } + + private func parseScreenshotResponse(_ data: Data, name: String) -> [String: Any]? { + do { + return try JSONSerialization.jsonObject(with: data) as? [String: Any] + } catch { + print("⚠️ Failed to parse response for \(name): \(error)") + return nil + } + } + private func discoverServerUrl() -> String? { // 1. Check VIZZLY_SERVER_URL environment variable (highest priority, set by CLI or test runner) if let envUrl = ProcessInfo.processInfo.environment["VIZZLY_SERVER_URL"] { @@ -353,7 +446,7 @@ public final class VizzlyClient { let semaphore = DispatchSemaphore(value: 0) var isReachable = false - let task = URLSession.shared.dataTask(with: request) { _, response, error in + let task = URLSession.shared.dataTask(with: request) { _, response, _ in defer { semaphore.signal() } if let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 200 { @@ -367,7 +460,6 @@ public final class VizzlyClient { return isReachable } - private func handleError(name: String, error: Error) { print("⚠️ Vizzly screenshot failed for \(name): \(error.localizedDescription)") diff --git a/clients/swift/Sources/Vizzly/XCTestExtensions.swift b/clients/swift/Sources/VizzlyXCTest/XCTestExtensions.swift similarity index 78% rename from clients/swift/Sources/Vizzly/XCTestExtensions.swift rename to clients/swift/Sources/VizzlyXCTest/XCTestExtensions.swift index 31cbe22e..0a4c0b17 100644 --- a/clients/swift/Sources/Vizzly/XCTestExtensions.swift +++ b/clients/swift/Sources/VizzlyXCTest/XCTestExtensions.swift @@ -1,4 +1,5 @@ import Foundation +import Vizzly import XCTest #if canImport(UIKit) @@ -18,7 +19,10 @@ extension XCTestCase { /// - name: Unique name for the screenshot /// - app: The XCUIApplication instance (iOS/macOS) /// - properties: Additional properties to attach - /// - threshold: Pixel difference threshold (0-100) + /// - threshold: Optional CIEDE2000 Delta E threshold. When nil, the + /// Vizzly server configuration is used. + /// - minClusterSize: Optional minimum changed-pixel cluster size to count + /// as a real difference. When nil, the server configuration is used. /// - fullPage: Whether this is a full page screenshot @available(iOS 13.0, macOS 10.15, *) @discardableResult @@ -26,7 +30,8 @@ extension XCTestCase { name: String, app: XCUIApplication, properties: [String: Any]? = nil, - threshold: Int = 0, + threshold: Double? = nil, + minClusterSize: Int? = nil, fullPage: Bool = false ) -> [String: Any]? { let screenshot = app.screenshot() @@ -62,6 +67,7 @@ extension XCTestCase { image: screenshot.pngRepresentation, properties: combinedProperties, threshold: threshold, + minClusterSize: minClusterSize, fullPage: fullPage ) } @@ -72,14 +78,18 @@ extension XCTestCase { /// - name: Unique name for the screenshot /// - element: The XCUIElement to screenshot /// - properties: Additional properties to attach - /// - threshold: Pixel difference threshold (0-100) + /// - threshold: Optional CIEDE2000 Delta E threshold. When nil, the + /// Vizzly server configuration is used. + /// - minClusterSize: Optional minimum changed-pixel cluster size to count + /// as a real difference. When nil, the server configuration is used. @available(iOS 13.0, macOS 10.15, *) @discardableResult public func vizzlyScreenshot( name: String, element: XCUIElement, properties: [String: Any]? = nil, - threshold: Int = 0 + threshold: Double? = nil, + minClusterSize: Int? = nil ) -> [String: Any]? { let screenshot = element.screenshot() @@ -99,6 +109,7 @@ extension XCTestCase { image: screenshot.pngRepresentation, properties: combinedProperties, threshold: threshold, + minClusterSize: minClusterSize, fullPage: false ) } @@ -113,13 +124,17 @@ extension XCUIApplication { /// - Parameters: /// - name: Unique name for the screenshot /// - properties: Additional properties to attach - /// - threshold: Pixel difference threshold (0-100) + /// - threshold: Optional CIEDE2000 Delta E threshold. When nil, the + /// Vizzly server configuration is used. + /// - minClusterSize: Optional minimum changed-pixel cluster size to count + /// as a real difference. When nil, the server configuration is used. /// - fullPage: Whether this is a full page screenshot @discardableResult public func vizzlyScreenshot( name: String, properties: [String: Any]? = nil, - threshold: Int = 0, + threshold: Double? = nil, + minClusterSize: Int? = nil, fullPage: Bool = false ) -> [String: Any]? { let screenshot = self.screenshot() @@ -147,6 +162,7 @@ extension XCUIApplication { image: screenshot.pngRepresentation, properties: combinedProperties, threshold: threshold, + minClusterSize: minClusterSize, fullPage: fullPage ) } @@ -161,12 +177,16 @@ extension XCUIElement { /// - Parameters: /// - name: Unique name for the screenshot /// - properties: Additional properties to attach - /// - threshold: Pixel difference threshold (0-100) + /// - threshold: Optional CIEDE2000 Delta E threshold. When nil, the + /// Vizzly server configuration is used. + /// - minClusterSize: Optional minimum changed-pixel cluster size to count + /// as a real difference. When nil, the server configuration is used. @discardableResult public func vizzlyScreenshot( name: String, properties: [String: Any]? = nil, - threshold: Int = 0 + threshold: Double? = nil, + minClusterSize: Int? = nil ) -> [String: Any]? { let screenshot = self.screenshot() @@ -186,6 +206,7 @@ extension XCUIElement { image: screenshot.pngRepresentation, properties: combinedProperties, threshold: threshold, + minClusterSize: minClusterSize, fullPage: false ) } diff --git a/clients/swift/Tests/VizzlyTests/VizzlyClientTests.swift b/clients/swift/Tests/VizzlyTests/VizzlyClientTests.swift index 51f9b48a..0c2741c5 100644 --- a/clients/swift/Tests/VizzlyTests/VizzlyClientTests.swift +++ b/clients/swift/Tests/VizzlyTests/VizzlyClientTests.swift @@ -1,5 +1,6 @@ import XCTest @testable import Vizzly +import VizzlyXCTest final class VizzlyClientTests: XCTestCase { @@ -56,7 +57,7 @@ final class VizzlyClientTests: XCTestCase { func testScreenshotWithNoServer() { // Create client with invalid server URL - let client = VizzlyClient(serverUrl: nil) + let client = VizzlyClient(serverUrl: nil, autoDiscover: false) let testImage = createTestImage() let result = client.screenshot(name: "test", image: testImage) @@ -80,25 +81,55 @@ final class VizzlyClientTests: XCTestCase { } func testUserPropertiesTakePrecedence() { - // This test verifies the merge behavior conceptually - // User-provided properties should override auto-detected ones - let client = VizzlyClient(serverUrl: "http://localhost:47392") + let payload = VizzlyClient.makeScreenshotPayload( + name: "test", + image: createTestImage(), + properties: [ + "platform": "custom-platform", + "customKey": "customValue" + ], + deviceInfo: [ + "platform": "iOS", + "deviceName": "iPhone" + ] + ) - // When user provides custom platform, it should be used - // (We can't easily test the actual HTTP payload without mocking, - // but we verify the client accepts properties) - let testImage = createTestImage() + let properties = payload["properties"] as? [String: Any] + XCTAssertEqual(properties?["platform"] as? String, "custom-platform") + XCTAssertEqual(properties?["customKey"] as? String, "customValue") + XCTAssertEqual(properties?["deviceName"] as? String, "iPhone") + } - // This should not crash and should accept custom properties - client.disable() // Disable to prevent actual network call - let result = client.screenshot( + func testComparisonOptionsAreSentAsProperties() { + let payload = VizzlyClient.makeScreenshotPayload( name: "test", - image: testImage, - properties: ["platform": "custom-platform", "customKey": "customValue"] + image: createTestImage(), + properties: ["theme": "dark"], + threshold: 1.5, + minClusterSize: 4, + fullPage: true ) - // Result is nil because client is disabled, but no crash means properties work - XCTAssertNil(result) + XCTAssertNil(payload["threshold"]) + XCTAssertNil(payload["minClusterSize"]) + + let properties = payload["properties"] as? [String: Any] + XCTAssertEqual(properties?["theme"] as? String, "dark") + XCTAssertEqual(properties?["threshold"] as? Double, 1.5) + XCTAssertEqual(properties?["minClusterSize"] as? Int, 4) + XCTAssertEqual(properties?["fullPage"] as? Bool, true) + } + + func testDefaultComparisonOptionsUseServerConfiguration() { + let payload = VizzlyClient.makeScreenshotPayload( + name: "test", + image: createTestImage() + ) + + XCTAssertNil(payload["threshold"]) + XCTAssertNil(payload["minClusterSize"]) + XCTAssertNil(payload["fullPage"]) + XCTAssertNil(payload["properties"]) } // MARK: - Helpers From 73e2d8652fc98e326f6f06b26688900a103cbb0e Mon Sep 17 00:00:00 2001 From: Robert DeLuca Date: Tue, 12 May 2026 00:20:34 -0500 Subject: [PATCH 2/4] =?UTF-8?q?=E2=9C=85=20Add=20Swift=20SDK=20E2E=20cover?= =?UTF-8?q?age?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the Swift client against a real local Vizzly TDD server in an isolated temp workspace. Cover server discovery, PNG upload, new baseline creation, and repeated screenshot matching. --- clients/swift/Package.swift | 3 + clients/swift/README.md | 12 +++ .../Tests/VizzlyE2ETests/VizzlyE2ETests.swift | 90 +++++++++++++++++++ clients/swift/scripts/run-e2e.js | 52 +++++++++++ package.json | 1 + 5 files changed, 158 insertions(+) create mode 100644 clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift create mode 100644 clients/swift/scripts/run-e2e.js diff --git a/clients/swift/Package.swift b/clients/swift/Package.swift index ba60bc95..5ab62e32 100644 --- a/clients/swift/Package.swift +++ b/clients/swift/Package.swift @@ -27,5 +27,8 @@ let package = Package( .testTarget( name: "VizzlyTests", dependencies: ["Vizzly", "VizzlyXCTest"]), + .testTarget( + name: "VizzlyE2ETests", + dependencies: ["Vizzly"]), ] ) diff --git a/clients/swift/README.md b/clients/swift/README.md index 9be912a1..2db4b862 100644 --- a/clients/swift/README.md +++ b/clients/swift/README.md @@ -458,6 +458,18 @@ Check out the `Example/` directory for: - Custom properties and thresholds - Direct client usage +## SDK E2E Tests + +The Swift SDK has an end-to-end test path that runs against a real local +Vizzly TDD server and uploads real PNG bytes through `VizzlyClient`: + +```bash +npm run test:swift:e2e +``` + +This command builds the CLI, starts an isolated TDD run in a temp directory, +and executes the `VizzlyE2ETests` SwiftPM suite. + ## Contributing Bug reports and pull requests are welcome at https://github.com/vizzly-testing/cli diff --git a/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift b/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift new file mode 100644 index 00000000..1ad57a78 --- /dev/null +++ b/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift @@ -0,0 +1,90 @@ +import XCTest +import Vizzly + +#if os(macOS) +import AppKit +#endif + +final class VizzlyE2ETests: XCTestCase { + override func setUpWithError() throws { + try super.setUpWithError() + + guard ProcessInfo.processInfo.environment["VIZZLY_E2E"] == "1" else { + throw XCTSkip("Set VIZZLY_E2E=1 to run Swift SDK E2E tests") + } + } + + func testUploadsScreenshotThroughRunningTddServer() throws { + let client = VizzlyClient() + XCTAssertTrue(client.isReady, "Expected VizzlyClient to discover the TDD server") + + let firstResult = try XCTUnwrap( + client.screenshot( + name: "swift-sdk-e2e-home", + image: try Self.makeTestPng(), + properties: [ + "platform": "swift-e2e", + "screen": "home" + ], + threshold: 1.5, + minClusterSize: 3, + fullPage: true + ) + ) + + XCTAssertTrue( + Self.successStatuses.contains(Self.status(from: firstResult)), + "Expected first upload to create or match a baseline, got: \(firstResult)" + ) + XCTAssertEqual(firstResult["name"] as? String, "swift-sdk-e2e-home") + XCTAssertNotNil(firstResult["current"]) + XCTAssertNotNil(firstResult["baseline"]) + } + + func testSecondUploadMatchesExistingBaseline() throws { + let client = VizzlyClient() + + let firstResult = try XCTUnwrap( + client.screenshot( + name: "swift-sdk-e2e-repeatable", + image: try Self.makeTestPng(), + properties: ["case": "repeatable"] + ) + ) + XCTAssertTrue( + Self.successStatuses.contains(Self.status(from: firstResult)), + "Expected first upload to create or match a baseline, got: \(firstResult)" + ) + + let secondResult = try XCTUnwrap( + client.screenshot( + name: "swift-sdk-e2e-repeatable", + image: try Self.makeTestPng(), + properties: ["case": "repeatable"] + ) + ) + XCTAssertEqual(Self.status(from: secondResult), "match") + } + + private static let successStatuses: Set = ["new", "match"] + + private static func status(from result: [String: Any]) -> String { + return result["status"] as? String ?? "" + } + + private static func makeTestPng() throws -> Data { + #if os(macOS) + let image = NSImage(size: NSSize(width: 2, height: 2)) + image.lockFocus() + NSColor.systemBlue.setFill() + NSRect(x: 0, y: 0, width: 2, height: 2).fill() + image.unlockFocus() + + let tiffData = try XCTUnwrap(image.tiffRepresentation) + let bitmap = try XCTUnwrap(NSBitmapImageRep(data: tiffData)) + return try XCTUnwrap(bitmap.representation(using: .png, properties: [:])) + #else + throw XCTSkip("Swift SDK E2E PNG fixture generation currently runs on macOS") + #endif + } +} diff --git a/clients/swift/scripts/run-e2e.js b/clients/swift/scripts/run-e2e.js new file mode 100644 index 00000000..00811cc4 --- /dev/null +++ b/clients/swift/scripts/run-e2e.js @@ -0,0 +1,52 @@ +import { spawn } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +let scriptDir = dirname(fileURLToPath(import.meta.url)); +let swiftPackageDir = resolve(scriptDir, '..'); +let repoRoot = resolve(swiftPackageDir, '../..'); +let cliPath = join(repoRoot, 'bin/vizzly.js'); +let distCliPath = join(repoRoot, 'dist/cli.js'); + +if (!existsSync(distCliPath)) { + console.error('Build the CLI first: npm run build'); + process.exit(1); +} + +let tempDir = mkdtempSync(join(tmpdir(), 'vizzly-swift-e2e-')); +let swiftTestCommand = [ + 'cd', + JSON.stringify(swiftPackageDir), + '&&', + 'swift', + 'test', + '--filter', + 'VizzlyE2ETests', +].join(' '); + +let child = spawn( + process.execPath, + [cliPath, 'tdd', 'run', swiftTestCommand, '--no-color'], + { + cwd: tempDir, + stdio: 'inherit', + env: { + ...process.env, + VIZZLY_E2E: '1', + VIZZLY_HOME: join(tempDir, '.vizzly-home'), + }, + } +); + +child.on('exit', code => { + rmSync(tempDir, { recursive: true, force: true }); + process.exit(code ?? 1); +}); + +child.on('error', error => { + rmSync(tempDir, { recursive: true, force: true }); + console.error(error); + process.exit(1); +}); diff --git a/package.json b/package.json index e347aad6..da845c10 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "test:watch": "node --test --test-reporter=spec --watch $(find tests -name '*.test.js')", "test:reporter": "playwright test --config=tests/reporter/playwright.config.js", "test:reporter:visual": "node bin/vizzly.js run \"npm run test:reporter\"", + "test:swift:e2e": "npm run build && node clients/swift/scripts/run-e2e.js", "test:tui": "node --test --test-reporter=spec tests/tui/*.test.js", "test:tui:docker": "./tests/tui/run-tui-tests.sh", "lint": "biome check src tests", From cf363aeef7c017b6e3c801fc2d728a14ffab8879 Mon Sep 17 00:00:00 2001 From: Robert DeLuca Date: Tue, 12 May 2026 00:51:33 -0500 Subject: [PATCH 3/4] =?UTF-8?q?=E2=9C=85=20Run=20Swift=20SDK=20in=20E2E=20?= =?UTF-8?q?workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the Swift package to the shared SDK E2E GitHub Actions workflow with both local TDD and cloud-mode coverage, using the Swift-specific project token secret. --- .github/workflows/sdk-e2e.yml | 46 ++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-e2e.yml b/.github/workflows/sdk-e2e.yml index 2cb3f6f2..5d4b042e 100644 --- a/.github/workflows/sdk-e2e.yml +++ b/.github/workflows/sdk-e2e.yml @@ -376,11 +376,51 @@ jobs: VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }} VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }} + # Swift SDK + swift: + name: Swift SDK + runs-on: macos-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: 'npm' + + - name: Install CLI dependencies + run: npm ci + + - name: Build CLI + run: npm run build + + - name: Build Swift package + working-directory: ./clients/swift + run: swift build + + - name: Run E2E tests (TDD mode) + working-directory: ./clients/swift + run: ../../bin/vizzly.js tdd run "VIZZLY_E2E=1 swift test --filter VizzlyE2ETests" + env: + CI: true + + - name: Run E2E tests (Cloud mode) + working-directory: ./clients/swift + run: ../../bin/vizzly.js run "VIZZLY_E2E=1 swift test --filter VizzlyE2ETests" + env: + CI: true + VIZZLY_TOKEN: ${{ secrets.VIZZLY_SWIFT_CLIENT_TOKEN }} + VIZZLY_COMMIT_MESSAGE: ${{ github.event.pull_request.head.commit.message || github.event.head_commit.message }} + VIZZLY_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.event.head_commit.id }} + # Status check for branch protection check: name: E2E Status runs-on: ubuntu-latest - needs: [core-js, vitest, storybook, static-site, ember, ruby] + needs: [core-js, vitest, storybook, static-site, ember, ruby, swift] if: always() steps: - name: Check all SDK E2E tests passed @@ -409,4 +449,8 @@ jobs: echo "Ruby SDK E2E tests failed" exit 1 fi + if [[ "${{ needs.swift.result }}" == "failure" ]]; then + echo "Swift SDK E2E tests failed" + exit 1 + fi echo "All SDK E2E tests passed" From 73e890b7929d1cd3b3ae06a3fc2dc152bd6a3f25 Mon Sep 17 00:00:00 2001 From: Robert DeLuca Date: Tue, 12 May 2026 00:54:35 -0500 Subject: [PATCH 4/4] =?UTF-8?q?=E2=9C=85=20Accept=20cloud=20Swift=20E2E=20?= =?UTF-8?q?responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeps the Swift E2E tests strict for TDD comparison metadata while accepting the leaner successful upload response returned by cloud mode. --- .../Tests/VizzlyE2ETests/VizzlyE2ETests.swift | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift b/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift index 1ad57a78..dc09622b 100644 --- a/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift +++ b/clients/swift/Tests/VizzlyE2ETests/VizzlyE2ETests.swift @@ -33,12 +33,15 @@ final class VizzlyE2ETests: XCTestCase { ) XCTAssertTrue( - Self.successStatuses.contains(Self.status(from: firstResult)), + Self.isSuccessfulUpload(firstResult), "Expected first upload to create or match a baseline, got: \(firstResult)" ) XCTAssertEqual(firstResult["name"] as? String, "swift-sdk-e2e-home") - XCTAssertNotNil(firstResult["current"]) - XCTAssertNotNil(firstResult["baseline"]) + + if Self.hasTddComparisonMetadata(firstResult) { + XCTAssertNotNil(firstResult["current"]) + XCTAssertNotNil(firstResult["baseline"]) + } } func testSecondUploadMatchesExistingBaseline() throws { @@ -52,7 +55,7 @@ final class VizzlyE2ETests: XCTestCase { ) ) XCTAssertTrue( - Self.successStatuses.contains(Self.status(from: firstResult)), + Self.isSuccessfulUpload(firstResult), "Expected first upload to create or match a baseline, got: \(firstResult)" ) @@ -63,13 +66,36 @@ final class VizzlyE2ETests: XCTestCase { properties: ["case": "repeatable"] ) ) - XCTAssertEqual(Self.status(from: secondResult), "match") + XCTAssertTrue( + Self.isSuccessfulUpload(secondResult), + "Expected second upload to succeed, got: \(secondResult)" + ) + + if let status = secondResult["status"] as? String { + XCTAssertEqual(status, "match") + } } private static let successStatuses: Set = ["new", "match"] - private static func status(from result: [String: Any]) -> String { - return result["status"] as? String ?? "" + private static func isSuccessfulUpload(_ result: [String: Any]) -> Bool { + if let status = result["status"] as? String { + return successStatuses.contains(status) + } + + if let success = result["success"] as? Bool { + return success + } + + if let success = result["success"] as? Int { + return success == 1 + } + + return false + } + + private static func hasTddComparisonMetadata(_ result: [String: Any]) -> Bool { + return result["status"] != nil } private static func makeTestPng() throws -> Data {