diff --git a/CHANGELOG.md b/CHANGELOG.md index a4f03d6..173dfb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to OpenVision will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- **OpenClaw backend can now reach a gateway on the local network.** Previously only a gateway on localhost worked: the handshake negotiated protocol 3 (current gateways speak 4), sent no role or scopes, and presented no device identity — so `chat.send` failed with `missing scope: operator.write` even though the connection itself succeeded. The app now signs an Ed25519 device identity (persisted in the Keychain) that the gateway holds as a pairing request; approve it once with `openclaw devices approve `. See SETUP.md +- Reconnecting to an OpenClaw gateway no longer fails with `device nonce mismatch`. Each socket gets its own `connect.challenge` nonce and the signature covers it, so caching the first one meant the initial connection after launch succeeded and every reconnect afterwards failed +- Added `NSLocalNetworkUsageDescription`, without which iOS silently blocks connections to a LAN gateway rather than prompting for consent + ## [2.11.0] - 2026-08-15 ### Added diff --git a/OpenVision.xcodeproj/project.pbxproj b/OpenVision.xcodeproj/project.pbxproj index 2255663..60f1895 100644 --- a/OpenVision.xcodeproj/project.pbxproj +++ b/OpenVision.xcodeproj/project.pbxproj @@ -35,6 +35,7 @@ 456B4E46EE405F5C8F45B04D /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1ABBF4AC21A442A2C3CF6750 /* Theme.swift */; }; 49DFBFFE52D341E96BCE382E /* AppleFoundationService.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7B97C063C947F53F6E209D1 /* AppleFoundationService.swift */; }; 4DFF630E56769253493B4B57 /* NotificationForegroundPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4831CE885B0320D9CAD71736 /* NotificationForegroundPresenter.swift */; }; + 529CAD92EEAB3567C51769E6 /* OpenClawDeviceIdentity.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E9830333CA08F4DA42A994 /* OpenClawDeviceIdentity.swift */; }; 52E3A16AB6227CDA3D67817D /* AppSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2FD4A9B156D92B468B28864 /* AppSettings.swift */; }; 53B484EF0A2017B9718AC548 /* WebSearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 71ACC88CE5E9E799ED607853 /* WebSearchSettingsView.swift */; }; 53C17BBA774E7D3D4E295584 /* LocalAgent.swift in Sources */ = {isa = PBXBuildFile; fileRef = CEE6F61B22736F87248FD0F5 /* LocalAgent.swift */; }; @@ -186,6 +187,7 @@ BE848862D72446F7BBDD3D89 /* KokoroTTSService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KokoroTTSService.swift; sourceTree = ""; }; BEEDCA2C462B77066AA0CD25 /* VoiceAgentViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceAgentViewModel.swift; sourceTree = ""; }; C05A9A6876CC108B7CEDFDFC /* TextChunkingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TextChunkingTests.swift; sourceTree = ""; }; + C0E9830333CA08F4DA42A994 /* OpenClawDeviceIdentity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenClawDeviceIdentity.swift; sourceTree = ""; }; C1BF0DC63344C51A046E37E0 /* OVExceptionCatcher.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = OVExceptionCatcher.h; sourceTree = ""; }; C1BFC8380B5BDA0778E04EAB /* GemmaSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GemmaSettingsView.swift; sourceTree = ""; }; C27052ECD2FFC79652B5CDB3 /* OnboardingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingView.swift; sourceTree = ""; }; @@ -463,6 +465,7 @@ 867CD5E802B9354B21C23D0B /* OpenClaw */ = { isa = PBXGroup; children = ( + C0E9830333CA08F4DA42A994 /* OpenClawDeviceIdentity.swift */, 7946C8232CEA5A362056AC2A /* OpenClawProtocol.swift */, C539AB340611B1A6CCC50F17 /* OpenClawService.swift */, ); @@ -702,7 +705,6 @@ }; }; buildConfigurationList = F2E191267E52B7178955EEDB /* Build configuration list for PBXProject "OpenVision" */; - compatibilityVersion = "Xcode 14.0"; developmentRegion = en; hasScannedForEncodings = 0; knownRegions = ( @@ -721,11 +723,12 @@ A8456994C4E280FE9D1FCB59 /* XCLocalSwiftPackageReference "Vendor/kokoro-ios" */, ); preferredProjectObjectVersion = 77; + productRefGroup = 36F672D5AB5BCE7D4C73AA30 /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( - 9418DE699CD106BA6A8A4B7E /* OpenVision */, 60AF4ED566AAE248296380F1 /* OpenVisionTests */, + 9418DE699CD106BA6A8A4B7E /* OpenVision */, ); }; /* End PBXProject section */ @@ -837,6 +840,7 @@ A17626C4B92966423AF06452 /* OpenAIRealtimeService.swift in Sources */, 223474C328BCC3C81ADCB93D /* OpenAIService.swift in Sources */, 800BD44B75446351A267E190 /* OpenAISettingsView.swift in Sources */, + 529CAD92EEAB3567C51769E6 /* OpenClawDeviceIdentity.swift in Sources */, C3B4CF9D8B6EC21957CC7BEF /* OpenClawProtocol.swift in Sources */, 186FD72F369F1962B15EB858 /* OpenClawService.swift in Sources */, 758F66B632E8A1CBDBB40094 /* OpenClawSettingsView.swift in Sources */, diff --git a/OpenVision/Resources/Info.plist b/OpenVision/Resources/Info.plist index 0206bed..7bb65a8 100644 --- a/OpenVision/Resources/Info.plist +++ b/OpenVision/Resources/Info.plist @@ -142,6 +142,19 @@ + + NSLocalNetworkUsageDescription + OpenVision connects to the OpenClaw assistant running on your own computer over your local network, and streams video from your glasses over Wi-Fi. + NSBonjourServices + + _bonjour._tcp + + MWDAT diff --git a/OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift b/OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift new file mode 100644 index 0000000..c937552 --- /dev/null +++ b/OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift @@ -0,0 +1,106 @@ +import CryptoKit +import Foundation + +/// Ed25519 device identity for the OpenClaw gateway handshake. +/// +/// The gateway trusts loopback connections implicitly, so a client running on +/// the same machine as the gateway connects with a token alone. A connection +/// from another host — which is every connection from this phone — is granted +/// no write scope on a token by itself: `chat.send` comes back +/// "missing scope: operator.write" even though the handshake succeeded. It has +/// to present a signed device identity, which the gateway then holds as a +/// pairing request until it is approved once with: +/// +/// openclaw devices approve +/// +/// The key is persisted in the Keychain so the phone presents the same device +/// on every launch; a fresh key each time would mean a new pairing request each +/// time, and a list full of stale devices. +enum OpenClawDeviceIdentity { + + private static let service = "ai.openclaw.openvision" + private static let account = "openclaw-device-key" + + private static let key: Curve25519.Signing.PrivateKey = { + if let stored = loadKey() { return stored } + let fresh = Curve25519.Signing.PrivateKey() + storeKey(fresh) + return fresh + }() + + /// Raw 32-byte public key, base64url — the form the gateway expects. + static var publicKeyBase64URL: String { base64URL(key.publicKey.rawRepresentation) } + + /// sha256 of the raw public key, hex. The gateway derives this itself and + /// compares, so it cannot be chosen freely. + static var deviceId: String { + SHA256.hash(data: key.publicKey.rawRepresentation) + .map { String(format: "%02x", $0) } + .joined() + } + + /// The `device` object for the connect params. `nonce` must be the nonce + /// from the server's `connect.challenge` event, and the signature covers + /// the exact field order below — any deviation fails verification. + static func deviceParams( + clientId: String, + clientMode: String, + role: String, + scopes: [String], + token: String, + nonce: String + ) -> [String: Any]? { + let signedAt = Int(Date().timeIntervalSince1970 * 1000) + let payload = [ + "v2", deviceId, clientId, clientMode, role, + scopes.joined(separator: ","), String(signedAt), token, nonce, + ].joined(separator: "|") + + guard let data = payload.data(using: .utf8), + let signature = try? key.signature(for: data) else { return nil } + + return [ + "id": deviceId, + "publicKey": publicKeyBase64URL, + "signature": base64URL(signature), + "signedAt": signedAt, + "nonce": nonce, + ] + } + + // MARK: - Helpers + + private static func base64URL(_ data: Data) -> String { + data.base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } + + private static func loadKey() -> Curve25519.Signing.PrivateKey? { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + kSecReturnData as String: true, + kSecMatchLimit as String: kSecMatchLimitOne, + ] + var item: CFTypeRef? + guard SecItemCopyMatching(query as CFDictionary, &item) == errSecSuccess, + let data = item as? Data else { return nil } + return try? Curve25519.Signing.PrivateKey(rawRepresentation: data) + } + + private static func storeKey(_ key: Curve25519.Signing.PrivateKey) { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + ] + SecItemDelete(query as CFDictionary) + var attributes = query + attributes[kSecValueData as String] = key.rawRepresentation + attributes[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlock + SecItemAdd(attributes as CFDictionary, nil) + } +} diff --git a/OpenVision/Services/OpenClaw/OpenClawService.swift b/OpenVision/Services/OpenClaw/OpenClawService.swift index 71fefbb..c730d92 100644 --- a/OpenVision/Services/OpenClaw/OpenClawService.swift +++ b/OpenVision/Services/OpenClaw/OpenClawService.swift @@ -85,6 +85,9 @@ final class OpenClawService: ObservableObject { private var requestCounter: Int = 0 private var pendingRequests: [String: CheckedContinuation] = [:] private var receiveTask: Task? + /// Nonce from the server's connect.challenge, needed to sign the device identity. + private var challengeNonce: String? + private var challengeWaiter: CheckedContinuation? // MARK: - Reconnection @@ -221,6 +224,11 @@ final class OpenClawService: ObservableObject { requestCounter = 0 failPendingRequests(error: AIBackendError.notConnected) + // Every socket gets its own connect.challenge nonce. Carrying one over + // from a previous socket makes the gateway reject the handshake with + // "device nonce mismatch", so each attempt must wait for a fresh one. + challengeNonce = nil + guard !Task.isCancelled else { return } do { @@ -358,12 +366,39 @@ final class OpenClawService: ObservableObject { // MARK: - Handshake + /// Wait briefly for the server's connect.challenge nonce. The socket is + /// already receiving by the time the handshake is built, so this normally + /// returns immediately. + private func waitForChallengeNonce() async -> String? { + if let nonce = challengeNonce { return nonce } + return await withTaskGroup(of: String?.self) { group in + group.addTask { [weak self] in + await withCheckedContinuation { (c: CheckedContinuation) in + Task { @MainActor in + if let existing = self?.challengeNonce { c.resume(returning: existing) } + else { self?.challengeWaiter = c } + } + } + } + group.addTask { + try? await Task.sleep(nanoseconds: 3_000_000_000) + return nil + } + let first = await group.next() ?? nil + group.cancelAll() + return first + } + } + /// Send initial connect handshake private func sendHandshake() async throws { // Match xmeta's handshake format exactly - let params: [String: Any] = [ - "minProtocol": 3, - "maxProtocol": 3, + var params: [String: Any] = [ + // Protocol 4: this gateway (OpenClaw 2026.7.1-2) sets + // MIN_CLIENT_PROTOCOL_VERSION = 4 and rejects a 3/3 offer outright with + // PROTOCOL_MISMATCH, so the shipped values cannot connect to it. + "minProtocol": 4, + "maxProtocol": 4, "client": [ "id": "cli", "displayName": "OpenVision", @@ -372,11 +407,30 @@ final class OpenClawService: ObservableObject { "mode": "cli" ], "caps": [String](), // Empty array like xmeta + // Without these the handshake still succeeds, but every chat.send is + // rejected with "missing scope: operator.write" -- the gateway grants + // no write scope to a connection that requests none. + "role": "operator", + "scopes": ["operator.read", "operator.write"], "auth": ["token": authToken], "locale": "en-US", "userAgent": "OpenVision/1.0.0" ] + // A connection from another host is granted no write scope on a token + // alone, so sign a device identity with the challenge nonce. First run + // returns PAIRING_REQUIRED until the device is approved once on the + // gateway host with: openclaw devices approve + if let nonce = await waitForChallengeNonce(), + let device = OpenClawDeviceIdentity.deviceParams( + clientId: "cli", clientMode: "cli", role: "operator", + scopes: ["operator.read", "operator.write"], + token: authToken, nonce: nonce) { + params["device"] = device + } else { + print("[OpenClaw] no challenge nonce — connecting without device identity") + } + let response = try await sendRequest(method: .connect, params: params) guard response.ok else { @@ -683,7 +737,16 @@ final class OpenClawService: ObservableObject { print("[OpenClaw] Chat state: \(state)") } - case "connect.challenge", "tick", "presence", "health": + case "connect.challenge": + // The nonce is required to sign the device identity; without it a + // remote connection gets no operator.write scope. + if let nonce = payload["nonce"]?.stringValue { + challengeNonce = nonce + challengeWaiter?.resume(returning: nonce) + challengeWaiter = nil + } + + case "tick", "presence", "health": // Ignore these events break diff --git a/SETUP.md b/SETUP.md index 1024225..d512f23 100644 --- a/SETUP.md +++ b/SETUP.md @@ -247,6 +247,21 @@ Once the app is running: - **Auth Token**: Your OpenClaw authentication token 5. Tap **Test Connection** to verify +**If the gateway is on your local network** (not localhost), it will not grant +the app write access on a token alone — `chat.send` fails with +`missing scope: operator.write`. The app presents a signed device identity, and +the gateway holds it as a pairing request until you approve it once, on the +machine running the gateway: + +```bash +openclaw devices list # find the pending request id +openclaw devices approve +``` + +The first connection is expected to fail with "pairing required: device is not +approved yet". Approve it, and the app connects on the next attempt. The app's +key is stored in the Keychain, so this is a one-time step per device. + ### Gemini Live Setup 1. Go to **Settings → AI Backend**