From d2de2c3f0b940b8b0e89359fd1631eb9176d5eab Mon Sep 17 00:00:00 2001 From: Orac Bot Date: Tue, 1 Sep 2026 22:50:32 +1000 Subject: [PATCH 1/2] Connect to an OpenClaw gateway over the LAN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OpenClaw backend could reach a gateway on localhost but not one on another host, which is every connection from the phone. Four changes were needed, each found by elimination against a live gateway (2026.8.1): Protocol 4. The handshake advertised min/max protocol 3; the gateway speaks 4 and rejects the older negotiation. Role and scopes. The connect params carried neither, so the connection was granted nothing and chat.send failed with "missing scope: operator.write" even though the handshake itself succeeded. Device identity. The gateway trusts loopback implicitly, so a token alone is enough there. A connection from another host is granted no write scope on a token by itself and must present a signed Ed25519 identity, which the gateway holds as a pairing request until it is approved once with `openclaw devices approve `. OpenClawDeviceIdentity persists the key in the Keychain so the phone presents the same device on every launch; a fresh key each time would mean a new pairing request each time. Fresh nonce per socket. Every socket gets its own connect.challenge nonce, and the signature covers it. Caching the first one made every reconnect fail with "device nonce mismatch" — the first connection after launch worked and no later one ever did. Info.plist gains NSLocalNetworkUsageDescription: ATS permits cleartext, but reaching a LAN address at all needs that separate consent, and without the string iOS blocks the connection outright rather than prompting. project.pbxproj is regenerated (xcodegen globs OpenVision/), so it also carries incidental churn from a newer xcodegen than generated the committed file. Verified end to end on an iPhone 16 Pro Max: pairing request approved, then a spoken "what is the capital of France" reached the gateway and streamed back "Paris." Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y1rhdYF6XjxjkeqjBM2zGM --- OpenVision.xcodeproj/project.pbxproj | 8 +- OpenVision/Resources/Info.plist | 13 +++ .../OpenClaw/OpenClawDeviceIdentity.swift | 106 ++++++++++++++++++ .../Services/OpenClaw/OpenClawService.swift | 71 +++++++++++- 4 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift diff --git a/OpenVision.xcodeproj/project.pbxproj b/OpenVision.xcodeproj/project.pbxproj index 2255663..60f1895 100644 --- a/OpenVision.xcodeproj/project.pbxproj +++ b/OpenVision.xcodeproj/project.pbxproj @@ -35,6 +35,7 @@ 456B4E46EE405F5C8F45B04D /* Theme.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1ABBF4AC21A442A2C3CF6750 /* Theme.swift */; }; 49DFBFFE52D341E96BCE382E /* AppleFoundationService.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7B97C063C947F53F6E209D1 /* AppleFoundationService.swift */; }; 4DFF630E56769253493B4B57 /* NotificationForegroundPresenter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4831CE885B0320D9CAD71736 /* NotificationForegroundPresenter.swift */; }; + 529CAD92EEAB3567C51769E6 /* OpenClawDeviceIdentity.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E9830333CA08F4DA42A994 /* OpenClawDeviceIdentity.swift */; }; 52E3A16AB6227CDA3D67817D /* AppSettings.swift in Sources */ = {isa = PBXBuildFile; fileRef = B2FD4A9B156D92B468B28864 /* AppSettings.swift */; }; 53B484EF0A2017B9718AC548 /* WebSearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 71ACC88CE5E9E799ED607853 /* WebSearchSettingsView.swift */; }; 53C17BBA774E7D3D4E295584 /* LocalAgent.swift in Sources */ = {isa = PBXBuildFile; fileRef = CEE6F61B22736F87248FD0F5 /* LocalAgent.swift */; }; @@ -186,6 +187,7 @@ BE848862D72446F7BBDD3D89 /* KokoroTTSService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KokoroTTSService.swift; sourceTree = ""; }; BEEDCA2C462B77066AA0CD25 /* VoiceAgentViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceAgentViewModel.swift; sourceTree = ""; }; C05A9A6876CC108B7CEDFDFC /* TextChunkingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TextChunkingTests.swift; sourceTree = ""; }; + C0E9830333CA08F4DA42A994 /* OpenClawDeviceIdentity.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OpenClawDeviceIdentity.swift; sourceTree = ""; }; C1BF0DC63344C51A046E37E0 /* OVExceptionCatcher.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = OVExceptionCatcher.h; sourceTree = ""; }; C1BFC8380B5BDA0778E04EAB /* GemmaSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GemmaSettingsView.swift; sourceTree = ""; }; C27052ECD2FFC79652B5CDB3 /* OnboardingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OnboardingView.swift; sourceTree = ""; }; @@ -463,6 +465,7 @@ 867CD5E802B9354B21C23D0B /* OpenClaw */ = { isa = PBXGroup; children = ( + C0E9830333CA08F4DA42A994 /* OpenClawDeviceIdentity.swift */, 7946C8232CEA5A362056AC2A /* OpenClawProtocol.swift */, C539AB340611B1A6CCC50F17 /* OpenClawService.swift */, ); @@ -702,7 +705,6 @@ }; }; buildConfigurationList = F2E191267E52B7178955EEDB /* Build configuration list for PBXProject "OpenVision" */; - compatibilityVersion = "Xcode 14.0"; developmentRegion = en; hasScannedForEncodings = 0; knownRegions = ( @@ -721,11 +723,12 @@ A8456994C4E280FE9D1FCB59 /* XCLocalSwiftPackageReference "Vendor/kokoro-ios" */, ); preferredProjectObjectVersion = 77; + productRefGroup = 36F672D5AB5BCE7D4C73AA30 /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( - 9418DE699CD106BA6A8A4B7E /* OpenVision */, 60AF4ED566AAE248296380F1 /* OpenVisionTests */, + 9418DE699CD106BA6A8A4B7E /* OpenVision */, ); }; /* End PBXProject section */ @@ -837,6 +840,7 @@ A17626C4B92966423AF06452 /* OpenAIRealtimeService.swift in Sources */, 223474C328BCC3C81ADCB93D /* OpenAIService.swift in Sources */, 800BD44B75446351A267E190 /* OpenAISettingsView.swift in Sources */, + 529CAD92EEAB3567C51769E6 /* OpenClawDeviceIdentity.swift in Sources */, C3B4CF9D8B6EC21957CC7BEF /* OpenClawProtocol.swift in Sources */, 186FD72F369F1962B15EB858 /* OpenClawService.swift in Sources */, 758F66B632E8A1CBDBB40094 /* OpenClawSettingsView.swift in Sources */, diff --git a/OpenVision/Resources/Info.plist b/OpenVision/Resources/Info.plist index 0206bed..7bb65a8 100644 --- a/OpenVision/Resources/Info.plist +++ b/OpenVision/Resources/Info.plist @@ -142,6 +142,19 @@ + + NSLocalNetworkUsageDescription + OpenVision connects to the OpenClaw assistant running on your own computer over your local network, and streams video from your glasses over Wi-Fi. + NSBonjourServices + + _bonjour._tcp + + MWDAT diff --git a/OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift b/OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift new file mode 100644 index 0000000..c937552 --- /dev/null +++ b/OpenVision/Services/OpenClaw/OpenClawDeviceIdentity.swift @@ -0,0 +1,106 @@ +import CryptoKit +import Foundation + +/// Ed25519 device identity for the OpenClaw gateway handshake. +/// +/// The gateway trusts loopback connections implicitly, so a client running on +/// the same machine as the gateway connects with a token alone. A connection +/// from another host — which is every connection from this phone — is granted +/// no write scope on a token by itself: `chat.send` comes back +/// "missing scope: operator.write" even though the handshake succeeded. It has +/// to present a signed device identity, which the gateway then holds as a +/// pairing request until it is approved once with: +/// +/// openclaw devices approve +/// +/// The key is persisted in the Keychain so the phone presents the same device +/// on every launch; a fresh key each time would mean a new pairing request each +/// time, and a list full of stale devices. +enum OpenClawDeviceIdentity { + + private static let service = "ai.openclaw.openvision" + private static let account = "openclaw-device-key" + + private static let key: Curve25519.Signing.PrivateKey = { + if let stored = loadKey() { return stored } + let fresh = Curve25519.Signing.PrivateKey() + storeKey(fresh) + return fresh + }() + + /// Raw 32-byte public key, base64url — the form the gateway expects. + static var publicKeyBase64URL: String { base64URL(key.publicKey.rawRepresentation) } + + /// sha256 of the raw public key, hex. The gateway derives this itself and + /// compares, so it cannot be chosen freely. + static var deviceId: String { + SHA256.hash(data: key.publicKey.rawRepresentation) + .map { String(format: "%02x", $0) } + .joined() + } + + /// The `device` object for the connect params. `nonce` must be the nonce + /// from the server's `connect.challenge` event, and the signature covers + /// the exact field order below — any deviation fails verification. + static func deviceParams( + clientId: String, + clientMode: String, + role: String, + scopes: [String], + token: String, + nonce: String + ) -> [String: Any]? { + let signedAt = Int(Date().timeIntervalSince1970 * 1000) + let payload = [ + "v2", deviceId, clientId, clientMode, role, + scopes.joined(separator: ","), String(signedAt), token, nonce, + ].joined(separator: "|") + + guard let data = payload.data(using: .utf8), + let signature = try? key.signature(for: data) else { return nil } + + return [ + "id": deviceId, + "publicKey": publicKeyBase64URL, + "signature": base64URL(signature), + "signedAt": signedAt, + "nonce": nonce, + ] + } + + // MARK: - Helpers + + private static func base64URL(_ data: Data) -> String { + data.base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: "") + } + + private static func loadKey() -> Curve25519.Signing.PrivateKey? { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + kSecReturnData as String: true, + kSecMatchLimit as String: kSecMatchLimitOne, + ] + var item: CFTypeRef? + guard SecItemCopyMatching(query as CFDictionary, &item) == errSecSuccess, + let data = item as? Data else { return nil } + return try? Curve25519.Signing.PrivateKey(rawRepresentation: data) + } + + private static func storeKey(_ key: Curve25519.Signing.PrivateKey) { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: account, + ] + SecItemDelete(query as CFDictionary) + var attributes = query + attributes[kSecValueData as String] = key.rawRepresentation + attributes[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlock + SecItemAdd(attributes as CFDictionary, nil) + } +} diff --git a/OpenVision/Services/OpenClaw/OpenClawService.swift b/OpenVision/Services/OpenClaw/OpenClawService.swift index 71fefbb..c730d92 100644 --- a/OpenVision/Services/OpenClaw/OpenClawService.swift +++ b/OpenVision/Services/OpenClaw/OpenClawService.swift @@ -85,6 +85,9 @@ final class OpenClawService: ObservableObject { private var requestCounter: Int = 0 private var pendingRequests: [String: CheckedContinuation] = [:] private var receiveTask: Task? + /// Nonce from the server's connect.challenge, needed to sign the device identity. + private var challengeNonce: String? + private var challengeWaiter: CheckedContinuation? // MARK: - Reconnection @@ -221,6 +224,11 @@ final class OpenClawService: ObservableObject { requestCounter = 0 failPendingRequests(error: AIBackendError.notConnected) + // Every socket gets its own connect.challenge nonce. Carrying one over + // from a previous socket makes the gateway reject the handshake with + // "device nonce mismatch", so each attempt must wait for a fresh one. + challengeNonce = nil + guard !Task.isCancelled else { return } do { @@ -358,12 +366,39 @@ final class OpenClawService: ObservableObject { // MARK: - Handshake + /// Wait briefly for the server's connect.challenge nonce. The socket is + /// already receiving by the time the handshake is built, so this normally + /// returns immediately. + private func waitForChallengeNonce() async -> String? { + if let nonce = challengeNonce { return nonce } + return await withTaskGroup(of: String?.self) { group in + group.addTask { [weak self] in + await withCheckedContinuation { (c: CheckedContinuation) in + Task { @MainActor in + if let existing = self?.challengeNonce { c.resume(returning: existing) } + else { self?.challengeWaiter = c } + } + } + } + group.addTask { + try? await Task.sleep(nanoseconds: 3_000_000_000) + return nil + } + let first = await group.next() ?? nil + group.cancelAll() + return first + } + } + /// Send initial connect handshake private func sendHandshake() async throws { // Match xmeta's handshake format exactly - let params: [String: Any] = [ - "minProtocol": 3, - "maxProtocol": 3, + var params: [String: Any] = [ + // Protocol 4: this gateway (OpenClaw 2026.7.1-2) sets + // MIN_CLIENT_PROTOCOL_VERSION = 4 and rejects a 3/3 offer outright with + // PROTOCOL_MISMATCH, so the shipped values cannot connect to it. + "minProtocol": 4, + "maxProtocol": 4, "client": [ "id": "cli", "displayName": "OpenVision", @@ -372,11 +407,30 @@ final class OpenClawService: ObservableObject { "mode": "cli" ], "caps": [String](), // Empty array like xmeta + // Without these the handshake still succeeds, but every chat.send is + // rejected with "missing scope: operator.write" -- the gateway grants + // no write scope to a connection that requests none. + "role": "operator", + "scopes": ["operator.read", "operator.write"], "auth": ["token": authToken], "locale": "en-US", "userAgent": "OpenVision/1.0.0" ] + // A connection from another host is granted no write scope on a token + // alone, so sign a device identity with the challenge nonce. First run + // returns PAIRING_REQUIRED until the device is approved once on the + // gateway host with: openclaw devices approve + if let nonce = await waitForChallengeNonce(), + let device = OpenClawDeviceIdentity.deviceParams( + clientId: "cli", clientMode: "cli", role: "operator", + scopes: ["operator.read", "operator.write"], + token: authToken, nonce: nonce) { + params["device"] = device + } else { + print("[OpenClaw] no challenge nonce — connecting without device identity") + } + let response = try await sendRequest(method: .connect, params: params) guard response.ok else { @@ -683,7 +737,16 @@ final class OpenClawService: ObservableObject { print("[OpenClaw] Chat state: \(state)") } - case "connect.challenge", "tick", "presence", "health": + case "connect.challenge": + // The nonce is required to sign the device identity; without it a + // remote connection gets no operator.write scope. + if let nonce = payload["nonce"]?.stringValue { + challengeNonce = nonce + challengeWaiter?.resume(returning: nonce) + challengeWaiter = nil + } + + case "tick", "presence", "health": // Ignore these events break From ebc56db8f4bd35e79edca6dcdbe2583addf3f696 Mon Sep 17 00:00:00 2001 From: Orac Bot Date: Tue, 1 Sep 2026 22:57:01 +1000 Subject: [PATCH 2/2] Document the LAN gateway pairing step The first connection from a device that is not on the gateway host is expected to fail with "pairing required", which reads like a bug unless you know an approval is waiting on the other machine. SETUP.md now spells out the approve step, and the changelog records the behaviour change. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Y1rhdYF6XjxjkeqjBM2zGM --- CHANGELOG.md | 7 +++++++ SETUP.md | 15 +++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4f03d6..173dfb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to OpenVision will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed +- **OpenClaw backend can now reach a gateway on the local network.** Previously only a gateway on localhost worked: the handshake negotiated protocol 3 (current gateways speak 4), sent no role or scopes, and presented no device identity — so `chat.send` failed with `missing scope: operator.write` even though the connection itself succeeded. The app now signs an Ed25519 device identity (persisted in the Keychain) that the gateway holds as a pairing request; approve it once with `openclaw devices approve `. See SETUP.md +- Reconnecting to an OpenClaw gateway no longer fails with `device nonce mismatch`. Each socket gets its own `connect.challenge` nonce and the signature covers it, so caching the first one meant the initial connection after launch succeeded and every reconnect afterwards failed +- Added `NSLocalNetworkUsageDescription`, without which iOS silently blocks connections to a LAN gateway rather than prompting for consent + ## [2.11.0] - 2026-08-15 ### Added diff --git a/SETUP.md b/SETUP.md index 1024225..d512f23 100644 --- a/SETUP.md +++ b/SETUP.md @@ -247,6 +247,21 @@ Once the app is running: - **Auth Token**: Your OpenClaw authentication token 5. Tap **Test Connection** to verify +**If the gateway is on your local network** (not localhost), it will not grant +the app write access on a token alone — `chat.send` fails with +`missing scope: operator.write`. The app presents a signed device identity, and +the gateway holds it as a pairing request until you approve it once, on the +machine running the gateway: + +```bash +openclaw devices list # find the pending request id +openclaw devices approve +``` + +The first connection is expected to fail with "pairing required: device is not +approved yet". Approve it, and the app connects on the next attempt. The app's +key is stored in the Keychain, so this is a one-time step per device. + ### Gemini Live Setup 1. Go to **Settings → AI Backend**