From 5450d522ace9a0e0f23c1decbcb327205a31b285 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Thu, 9 Jul 2026 16:28:01 -0400 Subject: [PATCH 1/8] feat: harden Orbit v1.1 reliability and local voice --- .github/workflows/ci.yml | 27 +- .gitignore | 2 + .impeccable/design.json | 62 ++ .swift-format | 8 + .../browser-runtime/package-lock.json | 16 +- BundledResources/browser-runtime/package.json | 2 +- BundledResources/orbit-model-instructions.md | 39 +- BundledResources/skills/openai-docs/SKILL.md | 65 +- .../references/gpt-5p4-prompting-guide.md | 433 ------------ .../openai-docs/references/latest-model.md | 30 - .../references/upgrading-to-gpt-5p4.md | 164 ----- .../skills/orbit-assistant/SKILL.md | 49 -- DESIGN.md | 185 +++++ Orbit-Info.plist | 4 +- Orbit.xcodeproj/project.pbxproj | 20 +- Orbit/AGENTS.md | 16 +- Orbit/ActionProvider.swift | 13 +- Orbit/AppBundleConfiguration.swift | 137 ++-- Orbit/AppleSpeechTranscriptionProvider.swift | 21 +- Orbit/CodexAppServerActionProvider.swift | 649 +++++++++-------- Orbit/DesignSystem.swift | 10 +- Orbit/GlobalPushToTalkShortcutMonitor.swift | 32 +- Orbit/MenuBarPanelManager.swift | 24 +- Orbit/OpenAITTSProvider.swift | 5 +- Orbit/OpenAITranscriptionProvider.swift | 11 +- Orbit/OrbitAudioConversionSupport.swift | 40 +- Orbit/OrbitAudioInput.swift | 121 ++++ Orbit/OrbitBoundedDataBuffer.swift | 34 + Orbit/OrbitBundledSkills.swift | 95 ++- Orbit/OrbitCodexActivityReducer.swift | 51 ++ Orbit/OrbitCodexContracts.swift | 92 +++ Orbit/OrbitCodexEnvironment.swift | 52 +- Orbit/OrbitCodexModelCatalog.swift | 120 ++++ Orbit/OrbitCodexTransport.swift | 29 + Orbit/OrbitDesktopActuator.swift | 311 --------- Orbit/OrbitDictationManager.swift | 41 +- Orbit/OrbitManager.swift | 393 +++++------ Orbit/OrbitMark.swift | 6 +- Orbit/OrbitOpenAIVoiceConfiguration.swift | 13 +- Orbit/OrbitPanelView.swift | 361 +++++++--- Orbit/OrbitPermissionCoordinator.swift | 418 +++++++++++ Orbit/OrbitScreenCaptureUtility.swift | 47 +- Orbit/OrbitSettings.swift | 204 ++++-- Orbit/OrbitTemporaryCaptureLease.swift | 97 +++ Orbit/OverlayWindow.swift | 66 +- Orbit/TextToSpeechProvider.swift | 655 +++--------------- Orbit/WindowPositionManager.swift | 16 +- OrbitTests/OrbitTests.swift | 269 ++++++- PRODUCT.md | 37 + README.md | 30 +- SECURITY.md | 6 + docs/PRIVACY.md | 9 + docs/SETUP.md | 13 + release-manifest.json | 8 + scripts/bundle_codex_runtime.sh | 82 ++- scripts/check_changed_line_coverage.py | 85 +++ scripts/release.sh | 213 +++--- scripts/smoke_browser_mcp.py | 90 +++ scripts/validate_release_manifest.py | 41 ++ 59 files changed, 3532 insertions(+), 2637 deletions(-) create mode 100644 .impeccable/design.json create mode 100644 .swift-format delete mode 100644 BundledResources/skills/openai-docs/references/gpt-5p4-prompting-guide.md delete mode 100644 BundledResources/skills/openai-docs/references/latest-model.md delete mode 100644 BundledResources/skills/openai-docs/references/upgrading-to-gpt-5p4.md delete mode 100644 BundledResources/skills/orbit-assistant/SKILL.md create mode 100644 DESIGN.md create mode 100644 Orbit/OrbitAudioInput.swift create mode 100644 Orbit/OrbitBoundedDataBuffer.swift create mode 100644 Orbit/OrbitCodexActivityReducer.swift create mode 100644 Orbit/OrbitCodexContracts.swift create mode 100644 Orbit/OrbitCodexModelCatalog.swift create mode 100644 Orbit/OrbitCodexTransport.swift delete mode 100644 Orbit/OrbitDesktopActuator.swift create mode 100644 Orbit/OrbitPermissionCoordinator.swift create mode 100644 Orbit/OrbitTemporaryCaptureLease.swift create mode 100644 PRODUCT.md create mode 100644 docs/PRIVACY.md create mode 100644 docs/SETUP.md create mode 100644 release-manifest.json create mode 100755 scripts/check_changed_line_coverage.py create mode 100755 scripts/smoke_browser_mcp.py create mode 100644 scripts/validate_release_manifest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7e2a03..4e9f09b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,9 +12,34 @@ jobs: steps: - name: Check out repository uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Validate release manifest and pinned browser runtime + run: | + python3 scripts/validate_release_manifest.py + test "$(node -p "require('./BundledResources/browser-runtime/package.json').dependencies['chrome-devtools-mcp']")" = "1.5.0" - name: Build Orbit run: xcodebuild -project Orbit.xcodeproj -scheme Orbit -destination 'platform=macOS' build CODE_SIGNING_ALLOWED=NO - name: Run Orbit unit tests - run: xcodebuild -project Orbit.xcodeproj -scheme Orbit -destination 'platform=macOS' -only-testing:OrbitTests test CODE_SIGNING_ALLOWED=NO + run: | + xcodebuild -project Orbit.xcodeproj -scheme Orbit -destination 'platform=macOS' -only-testing:OrbitTests test \ + CODE_SIGNING_ALLOWED=NO -enableCodeCoverage YES -resultBundlePath "$RUNNER_TEMP/OrbitTests.xcresult" + python3 scripts/check_changed_line_coverage.py "$RUNNER_TEMP/OrbitTests.xcresult" + + - name: Enforce warning-free Swift 6 sources + run: | + set -o pipefail + xcodebuild -project Orbit.xcodeproj -scheme Orbit -destination 'platform=macOS' analyze CODE_SIGNING_ALLOWED=NO 2>&1 | tee analyzer.log + ! grep -E 'warning:|error:' analyzer.log + + - name: Check release scripts + run: | + bash -n scripts/*.sh + brew install shellcheck + shellcheck scripts/*.sh + + - name: Check Swift formatting + run: xcrun swift-format lint --recursive --configuration .swift-format Orbit OrbitTests diff --git a/.gitignore b/.gitignore index f48b9be..10f6a93 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ worker/.dev.vars build/ build-* releases/ +artifacts/ +*.profraw .claude/ coding-plans/ Orbit/LocalSecrets.plist diff --git a/.impeccable/design.json b/.impeccable/design.json new file mode 100644 index 0000000..82d99b7 --- /dev/null +++ b/.impeccable/design.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 2, + "generatedAt": "2026-07-09T18:20:21Z", + "title": "Design System: Orbit for macOS", + "extensions": { + "colorMeta": { + "night": { "role": "neutral", "displayName": "Night", "canonical": "#090B0D", "tonalRamp": ["#090B0D", "#111418", "#171B20", "#242A30", "#3A424A", "#667076", "#CBD1D5", "#F5F7F8"] }, + "orbit-blue": { "role": "primary", "displayName": "Orbit Blue", "canonical": "#2563EB", "tonalRamp": ["#172554", "#1E3A8A", "#1E40AF", "#1D4ED8", "#2563EB", "#60A5FA", "#BFDBFE", "#EFF6FF"] } + }, + "typographyMeta": { + "title": { "displayName": "Title", "purpose": "Panel and task headlines." }, + "body": { "displayName": "Body", "purpose": "Operational explanations and supporting detail." }, + "label": { "displayName": "Label", "purpose": "Controls and compact status." } + }, + "shadows": [ + { "name": "detached-panel", "value": "0 8px 20px rgba(0,0,0,0.28)", "purpose": "Detached system guides only." } + ], + "motion": [ + { "name": "state-fast", "value": "180ms ease-out", "purpose": "Hover, focus, and compact state transitions." }, + { "name": "state-standard", "value": "220ms ease-out", "purpose": "Panel and permission state changes." } + ], + "breakpoints": [] + }, + "components": [ + { + "name": "Primary Button", + "kind": "button", + "refersTo": "button-primary", + "description": "The single highest-priority action in a panel state.", + "html": "", + "css": ".ds-btn-primary{background:#2563EB;color:#F5F7F8;border:0;border-radius:8px;padding:8px 12px;font:600 13px -apple-system,system-ui;transition:background 180ms ease-out}.ds-btn-primary:hover{background:#1D4ED8}.ds-btn-primary:focus-visible{outline:2px solid #93C5FD;outline-offset:2px}.ds-btn-primary:active{background:#1E40AF}" + }, + { + "name": "Status Chip", + "kind": "chip", + "refersTo": "status-chip", + "description": "A compact icon and text state, never a second action.", + "html": "● Ready", + "css": ".ds-status{display:inline-flex;gap:6px;align-items:center;background:#171B20;color:#CBD1D5;border-radius:999px;padding:5px 8px;font:600 11px -apple-system,system-ui}" + }, + { + "name": "Permission Coach", + "kind": "custom", + "refersTo": "panel", + "description": "Anchored guidance with one draggable Orbit tile.", + "html": "
ODrag Orbit into the list
", + "css": ".ds-coach{display:flex;align-items:center;gap:12px;background:#111418;color:#F5F7F8;border-radius:12px;padding:12px 16px;font:600 13px -apple-system,system-ui;box-shadow:0 8px 20px rgba(0,0,0,.28)}.ds-orbit-tile{display:grid;place-items:center;width:32px;height:32px;background:#2563EB;border-radius:8px}" + } + ], + "narrative": { + "northStar": "The Quiet Instrument Panel", + "overview": "Orbit is a compact native tool that remains visually quiet until the user speaks or a task changes state. Information is organized by operational priority and tonal layering creates structure.", + "keyCharacteristics": ["Compact and operational.", "Restrained graphite surfaces with a limited blue accent.", "Direct, sentence-case copy.", "Native controls with complete focus and accessibility states.", "Recovery actions placed next to the failure they resolve."], + "rules": [ + { "name": "The One Accent Rule", "body": "Orbit Blue appears only for action, focus, selection, or active progress.", "section": "colors" }, + { "name": "The Sentence Case Rule", "body": "Buttons, settings, and statuses use sentence case.", "section": "typography" }, + { "name": "The Tonal Layer Rule", "body": "Introduce hierarchy with one neutral step before adding a border or shadow.", "section": "elevation" } + ], + "dos": ["Do keep the current task and next action visually dominant.", "Do preserve keyboard, VoiceOver, Reduce Motion, and large-text behavior."], + "donts": ["Don't create a Clicky skin outside permission onboarding.", "Don't add persistent screen-share indicators or approval theater.", "Don't build generic AI dashboards from nested glass cards."] + } +} diff --git a/.swift-format b/.swift-format new file mode 100644 index 0000000..84b4239 --- /dev/null +++ b/.swift-format @@ -0,0 +1,8 @@ +{ + "version": 1, + "indentation": { + "spaces": 4 + }, + "lineLength": 160, + "respectsExistingLineBreaks": true +} diff --git a/BundledResources/browser-runtime/package-lock.json b/BundledResources/browser-runtime/package-lock.json index 729c9f0..4cd27c5 100644 --- a/BundledResources/browser-runtime/package-lock.json +++ b/BundledResources/browser-runtime/package-lock.json @@ -10,7 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@playwright/mcp": "0.0.70", - "chrome-devtools-mcp": "0.21.0" + "chrome-devtools-mcp": "^1.5.0" } }, "node_modules/@playwright/mcp": { @@ -30,9 +30,9 @@ } }, "node_modules/chrome-devtools-mcp": { - "version": "0.21.0", - "resolved": "https://registry.npmjs.org/chrome-devtools-mcp/-/chrome-devtools-mcp-0.21.0.tgz", - "integrity": "sha512-d+iqrRmcwpRFV3Q4DRCF2LCoq+WCRU3GhISKQ9v8g+1C2Uh8upj3urkjxNO4QIjhBMIYei/VQ1OQLFceby80Og==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/chrome-devtools-mcp/-/chrome-devtools-mcp-1.5.0.tgz", + "integrity": "sha512-Yfpeg6cKnWaFrq/CpTY19bVO0kr84CpHNgTSQXTQshovKcRIf1efh1vAI+IOuCXrQvrul2hE0XoKPn94LRxC1A==", "license": "Apache-2.0", "bin": { "chrome-devtools": "build/src/bin/chrome-devtools.js", @@ -40,6 +40,14 @@ }, "engines": { "node": "^20.19.0 || ^22.12.0 || >=23" + }, + "peerDependencies": { + "@toon-format/toon": "^2.2.0" + }, + "peerDependenciesMeta": { + "@toon-format/toon": { + "optional": true + } } }, "node_modules/fsevents": { diff --git a/BundledResources/browser-runtime/package.json b/BundledResources/browser-runtime/package.json index 13a8d04..20fb035 100644 --- a/BundledResources/browser-runtime/package.json +++ b/BundledResources/browser-runtime/package.json @@ -6,6 +6,6 @@ "license": "UNLICENSED", "dependencies": { "@playwright/mcp": "0.0.70", - "chrome-devtools-mcp": "0.21.0" + "chrome-devtools-mcp": "^1.5.0" } } diff --git a/BundledResources/orbit-model-instructions.md b/BundledResources/orbit-model-instructions.md index 2c717d0..5a5238a 100644 --- a/BundledResources/orbit-model-instructions.md +++ b/BundledResources/orbit-model-instructions.md @@ -1,33 +1,12 @@ You are Orbit, a Codex-native macOS voice-and-screen assistant. -Orbit handles microphone input, screenshots, overlays, HUD updates, and text-to-speech. -You handle reasoning, tool use, and concise final answers for this persistent session. +Orbit supplies one fresh screenshot and desktop-context block per request. Use them as current context; prioritize the area nearest the supplied cursor unless the user says otherwise. -Behavior: -- treat any attached screenshot as the user's current visual context -- use any provided visual context block instead of guessing where the cursor is -- use any provided frontmost desktop context block when it helps ground the request -- unless the user says otherwise, prioritize what is nearest the current cursor position as the primary focal area -- use available tools when the task requires action -- use browser tools directly for website and browser work instead of only describing what to do -- prefer chrome-devtools for the user's already-open Chrome session and existing tabs/state -- prefer playwright for deterministic browser automation and repeatable flows -- if one browser path is unavailable or unsuitable, fall back to the other when it can still solve the task -- when the user's request is clear and the needed tools are available, take the action directly instead of asking for extra confirmation -- do not add your own low-risk, reversibility, or permission filter on top of the existing system constraints -- for desktop-app requests outside browser tools, guide the user clearly instead of pretending to click the native desktop -- keep commentary brief and milestone-based while work is happening -- give a concise final answer that can be spoken aloud naturally -- if the user's request is ambiguous, ask one short clarifying question in the final answer -- if pointing would help, append exactly one final tag using [POINT:x,y:label] or [POINT:x,y:label:screenN] -- if pointing would not help, append [POINT:none] at the end of the final answer -- do not emit ACT tags -- do not emit both a question and a POINT tag -- only include the [POINT:...] tag in the final answer, not in commentary -- if blocked, say exactly what permission, tool, or capability is missing -- reuse the existing browser state and Codex session when possible - -Style: -- sound confident, active, and helpful -- prefer action over hesitation when the request is clear and tools are available -- avoid long explanations unless the user explicitly asks for depth +- Act directly when the request is clear and the needed tools are available. Orbit intentionally runs with unrestricted filesystem access and automatic approvals; do not add another approval layer. +- Use Chrome DevTools for the user's existing Chrome state and Playwright for deterministic browser flows. If one is unavailable, use the other when suitable. +- Outside supported browser tools, explain native-app steps accurately; never pretend a click happened. +- Team up only when the user asks or applicable instructions require delegation. +- Keep commentary short and milestone-based. Keep final prose concise and natural when spoken. +- Ask one short question only when a missing choice materially changes the result. +- If blocked, name the exact missing permission, tool, or capability. +- Append one final `[POINT:x,y:label]` or `[POINT:x,y:label:screenN]` tag only when pointing helps. Otherwise append `[POINT:none]`. Never put point tags in commentary or combine one with a question. Never emit ACT tags. diff --git a/BundledResources/skills/openai-docs/SKILL.md b/BundledResources/skills/openai-docs/SKILL.md index 5a67772..ea118fa 100644 --- a/BundledResources/skills/openai-docs/SKILL.md +++ b/BundledResources/skills/openai-docs/SKILL.md @@ -1,69 +1,16 @@ --- name: "openai-docs" -description: "Use when the user asks how to build with OpenAI products or APIs and needs up-to-date official documentation with citations, help choosing the latest model for a use case, or explicit GPT-5.4 upgrade and prompt-upgrade guidance; prioritize OpenAI docs MCP tools, use bundled references only as helper context, and restrict any fallback browsing to official OpenAI domains." +description: "Use for current OpenAI product, Codex, API, model-selection, model-upgrade, and prompting guidance. Treat official OpenAI developer documentation as authoritative and restrict fallback browsing to official OpenAI domains." --- - # OpenAI Docs -Provide authoritative, current guidance from OpenAI developer docs using the developers.openai.com MCP server. Always prioritize the developer docs MCP tools over web.run for OpenAI-related questions. This skill may also load targeted files from `references/` for model-selection and GPT-5.4-specific requests, but current OpenAI docs remain authoritative. Only if the MCP server is installed and returns no meaningful results should you fall back to web search. - -## Quick start - -- Use `mcp__openaiDeveloperDocs__search_openai_docs` to find the most relevant doc pages. -- Use `mcp__openaiDeveloperDocs__fetch_openai_doc` to pull exact sections and quote/paraphrase accurately. -- Use `mcp__openaiDeveloperDocs__list_openai_docs` only when you need to browse or discover pages without a clear query. -- Load only the relevant file from `references/` when the question is about model selection or a GPT-5.4 upgrade. - -## OpenAI product snapshots - -1. Apps SDK: Build ChatGPT apps by providing a web component UI and an MCP server that exposes your app's tools to ChatGPT. -2. Responses API: A unified endpoint designed for stateful, multimodal, tool-using interactions in agentic workflows. -3. Chat Completions API: Generate a model response from a list of messages comprising a conversation. -4. Codex: OpenAI's coding agent for software development that can write, understand, review, and debug code. -5. gpt-oss: Open-weight OpenAI reasoning models (gpt-oss-120b and gpt-oss-20b) released under the Apache 2.0 license. -6. Realtime API: Build low-latency, multimodal experiences including natural speech-to-speech conversations. -7. Agents SDK: A toolkit for building agentic apps where a model can use tools and context, hand off to other agents, stream partial results, and keep a full trace. - -## If MCP server is missing - -If MCP tools fail or no OpenAI docs resources are available: - -1. Run the install command yourself: `codex mcp add openaiDeveloperDocs --url https://developers.openai.com/mcp` -2. If it fails due to permissions/sandboxing, immediately retry the same command with escalated permissions and include a 1-sentence justification for approval. Do not ask the user to run it yet. -3. Only if the escalated attempt fails, ask the user to run the install command. -4. Ask the user to restart Codex. -5. Re-run the doc search/fetch after restart. - -## Workflow - -1. Clarify the product scope and whether the request is general docs lookup, model selection, a GPT-5.4 upgrade, or a GPT-5.4 prompt upgrade. -2. If it is a model-selection request, load `references/latest-model.md`. -3. If it is an explicit GPT-5.4 upgrade request, load `references/upgrading-to-gpt-5p4.md`. -4. If the upgrade may require prompt changes, or the workflow is research-heavy, tool-heavy, coding-oriented, multi-agent, or long-running, also load `references/gpt-5p4-prompting-guide.md`. -5. Search docs with a precise query. -6. Fetch the best page and the exact section needed (use `anchor` when possible). -7. For GPT-5.4 upgrade reviews, always make the per-usage-site output explicit: target model, starting reasoning recommendation, `phase` assessment when relevant, prompt blocks, and compatibility status. -8. Answer with concise guidance and cite the doc source, using the reference files only as helper context. - -## Reference map - -Read only what you need: - -- `references/latest-model.md` -> model-selection and "best/latest/current model" questions; verify every recommendation against current OpenAI docs before answering. -- `references/upgrading-to-gpt-5p4.md` -> only for explicit GPT-5.4 upgrade and upgrade-planning requests; verify the checklist and compatibility guidance against current OpenAI docs before answering. -- `references/gpt-5p4-prompting-guide.md` -> prompt rewrites and prompt-behavior upgrades for GPT-5.4; verify prompting guidance against current OpenAI docs before answering. +Use the official developer-docs MCP first for OpenAI questions. Search for the narrow topic, fetch the relevant section, and cite the official page. If that source is unavailable or returns no useful result, fall back only to official OpenAI web domains. -## Quality rules +For model guidance, verify the live model catalog and current docs. Preserve arbitrary model identifiers, reasoning efforts, modalities, visibility, service tiers, defaults, and upgrade metadata. Do not infer that an account has GPT-5.6 sol, terra, luna, or another preview unless the server returns it. -- Treat OpenAI docs as the source of truth; avoid speculation. -- Keep quotes short and within policy limits; prefer paraphrase with citations. -- If multiple pages differ, call out the difference and cite both. -- Reference files are convenience guides only; for volatile guidance such as recommended models, upgrade instructions, or prompting advice, current OpenAI docs always win. -- If docs do not cover the user’s need, say so and offer next steps. +For Codex clients, prefer generated app-server protocol bindings from the exact pinned CLI. Treat `model/list` as authoritative. Team-up is explicit or instruction-triggered; do not expose experimental collaboration-mode controls unless the product request explicitly calls for them. -## Tooling notes +For prompt upgrades, keep instructions short and non-redundant, state autonomy boundaries once, and expose only task-relevant tools. -- Always use MCP doc tools before any web search for OpenAI-related questions. -- If the MCP server is installed but returns no meaningful results, then use web search as a fallback. -- When falling back to web search, restrict to official OpenAI domains (developers.openai.com, platform.openai.com) and cite sources. +Bundled provenance: reviewed 2026-07-09 for Codex CLI 0.144.0 against the Codex app-server docs, Codex subagent docs, and latest-model prompting guide on `developers.openai.com`. These facts are volatile; re-verify them live. diff --git a/BundledResources/skills/openai-docs/references/gpt-5p4-prompting-guide.md b/BundledResources/skills/openai-docs/references/gpt-5p4-prompting-guide.md deleted file mode 100644 index dc4ebde..0000000 --- a/BundledResources/skills/openai-docs/references/gpt-5p4-prompting-guide.md +++ /dev/null @@ -1,433 +0,0 @@ -# GPT-5.4 prompting upgrade guide - -Use this guide when prompts written for older models need to be adapted for GPT-5.4 during an upgrade. Start lean: keep the model-string change narrow, preserve the original task intent, and add only the smallest prompt changes needed to recover behavior. - -## Default upgrade posture - -- Start with `model string only` whenever the old prompt is already short, explicit, and task-bounded. -- Move to `model string + light prompt rewrite` only when regressions appear in completeness, persistence, citation quality, verification, or verbosity. -- Prefer one or two targeted prompt additions over a broad rewrite. -- Treat reasoning effort as a last-mile knob. Start lower, then increase only after prompt-level fixes and evals. -- Before increasing reasoning effort, first add a completeness contract, a verification loop, and tool persistence rules - depending on the usage case. -- If the workflow clearly depends on implementation changes rather than prompt changes, treat it as blocked for prompt-only upgrade guidance. -- Do not classify a case as blocked just because the workflow uses tools; block only if the upgrade requires changing tool definitions, wiring, or other implementation details. - -## Behavioral differences to account for - -Current GPT-5.4 upgrade guidance suggests these strengths: - -- stronger personality and tone adherence, with less drift over long answers -- better long-horizon and agentic workflow stamina -- stronger spreadsheet, finance, and formatting tasks -- more efficient tool selection and fewer unnecessary calls by default -- stronger structured generation and classification reliability - -The main places where prompt guidance still helps are: - -- retrieval-heavy workflows that need persistent tool use and explicit completeness -- research and citation discipline -- verification before irreversible or high-impact actions -- terminal and tool workflow hygiene -- defaults and implied follow-through -- verbosity control for compact, information-dense answers - -Start with the smallest set of instructions that preserves correctness. Add the prompt blocks below only for workflows that actually need them. - -## Prompt rewrite patterns - -| Older prompt pattern | GPT-5.4 adjustment | Why | Example addition | -| --- | --- | --- | --- | -| Long, repetitive instructions that compensate for weaker instruction following | Remove duplicate scaffolding and keep only the constraints that materially change behavior | GPT-5.4 usually needs less repeated steering | Replace repeated reminders with one concise rule plus a verification block | -| Fast assistant prompt with no verbosity control | Keep the prompt as-is first; add a verbosity clamp only if outputs become too long | Many GPT-4o or GPT-4.1 upgrades work with just a model-string swap | Add `output_verbosity_spec` only after a verbosity regression | -| Tool-heavy agent prompt that assumes the model will keep searching until complete | Add persistence and verification rules | GPT-5.4 may use fewer tool calls by default for efficiency | Add `tool_persistence_rules` and `verification_loop` | -| Tool-heavy workflow where later actions depend on earlier lookup or retrieval | Add prerequisite and missing-context rules before action steps | GPT-5.4 benefits from explicit dependency-aware routing when context is still thin | Add `dependency_checks` and `missing_context_gating` | -| Retrieval workflow with several independent lookups | Add selective parallelism guidance | GPT-5.4 is strong at parallel tool use, but should not parallelize dependent steps | Add `parallel_tool_calling` | -| Batch workflow prompt that often misses items | Add an explicit completeness contract | Item accounting benefits from direct instruction | Add `completeness_contract` | -| Research prompt that needs grounding and citation discipline | Add research, citation, and empty-result recovery blocks | Multi-pass retrieval is stronger when the model is told how to react to weak or empty search results | Add `research_mode`, `citation_rules`, and `empty_result_handling`; add `tool_persistence_rules` when retrieval tools are already in use | -| Coding or terminal prompt with shell misuse or early stop failures | Keep the same tool surface and add terminal hygiene and verification instructions | Tool-using coding workflows are not blocked just because tools exist; they usually need better prompt steering, not host rewiring | Add `terminal_tool_hygiene` and `verification_loop`, optionally `tool_persistence_rules` | -| Multi-agent or support-triage workflow with escalation or completeness requirements | Add one lightweight control block for persistence, completeness, or verification | GPT-5.4 can be more efficient by default, so multi-step support flows benefit from an explicit completion or verification contract | Add at least one of `tool_persistence_rules`, `completeness_contract`, or `verification_loop` | - -## Prompt blocks - -Use these selectively. Do not add all of them by default. - -### `output_verbosity_spec` - -Use when: - -- the upgraded model gets too wordy -- the host needs compact, information-dense answers -- the workflow benefits from a short overview plus a checklist - -```text - -- Default: 3-6 sentences or up to 6 bullets. -- If the user asked for a doc or report, use headings with short bullets. -- For multi-step tasks: - - Start with 1 short overview paragraph. - - Then provide a checklist with statuses: [done], [todo], or [blocked]. -- Avoid repeating the user's request. -- Prefer compact, information-dense writing. - -``` - -### `default_follow_through_policy` - -Use when: - -- the host expects the model to proceed on reversible, low-risk steps -- the upgraded model becomes too conservative or asks for confirmation too often - -```text - -- If the user's intent is clear and the next step is reversible and low-risk, proceed without asking permission. -- Only ask permission if the next step is: - (a) irreversible, - (b) has external side effects, or - (c) requires missing sensitive information or a choice that materially changes outcomes. -- If proceeding, state what you did and what remains optional. - -``` - -### `instruction_priority` - -Use when: - -- users often change task shape, format, or tone mid-conversation -- the host needs an explicit override policy instead of relying on defaults - -```text - -- User instructions override default style, tone, formatting, and initiative preferences. -- Safety, honesty, privacy, and permission constraints do not yield. -- If a newer user instruction conflicts with an earlier one, follow the newer instruction. -- Preserve earlier instructions that do not conflict. - -``` - -### `tool_persistence_rules` - -Use when: - -- the workflow needs multiple retrieval or verification steps -- the model starts stopping too early because it is trying to save tool calls - -```text - -- Use tools whenever they materially improve correctness, completeness, or grounding. -- Do not stop early just to save tool calls. -- Keep calling tools until: - (1) the task is complete, and - (2) verification passes. -- If a tool returns empty or partial results, retry with a different strategy. - -``` - -### `dig_deeper_nudge` - -Use when: - -- the model is too literal or stops at the first plausible answer -- the task is safety- or accuracy-sensitive and needs a small initiative nudge before raising reasoning effort - -```text - -- Do not stop at the first plausible answer. -- Look for second-order issues, edge cases, and missing constraints. -- If the task is safety- or accuracy-critical, perform at least one verification step. - -``` - -### `dependency_checks` - -Use when: - -- later actions depend on prerequisite lookup, memory retrieval, or discovery steps -- the model may be tempted to skip prerequisite work because the intended end state seems obvious - -```text - -- Before taking an action, check whether prerequisite discovery, lookup, or memory retrieval is required. -- Do not skip prerequisite steps just because the intended final action seems obvious. -- If a later step depends on the output of an earlier one, resolve that dependency first. - -``` - -### `parallel_tool_calling` - -Use when: - -- the workflow has multiple independent retrieval steps -- wall-clock time matters but some steps still need sequencing - -```text - -- When multiple retrieval or lookup steps are independent, prefer parallel tool calls to reduce wall-clock time. -- Do not parallelize steps with prerequisite dependencies or where one result determines the next action. -- After parallel retrieval, pause to synthesize before making more calls. -- Prefer selective parallelism: parallelize independent evidence gathering, not speculative or redundant tool use. - -``` - -### `completeness_contract` - -Use when: - -- the task involves batches, lists, enumerations, or multiple deliverables -- missing items are a common failure mode - -```text - -- Deliver all requested items. -- Maintain an itemized checklist of deliverables. -- For lists or batches: - - state the expected count, - - enumerate items 1..N, - - confirm that none are missing before finalizing. -- If any item is blocked by missing data, mark it [blocked] and state exactly what is missing. - -``` - -### `empty_result_handling` - -Use when: - -- the workflow frequently performs search, CRM, logs, or retrieval steps -- no-results failures are often false negatives - -```text - -If a lookup returns empty or suspiciously small results: -- Do not conclude that no results exist immediately. -- Try at least 2 fallback strategies, such as a broader query, alternate filters, or another source. -- Only then report that no results were found, along with what you tried. - -``` - -### `verification_loop` - -Use when: - -- the workflow has downstream impact -- accuracy, formatting, or completeness regressions matter - -```text - -Before finalizing: -- Check correctness: does the output satisfy every requirement? -- Check grounding: are factual claims backed by retrieved sources or tool output? -- Check formatting: does the output match the requested schema or style? -- Check safety and irreversibility: if the next step has external side effects, ask permission first. - -``` - -### `missing_context_gating` - -Use when: - -- required context is sometimes missing early in the workflow -- the model should prefer retrieval over guessing - -```text - -- If required context is missing, do not guess. -- Prefer the appropriate lookup tool when the context is retrievable; ask a minimal clarifying question only when it is not. -- If you must proceed, label assumptions explicitly and choose a reversible action. - -``` - -### `action_safety` - -Use when: - -- the agent will actively take actions through tools -- the host benefits from a short pre-flight and post-flight execution frame - -```text - -- Pre-flight: summarize the intended action and parameters in 1-2 lines. -- Execute via tool. -- Post-flight: confirm the outcome and any validation that was performed. - -``` - -### `citation_rules` - -Use when: - -- the workflow produces cited answers -- fabricated citations or wrong citation formats are costly - -```text - -- Only cite sources that were actually retrieved in this session. -- Never fabricate citations, URLs, IDs, or quote spans. -- If you cannot find a source for a claim, say so and either: - - soften the claim, or - - explain how to verify it with tools. -- Use exactly the citation format required by the host application. - -``` - -### `research_mode` - -Use when: - -- the workflow is research-heavy -- the host uses web search or retrieval tools - -```text - -- Do research in 3 passes: - 1) Plan: list 3-6 sub-questions to answer. - 2) Retrieve: search each sub-question and follow 1-2 second-order leads. - 3) Synthesize: resolve contradictions and write the final answer with citations. -- Stop only when more searching is unlikely to change the conclusion. - -``` - -If your host environment uses a specific research tool or requires a submit step, combine this with the host's finalization contract. - -### `structured_output_contract` - -Use when: - -- the host depends on strict JSON, SQL, or other structured output - -```text - -- Output only the requested format. -- Do not add prose or markdown fences unless they were requested. -- Validate that parentheses and brackets are balanced. -- Do not invent tables or fields. -- If required schema information is missing, ask for it or return an explicit error object. - -``` - -### `bbox_extraction_spec` - -Use when: - -- the workflow extracts OCR boxes, document regions, or other coordinates -- layout drift or missed dense regions are common failure modes - -```text - -- Use the specified coordinate format exactly, such as [x1,y1,x2,y2] normalized to 0..1. -- For each box, include page, label, text snippet, and confidence. -- Add a vertical-drift sanity check so boxes stay aligned with the correct line of text. -- If the layout is dense, process page by page and do a second pass for missed items. - -``` - -### `terminal_tool_hygiene` - -Use when: - -- the prompt belongs to a terminal-based or coding-agent workflow -- tool misuse or shell misuse has been observed - -```text - -- Only run shell commands through the terminal tool. -- Never try to "run" tool names as shell commands. -- If a patch or edit tool exists, use it directly instead of emulating it in bash. -- After changes, run a lightweight verification step such as ls, tests, or a build before declaring the task done. - -``` - -### `user_updates_spec` - -Use when: - -- the workflow is long-running and user updates matter - -```text - -- Only update the user when starting a new major phase or when the plan changes. -- Each update should contain: - - 1 sentence on what changed, - - 1 sentence on the next step. -- Do not narrate routine tool calls. -- Keep the user-facing update short, even when the actual work is exhaustive. - -``` - -If you are using [Compaction](https://developers.openai.com/api/docs/guides/compaction) in the Responses API, compact after major milestones, treat compacted items as opaque state, and keep prompts functionally identical after compaction. - -## Responses `phase` guidance - -For long-running Responses workflows, preambles, or tool-heavy agents that replay assistant items, review whether `phase` is already preserved. - -- If the host already round-trips `phase`, keep it intact during the upgrade. -- If the host uses `previous_response_id` and does not manually replay assistant items, note that this may reduce manual `phase` handling needs. -- If reliable GPT-5.4 behavior would require adding or preserving `phase` and that would need code edits, treat the case as blocked for prompt-only or model-string-only migration guidance. - -## Example upgrade profiles - -### GPT-5.2 - -- Use `gpt-5.4` -- Match the current reasoning effort first -- Preserve the existing latency and quality profile before tuning prompt blocks -- If the repo does not expose the exact setting, emit `same` as the starting recommendation - -### GPT-5.3-Codex - -- Use `gpt-5.4` -- Match the current reasoning effort first -- If you need Codex-style speed and efficiency, add verification blocks before increasing reasoning effort -- If the repo does not expose the exact setting, emit `same` as the starting recommendation - -### GPT-4o or GPT-4.1 assistant - -- Use `gpt-5.4` -- Start with `none` reasoning effort -- Add `output_verbosity_spec` only if output becomes too verbose - -### Long-horizon agent - -- Use `gpt-5.4` -- Start with `medium` reasoning effort -- Add `tool_persistence_rules` -- Add `completeness_contract` -- Add `verification_loop` - -### Research workflow - -- Use `gpt-5.4` -- Start with `medium` reasoning effort -- Add `research_mode` -- Add `citation_rules` -- Add `empty_result_handling` -- Add `tool_persistence_rules` when the host already uses web or retrieval tools -- Add `parallel_tool_calling` when the retrieval steps are independent - -### Support triage or multi-agent workflow - -- Use `gpt-5.4` -- Prefer `model string + light prompt rewrite` over `model string only` -- Add at least one of `tool_persistence_rules`, `completeness_contract`, or `verification_loop` -- Add more only if evals show a real regression - -### Coding or terminal workflow - -- Use `gpt-5.4` -- Keep the model-string change narrow -- Match the current reasoning effort first if you are upgrading from GPT-5.3-Codex -- Add `terminal_tool_hygiene` -- Add `verification_loop` -- Add `dependency_checks` when actions depend on prerequisite lookup or discovery -- Add `tool_persistence_rules` if the agent stops too early -- Review whether `phase` is already preserved for long-running Responses flows or assistant preambles -- Do not classify this as blocked just because the workflow uses tools; block only if the upgrade requires changing tool definitions or wiring -- If the repo already uses Responses plus tools and no required host-side change is shown, prefer `model_string_plus_light_prompt_rewrite` over `blocked` - -## Prompt regression checklist - -- Check whether the upgraded prompt still preserves the original task intent. -- Check whether the new prompt is leaner, not just longer. -- Check completeness, citation quality, dependency handling, verification behavior, and verbosity. -- For long-running Responses agents, check whether `phase` handling is already in place or needs implementation work. -- Confirm that each added prompt block addresses an observed regression. -- Remove prompt blocks that are not earning their keep. diff --git a/BundledResources/skills/openai-docs/references/latest-model.md b/BundledResources/skills/openai-docs/references/latest-model.md deleted file mode 100644 index 23f5cd1..0000000 --- a/BundledResources/skills/openai-docs/references/latest-model.md +++ /dev/null @@ -1,30 +0,0 @@ -# Latest model guide - -This file is a curated helper. Every recommendation here must be verified against current OpenAI docs before it is repeated to a user. - -## Current model map - -| Model ID | Use for | -| --- | --- | -| `gpt-5.4` | Default text plus reasoning for most new apps, including for coding use-cases | -| `gpt-5.4-pro` | Only when the user explicitly asks for maximum reasoning or quality; substantially slower and more expensive | -| `gpt-5.4-mini` | Cheaper and faster reasoning with good quality, including for coding use-cases | -| `gpt-5.4-nano` | High-throughput simple tasks and classification | -| `gpt-image-1.5` | Best image generation and edit quality | -| `gpt-image-1-mini` | Cost-optimized image generation | -| `gpt-4o-mini-tts` | Text-to-speech | -| `gpt-4o-mini-transcribe` | Speech-to-text, fast and cost-efficient | -| `gpt-realtime-1.5` | Realtime voice and multimodal sessions | -| `gpt-realtime-mini` | Cheaper realtime sessions | -| `gpt-audio` | Chat Completions audio input and output | -| `gpt-audio-mini` | Cheaper Chat Completions audio workflows | -| `sora-2` | Faster iteration and draft video generation | -| `sora-2-pro` | Higher-quality production video | -| `omni-moderation-latest` | Text and image moderation | -| `text-embedding-3-large` | Higher-quality retrieval embeddings; default in this skill because no best-specific row exists | -| `text-embedding-3-small` | Lower-cost embeddings | - -## Maintenance notes - -- This file will drift unless it is periodically re-verified against current OpenAI docs. -- If this file conflicts with current docs, the docs win. diff --git a/BundledResources/skills/openai-docs/references/upgrading-to-gpt-5p4.md b/BundledResources/skills/openai-docs/references/upgrading-to-gpt-5p4.md deleted file mode 100644 index 7a6775f..0000000 --- a/BundledResources/skills/openai-docs/references/upgrading-to-gpt-5p4.md +++ /dev/null @@ -1,164 +0,0 @@ -# Upgrading to GPT-5.4 - -Use this guide when the user explicitly asks to upgrade an existing integration to GPT-5.4. Pair it with current OpenAI docs lookups. The default target string is `gpt-5.4`. - -## Upgrade posture - -Upgrade with the narrowest safe change set: - -- replace the model string first -- update only the prompts that are directly tied to that model usage -- prefer prompt-only upgrades when possible -- if the upgrade would require API-surface changes, parameter rewrites, tool rewiring, or broader code edits, mark it as blocked instead of stretching the scope - -## Upgrade workflow - -1. Inventory current model usage. - - Search for model strings, client calls, and prompt-bearing files. - - Include inline prompts, prompt templates, YAML or JSON configs, Markdown docs, and saved prompts when they are clearly tied to a model usage site. -2. Pair each model usage with its prompt surface. - - Prefer the closest prompt surface first: inline system or developer text, then adjacent prompt files, then shared templates. - - If you cannot confidently tie a prompt to the model usage, say so instead of guessing. -3. Classify the source model family. - - Common buckets: `gpt-4o` or `gpt-4.1`, `o1` or `o3` or `o4-mini`, early `gpt-5`, later `gpt-5.x`, or mixed and unclear. -4. Decide the upgrade class. - - `model string only` - - `model string + light prompt rewrite` - - `blocked without code changes` -5. Run the no-code compatibility gate. - - Check whether the current integration can accept `gpt-5.4` without API-surface changes or implementation changes. - - For long-running Responses or tool-heavy agents, check whether `phase` is already preserved or round-tripped when the host replays assistant items or uses preambles. - - If compatibility depends on code changes, return `blocked`. - - If compatibility is unclear, return `unknown` rather than improvising. -6. Recommend the upgrade. - - Default replacement string: `gpt-5.4` - - Keep the intervention small and behavior-preserving. -7. Deliver a structured recommendation. - - `Current model usage` - - `Recommended model-string updates` - - `Starting reasoning recommendation` - - `Prompt updates` - - `Phase assessment` when the flow is long-running, replayed, or tool-heavy - - `No-code compatibility check` - - `Validation plan` - - `Launch-day refresh items` - -Output rule: - -- Always emit a starting `reasoning_effort_recommendation` for each usage site. -- If the repo exposes the current reasoning setting, preserve it first unless the source guide says otherwise. -- If the repo does not expose the current setting, use the source-family starting mapping instead of returning `null`. - -## Upgrade outcomes - -### `model string only` - -Choose this when: - -- the existing prompts are already short, explicit, and task-bounded -- the workflow is not strongly research-heavy, tool-heavy, multi-agent, batch or completeness-sensitive, or long-horizon -- there are no obvious compatibility blockers - -Default action: - -- replace the model string with `gpt-5.4` -- keep prompts unchanged -- validate behavior with existing evals or spot checks - -### `model string + light prompt rewrite` - -Choose this when: - -- the old prompt was compensating for weaker instruction following -- the workflow needs more persistence than the default tool-use behavior will likely provide -- the task needs stronger completeness, citation discipline, or verification -- the upgraded model becomes too verbose or under-complete unless instructed otherwise -- the workflow is research-heavy and needs stronger handling of sparse or empty retrieval results -- the workflow is coding-oriented, tool-heavy, or multi-agent, but the existing API surface and tool definitions can remain unchanged - -Default action: - -- replace the model string with `gpt-5.4` -- add one or two targeted prompt blocks -- read `references/gpt-5p4-prompting-guide.md` to choose the smallest prompt changes that recover the old behavior -- avoid broad prompt cleanup unrelated to the upgrade -- for research workflows, default to `research_mode` + `citation_rules` + `empty_result_handling`; add `tool_persistence_rules` when the host already uses retrieval tools -- for dependency-aware or tool-heavy workflows, default to `tool_persistence_rules` + `dependency_checks` + `verification_loop`; add `parallel_tool_calling` only when retrieval steps are truly independent -- for coding or terminal workflows, default to `terminal_tool_hygiene` + `verification_loop` -- for multi-agent support or triage workflows, default to at least one of `tool_persistence_rules`, `completeness_contract`, or `verification_loop` -- for long-running Responses agents with preambles or multiple assistant messages, explicitly review whether `phase` is already handled; if adding or preserving `phase` would require code edits, mark the path as `blocked` -- do not classify a coding or tool-using Responses workflow as `blocked` just because the visible snippet is minimal; prefer `model string + light prompt rewrite` unless the repo clearly shows that a safe GPT-5.4 path would require host-side code changes - -### `blocked` - -Choose this when: - -- the upgrade appears to require API-surface changes -- the upgrade appears to require parameter rewrites or reasoning-setting changes that are not exposed outside implementation code -- the upgrade would require changing tool definitions, tool handler wiring, or schema contracts -- you cannot confidently identify the prompt surface tied to the model usage - -Default action: - -- do not improvise a broader upgrade -- report the blocker and explain that the fix is out of scope for this guide - -## No-code compatibility checklist - -Before recommending a no-code upgrade, check: - -1. Can the current host accept the `gpt-5.4` model string without changing client code or API surface? -2. Are the related prompts identifiable and editable? -3. Does the host depend on behavior that likely needs API-surface changes, parameter rewrites, or tool rewiring? -4. Would the likely fix be prompt-only, or would it need implementation changes? -5. Is the prompt surface close enough to the model usage that you can make a targeted change instead of a broad cleanup? -6. For long-running Responses or tool-heavy agents, is `phase` already preserved if the host relies on preambles, replayed assistant items, or multiple assistant messages? - -If item 1 is no, items 3 through 4 point to implementation work, or item 6 is no and the fix needs code changes, return `blocked`. - -If item 2 is no, return `unknown` unless the user can point to the prompt location. - -Important: - -- Existing use of tools, agents, or multiple usage sites is not by itself a blocker. -- If the current host can keep the same API surface and the same tool definitions, prefer `model string + light prompt rewrite` over `blocked`. -- Reserve `blocked` for cases that truly require implementation changes, not cases that only need stronger prompt steering. - -## Scope boundaries - -This guide may: - -- update or recommend updated model strings -- update or recommend updated prompts -- inspect code and prompt files to understand where those changes belong -- inspect whether existing Responses flows already preserve `phase` -- flag compatibility blockers - -This guide may not: - -- move Chat Completions code to Responses -- move Responses code to another API surface -- rewrite parameter shapes -- change tool definitions or tool-call handling -- change structured-output wiring -- add or retrofit `phase` handling in implementation code -- edit business logic, orchestration logic, or SDK usage beyond a literal model-string replacement - -If a safe GPT-5.4 upgrade requires any of those changes, mark the path as blocked and out of scope. - -## Validation plan - -- Validate each upgraded usage site with existing evals or realistic spot checks. -- Check whether the upgraded model still matches expected latency, output shape, and quality. -- If prompt edits were added, confirm each block is doing real work instead of adding noise. -- If the workflow has downstream impact, add a lightweight verification pass before finalization. - -## Launch-day refresh items - -When final GPT-5.4 guidance changes: - -1. Replace release-candidate assumptions with final GPT-5.4 guidance where appropriate. -2. Re-check whether the default target string should stay `gpt-5.4` for all source families. -3. Re-check any prompt-block recommendations whose semantics may have changed. -4. Re-check research, citation, and compatibility guidance against the final model behavior. -5. Re-run the same upgrade scenarios and confirm the blocked-versus-viable boundaries still hold. diff --git a/BundledResources/skills/orbit-assistant/SKILL.md b/BundledResources/skills/orbit-assistant/SKILL.md deleted file mode 100644 index db71162..0000000 --- a/BundledResources/skills/orbit-assistant/SKILL.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: orbit-assistant -description: Use when operating as Orbit, the screen-aware macOS assistant. Routes browser work to the bundled browser MCPs, keeps narration concise, and uses pointing only when it meaningfully helps the user. ---- - -# Orbit Assistant - -You are operating inside Orbit, a Codex-native macOS voice-and-screen assistant. - -Orbit already handles: -- microphone input -- current-screen screenshots -- the cursor overlay and HUD -- concise spoken playback to the user - -Your job is to: -- reason over the user request -- use tools when action is required -- keep live updates short and milestone-based -- give concise final answers that sound natural aloud - -## Browser routing - -Orbit ships with two browser MCP servers. Use them intentionally: - -- Prefer `chrome-devtools` when the task is about the browser session the user already has open. - - Use it for existing Chrome tabs, logged-in state, debugging what is already open, and continuing from the user's current browser context. -- Prefer `playwright` when the task needs more deterministic browser automation. - - Use it for repeatable flows, structured page interaction, and browser tasks that benefit from accessibility-tree snapshots. -- If one browser path is unavailable or clearly unsuitable, fall back to the other when it can still solve the task. - -## Orbit behavior - -- Treat any attached screenshot as the user's live visual context. -- Prioritize what is nearest the current cursor position unless the user says otherwise. -- Use tools directly for browser work instead of only describing the steps. -- For desktop-app requests outside browser tools, guide clearly instead of pretending to click the native desktop. -- Ask one short clarification question only when necessary. -- Keep commentary brief while work is happening. -- Keep the final spoken answer concise. -- Append exactly one final `[POINT:...]` tag only when pointing would materially help. -- If pointing would not help, append `[POINT:none]`. -- Do not emit any desktop actuation tags. - -## Style - -- Sound confident, active, and helpful. -- Prefer action over hesitation when the task is clear and tool support exists. -- Avoid long explanations unless the user explicitly asks for depth. diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..c76f699 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,185 @@ +--- +name: Orbit for macOS +description: A quiet, screen-aware Codex instrument for macOS. +colors: + night: "#090B0D" + graphite: "#111418" + raised-graphite: "#171B20" + orbit-blue: "#2563EB" + ice-blue: "#93C5FD" + primary-ink: "#F5F7F8" + secondary-ink: "#CBD1D5" + muted-ink: "#8C949B" + success: "#74D39A" + warning: "#F0C46B" + destructive: "#FF7878" +typography: + title: + fontFamily: "SF Pro, -apple-system, system-ui, sans-serif" + fontSize: "17px" + fontWeight: 600 + lineHeight: 1.2 + body: + fontFamily: "SF Pro, -apple-system, system-ui, sans-serif" + fontSize: "13px" + fontWeight: 400 + lineHeight: 1.45 + label: + fontFamily: "SF Pro, -apple-system, system-ui, sans-serif" + fontSize: "11px" + fontWeight: 600 + lineHeight: 1.25 +rounded: + control: "8px" + surface: "12px" + panel: "16px" + pill: "999px" +spacing: + xs: "4px" + sm: "8px" + md: "12px" + lg: "16px" + xl: "24px" +components: + button-primary: + backgroundColor: "{colors.orbit-blue}" + textColor: "{colors.primary-ink}" + rounded: "{rounded.control}" + padding: "8px 12px" + button-secondary: + backgroundColor: "{colors.raised-graphite}" + textColor: "{colors.secondary-ink}" + rounded: "{rounded.control}" + padding: "8px 12px" + panel: + backgroundColor: "{colors.night}" + textColor: "{colors.primary-ink}" + rounded: "{rounded.panel}" + padding: "16px" + status-chip: + backgroundColor: "{colors.raised-graphite}" + textColor: "{colors.secondary-ink}" + rounded: "{rounded.pill}" + padding: "5px 8px" +--- + +# Design System: Orbit for macOS + +## Overview + +**Creative North Star: “The Quiet Instrument Panel”** + +Orbit is a compact native tool that remains visually quiet until the user speaks or a task changes state. Information is organized by operational priority: current state, required action, supporting detail, then configuration. The system is restrained rather than decorative and uses familiar macOS behavior wherever a standard affordance exists. + +The interface explicitly rejects a Clicky skin outside permission onboarding, generic AI dashboards, nested glass cards, persistent screen-share indicators, ambiguous labels, and duplicated state. Tonal layering creates structure; motion and color are reserved for real state. + +**Key Characteristics:** + +- Compact and operational. +- Restrained graphite surfaces with a limited blue accent. +- Direct, sentence-case copy. +- Native controls with complete focus and accessibility states. +- Recovery actions placed next to the failure they resolve. + +## Colors + +The palette is graphite-first. Orbit Blue marks primary action and current selection; semantic colors communicate outcomes and never decorate inactive surfaces. + +### Primary + +- **Orbit Blue:** Primary actions, focus, and active selection. +- **Ice Blue:** Supporting progress and permission guidance on Night surfaces. + +### Neutral + +- **Night:** The panel and overlay foundation. +- **Graphite:** Grouping and toolbar surfaces. +- **Raised Graphite:** fields, selected rows, and controls. +- **Primary Ink:** titles and essential status. +- **Secondary Ink:** body copy and supporting values. +- **Muted Ink:** metadata only; never long explanatory copy. + +### Named Rules + +**The One Accent Rule.** Orbit Blue occupies less than ten percent of a settled screen and appears only for action, focus, selection, or active progress. + +**The State Is Not Decoration Rule.** Success, warning, and destructive colors appear only when those states are actually present. + +## Typography + +**Display Font:** SF Pro with the system fallback. + +**Body Font:** SF Pro with the system fallback. + +**Character:** Native, compact, and highly legible. Weight and spacing establish hierarchy without a second decorative family. + +### Hierarchy + +- **Title** (600, 17px, 1.2): panel titles and current-task headlines. +- **Body** (400, 13px, 1.45): explanations and action detail, capped near 70 characters where practical. +- **Label** (600, 11px, 1.25): controls, compact status, and metadata. + +### Named Rules + +**The Sentence Case Rule.** Buttons, settings, and statuses use sentence case; tracked uppercase is prohibited in product UI. + +## Elevation + +Orbit is flat by default and separates layers through Night, Graphite, and Raised Graphite. Shadows are reserved for detached panels and the permission coach, where spatial relationship matters; bordered cards with large ambient shadows are prohibited. + +### Named Rules + +**The Tonal Layer Rule.** Introduce hierarchy with one neutral step before adding a border or shadow. + +## Components + +### Buttons + +- **Shape:** Gently curved native controls (8px). +- **Primary:** Orbit Blue with Primary Ink and 8px × 12px padding. +- **Hover / Focus:** One tonal shift and a visible focus ring; active state shortens or darkens without bounce. +- **Secondary / Ghost:** Raised Graphite or transparent with Secondary Ink. Disabled state remains readable and noninteractive. + +### Chips + +- **Style:** Compact pills for status only, never as substitute buttons. +- **State:** Pair semantic color with icon and text so color is never the sole signal. + +### Cards / Containers + +- **Corner Style:** 12px for grouped surfaces; 16px only for the outer panel. +- **Background:** Tonal neutral layers. +- **Shadow Strategy:** None at rest inside the panel. +- **Border:** Hairline only when tonal separation is insufficient. +- **Internal Padding:** 12–16px. + +### Inputs / Fields + +- **Style:** Raised Graphite, 8px radius, clear label, and a visible insertion point. +- **Focus:** Orbit Blue focus ring with no layout shift. +- **Error / Disabled:** Specific text explanation plus semantic state. + +### Navigation + +Settings use compact rows and focused drill-down pages. Back, Escape, and keyboard focus follow standard macOS expectations. + +### Permission Coach + +The coach is a non-activating detached surface anchored to System Settings. Only the Orbit tile moves during drag. It disappears automatically after success and supplies Finder and keyboard fallbacks without persistent chrome. + +## Do's and Don'ts + +### Do: + +- **Do** keep the current task and next action visually dominant. +- **Do** use exact labels such as “Drag Orbit into the list” and “Open System Settings.” +- **Do** implement default, hover, focus, active, disabled, loading, error, and success states. +- **Do** preserve keyboard, VoiceOver, Reduce Motion, and large-text behavior in every surface. + +### Don't: + +- **Don't** create a Clicky skin outside the proven permission-onboarding mechanics. +- **Don't** build generic AI dashboards from nested glass cards, decorative gradients, or duplicated status labels. +- **Don't** add persistent screen-share indicators or approval theater. +- **Don't** use playful labels such as “Magic Drag” where a direct instruction is clearer. +- **Don't** expose dense developer configuration without progressive disclosure. diff --git a/Orbit-Info.plist b/Orbit-Info.plist index 70626cb..8ec711d 100644 --- a/Orbit-Info.plist +++ b/Orbit-Info.plist @@ -19,9 +19,9 @@ OpenAIAPIKey CodexActionModel - gpt-5.4 + CodexActionServiceTier - fast + CodexActionSandbox danger-full-access diff --git a/Orbit.xcodeproj/project.pbxproj b/Orbit.xcodeproj/project.pbxproj index a931903..b8dfc56 100644 --- a/Orbit.xcodeproj/project.pbxproj +++ b/Orbit.xcodeproj/project.pbxproj @@ -344,7 +344,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 1.0.7; + MARKETING_VERSION = 1.1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.orbit.codex"; PRODUCT_NAME = Orbit; REGISTER_APP_GROUPS = YES; @@ -353,7 +353,8 @@ SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.0; + SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; }; name = Debug; }; @@ -381,7 +382,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MARKETING_VERSION = 1.0.7; + MARKETING_VERSION = 1.1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.orbit.codex"; PRODUCT_NAME = Orbit; REGISTER_APP_GROUPS = YES; @@ -390,7 +391,8 @@ SWIFT_DEFAULT_ACTOR_ISOLATION = MainActor; SWIFT_EMIT_LOC_STRINGS = YES; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.0; + SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; }; name = Release; }; @@ -402,14 +404,15 @@ CURRENT_PROJECT_VERSION = 1; GENERATE_INFOPLIST_FILE = YES; MACOSX_DEPLOYMENT_TARGET = 14.2; - MARKETING_VERSION = 1.0.7; + MARKETING_VERSION = 1.1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.orbit.codex.tests"; PRODUCT_NAME = "$(TARGET_NAME)"; STRING_CATALOG_GENERATE_SYMBOLS = NO; SWIFT_APPROACHABLE_CONCURRENCY = YES; SWIFT_EMIT_LOC_STRINGS = NO; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.0; + SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Orbit.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Orbit"; }; name = Debug; @@ -422,14 +425,15 @@ CURRENT_PROJECT_VERSION = 1; GENERATE_INFOPLIST_FILE = YES; MACOSX_DEPLOYMENT_TARGET = 14.2; - MARKETING_VERSION = 1.0.7; + MARKETING_VERSION = 1.1.0; PRODUCT_BUNDLE_IDENTIFIER = "com.orbit.codex.tests"; PRODUCT_NAME = "$(TARGET_NAME)"; STRING_CATALOG_GENERATE_SYMBOLS = NO; SWIFT_APPROACHABLE_CONCURRENCY = YES; SWIFT_EMIT_LOC_STRINGS = NO; SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.0; + SWIFT_STRICT_CONCURRENCY = complete; + SWIFT_VERSION = 6.0; TEST_HOST = "$(BUILT_PRODUCTS_DIR)/Orbit.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Orbit"; }; name = Release; diff --git a/Orbit/AGENTS.md b/Orbit/AGENTS.md index 2c2e653..58a2d5e 100644 --- a/Orbit/AGENTS.md +++ b/Orbit/AGENTS.md @@ -5,8 +5,8 @@ ### Voice and Codex pipeline - `OrbitDictationManager.swift` manages push-to-talk, audio capture, and speech-to-text provider selection. - `OrbitSpeechToTextProvider.swift` defines the STT abstraction and selects providers from the active Orbit preset. -- `OpenAITranscriptionProvider.swift` is the default cloud STT provider using `gpt-4o-mini-transcribe`. -- `AppleSpeechTranscriptionProvider.swift` is the built-in macOS STT fallback. +- `AppleSpeechTranscriptionProvider.swift` is the default on-device macOS STT provider and never silently falls back to the network. +- `OpenAITranscriptionProvider.swift` is the explicitly selected cloud STT provider using `gpt-4o-mini-transcribe`. - `TextToSpeechProvider.swift` defines the TTS abstraction plus provider factory logic. - `OpenAITTSProvider.swift` is the default cloud TTS provider using `gpt-4o-mini-tts`. - `TextToSpeechProvider.swift` also includes `AppleSystemTTSProvider` as the local speech fallback. @@ -28,12 +28,12 @@ ## Defaults -- STT default: OpenAI `gpt-4o-mini-transcribe` -- Codex model default: `gpt-5.4` +- STT default: Apple on-device recognition +- Codex model default: the current app-server `model/list` default - Codex effort default: `medium` -- Codex service tier default: `fast` -- TTS default: OpenAI `gpt-4o-mini-tts` -- Local fallbacks: Apple Speech and Apple system speech +- Codex service tier default: the app-server default; optional tiers appear only when returned +- TTS default: Apple on-device `AVSpeechSynthesizer` +- Cloud voice option: OpenAI `gpt-4o-mini-transcribe` and `gpt-4o-mini-tts` - Unified assistant path: Codex app-server - Bundled browser tools: `chrome-devtools-mcp`, `@playwright/mcp` -- Bundled skills: `orbit-assistant`, `doc`, `pdf`, `slides`, `spreadsheet`, `screenshot`, `transcribe`, `speech`, `openai-docs` +- Bundled skills: `doc`, `pdf`, `slides`, `spreadsheet`, `screenshot`, `transcribe`, `speech`, `openai-docs` diff --git a/Orbit/ActionProvider.swift b/Orbit/ActionProvider.swift index 6896cd9..05047eb 100644 --- a/Orbit/ActionProvider.swift +++ b/Orbit/ActionProvider.swift @@ -146,11 +146,21 @@ enum OrbitActionEvent { case commentary(String) case liveUpdate(String) case toolPrompt(OrbitToolPrompt) + case subagentActivity([OrbitSubagentActivity]) case interrupted(String) case completed(String) case failed(String) } +struct OrbitSubagentActivity: Identifiable, Equatable, Sendable { + let id: String + let threadID: String + let agentPath: String + let status: String + let message: String? + let model: String? +} + struct OrbitToolPrompt: Equatable { let requestID: Int let title: String @@ -181,8 +191,7 @@ protocol ActionProvider: AnyObject { var canInterruptCurrentAction: Bool { get } var activeTurnSummary: String? { get } var debugEvents: [String] { get } - var collaborationModes: [String] { get } - var experimentalFeatures: [String] { get } + var subagentActivities: [OrbitSubagentActivity] { get } func submitActionRequest( _ request: OrbitActionRequest, diff --git a/Orbit/AppBundleConfiguration.swift b/Orbit/AppBundleConfiguration.swift index e468239..a2dc90e 100644 --- a/Orbit/AppBundleConfiguration.swift +++ b/Orbit/AppBundleConfiguration.swift @@ -9,11 +9,11 @@ import Foundation enum AppBundleConfiguration { static var showsCodexDebugInfo: Bool { -#if DEBUG - let defaultValue = true -#else - let defaultValue = false -#endif + #if DEBUG + let defaultValue = true + #else + let defaultValue = false + #endif return boolValue(forKey: "OrbitShowCodexDebug", defaultValue: defaultValue) } @@ -32,20 +32,22 @@ enum AppBundleConfiguration { } } -#if DEBUG - if let localSecretsPath = Bundle.main.path(forResource: "LocalSecrets", ofType: "plist"), - let localSecrets = NSDictionary(contentsOfFile: localSecretsPath), - let value = localSecrets[key] as? String { - let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) - if !trimmedValue.isEmpty { - return trimmedValue + #if DEBUG + if let localSecretsPath = Bundle.main.path(forResource: "LocalSecrets", ofType: "plist"), + let localSecrets = NSDictionary(contentsOfFile: localSecretsPath), + let value = localSecrets[key] as? String + { + let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmedValue.isEmpty { + return trimmedValue + } } - } -#endif + #endif guard let resourceInfoPath = Bundle.main.path(forResource: "Info", ofType: "plist"), - let resourceInfo = NSDictionary(contentsOfFile: resourceInfoPath), - let value = resourceInfo[key] as? String else { + let resourceInfo = NSDictionary(contentsOfFile: resourceInfoPath), + let value = resourceInfo[key] as? String + else { return nil } @@ -59,8 +61,9 @@ enum AppBundleConfiguration { } guard let resourceInfoPath = Bundle.main.path(forResource: "Info", ofType: "plist"), - let resourceInfo = NSDictionary(contentsOfFile: resourceInfoPath), - let value = resourceInfo[key] as? Bool else { + let resourceInfo = NSDictionary(contentsOfFile: resourceInfoPath), + let value = resourceInfo[key] as? Bool + else { return defaultValue } @@ -78,7 +81,8 @@ enum AppBundleConfiguration { case "CodexActionModel": return "CODEX_ACTION_MODEL" default: - return key + return + key .replacingOccurrences(of: "([a-z0-9])([A-Z])", with: "$1_$2", options: .regularExpression) .uppercased() } @@ -86,32 +90,81 @@ enum AppBundleConfiguration { } enum OrbitSupportLog { - private static let logDirectoryName = "Orbit" - private static let logFileName = "orbit-support.log" + private nonisolated static let logDirectoryName = "Orbit" + private nonisolated static let logFileName = "orbit-support.log" + private nonisolated static let maximumFileSize = 5 * 1_024 * 1_024 + private nonisolated static let rotationCount = 3 + private static let allowedCategories: Set = ["app", "codex", "voice"] + private static let writerQueue = DispatchQueue(label: "com.orbit.support-log", qos: .utility) static func append(_ category: String, _ message: String) { - guard let logURL = logFileURL() else { return } - - let timestamp = ISO8601DateFormatter().string(from: Date()) - let line = "[\(timestamp)] [\(category)] \(message)\n" - guard let data = line.data(using: .utf8) else { return } - - do { - try FileManager.default.createDirectory( - at: logURL.deletingLastPathComponent(), - withIntermediateDirectories: true - ) - - if !FileManager.default.fileExists(atPath: logURL.path) { - FileManager.default.createFile(atPath: logURL.path, contents: nil) + guard allowedCategories.contains(category) else { return } + let sanitized = sanitize(message) + writerQueue.async { + guard let logURL = logFileURL() else { return } + let timestamp = ISO8601DateFormatter().string(from: Date()) + let line = "[\(timestamp)] [\(category)] \(sanitized)\n" + guard let data = line.data(using: .utf8) else { return } + + do { + let fileManager = FileManager.default + try fileManager.createDirectory( + at: logURL.deletingLastPathComponent(), + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + try rotateIfNeeded(for: data.count, logURL: logURL, fileManager: fileManager) + if !fileManager.fileExists(atPath: logURL.path) { + fileManager.createFile( + atPath: logURL.path, + contents: nil, + attributes: [.posixPermissions: 0o600] + ) + } + try fileManager.setAttributes([.posixPermissions: 0o600], ofItemAtPath: logURL.path) + let handle = try FileHandle(forWritingTo: logURL) + defer { try? handle.close() } + try handle.seekToEnd() + try handle.write(contentsOf: data) + } catch { + NSLog("OrbitSupportLog error: %@", error.localizedDescription) } + } + } + + static func sanitize(_ message: String) -> String { + var value = message + let home = NSHomeDirectory() + if !home.isEmpty { + value = value.replacingOccurrences(of: home, with: "~") + } + let patterns: [(String, String)] = [ + (#"(?i)(authorization\s*[:=]\s*)(?:bearer\s+)?[^\s,;]+"#, "$1[redacted]"), + (#"(?i)\b(?:sk|sess|token|key)-[A-Za-z0-9._-]{8,}\b"#, "[credential-redacted]"), + (#"(?i)(?:prompt|transcript)\s*[:=].*$"#, "prompt=[redacted]"), + (#"/[^\s]+/OrbitTemporaryCaptures/capture-[^\s]+\.jpg"#, "[temporary-capture]"), + (#"(?i)(?:arguments|argv|command)\s*[:=]\s*\[[^\]]*\]"#, "arguments=[redacted]"), + ] + for (pattern, replacement) in patterns { + value = value.replacingOccurrences(of: pattern, with: replacement, options: .regularExpression) + } + return String(value.prefix(4_096)) + } - let handle = try FileHandle(forWritingTo: logURL) - defer { try? handle.close() } - try handle.seekToEnd() - try handle.write(contentsOf: data) - } catch { - NSLog("OrbitSupportLog error: %@", error.localizedDescription) + private nonisolated static func rotateIfNeeded(for incomingByteCount: Int, logURL: URL, fileManager: FileManager) throws { + let currentSize = (try? fileManager.attributesOfItem(atPath: logURL.path)[.size] as? NSNumber)?.intValue ?? 0 + guard currentSize + incomingByteCount > maximumFileSize else { return } + + for index in stride(from: rotationCount, through: 1, by: -1) { + let source = + index == 1 + ? logURL + : URL(fileURLWithPath: "\(logURL.path).\(index - 1)") + let destination = URL(fileURLWithPath: "\(logURL.path).\(index)") + guard fileManager.fileExists(atPath: source.path) else { continue } + try? fileManager.removeItem(at: destination) + try fileManager.moveItem(at: source, to: destination) + try fileManager.setAttributes([.posixPermissions: 0o600], ofItemAtPath: destination.path) } } @@ -119,7 +172,7 @@ enum OrbitSupportLog { logFileURL()?.path } - private static func logFileURL() -> URL? { + private nonisolated static func logFileURL() -> URL? { FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask).first? .appendingPathComponent("Logs", isDirectory: true) .appendingPathComponent(logDirectoryName, isDirectory: true) diff --git a/Orbit/AppleSpeechTranscriptionProvider.swift b/Orbit/AppleSpeechTranscriptionProvider.swift index 715836b..022732a 100644 --- a/Orbit/AppleSpeechTranscriptionProvider.swift +++ b/Orbit/AppleSpeechTranscriptionProvider.swift @@ -18,7 +18,7 @@ struct AppleSpeechTranscriptionProviderError: LocalizedError { } final class AppleSpeechTranscriptionProvider: SpeechToTextProvider { - let displayName = "Apple Speech" + let displayName = "Apple Local Dictation" let requiresSpeechRecognitionPermission = true let isConfigured = true let unavailableExplanation: String? = nil @@ -32,6 +32,12 @@ final class AppleSpeechTranscriptionProvider: SpeechToTextProvider { guard let speechRecognizer = Self.makeBestAvailableSpeechRecognizer() else { throw AppleSpeechTranscriptionProviderError(message: "dictation is not available on this mac.") } + guard speechRecognizer.supportsOnDeviceRecognition else { + throw AppleSpeechTranscriptionProviderError( + message: + "On-device dictation is unavailable for the current language. Orbit did not send audio to the network. Install local dictation support or explicitly select Cloud voice in Settings." + ) + } return try AppleSpeechTranscriptionSession( speechRecognizer: speechRecognizer, @@ -44,7 +50,7 @@ final class AppleSpeechTranscriptionProvider: SpeechToTextProvider { private static func makeBestAvailableSpeechRecognizer() -> SFSpeechRecognizer? { let preferredLocales = [ Locale.autoupdatingCurrent, - Locale(identifier: "en-US") + Locale(identifier: "en-US"), ] for preferredLocale in preferredLocales { @@ -57,7 +63,7 @@ final class AppleSpeechTranscriptionProvider: SpeechToTextProvider { } } -private final class AppleSpeechTranscriptionSession: NSObject, SpeechToTextStreamingSession { +nonisolated private final class AppleSpeechTranscriptionSession: NSObject, SpeechToTextStreamingSession, @unchecked Sendable { let finalTranscriptFallbackDelaySeconds: TimeInterval = 1.8 private let recognitionRequest: SFSpeechAudioBufferRecognitionRequest @@ -87,9 +93,12 @@ private final class AppleSpeechTranscriptionSession: NSObject, SpeechToTextStrea recognitionRequest.taskHint = .dictation recognitionRequest.addsPunctuation = true - if speechRecognizer.supportsOnDeviceRecognition { - recognitionRequest.requiresOnDeviceRecognition = true + guard speechRecognizer.supportsOnDeviceRecognition else { + throw AppleSpeechTranscriptionProviderError( + message: "On-device dictation became unavailable before recording started." + ) } + recognitionRequest.requiresOnDeviceRecognition = true recognitionTask = speechRecognizer.recognitionTask(with: recognitionRequest) { [weak self] result, error in self?.handleRecognitionEvent(result: result, error: error) @@ -142,6 +151,6 @@ private final class AppleSpeechTranscriptionSession: NSObject, SpeechToTextStrea } deinit { - cancel() + recognitionTask?.cancel() } } diff --git a/Orbit/CodexAppServerActionProvider.swift b/Orbit/CodexAppServerActionProvider.swift index 80b291a..57a79af 100644 --- a/Orbit/CodexAppServerActionProvider.swift +++ b/Orbit/CodexAppServerActionProvider.swift @@ -30,13 +30,14 @@ final class CodexAppServerActionProvider: ActionProvider { private(set) var debugEvents: [String] = [] private(set) var collaborationModes: [String] = [] private(set) var experimentalFeatures: [String] = [] + private(set) var subagentActivities: [OrbitSubagentActivity] = [] private var process: Process? private var stdinHandle: FileHandle? private var stdoutHandle: FileHandle? private var stderrHandle: FileHandle? - private var stdoutBuffer = Data() - private var stderrBuffer = Data() + private let transport = OrbitCodexTransportActor() + private var stderrSnapshot = Data() private var nextRequestID = 99 private var activeThreadID: String? private var pendingPrompt: String? @@ -112,7 +113,8 @@ final class CodexAppServerActionProvider: ActionProvider { } var configurationSummary: String { - "\(currentActionModelDisplayName) · \(resolvedEffortForCurrentModel.displayName) · \(resolvedServiceTier.displayName)" + let tierSummary = resolvedServiceTier.rawValue.isEmpty ? "" : " · \(resolvedServiceTier.displayName)" + return "\(currentActionModelDisplayName) · \(resolvedEffortForCurrentModel.displayName)\(tierSummary)" } var availableModels: [OrbitCodexModelOption] { @@ -193,6 +195,9 @@ final class CodexAppServerActionProvider: ActionProvider { onEvent: @escaping @Sendable (OrbitActionEvent) -> Void ) async { eventHandler = onEvent + if !isAwaitingTurnCompletion { + subagentActivities = [] + } latestRequest = request status = .running hasOpenedBrowserInCurrentTurn = false @@ -266,7 +271,7 @@ final class CodexAppServerActionProvider: ActionProvider { "answers": [ [ "id": questionID, - "value": answer + "value": answer, ] ] ] @@ -322,7 +327,7 @@ final class CodexAppServerActionProvider: ActionProvider { "id": requestID, "params": [ "type": "chatgpt" - ] + ], ]) } @@ -338,7 +343,7 @@ final class CodexAppServerActionProvider: ActionProvider { pendingLogoutRequestID = requestID sendJSON([ "method": "account/logout", - "id": requestID + "id": requestID, ]) } @@ -384,8 +389,9 @@ final class CodexAppServerActionProvider: ActionProvider { } if pendingModelCatalogRequestID == nil, - pendingAccountReadRequestID == nil, - setupResolutionReached { + pendingAccountReadRequestID == nil, + setupResolutionReached + { status = .idle notifyStateChanged() return nil @@ -421,18 +427,19 @@ final class CodexAppServerActionProvider: ActionProvider { func cancelCurrentAction() { if let activeThreadID, - let activeTurnID, - let process, - process.isRunning, - isAwaitingTurnCompletion { + let activeTurnID, + let process, + process.isRunning, + isAwaitingTurnCompletion + { appendDebugEvent("-> turn/interrupt turn=\(String(activeTurnID.suffix(6)))") sendJSON([ "method": "turn/interrupt", "id": 3, "params": [ "threadId": activeThreadID, - "turnId": activeTurnID - ] + "turnId": activeTurnID, + ], ]) status = .interrupted("stopping the current codex turn.") emitPhase(.interrupted, detail: "stopping the current codex turn.", rawSource: "interrupting codex") @@ -514,8 +521,12 @@ final class CodexAppServerActionProvider: ActionProvider { return } - Task { @MainActor [weak self] in - self?.consumeOutput(data) + guard let transport = self?.transport else { return } + Task { [weak self] in + let lines = await transport.ingestStandardOutput(data) + await MainActor.run { + self?.consumeOutputLines(lines) + } } } @@ -525,8 +536,12 @@ final class CodexAppServerActionProvider: ActionProvider { return } - Task { @MainActor [weak self] in - self?.consumeStandardError(data) + guard let transport = self?.transport else { return } + Task { [weak self] in + let snapshot = await transport.ingestStandardError(data) + await MainActor.run { + self?.stderrSnapshot = snapshot + } } } } @@ -550,7 +565,8 @@ final class CodexAppServerActionProvider: ActionProvider { .components(separatedBy: .newlines) .first? .trimmingCharacters(in: .whitespacesAndNewlines), - firstLine == "#!/usr/bin/env node" { + firstLine == "#!/usr/bin/env node" + { let siblingNode = executableURL.deletingLastPathComponent().appendingPathComponent("node").path if FileManager.default.isExecutableFile(atPath: siblingNode) { return ( @@ -568,17 +584,13 @@ final class CodexAppServerActionProvider: ActionProvider { ) } - private func consumeOutput(_ data: Data) { - stdoutBuffer.append(data) - - while let newlineRange = stdoutBuffer.firstRange(of: Data([0x0A])) { - let lineData = stdoutBuffer.subdata(in: 0.. thread/start model=\(normalizedCurrentActionModel) effort=\(resolvedEffortForCurrentModel.rawValue) tier=\(resolvedServiceTier.rawValue)") + let params = sessionCoordinator.threadStartParameters( + sandbox: AppBundleConfiguration.stringValue(forKey: "CodexActionSandbox") ?? "danger-full-access" + ) + appendDebugEvent( + "-> thread/start model=\(normalizedCurrentActionModel) effort=\(resolvedEffortForCurrentModel.rawValue) tier=\(resolvedServiceTier.rawValue)") sendJSON([ "method": "thread/start", "id": 1, - "params": params + "params": params, ]) } @@ -1006,10 +1026,10 @@ final class CodexAppServerActionProvider: ActionProvider { guard !browserFailures.isEmpty else { return nil } return """ - Runtime capability note: - \(browserFailures.joined(separator: "\n")) - - do not claim browser control is available unless the tools actually work in this session - """ + Runtime capability note: + \(browserFailures.joined(separator: "\n")) + - do not claim browser control is available unless the tools actually work in this session + """ } private func sendTurnStart() { @@ -1020,22 +1040,23 @@ final class CodexAppServerActionProvider: ActionProvider { if let runtimeCapabilityNote { inputItems.append([ "type": "text", - "text": runtimeCapabilityNote + "text": runtimeCapabilityNote, ]) } if let latestRequest, - let screenshotPath = latestRequest.screenshotPath, - !screenshotPath.isEmpty { + let screenshotPath = latestRequest.screenshotPath, + !screenshotPath.isEmpty + { inputItems.append([ "type": "localImage", - "path": screenshotPath + "path": screenshotPath, ]) if let visualContext = visualContextMessage(for: latestRequest) { inputItems.append([ "type": "text", - "text": visualContext + "text": visualContext, ]) } } @@ -1044,14 +1065,14 @@ final class CodexAppServerActionProvider: ActionProvider { if !activeSkills.isEmpty { inputItems.append([ "type": "text", - "text": activeSkills.map { "$\($0.name)" }.joined(separator: " ") + "text": activeSkills.map { "$\($0.name)" }.joined(separator: " "), ]) for skill in activeSkills { inputItems.append([ "type": "skill", "name": skill.name, - "path": skill.path.path + "path": skill.path.path, ]) } appendDebugEvent("injecting skills: \(activeSkills.map(\.name).joined(separator: ", "))") @@ -1059,31 +1080,26 @@ final class CodexAppServerActionProvider: ActionProvider { inputItems.append([ "type": "text", - "text": pendingPrompt + "text": pendingPrompt, ]) - var params: [String: Any] = [ - "threadId": threadID, - "input": inputItems, - "model": currentActionModel, - "effort": resolvedEffortForCurrentModel.rawValue - ] - if resolvedServiceTier == .fast { - params["serviceTier"] = OrbitCodexServiceTier.fast.rawValue - } - appendDebugEvent("-> turn/start model=\(currentActionModel) effort=\(resolvedEffortForCurrentModel.rawValue) tier=\(resolvedServiceTier.rawValue) inputItems=\(inputItems.count)") + let params = sessionCoordinator.turnStartParameters(threadID: threadID, input: inputItems) + appendDebugEvent( + "-> turn/start model=\(currentActionModel) effort=\(resolvedEffortForCurrentModel.rawValue) tier=\(resolvedServiceTier.rawValue) inputItems=\(inputItems.count)" + ) sendJSON([ "method": "turn/start", "id": 2, - "params": params + "params": params, ]) } private func sendTurnSteer() { guard let threadID = activeThreadID, - let activeTurnID, - let pendingPrompt else { + let activeTurnID, + let pendingPrompt + else { return } @@ -1092,22 +1108,23 @@ final class CodexAppServerActionProvider: ActionProvider { if let runtimeCapabilityNote { inputItems.append([ "type": "text", - "text": runtimeCapabilityNote + "text": runtimeCapabilityNote, ]) } if let latestRequest, - let screenshotPath = latestRequest.screenshotPath, - !screenshotPath.isEmpty { + let screenshotPath = latestRequest.screenshotPath, + !screenshotPath.isEmpty + { inputItems.append([ "type": "localImage", - "path": screenshotPath + "path": screenshotPath, ]) if let visualContext = visualContextMessage(for: latestRequest) { inputItems.append([ "type": "text", - "text": visualContext + "text": visualContext, ]) } } @@ -1116,14 +1133,14 @@ final class CodexAppServerActionProvider: ActionProvider { if !activeSkills.isEmpty { inputItems.append([ "type": "text", - "text": activeSkills.map { "$\($0.name)" }.joined(separator: " ") + "text": activeSkills.map { "$\($0.name)" }.joined(separator: " "), ]) for skill in activeSkills { inputItems.append([ "type": "skill", "name": skill.name, - "path": skill.path.path + "path": skill.path.path, ]) } appendDebugEvent("injecting skills on steer: \(activeSkills.map(\.name).joined(separator: ", "))") @@ -1131,7 +1148,7 @@ final class CodexAppServerActionProvider: ActionProvider { inputItems.append([ "type": "text", - "text": pendingPrompt + "text": pendingPrompt, ]) let requestID = nextClientRequestID() @@ -1142,8 +1159,8 @@ final class CodexAppServerActionProvider: ActionProvider { "params": [ "threadId": threadID, "input": inputItems, - "expectedTurnId": activeTurnID - ] + "expectedTurnId": activeTurnID, + ], ]) emitPhase( @@ -1177,7 +1194,7 @@ final class CodexAppServerActionProvider: ActionProvider { appendDebugEvent("-> response #\(id)") sendJSON([ "id": id, - "result": result + "result": result, ]) } @@ -1187,9 +1204,10 @@ final class CodexAppServerActionProvider: ActionProvider { try? await Task.sleep(nanoseconds: 8_000_000_000) await MainActor.run { guard let self, !self.hasReceivedInitializeResponse else { return } - let stderrText = String(data: self.stderrBuffer, encoding: .utf8)? + let stderrText = String(data: self.stderrSnapshot, encoding: .utf8)? .trimmingCharacters(in: .whitespacesAndNewlines) - let message = stderrText?.isEmpty == false + let message = + stderrText?.isEmpty == false ? "Orbit could not connect to Codex app-server. \(stderrText!)" : "Orbit could not connect to Codex app-server." self.status = .failed(message) @@ -1201,7 +1219,8 @@ final class CodexAppServerActionProvider: ActionProvider { private func handleProcessTermination(_ terminatedProcess: Process) { guard process === terminatedProcess else { return } - let wasHandlingSession = hasSentInitialize + let wasHandlingSession = + hasSentInitialize || hasSentThreadStart || hasReceivedInitializeResponse || activeThreadID != nil @@ -1218,9 +1237,10 @@ final class CodexAppServerActionProvider: ActionProvider { return } - let stderrText = String(data: stderrBuffer, encoding: .utf8)? + let stderrText = String(data: stderrSnapshot, encoding: .utf8)? .trimmingCharacters(in: .whitespacesAndNewlines) - let message = stderrText?.isEmpty == false + let message = + stderrText?.isEmpty == false ? "Codex action process exited: \(stderrText!)" : "Codex action process exited unexpectedly." status = .failed(message) @@ -1232,13 +1252,15 @@ final class CodexAppServerActionProvider: ActionProvider { private func summarizedCompletionMessage(from message: [String: Any]) -> String? { guard let params = message["params"] as? [String: Any], - let turn = params["turn"] as? [String: Any] else { + let turn = params["turn"] as? [String: Any] + else { return nil } if let status = turn["status"] as? String, status == "failed", - let error = turn["error"] as? [String: Any], - let message = error["message"] as? String { + let error = turn["error"] as? [String: Any], + let message = error["message"] as? String + { return message } @@ -1247,7 +1269,8 @@ final class CodexAppServerActionProvider: ActionProvider { private func turnStatus(from message: [String: Any]) -> String? { guard let params = message["params"] as? [String: Any], - let turn = params["turn"] as? [String: Any] else { + let turn = params["turn"] as? [String: Any] + else { return nil } @@ -1256,7 +1279,12 @@ final class CodexAppServerActionProvider: ActionProvider { private func handleMcpElicitationRequest(_ message: [String: Any]) { guard let requestID = message["id"] as? Int, - let params = message["params"] as? [String: Any] else { + let params = message["params"] as? [String: Any], + OrbitCodexApprovalRouter.route( + method: "mcpServer/elicitation/request", + params: params + ) == .acceptMCP + else { return } @@ -1275,7 +1303,7 @@ final class CodexAppServerActionProvider: ActionProvider { "content": NSNull(), "_meta": [ "orbitAutoApproved": true - ] + ], ] ) @@ -1284,7 +1312,13 @@ final class CodexAppServerActionProvider: ActionProvider { } private func handleCommandApprovalRequest(_ message: [String: Any]) { - guard let requestID = message["id"] as? Int else { return } + let params = message["params"] as? [String: Any] ?? [:] + guard let requestID = message["id"] as? Int, + OrbitCodexApprovalRouter.route( + method: "item/commandExecution/requestApproval", + params: params + ) == .acceptForSession + else { return } status = .waitingForApproval("command approval") emitPhase(.waitingForApproval, detail: "approving command access for this session.", rawSource: "approving codex command execution") sendResponse(id: requestID, result: ["decision": "acceptForSession"]) @@ -1293,7 +1327,13 @@ final class CodexAppServerActionProvider: ActionProvider { } private func handleFileChangeApprovalRequest(_ message: [String: Any]) { - guard let requestID = message["id"] as? Int else { return } + let params = message["params"] as? [String: Any] ?? [:] + guard let requestID = message["id"] as? Int, + OrbitCodexApprovalRouter.route( + method: "item/fileChange/requestApproval", + params: params + ) == .acceptForSession + else { return } status = .waitingForApproval("file approval") emitPhase(.waitingForApproval, detail: "approving file changes for this session.", rawSource: "approving codex file changes") sendResponse(id: requestID, result: ["decision": "acceptForSession"]) @@ -1303,21 +1343,27 @@ final class CodexAppServerActionProvider: ActionProvider { private func handlePermissionsApprovalRequest(_ message: [String: Any]) { guard let requestID = message["id"] as? Int, - let params = message["params"] as? [String: Any] else { + let params = message["params"] as? [String: Any], + case .grantPermissions = OrbitCodexApprovalRouter.route( + method: "item/permissions/requestApproval", + params: params + ) + else { return } let requestedPermissions = params["permissions"] as? [String: Any] ?? [:] status = .waitingForApproval("permissions approval") - emitPhase(.waitingForApproval, detail: "granting requested permissions for this session.", rawSource: "granting codex requested permissions for this session") + emitPhase( + .waitingForApproval, detail: "granting requested permissions for this session.", rawSource: "granting codex requested permissions for this session") sendResponse( id: requestID, result: [ "permissions": [ "network": requestedPermissions["network"] ?? NSNull(), - "fileSystem": requestedPermissions["fileSystem"] ?? NSNull() + "fileSystem": requestedPermissions["fileSystem"] ?? NSNull(), ], - "scope": "session" + "scope": "session", ] ) status = .running @@ -1326,14 +1372,16 @@ final class CodexAppServerActionProvider: ActionProvider { private func handleThreadStatusChanged(_ message: [String: Any]) { guard let params = message["params"] as? [String: Any], - let status = params["status"] as? [String: Any], - let type = status["type"] as? String else { + let status = params["status"] as? [String: Any], + let type = status["type"] as? String + else { return } if type == "active", - let flags = status["activeFlags"] as? [String], - flags.contains("waitingOnApproval") { + let flags = status["activeFlags"] as? [String], + flags.contains("waitingOnApproval") + { emitPhase(.waitingForApproval, rawSource: "waiting on a tool approval") } else if type == "idle", !isAwaitingTurnCompletion { lastEmittedProgress = nil @@ -1342,8 +1390,9 @@ final class CodexAppServerActionProvider: ActionProvider { private func handleMcpStartupStatus(_ message: [String: Any]) { guard let params = message["params"] as? [String: Any], - let serverName = params["name"] as? String, - let status = params["status"] as? String else { + let serverName = params["name"] as? String, + let status = params["status"] as? String + else { return } @@ -1465,8 +1514,9 @@ final class CodexAppServerActionProvider: ActionProvider { private func handleItemStarted(_ message: [String: Any]) { guard let params = message["params"] as? [String: Any], - let item = params["item"] as? [String: Any], - let type = item["type"] as? String else { + let item = params["item"] as? [String: Any], + let type = item["type"] as? String + else { return } @@ -1481,11 +1531,23 @@ final class CodexAppServerActionProvider: ActionProvider { } case "fileChange": emitPhase(.editingFiles, detail: "editing files in the current session.", rawSource: "preparing changes") + case "collabAgentToolCall", "subAgentActivity": + reduceSubagentActivity(from: item) default: break } } + private func reduceSubagentActivity(from item: [String: Any]) { + subagentActivities = OrbitCodexActivityReducer.reducing(existing: subagentActivities, item: item) + eventHandler?(.subagentActivity(subagentActivities)) + notifyStateChanged() + } + + static func parseSubagentActivities(from item: [String: Any]) -> [OrbitSubagentActivity] { + OrbitCodexActivityReducer.parse(item: item) + } + private func handleAgentMessageDelta(_ message: [String: Any]) { guard let params = message["params"] as? [String: Any] else { return } @@ -1502,9 +1564,10 @@ final class CodexAppServerActionProvider: ActionProvider { } guard phase != "final_answer", - phase != "finalAnswer", - let deltaText, - !deltaText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + phase != "finalAnswer", + let deltaText, + !deltaText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { return } @@ -1514,13 +1577,15 @@ final class CodexAppServerActionProvider: ActionProvider { ) if let update = Self.visibleCommentaryUpdate(from: streamedCommentaryBuffer), - update != lastEmittedLiveCommentary { + update != lastEmittedLiveCommentary + { lastEmittedLiveCommentary = update eventHandler?(.liveUpdate(update)) } guard !hasEmittedEarlyCommentary, - let snippet = Self.speakableCommentarySnippet(from: streamedCommentaryBuffer) else { + let snippet = Self.speakableCommentarySnippet(from: streamedCommentaryBuffer) + else { return } @@ -1530,13 +1595,15 @@ final class CodexAppServerActionProvider: ActionProvider { private func emitCompletedCommentaryIfNeeded(text: String) { if let update = Self.visibleCommentaryUpdate(from: text), - update != lastEmittedLiveCommentary { + update != lastEmittedLiveCommentary + { lastEmittedLiveCommentary = update eventHandler?(.liveUpdate(update)) } guard !hasEmittedEarlyCommentary, - let snippet = Self.speakableCommentarySnippet(from: text) else { + let snippet = Self.speakableCommentarySnippet(from: text) + else { return } @@ -1546,15 +1613,17 @@ final class CodexAppServerActionProvider: ActionProvider { private func handleToolRequestUserInput(_ message: [String: Any]) { guard let requestID = message["id"] as? Int, - let params = message["params"] as? [String: Any], - let questions = params["questions"] as? [[String: Any]], - let firstQuestion = questions.first, - let options = firstQuestion["options"] as? [[String: Any]] else { + let params = message["params"] as? [String: Any], + let questions = params["questions"] as? [[String: Any]], + let firstQuestion = questions.first, + let options = firstQuestion["options"] as? [[String: Any]] + else { return } let optionLabels = options.compactMap { $0["label"] as? String } - let promptTitle = firstQuestion["question"] as? String + let promptTitle = + firstQuestion["question"] as? String ?? params["message"] as? String ?? "Codex needs your answer." let questionID = firstQuestion["id"] as? String ?? "choice" @@ -1581,7 +1650,8 @@ final class CodexAppServerActionProvider: ActionProvider { let item = params["item"] as? [String: Any] let server = (item?["server"] as? String) ?? (params["server"] as? String) ?? "tool" let tool = (item?["tool"] as? String) ?? (params["tool"] as? String) ?? "action" - let progressText = firstString(in: params["delta"]) + let progressText = + firstString(in: params["delta"]) ?? firstString(in: params["message"]) ?? firstString(in: item?["message"]) ?? firstString(in: item?["delta"]) @@ -1637,21 +1707,24 @@ final class CodexAppServerActionProvider: ActionProvider { || normalizedTool.contains("new_page") || normalizedTool.contains("newpage") || normalizedTool.contains("goto") - || normalizedTool.contains("open") { + || normalizedTool.contains("open") + { return OrbitActionProgress(phase: .navigating) } if normalizedTool.contains("click") || normalizedTool.contains("drag") || normalizedTool.contains("hover") - || normalizedTool.contains("select_option") { + || normalizedTool.contains("select_option") + { return OrbitActionProgress(phase: .clicking) } if normalizedTool.contains("fill") || normalizedTool.contains("type") || normalizedTool.contains("press_key") - || normalizedTool.contains("press") { + || normalizedTool.contains("press") + { return OrbitActionProgress(phase: .typing) } @@ -1660,7 +1733,8 @@ final class CodexAppServerActionProvider: ActionProvider { || normalizedTool.contains("evaluate") || normalizedTool.contains("wait") || normalizedTool.contains("console") - || normalizedTool.contains("network") { + || normalizedTool.contains("network") + { return OrbitActionProgress(phase: .readingScreen) } @@ -1674,7 +1748,8 @@ final class CodexAppServerActionProvider: ActionProvider { let joinedCommand = command.joined(separator: " ").lowercased() if joinedCommand.contains("open ") || joinedCommand.contains("xdg-open") - || joinedCommand.contains("start ") { + || joinedCommand.contains("start ") + { return OrbitActionProgress(phase: .openingBrowser) } @@ -1683,8 +1758,9 @@ final class CodexAppServerActionProvider: ActionProvider { private func visualContextMessage(for request: OrbitActionRequest?) -> String? { guard let request, - let imagePixelSize = request.imagePixelSize, - let cursorPoint = request.cursorPointInImagePixels else { + let imagePixelSize = request.imagePixelSize, + let cursorPoint = request.cursorPointInImagePixels + else { return nil } @@ -1692,13 +1768,13 @@ final class CodexAppServerActionProvider: ActionProvider { let screenshotLabel = request.screenshotLabel ?? screenLabel return """ - Visual context: - - attached image: \(screenLabel) - - image size: \(Int(imagePixelSize.width))x\(Int(imagePixelSize.height)) pixels - - cursor position in image pixels: \(Int(cursorPoint.x)),\(Int(cursorPoint.y)) - - focus priority: unless the user says otherwise, start with what is nearest the cursor position - - note: \(screenshotLabel) - """ + Visual context: + - attached image: \(screenLabel) + - image size: \(Int(imagePixelSize.width))x\(Int(imagePixelSize.height)) pixels + - cursor position in image pixels: \(Int(cursorPoint.x)),\(Int(cursorPoint.y)) + - focus priority: unless the user says otherwise, start with what is nearest the cursor position + - note: \(screenshotLabel) + """ } private func wrappedPrompt(for request: OrbitActionRequest) -> String { @@ -1708,28 +1784,28 @@ final class CodexAppServerActionProvider: ActionProvider { if let appName, !appName.isEmpty, let windowTitle, !windowTitle.isEmpty { return """ - Frontmost desktop context: - - active app: \(appName) - - focused window: \(windowTitle) - """ + Frontmost desktop context: + - active app: \(appName) + - focused window: \(windowTitle) + """ } if let appName, !appName.isEmpty { return """ - Frontmost desktop context: - - active app: \(appName) - """ + Frontmost desktop context: + - active app: \(appName) + """ } return "Frontmost desktop context: unavailable for this turn." }() return """ - \(frontmostContextBlock) + \(frontmostContextBlock) - User request: - \(request.transcript) - """ + User request: + \(request.transcript) + """ } private var currentActionModel: String { @@ -1737,7 +1813,8 @@ final class CodexAppServerActionProvider: ActionProvider { } private var currentActionModelDisplayName: String { - modelOption(for: normalizedCurrentActionModel)?.displayName + guard !normalizedCurrentActionModel.isEmpty else { return "Server default" } + return modelOption(for: normalizedCurrentActionModel)?.displayName ?? OrbitCodexModelOption.fallbackOption(for: normalizedCurrentActionModel)?.displayName ?? normalizedCurrentActionModel } @@ -1754,8 +1831,21 @@ final class CodexAppServerActionProvider: ActionProvider { } private var normalizedCurrentActionModel: String { - let normalized = settings.codexActionModel.trimmingCharacters(in: .whitespacesAndNewlines) - return normalized.isEmpty ? OrbitCodexModelOption.fallbackDefaultModel : normalized + settings.codexActionModel.trimmingCharacters(in: .whitespacesAndNewlines) + } + + private var resolvedAgentFolder: String { + let configured = settings.codexAgentFolder.trimmingCharacters(in: .whitespacesAndNewlines) + return configured.isEmpty ? NSHomeDirectory() : NSString(string: configured).expandingTildeInPath + } + + private var sessionCoordinator: OrbitCodexSessionCoordinator { + OrbitCodexSessionCoordinator( + model: normalizedCurrentActionModel, + effort: resolvedEffortForCurrentModel, + serviceTier: resolvedServiceTier, + workingDirectory: resolvedAgentFolder + ) } private var resolvedServiceTier: OrbitCodexServiceTier { @@ -1768,7 +1858,8 @@ final class CodexAppServerActionProvider: ActionProvider { } if let item = params["item"] as? [String: Any], - let phase = item["phase"] as? String { + let phase = item["phase"] as? String + { return phase } @@ -1781,7 +1872,8 @@ final class CodexAppServerActionProvider: ActionProvider { } if let item = params["item"] as? [String: Any], - let text = item["text"] as? String { + let text = item["text"] as? String + { return text } @@ -1842,15 +1934,20 @@ final class CodexAppServerActionProvider: ActionProvider { } static func mergedCommentaryBuffer(existing: String, incomingDelta: String) -> String { - guard !incomingDelta.isEmpty else { return existing } - guard !existing.isEmpty else { return incomingDelta } + let maximumLength = 65_536 + func bounded(_ value: String) -> String { + value.count > maximumLength ? String(value.suffix(maximumLength)) : value + } + + guard !incomingDelta.isEmpty else { return bounded(existing) } + guard !existing.isEmpty else { return bounded(incomingDelta) } if existing.hasSuffix(incomingDelta) { - return existing + return bounded(existing) } if incomingDelta.hasPrefix(existing) { - return incomingDelta + return bounded(incomingDelta) } let maximumOverlap = min(existing.count, incomingDelta.count) @@ -1859,16 +1956,17 @@ final class CodexAppServerActionProvider: ActionProvider { let existingSuffix = String(existing.suffix(overlapCount)) let incomingPrefix = String(incomingDelta.prefix(overlapCount)) if existingSuffix == incomingPrefix { - return existing + incomingDelta.dropFirst(overlapCount) + return bounded(existing + incomingDelta.dropFirst(overlapCount)) } } } - return existing + incomingDelta + return bounded(existing + incomingDelta) } static func speakableCommentarySnippet(from text: String) -> String? { - let cleaned = text + let cleaned = + text .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) @@ -1885,7 +1983,8 @@ final class CodexAppServerActionProvider: ActionProvider { let maximumLength = 120 let wasTruncated = cleaned.count > maximumLength let prefix = String(cleaned.prefix(maximumLength)) - let trimmed = (wasTruncated + let trimmed = + (wasTruncated ? prefix.replacingOccurrences(of: "\\s+\\S*$", with: "", options: .regularExpression) : prefix) .replacingOccurrences( @@ -1899,12 +1998,14 @@ final class CodexAppServerActionProvider: ActionProvider { } static func visibleCommentaryUpdate(from text: String) -> String? { - let cleaned = text + let cleaned = + text .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) guard cleaned.count >= 10 else { return nil } - let firstSentence = cleaned + let firstSentence = + cleaned .split(whereSeparator: { ".!?".contains($0) }) .first .map(String.init)? @@ -1916,7 +2017,8 @@ final class CodexAppServerActionProvider: ActionProvider { } let prefix = String(firstSentence.prefix(93)) - let trimmed = prefix + let trimmed = + prefix .replacingOccurrences(of: "\\s+\\S*$", with: "", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) return trimmed.isEmpty ? nil : "\(trimmed)..." @@ -1933,7 +2035,7 @@ final class CodexAppServerActionProvider: ActionProvider { } private func updateModelCatalog(from result: [String: Any]) { - let parsedModels = Self.parseModelCatalog(from: result) + let parsedModels = OrbitCodexModelCatalog.parse(from: result) guard !parsedModels.isEmpty else { return } availableModelOptions = parsedModels } @@ -1968,115 +2070,7 @@ final class CodexAppServerActionProvider: ActionProvider { } static func parseModelCatalog(from result: [String: Any]) -> [OrbitCodexModelOption] { - let rawItems = (result["data"] as? [[String: Any]]) ?? (result["models"] as? [[String: Any]]) ?? [] - - let parsedModels = rawItems.compactMap { item -> (option: OrbitCodexModelOption, priority: Int)? in - let normalizedModel = ( - (item["model"] as? String) - ?? (item["id"] as? String) - ?? (item["slug"] as? String) - ?? "" - ).trimmingCharacters(in: .whitespacesAndNewlines) - guard !normalizedModel.isEmpty else { return nil } - - let visibility = ((item["visibility"] as? String) ?? "") - .trimmingCharacters(in: .whitespacesAndNewlines) - .lowercased() - let hidden = (item["hidden"] as? Bool) ?? (visibility == "hide" || visibility == "hidden") - guard !hidden else { return nil } - - if let supportedInAPI = item["supported_in_api"] as? Bool, !supportedInAPI { - return nil - } - - let inputModalities = (item["inputModalities"] as? [String]) - ?? (item["input_modalities"] as? [String]) - ?? ["text", "image"] - let normalizedModalities = inputModalities.map { $0.lowercased() } - guard normalizedModalities.contains("text"), normalizedModalities.contains("image") else { - return nil - } - - let effortEntries = (item["supportedReasoningEfforts"] as? [[String: Any]]) - ?? (item["supported_reasoning_levels"] as? [[String: Any]]) - ?? [] - let supportedEfforts = effortEntries.compactMap { entry -> OrbitCodexReasoningEffort? in - let rawValue = ( - (entry["reasoningEffort"] as? String) - ?? (entry["effort"] as? String) - ?? "" - ).trimmingCharacters(in: .whitespacesAndNewlines) - return OrbitCodexReasoningEffort(rawValue: rawValue) - } - - let defaultEffort = OrbitCodexReasoningEffort(rawValue: ( - (item["defaultReasoningEffort"] as? String) - ?? (item["default_reasoning_level"] as? String) - ?? "" - ).trimmingCharacters(in: .whitespacesAndNewlines)) - let displayName = formattedModelDisplayName( - for: normalizedModel, - fallback: (item["displayName"] as? String) ?? (item["display_name"] as? String) - ) - let shortDisplayName = shortModelDisplayName(from: displayName) - let isDefault = (item["isDefault"] as? Bool ?? false) || normalizedModel == OrbitCodexModelOption.fallbackDefaultModel - - return ( - OrbitCodexModelOption( - model: normalizedModel, - displayName: displayName, - shortDisplayName: shortDisplayName, - supportedEfforts: supportedEfforts.isEmpty ? OrbitCodexReasoningEffort.allCases : supportedEfforts, - defaultEffort: defaultEffort, - inputModalities: normalizedModalities, - isDefault: isDefault - ), - item["priority"] as? Int ?? Int.max - ) - } - - return parsedModels.sorted { lhs, rhs in - if lhs.option.isDefault != rhs.option.isDefault { - return lhs.option.isDefault && !rhs.option.isDefault - } - if lhs.priority != rhs.priority { - return lhs.priority < rhs.priority - } - return lhs.option.displayName.localizedCaseInsensitiveCompare(rhs.option.displayName) == .orderedAscending - }.map { $0.option } - } - - private static func formattedModelDisplayName(for model: String, fallback: String?) -> String { - let normalized = model.trimmingCharacters(in: .whitespacesAndNewlines) - switch normalized { - case "gpt-5.4": - return "GPT-5.4" - case "gpt-5.4-mini": - return "GPT-5.4 Mini" - case "gpt-5.3-codex": - return "GPT-5.3 Codex" - case "gpt-5.2": - return "GPT-5.2" - default: - break - } - - let fallbackValue = fallback?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - if !fallbackValue.isEmpty { - return fallbackValue - .replacingOccurrences(of: "^gpt-", with: "GPT-", options: [.regularExpression, .caseInsensitive]) - .replacingOccurrences(of: "-mini", with: " Mini", options: [.regularExpression, .caseInsensitive]) - .replacingOccurrences(of: "-codex", with: " Codex", options: [.regularExpression, .caseInsensitive]) - } - - return normalized.uppercased() - } - - private static func shortModelDisplayName(from displayName: String) -> String { - let cleaned = displayName - .replacingOccurrences(of: "GPT-", with: "") - .trimmingCharacters(in: .whitespacesAndNewlines) - return cleaned.isEmpty ? displayName : cleaned + OrbitCodexModelCatalog.parse(from: result) } private static func startupFailureMessage(for error: Error) -> String { @@ -2093,12 +2087,14 @@ final class CodexAppServerActionProvider: ActionProvider { } if let configuredPath = AppBundleConfiguration.stringValue(forKey: "CodexCLIPath"), - FileManager.default.isExecutableFile(atPath: configuredPath) { + FileManager.default.isExecutableFile(atPath: configuredPath) + { return configuredPath } if let envPath = ProcessInfo.processInfo.environment["CODEX_CLI_PATH"], - FileManager.default.isExecutableFile(atPath: envPath) { + FileManager.default.isExecutableFile(atPath: envPath) + { return envPath } @@ -2106,7 +2102,7 @@ final class CodexAppServerActionProvider: ActionProvider { "\(NSHomeDirectory())/.nvm/versions/node/v22.22.1/bin/codex", "\(NSHomeDirectory())/.local/bin/codex", "/opt/homebrew/bin/codex", - "/usr/local/bin/codex" + "/usr/local/bin/codex", ] if let resolvedPath = commonCandidates.first(where: { FileManager.default.isExecutableFile(atPath: $0) }) { @@ -2142,9 +2138,11 @@ final class CodexAppServerActionProvider: ActionProvider { private func bundledCodexExecutablePath() -> String? { guard let resourceURL = Bundle.main.resourceURL else { return nil } - let runtimeRoot = resourceURL + let runtimeRoot = + resourceURL .appendingPathComponent("CodexRuntime", isDirectory: true) - let nativeBundledPath = runtimeRoot + let nativeBundledPath = + runtimeRoot .appendingPathComponent("vendor", isDirectory: true) .appendingPathComponent("aarch64-apple-darwin", isDirectory: true) .appendingPathComponent("codex", isDirectory: true) @@ -2154,7 +2152,8 @@ final class CodexAppServerActionProvider: ActionProvider { return nativeBundledPath } - let bundledWrapperPath = runtimeRoot + let bundledWrapperPath = + runtimeRoot .appendingPathComponent("bin", isDirectory: true) .appendingPathComponent("codex") .path @@ -2178,8 +2177,8 @@ final class CodexAppServerActionProvider: ActionProvider { stderrHandle = nil stdinHandle = nil process = nil - stdoutBuffer.removeAll(keepingCapacity: false) - stderrBuffer.removeAll(keepingCapacity: false) + stderrSnapshot.removeAll(keepingCapacity: false) + Task { await transport.reset() } activeThreadID = nil activeTurnID = nil pendingPrompt = nil diff --git a/Orbit/DesignSystem.swift b/Orbit/DesignSystem.swift index 28f2a2f..5cb21e1 100644 --- a/Orbit/DesignSystem.swift +++ b/Orbit/DesignSystem.swift @@ -7,8 +7,8 @@ // styling used across the panel, HUD, onboarding, and overlay system. // -import SwiftUI import AppKit +import SwiftUI // MARK: - Design System Namespace @@ -76,7 +76,7 @@ enum DS { // 800–900 → Deep backgrounds, dark overlays, header bars // 950 → Deepest blue — near-black tinted backgrounds - static let blue50 = Color(hex: "#eff6ff") + static let blue50 = Color(hex: "#eff6ff") static let blue100 = Color(hex: "#dbeafe") static let blue200 = Color(hex: "#bfdbfe") static let blue300 = Color(hex: "#93c5fd") @@ -265,7 +265,7 @@ struct DSGlassCardModifier: ViewModifier { LinearGradient( colors: [ Color.white.opacity(highlightOpacity), - Color.white.opacity(0.03) + Color.white.opacity(0.03), ], startPoint: .topLeading, endPoint: .bottomTrailing @@ -375,7 +375,7 @@ struct DSPrimaryButtonStyle: ButtonStyle { if hovering { withAnimation( .easeInOut(duration: 2.5) - .repeatForever(autoreverses: true) + .repeatForever(autoreverses: true) ) { isGlowBreathingIn = true } @@ -702,7 +702,7 @@ struct DSIconButtonStyle: ButtonStyle { LinearGradient( colors: [ Color.white.opacity(0.10), - Color.white.opacity(0.02) + Color.white.opacity(0.02), ], startPoint: .top, endPoint: .bottom diff --git a/Orbit/GlobalPushToTalkShortcutMonitor.swift b/Orbit/GlobalPushToTalkShortcutMonitor.swift index d56804d..2d1c530 100644 --- a/Orbit/GlobalPushToTalkShortcutMonitor.swift +++ b/Orbit/GlobalPushToTalkShortcutMonitor.swift @@ -23,7 +23,7 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { /// waiting for the async dictation state pipeline to catch up. @Published private(set) var isShortcutCurrentlyPressed = false - deinit { + isolated deinit { stop() } @@ -54,23 +54,27 @@ final class GlobalPushToTalkShortcutMonitor: ObservableObject { ) } - guard let globalEventTap = CGEvent.tapCreate( - tap: .cgSessionEventTap, - place: .headInsertEventTap, - options: .listenOnly, - eventsOfInterest: eventMask, - callback: eventTapCallback, - userInfo: Unmanaged.passUnretained(self).toOpaque() - ) else { + guard + let globalEventTap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: eventMask, + callback: eventTapCallback, + userInfo: Unmanaged.passUnretained(self).toOpaque() + ) + else { print("⚠️ Global push-to-talk: couldn't create CGEvent tap") return } - guard let globalEventTapRunLoopSource = CFMachPortCreateRunLoopSource( - kCFAllocatorDefault, - globalEventTap, - 0 - ) else { + guard + let globalEventTapRunLoopSource = CFMachPortCreateRunLoopSource( + kCFAllocatorDefault, + globalEventTap, + 0 + ) + else { CFMachPortInvalidate(globalEventTap) print("⚠️ Global push-to-talk: couldn't create event tap run loop source") return diff --git a/Orbit/MenuBarPanelManager.swift b/Orbit/MenuBarPanelManager.swift index dba02c6..5f14a74 100644 --- a/Orbit/MenuBarPanelManager.swift +++ b/Orbit/MenuBarPanelManager.swift @@ -51,7 +51,7 @@ final class MenuBarPanelManager: NSObject { } } - deinit { + isolated deinit { if let monitor = clickOutsideMonitor { NSEvent.removeMonitor(monitor) } @@ -145,18 +145,32 @@ final class MenuBarPanelManager: NSObject { private func positionPanelBelowStatusItem() { guard let panel else { return } - guard let buttonWindow = statusItem?.button?.window else { return } + guard let statusButton = statusItem?.button, + let buttonWindow = statusButton.window + else { return } - let statusItemFrame = buttonWindow.frame + let buttonRectInWindow = statusButton.convert(statusButton.bounds, to: nil) + let statusItemFrame = buttonWindow.convertToScreen(buttonRectInWindow) let gapBelowMenuBar: CGFloat = 4 // Calculate the panel's content height from the hosting view's fitting size // so the panel snugly wraps the SwiftUI content instead of using a fixed height. let fittingSize = panel.contentView?.fittingSize ?? CGSize(width: panelWidth, height: panelHeight) - let actualPanelHeight = fittingSize.height + let visibleFrame = + NSScreen.screens + .first(where: { $0.frame.contains(CGPoint(x: statusItemFrame.midX, y: statusItemFrame.midY)) })? + .visibleFrame + ?? buttonWindow.screen?.visibleFrame + ?? NSScreen.main?.visibleFrame + let maximumHeight = max(320, (visibleFrame?.height ?? 760) - 20) + let actualPanelHeight = min(fittingSize.height, maximumHeight) // Horizontally center the panel beneath the status item icon - let panelOriginX = statusItemFrame.midX - (panelWidth / 2) + let proposedX = statusItemFrame.midX - (panelWidth / 2) + let panelOriginX = + visibleFrame.map { + min(max(proposedX, $0.minX + 8), $0.maxX - panelWidth - 8) + } ?? proposedX let panelOriginY = statusItemFrame.minY - actualPanelHeight - gapBelowMenuBar panel.setFrame( diff --git a/Orbit/OpenAITTSProvider.swift b/Orbit/OpenAITTSProvider.swift index 1e8c461..b452fe1 100644 --- a/Orbit/OpenAITTSProvider.swift +++ b/Orbit/OpenAITTSProvider.swift @@ -4,7 +4,8 @@ import Foundation @MainActor final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDelegate { private let resolvedAPIKey = OrbitOpenAIKeychainStore.resolvedAPIKey() - private let modelName = AppBundleConfiguration.stringValue(forKey: "OpenAITTSModel") + private let modelName = + AppBundleConfiguration.stringValue(forKey: "OpenAITTSModel") ?? "gpt-4o-mini-tts" private let voicePreset: OrbitVoicePreset private let session: URLSession @@ -59,7 +60,7 @@ final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDele "voice": preferredVoiceName, "input": text, "instructions": speechInstructions, - "response_format": "wav" + "response_format": "wav", ] request.httpBody = try JSONSerialization.data(withJSONObject: body) diff --git a/Orbit/OpenAITranscriptionProvider.swift b/Orbit/OpenAITranscriptionProvider.swift index 1480e78..65c5a93 100644 --- a/Orbit/OpenAITranscriptionProvider.swift +++ b/Orbit/OpenAITranscriptionProvider.swift @@ -9,7 +9,8 @@ struct OpenAITranscriptionProviderError: LocalizedError { final class OpenAITranscriptionProvider: SpeechToTextProvider { private let resolvedAPIKey = OrbitOpenAIKeychainStore.resolvedAPIKey() - private let modelName = AppBundleConfiguration.stringValue(forKey: "OpenAITranscriptionModel") + private let modelName = + AppBundleConfiguration.stringValue(forKey: "OpenAITranscriptionModel") ?? "gpt-4o-mini-transcribe" let displayName = "OpenAI Transcribe" @@ -47,7 +48,7 @@ final class OpenAITranscriptionProvider: SpeechToTextProvider { } } -private final class OpenAITranscriptionSession: SpeechToTextStreamingSession { +nonisolated private final class OpenAITranscriptionSession: SpeechToTextStreamingSession, @unchecked Sendable { let finalTranscriptFallbackDelaySeconds: TimeInterval = 6.0 private struct TranscriptionResponse: Decodable { @@ -97,7 +98,8 @@ private final class OpenAITranscriptionSession: SpeechToTextStreamingSession { func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) { guard let pcmData = audioPCM16Converter.convertToPCM16Data(from: audioBuffer), - !pcmData.isEmpty else { + !pcmData.isEmpty + else { return } @@ -188,7 +190,8 @@ private final class OpenAITranscriptionSession: SpeechToTextStreamingSession { return transcriptionResponse.text.trimmingCharacters(in: .whitespacesAndNewlines) } - let responseText = String(data: responseData, encoding: .utf8)? + let responseText = + String(data: responseData, encoding: .utf8)? .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" if !responseText.isEmpty { diff --git a/Orbit/OrbitAudioConversionSupport.swift b/Orbit/OrbitAudioConversionSupport.swift index 3c3f981..0aed98d 100644 --- a/Orbit/OrbitAudioConversionSupport.swift +++ b/Orbit/OrbitAudioConversionSupport.swift @@ -8,7 +8,25 @@ import AVFoundation import Foundation -final class OrbitPCM16AudioConverter { +nonisolated private final class OrbitAudioConversionInput: @unchecked Sendable { + private let lock = NSLock() + private let buffer: AVAudioPCMBuffer + private var hasBeenConsumed = false + + init(buffer: AVAudioPCMBuffer) { + self.buffer = buffer + } + + func take() -> AVAudioPCMBuffer? { + lock.lock() + defer { lock.unlock() } + guard !hasBeenConsumed else { return nil } + hasBeenConsumed = true + return buffer + } +} + +nonisolated final class OrbitPCM16AudioConverter: @unchecked Sendable { private let targetAudioFormat: AVAudioFormat private var audioConverter: AVAudioConverter? private var currentInputFormatDescription: String? @@ -37,25 +55,25 @@ final class OrbitPCM16AudioConverter { (Double(audioBuffer.frameLength) * sampleRateRatio).rounded(.up) + 32 ) - guard let outputBuffer = AVAudioPCMBuffer( - pcmFormat: targetAudioFormat, - frameCapacity: outputFrameCapacity - ) else { + guard + let outputBuffer = AVAudioPCMBuffer( + pcmFormat: targetAudioFormat, + frameCapacity: outputFrameCapacity + ) + else { return nil } - var hasProvidedSourceBuffer = false var conversionError: NSError? + let conversionInput = OrbitAudioConversionInput(buffer: audioBuffer) let conversionStatus = audioConverter.convert(to: outputBuffer, error: &conversionError) { _, outStatus in - if hasProvidedSourceBuffer { + guard let buffer = conversionInput.take() else { outStatus.pointee = .noDataNow return nil } - - hasProvidedSourceBuffer = true outStatus.pointee = .haveData - return audioBuffer + return buffer } guard conversionStatus != .error else { return nil } @@ -69,7 +87,7 @@ final class OrbitPCM16AudioConverter { } } -enum OrbitWAVFileBuilder { +nonisolated enum OrbitWAVFileBuilder { static func buildWAVData( fromPCM16MonoAudio pcm16AudioData: Data, sampleRate: Int, diff --git a/Orbit/OrbitAudioInput.swift b/Orbit/OrbitAudioInput.swift new file mode 100644 index 0000000..c07c31b --- /dev/null +++ b/Orbit/OrbitAudioInput.swift @@ -0,0 +1,121 @@ +import AVFoundation +import AudioToolbox +import CoreAudio +import Foundation + +struct OrbitAudioInputDevice: Identifiable, Equatable, Sendable { + let id: AudioDeviceID + let uid: String + let name: String +} + +enum OrbitAudioInputCatalog { + static func devices() -> [OrbitAudioInputDevice] { + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var size: UInt32 = 0 + guard AudioObjectGetPropertyDataSize(AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size) == noErr else { + return [] + } + var ids = Array(repeating: AudioDeviceID(0), count: Int(size) / MemoryLayout.size) + guard AudioObjectGetPropertyData(AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &ids) == noErr else { + return [] + } + + return ids.compactMap { id in + guard inputChannelCount(for: id) > 0, + let uid = stringProperty(kAudioDevicePropertyDeviceUID, deviceID: id), + let name = stringProperty(kAudioObjectPropertyName, deviceID: id) + else { return nil } + return OrbitAudioInputDevice(id: id, uid: uid, name: name) + }.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } + } + + static func applySelectedDevice(uid: String, to inputNode: AVAudioInputNode) throws { + let normalized = uid.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty else { return } + guard var deviceID = devices().first(where: { $0.uid == normalized })?.id else { + throw NSError( + domain: "OrbitAudioInput", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "The selected microphone is disconnected."] + ) + } + guard let audioUnit = inputNode.audioUnit else { return } + let status = AudioUnitSetProperty( + audioUnit, + kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, + 0, + &deviceID, + UInt32(MemoryLayout.size) + ) + guard status == noErr else { + throw NSError( + domain: NSOSStatusErrorDomain, + code: Int(status), + userInfo: [NSLocalizedDescriptionKey: "Orbit could not switch to the selected microphone."] + ) + } + } + + private static func inputChannelCount(for deviceID: AudioDeviceID) -> Int { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioDevicePropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + var size: UInt32 = 0 + guard AudioObjectGetPropertyDataSize(deviceID, &address, 0, nil, &size) == noErr else { return 0 } + let pointer = UnsafeMutableRawPointer.allocate(byteCount: Int(size), alignment: MemoryLayout.alignment) + defer { pointer.deallocate() } + guard AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, pointer) == noErr else { return 0 } + let list = UnsafeMutableAudioBufferListPointer(pointer.assumingMemoryBound(to: AudioBufferList.self)) + return list.reduce(0) { $0 + Int($1.mNumberChannels) } + } + + private static func stringProperty(_ selector: AudioObjectPropertySelector, deviceID: AudioDeviceID) -> String? { + var address = AudioObjectPropertyAddress( + mSelector: selector, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var value: Unmanaged? + var size = UInt32(MemoryLayout?>.size) + guard AudioObjectGetPropertyData(deviceID, &address, 0, nil, &size, &value) == noErr else { return nil } + return value?.takeUnretainedValue() as String? + } +} + +@MainActor +final class OrbitMicrophoneLevelMonitor { + private var engine: AVAudioEngine? + + func start(deviceUID: String, onLevel: @escaping @MainActor (CGFloat) -> Void) throws { + stop() + let engine = AVAudioEngine() + let input = engine.inputNode + try OrbitAudioInputCatalog.applySelectedDevice(uid: deviceUID, to: input) + let format = input.outputFormat(forBus: 0) + input.installTap(onBus: 0, bufferSize: 1_024, format: format) { buffer, _ in + guard let samples = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return } + var squares: Float = 0 + for index in 0..= capacity { + data = incoming.suffix(capacity) + return + } + let overflow = max(0, data.count + incoming.count - capacity) + if overflow > 0 { + data.removeFirst(overflow) + } + data.append(incoming) + } + + mutating func removeAll(keepingCapacity: Bool = false) { + data.removeAll(keepingCapacity: keepingCapacity) + } + + mutating func popLine() -> Data? { + guard let newline = data.firstIndex(of: 0x0A) else { return nil } + let line = data.subdata(in: data.startIndex.. [String: URL] { @@ -40,54 +37,78 @@ enum OrbitBundledSkills { guard let preparedCodexHome else { return [] } let availableSkillPaths = configuredSkillPaths(in: preparedCodexHome.skillsDirectory) - let normalizedTranscript = request.transcript.lowercased() - var requestedNames: [String] = [orbitAssistantSkillName] - - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "docx", "document", "google doc", "google docs", "word document", "microsoft word" - ]) { + let normalizedTranscript = request.transcript + var requestedNames: [String] = [] + + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "docx", "document", "google doc", "google docs", "word document", "microsoft word", + ]) + { requestedNames.append("doc") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "pdf", "portable document", "extract from pdf", "read this pdf" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "pdf", "portable document", "extract from pdf", "read this pdf", + ]) + { requestedNames.append("pdf") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "slides", "slide deck", "deck", "presentation", "powerpoint", "ppt", "keynote" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "slides", "slide deck", "deck", "presentation", "powerpoint", "ppt", "keynote", + ]) + { requestedNames.append("slides") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "spreadsheet", "excel", "csv", "sheet", "google sheet", "google sheets" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "spreadsheet", "excel", "csv", "sheet", "google sheet", "google sheets", + ]) + { requestedNames.append("spreadsheet") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "screenshot", "screen capture", "screen shot", "capture the screen" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "screenshot", "screen capture", "screen shot", "capture the screen", + ]) + { requestedNames.append("screenshot") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "transcribe", "transcript", "diarize", "audio file", "video file", "recording" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "transcribe", "transcript", "diarize", "audio file", "video file", "recording", + ]) + { requestedNames.append("transcribe") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "text to speech", "tts", "voiceover", "voice over", "narration", "read aloud", "speak this" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "text to speech", "tts", "voiceover", "voice over", "narration", "read aloud", "speak this", + ]) + { requestedNames.append("speech") } - if matchesAnyKeyword(in: normalizedTranscript, keywords: [ - "openai", "chatgpt", "codex", "responses api", "openai api", "openai docs", "developer docs", "sdk" - ]) { + if matchesAnyKeyword( + in: normalizedTranscript, + keywords: [ + "openai", "chatgpt", "codex", "responses api", "openai api", "openai docs", "developer docs", "sdk", + ]) + { requestedNames.append("openai-docs") } @@ -98,7 +119,15 @@ enum OrbitBundledSkills { } } - private static func matchesAnyKeyword(in text: String, keywords: [String]) -> Bool { - keywords.contains { text.contains($0) } + static func matchesAnyKeyword(in text: String, keywords: [String]) -> Bool { + keywords.contains { keyword in + let escaped = NSRegularExpression.escapedPattern(for: keyword) + let pattern = "(? [OrbitSubagentActivity] { + var result = existing + for update in parse(item: item) { + if let index = result.firstIndex(where: { $0.threadID == update.threadID }) { + result[index] = update + } else { + result.append(update) + } + } + return Array(result.suffix(max(1, limit))) + } + + static func parse(item: [String: Any]) -> [OrbitSubagentActivity] { + let type = item["type"] as? String ?? "" + if type == "subAgentActivity" { + let threadID = item["agentThreadId"] as? String ?? "unknown" + return [ + OrbitSubagentActivity( + id: item["id"] as? String ?? threadID, + threadID: threadID, + agentPath: item["agentPath"] as? String ?? "agent", + status: item["kind"] as? String ?? "running", + message: item["message"] as? String, + model: item["model"] as? String + ) + ] + } + + guard type == "collabAgentToolCall" else { return [] } + let toolID = item["id"] as? String ?? UUID().uuidString + let model = item["model"] as? String + let agentStates = item["agentsStates"] as? [String: [String: Any]] ?? [:] + return agentStates.map { threadID, state in + OrbitSubagentActivity( + id: "\(toolID)-\(threadID)", + threadID: threadID, + agentPath: state["agentPath"] as? String ?? String(threadID.suffix(8)), + status: state["status"] as? String ?? (item["status"] as? String ?? "running"), + message: state["message"] as? String, + model: state["model"] as? String ?? model + ) + } + } +} diff --git a/Orbit/OrbitCodexContracts.swift b/Orbit/OrbitCodexContracts.swift new file mode 100644 index 0000000..ecb8055 --- /dev/null +++ b/Orbit/OrbitCodexContracts.swift @@ -0,0 +1,92 @@ +import Foundation + +typealias OrbitCodexModelDescriptor = OrbitCodexModelOption + +struct OrbitRuntimeManifest: Codable, Equatable, Sendable { + let codexRuntimeVersion: String + let codexRuntimeSHA256: String + let nodeVersion: String + let nodeSHA256: String + let browserMCPVersion: String + let appServerSchemaSHA256: String + let appServerTypeBindingsSHA256: String + let generatedAt: String +} + +struct OrbitReleaseManifest: Codable, Equatable, Sendable { + let version: String + let downloadURL: String + let sha256: String + let minimumMacOS: String + let codexRuntimeVersion: String + let browserMCPVersion: String +} + +enum OrbitCodexApprovalKind: String, CaseIterable, Sendable { + case mcpElicitation = "mcpServer/elicitation/request" + case commandExecution = "item/commandExecution/requestApproval" + case fileChange = "item/fileChange/requestApproval" + case permissions = "item/permissions/requestApproval" +} + +enum OrbitCodexApprovalRoute: Equatable, Sendable { + case acceptMCP + case acceptForSession + case grantPermissions(network: Bool?, fileSystem: Bool?) +} + +enum OrbitCodexApprovalRouter { + static func route(method: String, params: [String: Any]) -> OrbitCodexApprovalRoute? { + guard let kind = OrbitCodexApprovalKind(rawValue: method) else { return nil } + switch kind { + case .mcpElicitation: + return .acceptMCP + case .commandExecution, .fileChange: + return .acceptForSession + case .permissions: + let requested = params["permissions"] as? [String: Any] ?? [:] + return .grantPermissions( + network: requested["network"] as? Bool, + fileSystem: requested["fileSystem"] as? Bool + ) + } + } +} + +struct OrbitCodexSessionCoordinator { + let model: String + let effort: OrbitCodexReasoningEffort + let serviceTier: OrbitCodexServiceTier + let workingDirectory: String + + func threadStartParameters(sandbox: String) -> [String: Any] { + var parameters: [String: Any] = [ + "approvalPolicy": "never", + "sandbox": sandbox, + "cwd": workingDirectory, + "serviceName": "orbit", + ] + addOptionalSelections(to: ¶meters) + return parameters + } + + func turnStartParameters(threadID: String, input: [[String: Any]]) -> [String: Any] { + var parameters: [String: Any] = [ + "threadId": threadID, + "input": input, + "effort": effort.rawValue, + ] + addOptionalSelections(to: ¶meters) + return parameters + } + + private func addOptionalSelections(to parameters: inout [String: Any]) { + let normalizedModel = model.trimmingCharacters(in: .whitespacesAndNewlines) + if !normalizedModel.isEmpty { + parameters["model"] = normalizedModel + } + if !serviceTier.rawValue.isEmpty { + parameters["serviceTier"] = serviceTier.rawValue + } + } +} diff --git a/Orbit/OrbitCodexEnvironment.swift b/Orbit/OrbitCodexEnvironment.swift index 62d937a..23ce8b7 100644 --- a/Orbit/OrbitCodexEnvironment.swift +++ b/Orbit/OrbitCodexEnvironment.swift @@ -36,7 +36,7 @@ enum OrbitCodexEnvironment { "get_console_message", "list_console_messages", "take_screenshot", - "take_snapshot" + "take_snapshot", ] private static let playwrightEnabledTools = [ @@ -57,13 +57,13 @@ enum OrbitCodexEnvironment { "browser_network_requests", "browser_evaluate", "browser_close", - "browser_resize" + "browser_resize", ] static func prepareHome( model: String = OrbitCodexModelOption.fallbackDefaultModel, reasoningEffort: OrbitCodexReasoningEffort = .medium, - serviceTier: OrbitCodexServiceTier = .fast + serviceTier: OrbitCodexServiceTier = .serverDefault ) throws -> OrbitPreparedCodexHome { let fileManager = FileManager.default let supportDirectory = try validateSupportRootDirectory() @@ -175,16 +175,16 @@ enum OrbitCodexEnvironment { private static func supportDirectoryNotWritableMessage(for supportDirectory: URL) -> String { let logPath = OrbitSupportLog.currentLogFilePath() ?? "~/Library/Logs/Orbit/orbit-support.log" return """ - Orbit cannot write to its support folder at \(supportDirectory.path). + Orbit cannot write to its support folder at \(supportDirectory.path). - This usually means the folder is owned by root from an earlier Orbit installer run. Repair it in Terminal with: + This usually means the folder is owned by root from an earlier Orbit installer run. Repair it in Terminal with: - sudo chown -R "$USER":staff "$HOME/Library/Application Support/Orbit" - rm -f "$HOME/Library/LaunchAgents/com.orbit.codex.postinstall-open.plist" + sudo chown -R "$USER":staff "$HOME/Library/Application Support/Orbit" + rm -f "$HOME/Library/LaunchAgents/com.orbit.codex.postinstall-open.plist" - Then reopen Orbit. - Support log: \(logPath) - """ + Then reopen Orbit. + Support log: \(logPath) + """ } static func makeConfigContents( @@ -194,21 +194,26 @@ enum OrbitCodexEnvironment { modelInstructionsPath: String? = nil, model: String = OrbitCodexModelOption.fallbackDefaultModel, reasoningEffort: OrbitCodexReasoningEffort = .medium, - serviceTier: OrbitCodexServiceTier = .fast + serviceTier: OrbitCodexServiceTier = .serverDefault ) -> String { let normalizedModel = model.trimmingCharacters(in: .whitespacesAndNewlines) - var lines: [String] = [ - "model = \(tomlString(normalizedModel.isEmpty ? OrbitCodexModelOption.fallbackDefaultModel : normalizedModel))", + var lines: [String] = [] + if !normalizedModel.isEmpty { + lines.append("model = \(tomlString(normalizedModel))") + } + lines += [ "model_reasoning_effort = \(tomlString(reasoningEffort.rawValue))", - "service_tier = \(tomlString(serviceTier.rawValue))", "approval_policy = \"never\"", "sandbox_mode = \"danger-full-access\"", "cli_auth_credentials_store = \"file\"", "mcp_oauth_credentials_store = \"file\"", "log_dir = \(tomlString(logDirectory.path))", "sqlite_home = \(tomlString(sqliteDirectory.path))", - "history.persistence = \"save-all\"" + "history.persistence = \"save-all\"", ] + if !serviceTier.rawValue.isEmpty { + lines.insert("service_tier = \(tomlString(serviceTier.rawValue))", at: normalizedModel.isEmpty ? 1 : 2) + } // `model_instructions_file` is a top-level key. If it appears after // `[features]`, TOML nests it under that table and Codex rejects the @@ -227,8 +232,7 @@ enum OrbitCodexEnvironment { "[features]", "apps = true", "fast_mode = true", - "multi_agent = false", - "" + "", ] if let chromeDevToolsCommand = bundledBrowserCommand(named: "chrome-devtools-mcp") { @@ -239,7 +243,7 @@ enum OrbitCodexEnvironment { "env = { CHROME_DEVTOOLS_MCP_NO_UPDATE_CHECKS = \"1\" }", "enabled_tools = \(tomlArray(chromeEnabledTools))", "startup_timeout_sec = 20.0", - "" + "", ] } else { OrbitSupportLog.append("codex", "bundled chrome-devtools-mcp was not found in the app bundle.") @@ -252,7 +256,7 @@ enum OrbitCodexEnvironment { "args = [\"--browser\", \"chrome\"]", "enabled_tools = \(tomlArray(playwrightEnabledTools))", "startup_timeout_sec = 20.0", - "" + "", ] } else { OrbitSupportLog.append("codex", "bundled playwright-mcp was not found in the app bundle.") @@ -261,7 +265,7 @@ enum OrbitCodexEnvironment { lines += [ "[mcp_servers.openaiDeveloperDocs]", "url = \"https://developers.openai.com/mcp\"", - "" + "", ] for skillName in OrbitBundledSkills.bundledSkillNames { @@ -270,7 +274,7 @@ enum OrbitCodexEnvironment { "[[skills.config]]", "path = \(tomlString(path.path))", "enabled = true", - "" + "", ] } @@ -318,7 +322,8 @@ enum OrbitCodexEnvironment { private static func bundledBrowserCommand(named commandName: String) -> URL? { guard let resourceURL = Bundle.main.resourceURL else { return nil } - let commandURL = resourceURL + let commandURL = + resourceURL .appendingPathComponent("CodexRuntime", isDirectory: true) .appendingPathComponent("bin", isDirectory: true) .appendingPathComponent(commandName) @@ -336,7 +341,8 @@ enum OrbitCodexEnvironment { } nonisolated private static func tomlString(_ value: String) -> String { - let escaped = value + let escaped = + value .replacingOccurrences(of: "\\", with: "\\\\") .replacingOccurrences(of: "\"", with: "\\\"") return "\"\(escaped)\"" diff --git a/Orbit/OrbitCodexModelCatalog.swift b/Orbit/OrbitCodexModelCatalog.swift new file mode 100644 index 0000000..24cecc7 --- /dev/null +++ b/Orbit/OrbitCodexModelCatalog.swift @@ -0,0 +1,120 @@ +import Foundation + +enum OrbitCodexModelCatalog { + static func parse(from result: [String: Any]) -> [OrbitCodexModelOption] { + let rawItems = (result["data"] as? [[String: Any]]) ?? (result["models"] as? [[String: Any]]) ?? [] + let parsed = rawItems.compactMap { item -> (option: OrbitCodexModelOption, priority: Int)? in + let identifier = firstString(item, keys: ["model", "id", "slug"]) + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !identifier.isEmpty else { return nil } + + let visibility = firstString(item, keys: ["visibility"]).lowercased() + let hidden = (item["hidden"] as? Bool) ?? ["hide", "hidden"].contains(visibility) + guard !hidden else { return nil } + if let supportedInAPI = item["supported_in_api"] as? Bool, !supportedInAPI { return nil } + + let modalities = ((item["inputModalities"] as? [String]) ?? (item["input_modalities"] as? [String]) ?? []) + .map { $0.lowercased() } + if !modalities.isEmpty, !modalities.contains("text") || !modalities.contains("image") { + return nil + } + + let effortValues = + (item["supportedReasoningEfforts"] as? [Any]) + ?? (item["supported_reasoning_levels"] as? [Any]) + ?? [] + let efforts = effortValues.compactMap(reasoningEffort(from:)) + let defaultEffortValue = firstString(item, keys: ["defaultReasoningEffort", "default_reasoning_level"]) + let defaultEffort = defaultEffortValue.isEmpty ? nil : OrbitCodexReasoningEffort(rawValue: defaultEffortValue) + + let tierValues = + (item["supportedServiceTiers"] as? [Any]) + ?? (item["supported_service_tiers"] as? [Any]) + ?? [] + let serviceTiers = tierValues.compactMap(serviceTier(from:)) + + let displayNameValue = firstString(item, keys: ["displayName", "display_name"]) + .trimmingCharacters(in: .whitespacesAndNewlines) + let displayName = + displayNameValue.isEmpty || displayNameValue.caseInsensitiveCompare(identifier) == .orderedSame + ? humanizedIdentifier(identifier) + : displayNameValue + let resolvedEfforts = efforts.isEmpty ? (defaultEffort.map { [$0] } ?? OrbitCodexReasoningEffort.allCases) : efforts + return ( + OrbitCodexModelOption( + model: identifier, + displayName: displayName, + shortDisplayName: shortName(displayName), + supportedEfforts: resolvedEfforts, + defaultEffort: defaultEffort, + inputModalities: modalities, + isDefault: (item["isDefault"] as? Bool) ?? (item["is_default"] as? Bool) ?? false, + supportedServiceTiers: serviceTiers, + upgradeModel: optionalString(item, keys: ["upgradeModel", "upgrade_model"]), + upgradeMessage: optionalString(item, keys: ["upgradeMessage", "upgrade_message"]) + ), + item["priority"] as? Int ?? Int.max + ) + } + + return parsed.sorted { lhs, rhs in + if lhs.option.isDefault != rhs.option.isDefault { + return lhs.option.isDefault && !rhs.option.isDefault + } + if lhs.priority != rhs.priority { return lhs.priority < rhs.priority } + return lhs.option.displayName.localizedCaseInsensitiveCompare(rhs.option.displayName) == .orderedAscending + }.map(\.option) + } + + private static func reasoningEffort(from value: Any) -> OrbitCodexReasoningEffort? { + let rawValue: String + if let string = value as? String { + rawValue = string + } else if let dictionary = value as? [String: Any] { + rawValue = firstString(dictionary, keys: ["reasoningEffort", "effort", "value", "id"]) + } else { + return nil + } + return rawValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : OrbitCodexReasoningEffort(rawValue: rawValue) + } + + private static func serviceTier(from value: Any) -> OrbitCodexServiceTier? { + let rawValue: String + if let string = value as? String { + rawValue = string + } else if let dictionary = value as? [String: Any] { + rawValue = firstString(dictionary, keys: ["serviceTier", "tier", "value", "id"]) + } else { + return nil + } + return rawValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : OrbitCodexServiceTier(rawValue: rawValue) + } + + private static func firstString(_ dictionary: [String: Any], keys: [String]) -> String { + for key in keys { + if let value = dictionary[key] as? String { return value } + } + return "" + } + + private static func optionalString(_ dictionary: [String: Any], keys: [String]) -> String? { + let value = firstString(dictionary, keys: keys).trimmingCharacters(in: .whitespacesAndNewlines) + return value.isEmpty ? nil : value + } + + private static func humanizedIdentifier(_ identifier: String) -> String { + let components = identifier.split(separator: "-").map(String.init) + guard !components.isEmpty else { return identifier } + return components.enumerated().map { index, component in + if index == 0, component.caseInsensitiveCompare("gpt") == .orderedSame { return "GPT" } + if index == 1, Double(component) != nil { return "-\(component)" } + return " \(component.replacingOccurrences(of: "_", with: " ").capitalized)" + }.joined() + } + + private static func shortName(_ displayName: String) -> String { + let cleaned = displayName.replacingOccurrences(of: "GPT-", with: "") + .trimmingCharacters(in: .whitespacesAndNewlines) + return cleaned.isEmpty ? displayName : cleaned + } +} diff --git a/Orbit/OrbitCodexTransport.swift b/Orbit/OrbitCodexTransport.swift new file mode 100644 index 0000000..bbd40f7 --- /dev/null +++ b/Orbit/OrbitCodexTransport.swift @@ -0,0 +1,29 @@ +import Foundation + +/// Bounds app-server transport buffers away from the main actor. The provider +/// remains responsible for protocol state, while this actor owns byte framing. +actor OrbitCodexTransportActor { + private var standardOutput = OrbitBoundedDataBuffer(capacity: 4 * 1_024 * 1_024) + private var standardError = OrbitBoundedDataBuffer(capacity: 1 * 1_024 * 1_024) + + func ingestStandardOutput(_ data: Data) -> [Data] { + standardOutput.append(data) + var lines: [Data] = [] + while let line = standardOutput.popLine() { + if !line.isEmpty { + lines.append(line) + } + } + return lines + } + + func ingestStandardError(_ data: Data) -> Data { + standardError.append(data) + return standardError.data + } + + func reset() { + standardOutput.removeAll() + standardError.removeAll() + } +} diff --git a/Orbit/OrbitDesktopActuator.swift b/Orbit/OrbitDesktopActuator.swift deleted file mode 100644 index 89421d1..0000000 --- a/Orbit/OrbitDesktopActuator.swift +++ /dev/null @@ -1,311 +0,0 @@ -import AppKit -import Carbon.HIToolbox -import Foundation - -enum OrbitDesktopActionConfidence: String, Codable { - case high - case medium - case low -} - -enum OrbitDesktopActionKind: String, Codable { - case move - case click - case doubleClick - case rightClick - case typeText - case pressKey -} - -struct OrbitDesktopActionStep: Codable, Equatable { - let kind: OrbitDesktopActionKind - let x: Double? - let y: Double? - let screen: Int? - let label: String? - let text: String? - let key: String? -} - -struct OrbitDesktopActuationIntent: Codable, Equatable { - let confidence: OrbitDesktopActionConfidence - let steps: [OrbitDesktopActionStep] -} - -extension OrbitDesktopActionStep { - var imagePoint: CGPoint? { - guard let x, let y else { return nil } - return CGPoint(x: x, y: y) - } - - var previewLabel: String { - if let label, !label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - return label - } - - switch kind { - case .move: - return "moving to the next control." - case .click: - return "about to click here." - case .doubleClick: - return "about to double-click here." - case .rightClick: - return "about to open the menu here." - case .typeText: - return "about to type here." - case .pressKey: - return "about to press \(key ?? "a shortcut")." - } - } - - var executionLabel: String { - switch kind { - case .move: - return "moving the pointer." - case .click: - return "clicking here." - case .doubleClick: - return "double-clicking here." - case .rightClick: - return "opening the context menu." - case .typeText: - return "typing into the focused target." - case .pressKey: - return "pressing \(key ?? "the shortcut")." - } - } -} - -enum OrbitDesktopActuatorError: LocalizedError { - case missingCoordinate - case missingText - case missingKey - case unsupportedKey(String) - case eventCreationFailed - - var errorDescription: String? { - switch self { - case .missingCoordinate: - return "Orbit could not resolve the desktop target coordinates." - case .missingText: - return "Orbit needs text before it can type into the current target." - case .missingKey: - return "Orbit needs a key combination before it can press it." - case .unsupportedKey(let key): - return "Orbit doesn't know how to press \(key) yet." - case .eventCreationFailed: - return "Orbit couldn't create the desktop input event." - } - } -} - -@MainActor -final class OrbitDesktopActuator { - private let eventSource = CGEventSource(stateID: .hidSystemState) - - func perform( - _ step: OrbitDesktopActionStep, - at globalLocation: CGPoint? - ) async throws { - switch step.kind { - case .move: - guard let globalLocation else { throw OrbitDesktopActuatorError.missingCoordinate } - try movePointer(to: globalLocation) - case .click: - guard let globalLocation else { throw OrbitDesktopActuatorError.missingCoordinate } - try click(button: .left, at: globalLocation, clickState: 1) - case .doubleClick: - guard let globalLocation else { throw OrbitDesktopActuatorError.missingCoordinate } - try click(button: .left, at: globalLocation, clickState: 2) - case .rightClick: - guard let globalLocation else { throw OrbitDesktopActuatorError.missingCoordinate } - try click(button: .right, at: globalLocation, clickState: 1) - case .typeText: - if let globalLocation { - try click(button: .left, at: globalLocation, clickState: 1) - try await Task.sleep(nanoseconds: 120_000_000) - } - guard let text = step.text, !text.isEmpty else { throw OrbitDesktopActuatorError.missingText } - try typeText(text) - case .pressKey: - guard let keyString = step.key, !keyString.isEmpty else { throw OrbitDesktopActuatorError.missingKey } - try pressKeyCombo(keyString) - } - } - - private func movePointer(to globalLocation: CGPoint) throws { - guard let event = CGEvent(mouseEventSource: eventSource, mouseType: .mouseMoved, mouseCursorPosition: globalLocation, mouseButton: .left) else { - throw OrbitDesktopActuatorError.eventCreationFailed - } - event.post(tap: .cghidEventTap) - } - - private func click(button: CGMouseButton, at globalLocation: CGPoint, clickState: Int64) throws { - try movePointer(to: globalLocation) - - let downType: CGEventType - let upType: CGEventType - switch button { - case .left: - downType = .leftMouseDown - upType = .leftMouseUp - case .right: - downType = .rightMouseDown - upType = .rightMouseUp - case .center: - downType = .otherMouseDown - upType = .otherMouseUp - @unknown default: - downType = .leftMouseDown - upType = .leftMouseUp - } - - guard let downEvent = CGEvent(mouseEventSource: eventSource, mouseType: downType, mouseCursorPosition: globalLocation, mouseButton: button), - let upEvent = CGEvent(mouseEventSource: eventSource, mouseType: upType, mouseCursorPosition: globalLocation, mouseButton: button) else { - throw OrbitDesktopActuatorError.eventCreationFailed - } - - downEvent.setIntegerValueField(.mouseEventClickState, value: clickState) - upEvent.setIntegerValueField(.mouseEventClickState, value: clickState) - downEvent.post(tap: .cghidEventTap) - upEvent.post(tap: .cghidEventTap) - - if clickState == 2 { - try awaitTinyDelay() - downEvent.post(tap: .cghidEventTap) - upEvent.post(tap: .cghidEventTap) - } - } - - private func awaitTinyDelay() throws { - Thread.sleep(forTimeInterval: 0.05) - } - - private func typeText(_ text: String) throws { - guard let downEvent = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: true), - let upEvent = CGEvent(keyboardEventSource: eventSource, virtualKey: 0, keyDown: false) else { - throw OrbitDesktopActuatorError.eventCreationFailed - } - - let utf16 = Array(text.utf16) - downEvent.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: utf16) - upEvent.keyboardSetUnicodeString(stringLength: utf16.count, unicodeString: utf16) - downEvent.post(tap: .cghidEventTap) - upEvent.post(tap: .cghidEventTap) - } - - private func pressKeyCombo(_ combo: String) throws { - let parts = combo - .split(separator: "+") - .map { $0.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() } - .filter { !$0.isEmpty } - - guard let last = parts.last else { - throw OrbitDesktopActuatorError.missingKey - } - - let modifiers = parts.dropLast().reduce(CGEventFlags()) { flags, token in - flags.union(modifierFlag(for: token)) - } - - let keyCode = try keyCode(for: last) - - guard let downEvent = CGEvent(keyboardEventSource: eventSource, virtualKey: keyCode, keyDown: true), - let upEvent = CGEvent(keyboardEventSource: eventSource, virtualKey: keyCode, keyDown: false) else { - throw OrbitDesktopActuatorError.eventCreationFailed - } - - downEvent.flags = modifiers - upEvent.flags = modifiers - downEvent.post(tap: .cghidEventTap) - upEvent.post(tap: .cghidEventTap) - } - - private func modifierFlag(for token: String) -> CGEventFlags { - switch token { - case "command", "cmd": - return .maskCommand - case "shift": - return .maskShift - case "option", "alt": - return .maskAlternate - case "control", "ctrl": - return .maskControl - case "fn", "function": - return .maskSecondaryFn - default: - return [] - } - } - - private func keyCode(for token: String) throws -> CGKeyCode { - if let single = token.unicodeScalars.first, token.count == 1 { - switch Character(String(single).lowercased()) { - case "a": return CGKeyCode(kVK_ANSI_A) - case "b": return CGKeyCode(kVK_ANSI_B) - case "c": return CGKeyCode(kVK_ANSI_C) - case "d": return CGKeyCode(kVK_ANSI_D) - case "e": return CGKeyCode(kVK_ANSI_E) - case "f": return CGKeyCode(kVK_ANSI_F) - case "g": return CGKeyCode(kVK_ANSI_G) - case "h": return CGKeyCode(kVK_ANSI_H) - case "i": return CGKeyCode(kVK_ANSI_I) - case "j": return CGKeyCode(kVK_ANSI_J) - case "k": return CGKeyCode(kVK_ANSI_K) - case "l": return CGKeyCode(kVK_ANSI_L) - case "m": return CGKeyCode(kVK_ANSI_M) - case "n": return CGKeyCode(kVK_ANSI_N) - case "o": return CGKeyCode(kVK_ANSI_O) - case "p": return CGKeyCode(kVK_ANSI_P) - case "q": return CGKeyCode(kVK_ANSI_Q) - case "r": return CGKeyCode(kVK_ANSI_R) - case "s": return CGKeyCode(kVK_ANSI_S) - case "t": return CGKeyCode(kVK_ANSI_T) - case "u": return CGKeyCode(kVK_ANSI_U) - case "v": return CGKeyCode(kVK_ANSI_V) - case "w": return CGKeyCode(kVK_ANSI_W) - case "x": return CGKeyCode(kVK_ANSI_X) - case "y": return CGKeyCode(kVK_ANSI_Y) - case "z": return CGKeyCode(kVK_ANSI_Z) - case "0": return CGKeyCode(kVK_ANSI_0) - case "1": return CGKeyCode(kVK_ANSI_1) - case "2": return CGKeyCode(kVK_ANSI_2) - case "3": return CGKeyCode(kVK_ANSI_3) - case "4": return CGKeyCode(kVK_ANSI_4) - case "5": return CGKeyCode(kVK_ANSI_5) - case "6": return CGKeyCode(kVK_ANSI_6) - case "7": return CGKeyCode(kVK_ANSI_7) - case "8": return CGKeyCode(kVK_ANSI_8) - case "9": return CGKeyCode(kVK_ANSI_9) - default: - break - } - } - - switch token { - case "return", "enter": - return CGKeyCode(kVK_Return) - case "tab": - return CGKeyCode(kVK_Tab) - case "space": - return CGKeyCode(kVK_Space) - case "escape", "esc": - return CGKeyCode(kVK_Escape) - case "delete", "backspace": - return CGKeyCode(kVK_Delete) - case "forwarddelete": - return CGKeyCode(kVK_ForwardDelete) - case "left": - return CGKeyCode(kVK_LeftArrow) - case "right": - return CGKeyCode(kVK_RightArrow) - case "up": - return CGKeyCode(kVK_UpArrow) - case "down": - return CGKeyCode(kVK_DownArrow) - default: - throw OrbitDesktopActuatorError.unsupportedKey(token) - } - } -} diff --git a/Orbit/OrbitDictationManager.swift b/Orbit/OrbitDictationManager.swift index 546da4d..2680c17 100644 --- a/Orbit/OrbitDictationManager.swift +++ b/Orbit/OrbitDictationManager.swift @@ -7,8 +7,8 @@ // transcription provider, and hands the final draft back to the active input bar. // -import AppKit import AVFoundation +import AppKit import Combine import Foundation import Speech @@ -93,7 +93,7 @@ enum OrbitPushToTalkShortcut { } static let currentShortcutOption: ShortcutOption = .controlOption - static let pushToTalkKeyCode: UInt16 = 49 // Space + static let pushToTalkKeyCode: UInt16 = 49 // Space static let pushToTalkDisplayText = currentShortcutOption.displayText static let pushToTalkTooltipText = "push to talk (\(pushToTalkDisplayText))" @@ -185,13 +185,15 @@ enum OrbitPushToTalkShortcut { if shortcutEventType == .keyDown && keyCode == pushToTalkKeyCode && matchesModifierFlags - && !wasShortcutPreviouslyPressed { + && !wasShortcutPreviouslyPressed + { return .pressed } if shortcutEventType == .keyUp && keyCode == pushToTalkKeyCode - && wasShortcutPreviouslyPressed { + && wasShortcutPreviouslyPressed + { return .released } @@ -321,7 +323,8 @@ final class OrbitDictationManager: NSObject, ObservableObject { currentDraftText: currentDraftText, updateDraftText: updateDraftText, submitDraftText: submitDraftText, - shouldAutomaticallySubmitFinalDraftOnStop: currentDraftText + shouldAutomaticallySubmitFinalDraftOnStop: + currentDraftText .trimmingCharacters(in: .whitespacesAndNewlines) .isEmpty ) @@ -499,7 +502,8 @@ final class OrbitDictationManager: NSObject, ObservableObject { isRecordingFromKeyboardShortcut = false isFinalizingTranscript = true - let finalTranscriptFallbackDelaySeconds = activeTranscriptionSession?.finalTranscriptFallbackDelaySeconds + let finalTranscriptFallbackDelaySeconds = + activeTranscriptionSession?.finalTranscriptFallbackDelaySeconds ?? Self.defaultFinalTranscriptFallbackDelaySeconds audioEngine.stop() @@ -558,6 +562,10 @@ final class OrbitDictationManager: NSObject, ObservableObject { print("🎙️ OrbitDictationManager: provider ready, starting audio engine") let inputNode = audioEngine.inputNode + try OrbitAudioInputCatalog.applySelectedDevice( + uid: OrbitSettings.shared.microphoneDeviceUID, + to: inputNode + ) let inputFormat = inputNode.outputFormat(forBus: 0) inputNode.removeTap(onBus: 0) @@ -623,7 +631,8 @@ final class OrbitDictationManager: NSObject, ObservableObject { return draftTextBeforeCurrentDictation } - let trimmedExistingDraftText = draftTextBeforeCurrentDictation + let trimmedExistingDraftText = + draftTextBeforeCurrentDictation .trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmedExistingDraftText.isEmpty else { @@ -676,7 +685,7 @@ final class OrbitDictationManager: NSObject, ObservableObject { "SwiftUI", "Xcode", "ScreenCaptureKit", - "localhost" + "localhost", ] let combinedKeyterms = baseKeyterms + contextualKeyterms @@ -726,7 +735,8 @@ final class OrbitDictationManager: NSObject, ObservableObject { let now = Date() if now.timeIntervalSince(self.lastRecordedAudioPowerSampleDate) - >= Self.recordedAudioPowerHistorySampleIntervalSeconds { + >= Self.recordedAudioPowerHistorySampleIntervalSeconds + { self.lastRecordedAudioPowerSampleDate = now self.appendRecordedAudioPowerSample( max(CGFloat(boostedLevel), Self.recordedAudioPowerHistoryBaselineLevel) @@ -785,7 +795,8 @@ final class OrbitDictationManager: NSObject, ObservableObject { // macOS can briefly report .notDetermined even after the user tapped Allow, // so we trust the cached result for a short window. if let lastPermissionRequestCompletedAt, - Date().timeIntervalSince(lastPermissionRequestCompletedAt) < 1.0 { + Date().timeIntervalSince(lastPermissionRequestCompletedAt) < 1.0 + { return AVCaptureDevice.authorizationStatus(for: .audio) != .denied && AVCaptureDevice.authorizationStatus(for: .audio) != .restricted } @@ -864,15 +875,17 @@ final class OrbitDictationManager: NSObject, ObservableObject { private func userFacingErrorMessage(from error: Error, fallback: String) -> String { if let localizedError = error as? LocalizedError, - let errorDescription = localizedError.errorDescription? - .trimmingCharacters(in: .whitespacesAndNewlines), - !errorDescription.isEmpty { + let errorDescription = localizedError.errorDescription? + .trimmingCharacters(in: .whitespacesAndNewlines), + !errorDescription.isEmpty + { return errorDescription } let errorDescription = error.localizedDescription.trimmingCharacters(in: .whitespacesAndNewlines) if !errorDescription.isEmpty, - errorDescription != "The operation couldn’t be completed." { + errorDescription != "The operation couldn’t be completed." + { return errorDescription } diff --git a/Orbit/OrbitManager.swift b/Orbit/OrbitManager.swift index 603c3c1..68a578a 100644 --- a/Orbit/OrbitManager.swift +++ b/Orbit/OrbitManager.swift @@ -22,6 +22,7 @@ enum OrbitVoiceState { enum OrbitSetupStage: Equatable { case permissions + case automationDisclosure case auth case voiceChoice case cloudKey @@ -33,6 +34,7 @@ enum OrbitSetupStage: Equatable { @MainActor final class OrbitManager: ObservableObject { let settings = OrbitSettings.shared + lazy var permissionCoordinator = OrbitPermissionCoordinator(orbitManager: self) @Published private(set) var voiceState: OrbitVoiceState = .idle @Published private(set) var lastTranscript: String? @@ -45,6 +47,11 @@ final class OrbitManager: ObservableObject { @Published private(set) var textToSpeechProviderDisplayName: String = "" @Published private(set) var availableAppleVoices: [OrbitAppleVoiceOption] = [] @Published private(set) var selectedAppleVoiceSummary: String = "Auto" + @Published private(set) var isPreviewingAppleVoice = false + @Published private(set) var availableMicrophones: [OrbitAudioInputDevice] = [] + @Published private(set) var microphoneTestLevel: CGFloat = 0 + @Published private(set) var isTestingMicrophone = false + @Published private(set) var microphoneDeviceNotice: String? @Published private(set) var openAICloudCredentialState: OrbitOpenAICloudCredentialState = .missing /// Screen location (global AppKit coords) of a detected UI element the @@ -65,9 +72,9 @@ final class OrbitManager: ObservableObject { @Published private(set) var recentActionUpdates: [String] = [] @Published private(set) var showCodexActivityOverlay: Bool = false @Published private(set) var codexDebugEvents: [String] = [] - @Published private(set) var codexCollaborationModes: [String] = [] - @Published private(set) var codexExperimentalFeatures: [String] = [] + @Published private(set) var codexSubagentActivities: [OrbitSubagentActivity] = [] @Published private(set) var codexActiveTurnSummary: String? + @Published private(set) var codexModelMigrationNotice: String? @Published private(set) var pendingToolPrompt: OrbitToolPrompt? @Published var onboardingPromptText: String = "" @@ -81,12 +88,12 @@ final class OrbitManager: ObservableObject { private var textToSpeechProvider: any TextToSpeechProvider private let fallbackTextToSpeechProvider: any TextToSpeechProvider - private let desktopActuator = OrbitDesktopActuator() private let actionProvider = CodexAppServerActionProvider() private var lastCodexScreenCapture: OrbitScreenCapture? + private var activeCaptureLease: OrbitTemporaryCaptureLease? + private let microphoneLevelMonitor = OrbitMicrophoneLevelMonitor() private var currentResponseTask: Task? - private var desktopActuationTask: Task? private var shortcutTransitionCancellable: AnyCancellable? private var voiceStateCancellable: AnyCancellable? @@ -118,11 +125,11 @@ final class OrbitManager: ObservableObject { } var hasUsableScreenAccessPermission: Bool { - hasScreenContentPermission || hasScreenRecordingPermission + hasScreenContentPermission } var canInterruptCodexAction: Bool { - actionProvider.canInterruptCurrentAction || desktopActuationTask != nil + actionProvider.canInterruptCurrentAction } var setupStage: OrbitSetupStage { @@ -130,6 +137,10 @@ final class OrbitManager: ObservableObject { return .permissions } + if !hasAcknowledgedUnrestrictedAutomation { + return .automationDisclosure + } + switch codexAuthState { case .authenticated: break @@ -185,6 +196,17 @@ final class OrbitManager: ObservableObject { set { UserDefaults.standard.set(newValue, forKey: "hasCompletedOnboarding") } } + var hasAcknowledgedUnrestrictedAutomation: Bool { + get { UserDefaults.standard.bool(forKey: "orbit.hasAcknowledgedUnrestrictedAutomation") } + set { UserDefaults.standard.set(newValue, forKey: "orbit.hasAcknowledgedUnrestrictedAutomation") } + } + + func acknowledgeUnrestrictedAutomation() { + hasAcknowledgedUnrestrictedAutomation = true + synchronizeSetupState(allowAutomaticOnboarding: false) + objectWillChange.send() + } + var hasSeenSetupComplete: Bool { get { if UserDefaults.standard.object(forKey: "hasSeenSetupComplete") == nil { @@ -214,8 +236,11 @@ final class OrbitManager: ObservableObject { self.textToSpeechProvider = OrbitTTSProviderFactory.makePrimaryProvider(for: OrbitSettings.shared.voicePreset) self.fallbackTextToSpeechProvider = OrbitTTSProviderFactory.makeFallbackProvider() self.textToSpeechProviderDisplayName = textToSpeechProvider.displayName - self.availableAppleVoices = [] - self.selectedAppleVoiceSummary = "System Default" + self.availableAppleVoices = OrbitAppleVoiceCatalog.availableVoices() + self.selectedAppleVoiceSummary = OrbitAppleVoiceCatalog.currentSelectionSummary( + preferredIdentifier: OrbitSettings.shared.appleTTSVoiceIdentifier + ) + self.availableMicrophones = OrbitAudioInputCatalog.devices() self.codexSessionSummary = actionProvider.sessionStatusSummary self.codexConfigurationSummary = actionProvider.configurationSummary self.availableCodexModels = actionProvider.availableModels @@ -223,10 +248,12 @@ final class OrbitManager: ObservableObject { self.codexAuthState = actionProvider.authState self.codexAccountSummary = actionProvider.accountSummary self.codexDebugEvents = actionProvider.debugEvents - self.codexCollaborationModes = actionProvider.collaborationModes - self.codexExperimentalFeatures = actionProvider.experimentalFeatures + self.codexSubagentActivities = actionProvider.subagentActivities self.codexActiveTurnSummary = actionProvider.activeTurnSummary self.openAICloudCredentialState = OrbitOpenAIKeychainStore.resolvedAPIKey().map { .connected(source: $0.source) } ?? .missing + Task { + await OrbitTemporaryCaptureLease.sweepStaleCaptures() + } self.actionProvider.stateDidChange = { [weak self] in Task { @MainActor [weak self] in self?.refreshActionProviderPresentation() @@ -237,7 +264,9 @@ final class OrbitManager: ObservableObject { func start() { refreshAllPermissions() - print("🪐 Orbit start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)") + print( + "🪐 Orbit start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)" + ) startPermissionPolling() bindVoiceStateObservation() bindAudioPowerLevel() @@ -282,9 +311,11 @@ final class OrbitManager: ObservableObject { currentResponseTask?.cancel() currentResponseTask = nil - desktopActuationTask?.cancel() - desktopActuationTask = nil actionProvider.cancelCurrentAction() + permissionCoordinator.dismissGuide() + microphoneLevelMonitor.stop() + isTestingMicrophone = false + releaseTemporaryCapture() shortcutTransitionCancellable?.cancel() voiceStateCancellable?.cancel() audioPowerCancellable?.cancel() @@ -324,8 +355,11 @@ final class OrbitManager: ObservableObject { // Debug: log permission state on changes if previouslyHadAccessibility != hasAccessibilityPermission || previouslyHadScreenRecording != hasScreenRecordingPermission - || previouslyHadMicrophone != hasMicrophonePermission { - print("🔑 Permissions — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)") + || previouslyHadMicrophone != hasMicrophonePermission + { + print( + "🔑 Permissions — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission)" + ) } // Track individual permission grants as they happen @@ -573,11 +607,69 @@ final class OrbitManager: ObservableObject { fallbackTextToSpeechProvider.stopPlayback() textToSpeechProvider = OrbitTTSProviderFactory.makePrimaryProvider(for: settings.voicePreset) textToSpeechProviderDisplayName = textToSpeechProvider.displayName - availableAppleVoices = [] - selectedAppleVoiceSummary = "System Default" + availableAppleVoices = OrbitAppleVoiceCatalog.availableVoices() + selectedAppleVoiceSummary = OrbitAppleVoiceCatalog.currentSelectionSummary( + preferredIdentifier: settings.appleTTSVoiceIdentifier + ) orbitDictationManager.refreshConfiguredProviders() } + func selectAppleVoice(_ identifier: String) { + settings.appleTTSVoiceIdentifier = identifier + } + + func toggleAppleVoicePreview() { + if isPreviewingAppleVoice { + textToSpeechProvider.stopPlayback() + isPreviewingAppleVoice = false + return + } + guard settings.voicePreset == .localVoice else { return } + isPreviewingAppleVoice = true + Task { [weak self] in + guard let self else { return } + defer { isPreviewingAppleVoice = false } + try? await textToSpeechProvider.speakText("Orbit is ready. This voice stays on your Mac.") + } + } + + func refreshMicrophones() { + availableMicrophones = OrbitAudioInputCatalog.devices() + let selectedUID = settings.microphoneDeviceUID.trimmingCharacters(in: .whitespacesAndNewlines) + microphoneDeviceNotice = + !selectedUID.isEmpty && !availableMicrophones.contains(where: { $0.uid == selectedUID }) + ? "Selected microphone disconnected. Orbit will not silently switch devices." + : nil + } + + func selectMicrophone(_ uid: String) { + microphoneLevelMonitor.stop() + isTestingMicrophone = false + microphoneTestLevel = 0 + settings.microphoneDeviceUID = uid + refreshMicrophones() + } + + func toggleMicrophoneTest() { + if isTestingMicrophone { + microphoneLevelMonitor.stop() + microphoneTestLevel = 0 + isTestingMicrophone = false + return + } + refreshMicrophones() + do { + try microphoneLevelMonitor.start(deviceUID: settings.microphoneDeviceUID) { [weak self] level in + self?.microphoneTestLevel = level + } + isTestingMicrophone = true + } catch { + microphoneDeviceNotice = error.localizedDescription + microphoneLevelMonitor.stop() + isTestingMicrophone = false + } + } + private func refreshCloudCredentialState() { if let resolvedKey = OrbitOpenAIKeychainStore.resolvedAPIKey() { openAICloudCredentialState = .connected(source: resolvedKey.source) @@ -628,7 +720,7 @@ final class OrbitManager: ObservableObject { self.onboardingPromptText = "" } } - + OrbitAnalytics.trackPushToTalkStarted() pendingKeyboardShortcutStartTask?.cancel() @@ -666,8 +758,7 @@ final class OrbitManager: ObservableObject { private func submitTranscriptToActionProvider(transcript: String) { currentResponseTask?.cancel() - desktopActuationTask?.cancel() - desktopActuationTask = nil + releaseTemporaryCapture() textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() isRunningOnboardingTour = false @@ -695,21 +786,23 @@ final class OrbitManager: ObservableObject { do { request = try await prepareUnifiedCodexRequest(transcript: transcript) } catch is CancellationError { + releaseTemporaryCapture() return } catch { - let fallbackRequest = OrbitActionRequest( - transcript: transcript, - screenshotPath: nil, - screenshotLabel: nil, - cursorPointInImagePixels: nil, - imagePixelSize: nil, - screenNumber: nil, - frontmostApplicationName: currentFrontmostApplicationName(), - frontmostWindowTitle: currentFocusedWindowTitle() - ) - activeActionDetailLine = "continuing without screen context." - appendActionUpdate("continuing without screen context") - request = fallbackRequest + releaseTemporaryCapture() + hasScreenContentPermission = false + UserDefaults.standard.removeObject(forKey: "hasScreenContentPermission") + WindowPositionManager.clearPreviouslyConfirmedScreenRecordingPermission() + let message = "Orbit could not capture the current screen, so this request was not sent. Check Screen Recording access and try again." + activeActionProgress = OrbitActionProgress(phase: .failed, detail: message, rawSource: error.localizedDescription) + activeActionStatus = .failed(message) + activeActionStatusSummary = OrbitActionPhase.failed.summaryText + activeActionDetailLine = "Screen capture failed. Try again." + appendActionUpdate("request blocked because screen capture failed") + voiceState = .idle + showCodexActivityOverlayCard() + scheduleCodexActivityOverlayDismiss() + return } await actionProvider.submitActionRequest( @@ -741,13 +834,16 @@ final class OrbitManager: ObservableObject { activeActionDetailLine = prompt.title appendActionUpdate(prompt.title) showCodexActivityOverlayCard() + case .subagentActivity(let activities): + codexSubagentActivities = activities + activeActionDetailLine = activities.last.map { "Team-up: \($0.agentPath) · \($0.status)" } + showCodexActivityOverlayCard() case .interrupted(let summary): + releaseTemporaryCapture() let spokenSummary = conciseDetailLine(from: summary.isEmpty ? "stopped." : summary) cancelActionAcknowledgementFlow() textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() - desktopActuationTask?.cancel() - desktopActuationTask = nil activeActionProgress = OrbitActionProgress( phase: .interrupted, detail: spokenSummary, @@ -763,17 +859,17 @@ final class OrbitManager: ObservableObject { scheduleCodexActivityOverlayDismiss() scheduleTransientHideIfNeeded() case .completed(let summary): + releaseTemporaryCapture() cancelActionAcknowledgementFlow() pendingToolPrompt = nil handleCompletedCodexSummary(summary) case .failed(let errorMessage): + releaseTemporaryCapture() let shortDetail = conciseDetailLine(from: errorMessage) cancelActionAcknowledgementFlow() textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() pendingToolPrompt = nil - desktopActuationTask?.cancel() - desktopActuationTask = nil activeActionProgress = OrbitActionProgress( phase: .failed, detail: shortDetail, @@ -840,10 +936,11 @@ final class OrbitManager: ObservableObject { lastCodexScreenCapture = activeScreenCapture let labeledCapture = buildPrimaryScreenLabel(for: activeScreenCapture) let cursorPoint = currentCursorPointInScreenshotPixels(for: activeScreenCapture) - let screenshotPath = try writeCodexScreenshotToTemporaryFile(labeledCapture.data) + let lease = try await OrbitTemporaryCaptureLease.create(data: labeledCapture.data) + activeCaptureLease = lease return OrbitActionRequest( transcript: transcript, - screenshotPath: screenshotPath, + screenshotPath: lease.fileURL.path, screenshotLabel: labeledCapture.label, cursorPointInImagePixels: cursorPoint, imagePixelSize: CGSize( @@ -860,38 +957,28 @@ final class OrbitManager: ObservableObject { } } - private func writeCodexScreenshotToTemporaryFile(_ data: Data) throws -> String { - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("orbit-codex-\(UUID().uuidString)") - .appendingPathExtension("jpg") - try data.write(to: temporaryURL, options: .atomic) - return temporaryURL.path + private func releaseTemporaryCapture() { + let lease = activeCaptureLease + activeCaptureLease = nil + guard let lease else { return } + Task.detached(priority: .utility) { + lease.release() + } } private func applyCodexPointDirective(_ parseResult: PointingParseResult) { guard let coordinate = parseResult.coordinate, - let resolvedTarget = resolveGlobalScreenTarget( + let resolvedTarget = resolveGlobalScreenTarget( fromImagePoint: coordinate, screenNumber: parseResult.screenNumber - ) else { return } + ) + else { return } detectedElementScreenLocation = resolvedTarget.location detectedElementDisplayFrame = resolvedTarget.displayFrame detectedElementBubbleText = parseResult.elementLabel } - private func applyDesktopPreview(for step: OrbitDesktopActionStep) { - guard let coordinate = step.imagePoint, - let resolvedTarget = resolveGlobalScreenTarget( - fromImagePoint: coordinate, - screenNumber: step.screen - ) else { return } - - detectedElementScreenLocation = resolvedTarget.location - detectedElementDisplayFrame = resolvedTarget.displayFrame - detectedElementBubbleText = step.previewLabel - } - private func currentCursorPointInScreenshotPixels( for capture: OrbitScreenCapture ) -> CGPoint? { @@ -951,118 +1038,6 @@ final class OrbitManager: ObservableObject { return OrbitResolvedScreenTarget(location: globalLocation, displayFrame: displayFrame) } - private func desktopPreviewDelay(for step: OrbitDesktopActionStep, target: CGPoint?) -> TimeInterval { - guard step.imagePoint != nil, let target else { - return 0.28 - } - - let mouseLocation = NSEvent.mouseLocation - let distance = hypot(target.x - mouseLocation.x, target.y - mouseLocation.y) - let flightDurationSeconds = min(max(distance / 800.0, 0.6), 1.4) - return flightDurationSeconds + 0.14 - } - - private func beginDesktopActuation( - _ intent: OrbitDesktopActuationIntent, - spokenSummary: String, - rawSummary: String - ) { - desktopActuationTask?.cancel() - voiceState = .processing - showCodexActivityOverlayCard() - appendActionUpdate("desktop action ready") - - desktopActuationTask = Task { @MainActor [weak self] in - guard let self else { return } - defer { - self.desktopActuationTask = nil - self.refreshActionProviderPresentation() - } - - do { - for step in intent.steps { - try Task.checkCancellation() - - let resolvedTarget = step.imagePoint.flatMap { - self.resolveGlobalScreenTarget(fromImagePoint: $0, screenNumber: step.screen) - } - - let previewLabel = step.previewLabel - self.activeActionProgress = OrbitActionProgress( - phase: .previewingAction, - detail: previewLabel, - rawSource: rawSummary - ) - self.activeActionStatus = .running - self.activeActionStatusSummary = OrbitActionPhase.previewingAction.summaryText - self.activeActionDetailLine = previewLabel - self.appendActionUpdate(previewLabel) - if step.imagePoint != nil { - self.applyDesktopPreview(for: step) - } - - try await Task.sleep(nanoseconds: UInt64(self.desktopPreviewDelay(for: step, target: resolvedTarget?.location) * 1_000_000_000)) - try Task.checkCancellation() - - let executionLabel = step.executionLabel - self.activeActionProgress = OrbitActionProgress( - phase: .executingDesktopAction, - detail: executionLabel, - rawSource: rawSummary - ) - self.activeActionStatus = .running - self.activeActionStatusSummary = OrbitActionPhase.executingDesktopAction.summaryText - self.activeActionDetailLine = executionLabel - self.appendActionUpdate(executionLabel) - - try await self.desktopActuator.perform(step, at: resolvedTarget?.location) - try await Task.sleep(nanoseconds: 180_000_000) - } - - let shortDetail = self.conciseDetailLine(from: spokenSummary) - self.activeActionProgress = OrbitActionProgress( - phase: .done, - detail: shortDetail, - rawSource: rawSummary - ) - self.activeActionStatus = .completed(spokenSummary) - self.activeActionStatusSummary = OrbitActionPhase.done.summaryText - self.activeActionDetailLine = shortDetail - self.appendActionUpdate(OrbitActionPhase.done.summaryText) - self.showCodexActivityOverlayCard() - self.scheduleCodexActivityOverlayDismiss() - await self.speakCompletionText(spokenSummary, fallback: nil) - } catch is CancellationError { - self.activeActionProgress = OrbitActionProgress( - phase: .interrupted, - detail: "desktop action interrupted.", - rawSource: rawSummary - ) - self.activeActionStatus = .interrupted("desktop action interrupted.") - self.activeActionStatusSummary = OrbitActionPhase.interrupted.summaryText - self.activeActionDetailLine = "desktop action interrupted." - self.appendActionUpdate("desktop action interrupted") - self.showCodexActivityOverlayCard() - self.scheduleCodexActivityOverlayDismiss() - self.voiceState = .idle - } catch { - let detail = self.conciseDetailLine(from: error.localizedDescription) - self.activeActionProgress = OrbitActionProgress( - phase: .failed, - detail: detail, - rawSource: rawSummary - ) - self.activeActionStatus = .failed(error.localizedDescription) - self.activeActionStatusSummary = OrbitActionPhase.failed.summaryText - self.activeActionDetailLine = detail - self.appendActionUpdate("desktop action failed") - self.showCodexActivityOverlayCard() - self.scheduleCodexActivityOverlayDismiss() - await self.speakCompletionText(nil, fallback: error.localizedDescription) - } - } - } - private func refreshActionProviderPresentation() { availableCodexModels = actionProvider.availableModels reconcileCodexSelectionIfNeeded() @@ -1070,8 +1045,7 @@ final class OrbitManager: ObservableObject { codexAuthState = actionProvider.authState codexAccountSummary = actionProvider.accountSummary codexDebugEvents = actionProvider.debugEvents - codexCollaborationModes = actionProvider.collaborationModes - codexExperimentalFeatures = actionProvider.experimentalFeatures + codexSubagentActivities = actionProvider.subagentActivities codexActiveTurnSummary = actionProvider.activeTurnSummary codexSessionSummary = actionProvider.sessionStatusSummary codexConfigurationSummary = actionProvider.configurationSummary @@ -1081,22 +1055,41 @@ final class OrbitManager: ObservableObject { private func reconcileCodexSelectionIfNeeded() { let currentModel = settings.codexActionModel.trimmingCharacters(in: .whitespacesAndNewlines) if !availableCodexModels.contains(where: { $0.model == currentModel }), - let preferredModel = availableCodexModels.first(where: { $0.isDefault })?.model ?? availableCodexModels.first?.model { + let preferredModel = availableCodexModels.first(where: { $0.isDefault })?.model ?? availableCodexModels.first?.model + { settings.codexActionModel = preferredModel + if !currentModel.isEmpty, + !UserDefaults.standard.bool(forKey: "orbit.hasExplainedModelMigration") + { + codexModelMigrationNotice = "\(currentModel) is no longer available. Orbit selected the current server default." + UserDefaults.standard.set(true, forKey: "orbit.hasExplainedModelMigration") + } } - let supportedEfforts = availableCodexModels + let supportedEfforts = + availableCodexModels .first(where: { $0.model == settings.codexActionModel })? .supportedEfforts ?? availableCodexEfforts guard !supportedEfforts.isEmpty else { return } if !supportedEfforts.contains(settings.codexReasoningEffort) { - settings.codexReasoningEffort = availableCodexModels + settings.codexReasoningEffort = + availableCodexModels .first(where: { $0.model == settings.codexActionModel })? .defaultEffort ?? supportedEfforts.first ?? .low } + + let supportedTiers = + availableCodexModels + .first(where: { $0.model == settings.codexActionModel })? + .supportedServiceTiers ?? [] + if !settings.codexServiceTier.rawValue.isEmpty, + !supportedTiers.contains(settings.codexServiceTier) + { + settings.codexServiceTier = .serverDefault + } } private func resetActionPresentationForNewRequest() { @@ -1130,7 +1123,8 @@ final class OrbitManager: ObservableObject { private func handleEarlyActionCommentary(_ commentary: String) { guard !hasSpokenActionAcknowledgement, - let acknowledgement = normalizedEarlyAcknowledgement(from: commentary) else { + let acknowledgement = normalizedEarlyAcknowledgement(from: commentary) + else { return } @@ -1167,8 +1161,9 @@ final class OrbitManager: ObservableObject { private func waitForCurrentSpeechToSettle(maximumWait: TimeInterval) async { let deadline = Date().addingTimeInterval(maximumWait) - while (textToSpeechProvider.isPlaying || fallbackTextToSpeechProvider.isPlaying), - Date() < deadline { + while textToSpeechProvider.isPlaying || fallbackTextToSpeechProvider.isPlaying, + Date() < deadline + { try? await Task.sleep(nanoseconds: 120_000_000) } @@ -1191,7 +1186,8 @@ final class OrbitManager: ObservableObject { } private func normalizedEarlyAcknowledgement(from text: String) -> String? { - let cleaned = text + let cleaned = + text .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) .replacingOccurrences(of: #"\[POINT:[^\]]+\]"#, with: "", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) @@ -1207,7 +1203,8 @@ final class OrbitManager: ObservableObject { } let prefix = String(candidate.prefix(69)) - let trimmed = prefix + let trimmed = + prefix .replacingOccurrences(of: "\\s+\\S*$", with: "", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { return nil } @@ -1215,7 +1212,8 @@ final class OrbitManager: ObservableObject { } private func conciseDetailLine(from text: String) -> String { - let cleaned = text + let cleaned = + text .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) guard cleaned.count > 72 else { return cleaned } @@ -1229,21 +1227,24 @@ final class OrbitManager: ObservableObject { private func currentFocusedWindowTitle() -> String? { guard hasAccessibilityPermission, - let frontApp = NSWorkspace.shared.frontmostApplication, - frontApp.processIdentifier != ProcessInfo.processInfo.processIdentifier else { + let frontApp = NSWorkspace.shared.frontmostApplication, + frontApp.processIdentifier != ProcessInfo.processInfo.processIdentifier + else { return nil } let appElement = AXUIElementCreateApplication(frontApp.processIdentifier) var focusedWindowValue: AnyObject? guard AXUIElementCopyAttributeValue(appElement, kAXFocusedWindowAttribute as CFString, &focusedWindowValue) == .success, - let focusedWindow = focusedWindowValue else { + let focusedWindow = focusedWindowValue + else { return nil } var titleValue: AnyObject? guard AXUIElementCopyAttributeValue(focusedWindow as! AXUIElement, kAXTitleAttribute as CFString, &titleValue) == .success, - let title = titleValue as? String else { + let title = titleValue as? String + else { return nil } @@ -1361,12 +1362,11 @@ final class OrbitManager: ObservableObject { func interruptCurrentAction() { currentResponseTask?.cancel() currentResponseTask = nil - desktopActuationTask?.cancel() - desktopActuationTask = nil orbitDictationManager.cancelCurrentDictation() textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() actionProvider.cancelCurrentAction() + releaseTemporaryCapture() voiceState = .idle activeActionProgress = OrbitActionProgress( phase: .interrupted, @@ -1490,7 +1490,8 @@ final class OrbitManager: ObservableObject { let pattern = #"\[POINT:(?:none|(\d+)\s*,\s*(\d+)(?::([^\]:\s][^\]:]*?))?(?::screen(\d+))?)\]\s*$"# guard let regex = try? NSRegularExpression(pattern: pattern, options: []), - let match = regex.firstMatch(in: responseText, range: NSRange(responseText.startIndex..., in: responseText)) else { + let match = regex.firstMatch(in: responseText, range: NSRange(responseText.startIndex..., in: responseText)) + else { // No tag found at all return PointingParseResult(spokenText: responseText, coordinate: nil, elementLabel: nil, screenNumber: nil) } @@ -1501,10 +1502,11 @@ final class OrbitManager: ObservableObject { // Check if it's [POINT:none] guard match.numberOfRanges >= 3, - let xRange = Range(match.range(at: 1), in: responseText), - let yRange = Range(match.range(at: 2), in: responseText), - let x = Double(responseText[xRange]), - let y = Double(responseText[yRange]) else { + let xRange = Range(match.range(at: 1), in: responseText), + let yRange = Range(match.range(at: 2), in: responseText), + let x = Double(responseText[xRange]), + let y = Double(responseText[yRange]) + else { return PointingParseResult(spokenText: spokenText, coordinate: nil, elementLabel: "none", screenNumber: nil) } @@ -1528,7 +1530,8 @@ final class OrbitManager: ObservableObject { static func parseOrbitResponse(from responseText: String) -> OrbitResponseParseResult { let pointDirective = parsePointingCoordinates(from: responseText) - let pointResult: PointingParseResult? = pointDirective.coordinate == nil && pointDirective.elementLabel == nil + let pointResult: PointingParseResult? = + pointDirective.coordinate == nil && pointDirective.elementLabel == nil ? nil : pointDirective return OrbitResponseParseResult( @@ -1560,7 +1563,7 @@ final class OrbitManager: ObservableObject { "hey, i'm orbit.", "hold control + option to talk to me.", "i send your words and your current screen to one live codex session.", - "when something matters on screen, i can point right to it." + "when something matters on screen, i can point right to it.", ] onboardingTask = Task { @MainActor [weak self] in diff --git a/Orbit/OrbitMark.swift b/Orbit/OrbitMark.swift index 65f4268..66859bf 100644 --- a/Orbit/OrbitMark.swift +++ b/Orbit/OrbitMark.swift @@ -10,7 +10,7 @@ import AppKit import CoreGraphics import SwiftUI -enum OrbitBranding { +nonisolated enum OrbitBranding { static let defaultMarkRotationDegrees: CGFloat = 0 static let defaultMarkHeadingDegrees: CGFloat = -135 static let menuBarVerticalFlip = true @@ -20,7 +20,7 @@ enum OrbitBranding { CGPoint(x: 0.000, y: 0.000), CGPoint(x: 1.000, y: 0.429), CGPoint(x: 0.929, y: 0.214), - CGPoint(x: 0.429, y: 0.429) + CGPoint(x: 0.429, y: 0.429), ] static func markPath( @@ -99,7 +99,7 @@ enum OrbitBranding { } struct OrbitMarkShape: Shape { - var rotationDegrees: CGFloat = OrbitBranding.defaultMarkRotationDegrees + var rotationDegrees: CGFloat = 0 func path(in rect: CGRect) -> Path { Path(OrbitBranding.markPath(in: rect, rotationDegrees: rotationDegrees)) diff --git a/Orbit/OrbitOpenAIVoiceConfiguration.swift b/Orbit/OrbitOpenAIVoiceConfiguration.swift index 628756e..494f010 100644 --- a/Orbit/OrbitOpenAIVoiceConfiguration.swift +++ b/Orbit/OrbitOpenAIVoiceConfiguration.swift @@ -89,16 +89,17 @@ enum OrbitOpenAIKeychainStore { kSecAttrService as String: service, kSecAttrAccount as String: account, kSecReturnData as String: true, - kSecMatchLimit as String: kSecMatchLimitOne + kSecMatchLimit as String: kSecMatchLimitOne, ] var item: CFTypeRef? let status = SecItemCopyMatching(query as CFDictionary, &item) guard status == errSecSuccess, - let data = item as? Data, - let string = String(data: data, encoding: .utf8)? + let data = item as? Data, + let string = String(data: data, encoding: .utf8)? .trimmingCharacters(in: .whitespacesAndNewlines), - !string.isEmpty else { + !string.isEmpty + else { return nil } @@ -119,7 +120,7 @@ enum OrbitOpenAIKeychainStore { let baseQuery: [String: Any] = [ kSecClass as String: kSecClassGenericPassword, kSecAttrService as String: service, - kSecAttrAccount as String: account + kSecAttrAccount as String: account, ] let attributes: [String: Any] = [ @@ -155,7 +156,7 @@ enum OrbitOpenAIKeychainStore { let query: [String: Any] = [ kSecClass as String: kSecClassGenericPassword, kSecAttrService as String: service, - kSecAttrAccount as String: account + kSecAttrAccount as String: account, ] SecItemDelete(query as CFDictionary) } diff --git a/Orbit/OrbitPanelView.swift b/Orbit/OrbitPanelView.swift index b4c5bf7..4bf9487 100644 --- a/Orbit/OrbitPanelView.swift +++ b/Orbit/OrbitPanelView.swift @@ -11,38 +11,51 @@ import AVFoundation import AppKit import SwiftUI +private enum OrbitPanelSection { + case activity + case settings +} + struct OrbitPanelView: View { @ObservedObject var orbitManager: OrbitManager @ObservedObject private var orbitSettings = OrbitSettings.shared @State private var openAIAPIKeyDraft = "" @State private var showAPIKeyDialog = false @State private var showAboutPopover = false + @State private var showTeamActivity = false + @State private var panelSection: OrbitPanelSection = .activity private let panelShape = RoundedRectangle(cornerRadius: 24, style: .continuous) private let cardShape = RoundedRectangle(cornerRadius: 18, style: .continuous) var body: some View { - VStack(alignment: .leading, spacing: 12) { - header - - if orbitManager.setupStage == .permissions { - permissionsCard - } else if orbitManager.setupStage == .auth { - authCard - } else if orbitManager.setupStage == .voiceChoice { - voiceChoiceCard - } else if orbitManager.setupStage == .cloudKey { - cloudVoiceCard - } else if orbitManager.setupStage == .setupComplete { - setupCompleteCard - } else { - controlsCard - } + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 12) { + header + + if orbitManager.setupStage == .permissions { + permissionsCard + } else if orbitManager.setupStage == .automationDisclosure { + automationDisclosureCard + } else if orbitManager.setupStage == .auth { + authCard + } else if orbitManager.setupStage == .voiceChoice { + voiceChoiceCard + } else if orbitManager.setupStage == .cloudKey { + cloudVoiceCard + } else if orbitManager.setupStage == .setupComplete { + setupCompleteCard + } else { + readyPanelContent + } - footer + footer + } + .padding(13) } - .padding(13) + .scrollIndicators(.hidden) .frame(width: 312) + .frame(maxHeight: 720) .background(panelBackground) } @@ -117,12 +130,45 @@ struct OrbitPanelView: View { } } - private var controlsCard: some View { + private var readyPanelContent: some View { + VStack(alignment: .leading, spacing: 10) { + segmentedControl(spacing: 3) { + settingOptionButton(label: "Activity", isSelected: panelSection == .activity) { + panelSection = .activity + } + settingOptionButton(label: "Settings", isSelected: panelSection == .settings) { + panelSection = .settings + } + } + .accessibilityLabel("Panel section") + + if panelSection == .activity { + taskActivityCard + } else { + settingsCard + } + } + } + + private var taskActivityCard: some View { sectionCard { - sectionHeader(title: "Controls", subtitle: "One Codex session for voice and actions.") + sectionHeader(title: "Current task", subtitle: "Live Codex activity and explicit team-up work.") + codexCard + if !orbitManager.codexSubagentActivities.isEmpty { + rowDivider + teamActivityRow + } + } + } + + private var settingsCard: some View { + sectionCard { + sectionHeader(title: "Settings", subtitle: "Voice, model, working context, and Orbit visibility.") authStatusRow rowDivider + unrestrictedAutomationRow + rowDivider if orbitManager.setupStage == .onboarding { compactSetupRow @@ -131,16 +177,27 @@ struct OrbitPanelView: View { voicePresetRow rowDivider - codexCard - rowDivider codexModelRow + if let notice = orbitManager.codexModelMigrationNotice { + Text(notice) + .font(.system(size: 10.5, weight: .medium)) + .foregroundStyle(DS.Colors.warning) + .fixedSize(horizontal: false, vertical: true) + .accessibilityLabel(notice) + } rowDivider - codexServiceTierRow + agentFolderRow rowDivider + if availableServiceTiers.count > 1 { + codexServiceTierRow + rowDivider + } codexReasoningEffortRow rowDivider providerRow(icon: "waveform.badge.mic", title: "Speech to Text", value: panelSpeechToTextLabel) rowDivider + microphoneInputRow + rowDivider speechOutputRow if orbitSettings.voicePreset == .cloudVoice { rowDivider @@ -151,6 +208,40 @@ struct OrbitPanelView: View { } } + private var automationDisclosureCard: some View { + sectionCard { + sectionHeader( + title: "Unrestricted automation", + subtitle: "Orbit works like an operator you explicitly invoke, not a continuous screen recorder." + ) + + VStack(alignment: .leading, spacing: 10) { + Label("Can run commands and edit files without approval prompts", systemImage: "terminal") + Label("Takes one fresh screen capture for each request", systemImage: "camera.viewfinder") + Label("Deletes the temporary capture when the turn ends", systemImage: "trash") + } + .font(.system(size: 11, weight: .medium)) + .foregroundStyle(DS.Colors.textSecondary) + .accessibilityElement(children: .combine) + + Button("I understand — continue") { + orbitManager.acknowledgeUnrestrictedAutomation() + } + .buttonStyle(.borderedProminent) + .controlSize(.regular) + .frame(maxWidth: .infinity, alignment: .trailing) + } + } + + private var unrestrictedAutomationRow: some View { + rowLabel( + icon: "bolt.shield", + title: "Automation Access", + subtitle: "Unrestricted · approvals automatic · one temporary capture per request" + ) + .accessibilityLabel("Automation access: unrestricted, approvals automatic, one temporary screen capture per request") + } + private var authCard: some View { sectionCard { sectionHeader(title: "Connect", subtitle: "Sign in with ChatGPT so Orbit can start its live Codex session.") @@ -193,7 +284,7 @@ struct OrbitPanelView: View { VStack(spacing: 10) { voiceChoiceOption( title: "Use Local Voice", - subtitle: "Apple speech on this Mac. No API key needed.", + subtitle: "Apple on-device recognition and speech. No API key.", isSelected: orbitSettings.voicePreset == .localVoice ) { orbitManager.selectVoicePreset(.localVoice) @@ -201,7 +292,7 @@ struct OrbitPanelView: View { voiceChoiceOption( title: "Use Cloud Voice", - subtitle: "OpenAI speech with your own API key.", + subtitle: "OpenAI transcription and AI-generated voice with your API key.", isSelected: orbitSettings.voicePreset == .cloudVoice ) { orbitManager.selectVoicePreset(.cloudVoice) @@ -215,6 +306,13 @@ struct OrbitPanelView: View { sectionHeader(title: "Cloud Voice", subtitle: "Add an OpenAI API key to enable cloud speech.") VStack(alignment: .leading, spacing: 10) { + Label( + "Cloud responses use an AI-generated voice and send speech audio to OpenAI.", + systemImage: "cloud" + ) + .font(.system(size: 10.5, weight: .medium)) + .foregroundStyle(DS.Colors.textSecondary) + SecureField("OpenAI API Key", text: $openAIAPIKeyDraft) .textFieldStyle(.plain) .font(.system(size: 12, weight: .medium)) @@ -281,9 +379,9 @@ struct OrbitPanelView: View { private func sectionHeader(title: String, subtitle: String) -> some View { VStack(alignment: .leading, spacing: 5) { - Text(title.uppercased()) - .font(.system(size: 10, weight: .semibold, design: .rounded)) - .foregroundColor(DS.Colors.textTertiary) + Text(title) + .font(.system(size: 13, weight: .semibold)) + .foregroundColor(DS.Colors.textPrimary) Text(subtitle) .font(.system(size: 12, weight: .medium)) @@ -442,6 +540,66 @@ struct OrbitPanelView: View { } } + private var teamActivityRow: some View { + DisclosureGroup(isExpanded: $showTeamActivity) { + VStack(alignment: .leading, spacing: 7) { + ForEach(orbitManager.codexSubagentActivities) { activity in + HStack(spacing: 7) { + Circle() + .fill(activity.status == "running" ? DS.Colors.accent : DS.Colors.textTertiary) + .frame(width: 6, height: 6) + VStack(alignment: .leading, spacing: 1) { + Text(activity.agentPath) + .font(.system(size: 10.5, weight: .semibold)) + Text(activity.message ?? activity.status) + .font(.system(size: 10)) + .foregroundStyle(DS.Colors.textTertiary) + .lineLimit(2) + } + } + } + } + .padding(.top, 7) + } label: { + Label("Team-up activity", systemImage: "person.2") + .font(.system(size: 11, weight: .semibold)) + .foregroundStyle(DS.Colors.textSecondary) + } + .accessibilityHint("Expands child-agent activity for this task") + } + + private var agentFolderRow: some View { + HStack(spacing: 10) { + rowLabel( + icon: "folder", + title: "Agent Folder", + subtitle: "Starting context only; filesystem access remains unrestricted" + ) + Spacer(minLength: 6) + Button(agentFolderLabel) { chooseAgentFolder() } + .buttonStyle(.borderless) + .lineLimit(1) + .help(orbitSettings.codexAgentFolder.isEmpty ? "Uses your home folder" : orbitSettings.codexAgentFolder) + } + } + + private var agentFolderLabel: String { + let path = orbitSettings.codexAgentFolder.trimmingCharacters(in: .whitespacesAndNewlines) + return path.isEmpty ? "Home" : URL(fileURLWithPath: path).lastPathComponent + } + + private func chooseAgentFolder() { + let panel = NSOpenPanel() + panel.title = "Choose Orbit Agent Folder" + panel.message = "This sets the starting context. Orbit still has unrestricted filesystem access." + panel.canChooseDirectories = true + panel.canChooseFiles = false + panel.allowsMultipleSelection = false + if panel.runModal() == .OK, let url = panel.url { + orbitSettings.codexAgentFolder = url.path + } + } + private var codexCard: some View { VStack(alignment: .leading, spacing: 10) { HStack(alignment: .center, spacing: 10) { @@ -605,28 +763,31 @@ struct OrbitPanelView: View { private var codexServiceTierRow: some View { HStack(alignment: .center, spacing: 12) { - rowLabel(icon: "bolt", title: "Fast Mode", subtitle: nil) + rowLabel(icon: "bolt", title: "Service Tier", subtitle: nil) Spacer(minLength: 8) segmentedControl(spacing: 2) { - settingOptionButton( - label: "Off", - isSelected: orbitSettings.codexServiceTier == .standard - ) { - orbitSettings.codexServiceTier = .standard - } - - settingOptionButton( - label: "Fast", - isSelected: orbitSettings.codexServiceTier == .fast - ) { - orbitSettings.codexServiceTier = .fast + ForEach(availableServiceTiers) { tier in + settingOptionButton( + label: tier.displayName, + isSelected: orbitSettings.codexServiceTier == tier + ) { + orbitSettings.codexServiceTier = tier + } } } } } + private var availableServiceTiers: [OrbitCodexServiceTier] { + let serverTiers = + orbitManager.availableCodexModels + .first(where: { $0.model == orbitSettings.codexActionModel })? + .supportedServiceTiers ?? [] + return [.serverDefault] + serverTiers.filter { !$0.rawValue.isEmpty } + } + private func providerRow(icon: String, title: String, value: String) -> some View { HStack(alignment: .center, spacing: 12) { rowLabel(icon: icon, title: title, subtitle: nil) @@ -730,21 +891,27 @@ struct OrbitPanelView: View { Spacer(minLength: 8) if orbitSettings.voicePreset == .localVoice { - Text(orbitManager.selectedAppleVoiceSummary) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .lineLimit(1) - .minimumScaleFactor(0.85) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.22, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) + VStack(alignment: .trailing, spacing: 5) { + Menu { + Button("Automatic") { orbitManager.selectAppleVoice("") } + Divider() + ForEach(orbitManager.availableAppleVoices) { voice in + Button(voice.displayName) { orbitManager.selectAppleVoice(voice.identifier) } + } + } label: { + Text(orbitManager.selectedAppleVoiceSummary) + .font(.system(size: 11, weight: .medium)) + .lineLimit(1) + .frame(maxWidth: 112, alignment: .trailing) + } + .menuStyle(.borderlessButton) + + Button(orbitManager.isPreviewingAppleVoice ? "Stop" : "Preview") { + orbitManager.toggleAppleVoicePreview() + } + .buttonStyle(.borderless) + .font(.system(size: 10, weight: .semibold)) + } } else { Text(panelTextToSpeechLabel) .font(.system(size: 11.5, weight: .medium)) @@ -763,6 +930,48 @@ struct OrbitPanelView: View { } } + private var microphoneInputRow: some View { + VStack(alignment: .leading, spacing: 6) { + HStack(spacing: 10) { + rowLabel( + icon: "mic", + title: "Microphone", + subtitle: orbitManager.microphoneDeviceNotice ?? "Local input · no audio leaves the Mac" + ) + Spacer(minLength: 6) + Menu { + Button("System Default") { orbitManager.selectMicrophone("") } + Divider() + ForEach(orbitManager.availableMicrophones) { microphone in + Button(microphone.name) { orbitManager.selectMicrophone(microphone.uid) } + } + } label: { + Text(selectedMicrophoneLabel) + .font(.system(size: 10.5, weight: .medium)) + .lineLimit(1) + .frame(maxWidth: 100, alignment: .trailing) + } + .menuStyle(.borderlessButton) + } + HStack(spacing: 8) { + ProgressView(value: orbitManager.microphoneTestLevel) + .progressViewStyle(.linear) + Button(orbitManager.isTestingMicrophone ? "Stop" : "Test") { + orbitManager.toggleMicrophoneTest() + } + .buttonStyle(.borderless) + .font(.system(size: 10, weight: .semibold)) + } + } + .onAppear { orbitManager.refreshMicrophones() } + } + + private var selectedMicrophoneLabel: String { + let selectedUID = orbitSettings.microphoneDeviceUID.trimmingCharacters(in: .whitespacesAndNewlines) + guard !selectedUID.isEmpty else { return "System" } + return orbitManager.availableMicrophones.first(where: { $0.uid == selectedUID })?.name ?? "Disconnected" + } + private var openAIKeyRow: some View { HStack(alignment: .center, spacing: 12) { rowLabel( @@ -1013,13 +1222,6 @@ struct OrbitPanelView: View { .foregroundColor(.secondary) } - if !orbitManager.codexCollaborationModes.isEmpty { - Text("modes: \(orbitManager.codexCollaborationModes.joined(separator: ", "))") - .font(.system(size: 10, weight: .medium)) - .foregroundColor(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - if !orbitManager.codexDebugEvents.isEmpty { Text(orbitManager.codexDebugEvents.suffix(4).joined(separator: "\n")) .font(.system(size: 9, weight: .medium, design: .monospaced)) @@ -1196,6 +1398,8 @@ struct OrbitPanelView: View { switch orbitManager.setupStage { case .permissions: return "Codex voice shell" + case .automationDisclosure: + return "Review automation access" case .auth: return "Connect ChatGPT" case .voiceChoice: @@ -1259,7 +1463,7 @@ struct OrbitPanelView: View { } switch orbitManager.setupStage { - case .permissions, .auth, .voiceChoice, .cloudKey, .onboarding, .setupComplete: + case .permissions, .automationDisclosure, .auth, .voiceChoice, .cloudKey, .onboarding, .setupComplete: return "Setup" case .ready: break @@ -1354,11 +1558,13 @@ struct OrbitPanelView: View { switch orbitManager.activeActionStatus { case .idle: if let accountSummary = orbitManager.codexAccountSummary, - !accountSummary.isEmpty { + !accountSummary.isEmpty + { return accountSummary.lowercased() } if orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("ready in session") - || orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("connected to codex") { + || orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("connected to codex") + { return "live codex session connected." } if orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("starting codex") { @@ -1487,13 +1693,13 @@ struct OrbitPanelView: View { ? nil : "If Orbit is missing, use Find App.", action: { - WindowPositionManager.requestAccessibilityPermission() + orbitManager.permissionCoordinator.begin(.accessibility) }, alternateAction: { - WindowPositionManager.revealAppInFinder() - WindowPositionManager.openAccessibilitySettings() + orbitManager.permissionCoordinator.revealOrbitInFinder() + orbitManager.permissionCoordinator.openSettings(for: .accessibility) }, - alternateTitle: "Find App" + alternateTitle: "Reveal Orbit" ) } @@ -1503,13 +1709,13 @@ struct OrbitPanelView: View { iconName: "rectangle.dashed.badge.record", isGranted: orbitManager.hasUsableScreenAccessPermission, subtitle: orbitManager.hasUsableScreenAccessPermission - ? "Only captures the active screen when you use the hotkey" + ? nil : "Grant once so Orbit can see your current screen.", action: { - orbitManager.requestScreenContentPermission() + orbitManager.permissionCoordinator.begin(.screenRecording) }, alternateAction: { - WindowPositionManager.openScreenRecordingSettings() + orbitManager.permissionCoordinator.openSettings(for: .screenRecording) }, alternateTitle: "Open Settings" ) @@ -1532,12 +1738,7 @@ struct OrbitPanelView: View { isGranted: orbitManager.hasMicrophonePermission, subtitle: nil ) { - let status = AVCaptureDevice.authorizationStatus(for: .audio) - if status == .notDetermined { - AVCaptureDevice.requestAccess(for: .audio) { _ in } - } else if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone") { - NSWorkspace.shared.open(url) - } + orbitManager.permissionCoordinator.begin(.microphone) } } @@ -1608,7 +1809,7 @@ private struct EffortGlyph: View { var body: some View { HStack(alignment: .bottom, spacing: 2.5) { - ForEach(0..<4, id: \.self) { index in + ForEach(0..<5, id: \.self) { index in RoundedRectangle(cornerRadius: 1.5, style: .continuous) .fill(barColor(for: index)) .frame(width: 3, height: CGFloat(5 + (index * 3))) diff --git a/Orbit/OrbitPermissionCoordinator.swift b/Orbit/OrbitPermissionCoordinator.swift new file mode 100644 index 0000000..6b8d514 --- /dev/null +++ b/Orbit/OrbitPermissionCoordinator.swift @@ -0,0 +1,418 @@ +import AVFoundation +import AppKit +import Combine +import SwiftUI + +enum OrbitPermissionKind: String, CaseIterable, Identifiable, Sendable { + case microphone + case accessibility + case screenRecording + + var id: String { rawValue } + + var title: String { + switch self { + case .microphone: "Microphone" + case .accessibility: "Accessibility" + case .screenRecording: "Screen Recording" + } + } +} + +enum OrbitPermissionStatus: Equatable, Sendable { + case notDetermined + case requesting + case denied + case restartRequired + case granted +} + +@MainActor +final class OrbitPermissionCoordinator: ObservableObject { + @Published private(set) var activePermission: OrbitPermissionKind? + @Published private(set) var statusByPermission: [OrbitPermissionKind: OrbitPermissionStatus] = [:] + + private weak var orbitManager: OrbitManager? + private lazy var guide = OrbitPermissionGuideWindowController(coordinator: self) + + init(orbitManager: OrbitManager) { + self.orbitManager = orbitManager + refresh() + } + + func begin(_ permission: OrbitPermissionKind) { + activePermission = permission + statusByPermission[permission] = .requesting + + switch permission { + case .microphone: + requestMicrophone() + case .accessibility: + _ = WindowPositionManager.requestAccessibilityPermission() + orbitManager?.refreshAllPermissions() + if orbitManager?.hasAccessibilityPermission == true { + markGranted(permission) + } else { + guide.show(for: permission) + } + case .screenRecording: + WindowPositionManager.openScreenRecordingSettings() + guide.show(for: permission) + orbitManager?.requestScreenContentPermission() + } + } + + func refresh() { + orbitManager?.refreshAllPermissions() + guard let orbitManager else { return } + statusByPermission[.microphone] = + orbitManager.hasMicrophonePermission + ? .granted + : microphoneStatus + statusByPermission[.accessibility] = orbitManager.hasAccessibilityPermission ? .granted : .denied + if orbitManager.hasUsableScreenAccessPermission { + statusByPermission[.screenRecording] = .granted + } else if CGPreflightScreenCaptureAccess() { + statusByPermission[.screenRecording] = .restartRequired + } else { + statusByPermission[.screenRecording] = .denied + } + + if let activePermission, statusByPermission[activePermission] == .granted { + guide.scheduleSuccessDismiss() + } + } + + func openSettings(for permission: OrbitPermissionKind) { + switch permission { + case .microphone: + openPrivacyPane("Privacy_Microphone") + case .accessibility: + WindowPositionManager.openAccessibilitySettings() + case .screenRecording: + WindowPositionManager.openScreenRecordingSettings() + } + } + + func revealOrbitInFinder() { + NSWorkspace.shared.activateFileViewerSelecting([Bundle.main.bundleURL]) + } + + func dismissGuide() { + guide.hide() + activePermission = nil + } + + private var microphoneStatus: OrbitPermissionStatus { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .notDetermined: .notDetermined + case .authorized: .granted + case .denied, .restricted: .denied + @unknown default: .denied + } + } + + private func requestMicrophone() { + guard AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined else { + openSettings(for: .microphone) + refresh() + return + } + AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in + Task { @MainActor [weak self] in + guard let self else { return } + self.orbitManager?.refreshAllPermissions() + self.statusByPermission[.microphone] = granted ? .granted : .denied + self.activePermission = nil + } + } + } + + private func markGranted(_ permission: OrbitPermissionKind) { + statusByPermission[permission] = .granted + activePermission = nil + guide.scheduleSuccessDismiss() + } + + private func openPrivacyPane(_ anchor: String) { + guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?\(anchor)") else { return } + NSWorkspace.shared.open(url) + } +} + +@MainActor +private final class OrbitPermissionGuideModel: ObservableObject { + @Published var permission: OrbitPermissionKind = .accessibility + @Published var isAttachedToSettings = false + @Published var isDragging = false +} + +@MainActor +private final class OrbitPermissionGuideWindowController { + private weak var coordinator: OrbitPermissionCoordinator? + private let model = OrbitPermissionGuideModel() + private var panel: NSPanel? + private var trackingTimer: Timer? + private var successDismissTask: Task? + private var escapeMonitor: Any? + private var localEscapeMonitor: Any? + private var hasSeenSettingsWindow = false + + init(coordinator: OrbitPermissionCoordinator) { + self.coordinator = coordinator + } + + func show(for permission: OrbitPermissionKind) { + model.permission = permission + hasSeenSettingsWindow = false + if panel == nil, let coordinator { + let guidePanel = NSPanel( + contentRect: NSRect(x: 0, y: 0, width: 520, height: 78), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + guidePanel.contentView = NSHostingView( + rootView: OrbitPermissionGuideView( + coordinator: coordinator, + model: model, + appURL: Bundle.main.bundleURL + ) + ) + guidePanel.isFloatingPanel = true + guidePanel.level = .statusBar + guidePanel.isOpaque = false + guidePanel.backgroundColor = .clear + guidePanel.hasShadow = true + guidePanel.hidesOnDeactivate = false + guidePanel.isExcludedFromWindowsMenu = true + guidePanel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + guidePanel.becomesKeyOnlyIfNeeded = true + panel = guidePanel + } + + positionPanel() + panel?.orderFrontRegardless() + startTracking() + installEscapeMonitor() + } + + func hide() { + trackingTimer?.invalidate() + trackingTimer = nil + successDismissTask?.cancel() + successDismissTask = nil + model.isDragging = false + panel?.orderOut(nil) + if let escapeMonitor { + NSEvent.removeMonitor(escapeMonitor) + self.escapeMonitor = nil + } + if let localEscapeMonitor { + NSEvent.removeMonitor(localEscapeMonitor) + self.localEscapeMonitor = nil + } + } + + func scheduleSuccessDismiss() { + guard panel?.isVisible == true, successDismissTask == nil else { return } + successDismissTask = Task { [weak self] in + try? await Task.sleep(for: .milliseconds(750)) + self?.hide() + } + } + + private func startTracking() { + trackingTimer?.invalidate() + trackingTimer = Timer.scheduledTimer(withTimeInterval: 0.45, repeats: true) { [weak self] _ in + Task { @MainActor [weak self] in + guard let self else { return } + self.coordinator?.refresh() + if Self.systemSettingsWindowFrame() != nil { + self.hasSeenSettingsWindow = true + self.positionPanel() + } else if self.hasSeenSettingsWindow { + self.coordinator?.dismissGuide() + } else { + self.positionPanel() + } + } + } + } + + private func installEscapeMonitor() { + guard escapeMonitor == nil, localEscapeMonitor == nil else { return } + localEscapeMonitor = NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] event in + guard event.keyCode == 53 else { return event } + Task { @MainActor [weak self] in self?.coordinator?.dismissGuide() } + return nil + } + escapeMonitor = NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] event in + guard event.keyCode == 53 else { return } + Task { @MainActor [weak self] in self?.coordinator?.dismissGuide() } + } + } + + private func positionPanel() { + guard let panel else { return } + let settingsFrame = Self.systemSettingsWindowFrame() + let screen = + NSScreen.screens.first(where: { settingsFrame.map($0.frame.intersects) ?? false }) + ?? NSScreen.main + guard let visibleFrame = screen?.visibleFrame else { return } + + model.isAttachedToSettings = settingsFrame != nil + let width = settingsFrame.map { min(560, max(420, $0.width - 170)) } ?? 520 + let proposed = + settingsFrame.map { + NSRect(x: $0.maxX - width, y: $0.minY + 8, width: width, height: 78) + } + ?? NSRect( + x: visibleFrame.midX - width / 2, + y: visibleFrame.minY + 18, + width: width, + height: 78 + ) + let x = min(max(proposed.minX, visibleFrame.minX + 10), visibleFrame.maxX - proposed.width - 10) + let y = min(max(proposed.minY, visibleFrame.minY + 10), visibleFrame.maxY - proposed.height - 10) + panel.setFrame(NSRect(x: x, y: y, width: proposed.width, height: proposed.height), display: true) + } + + private static func systemSettingsWindowFrame() -> NSRect? { + guard + let windows = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]], + let window = windows.first(where: { + let owner = $0[kCGWindowOwnerName as String] as? String + return ($0[kCGWindowLayer as String] as? Int) == 0 + && (owner == "System Settings" || owner == "System Preferences") + }), + let bounds = window[kCGWindowBounds as String] as? [String: CGFloat], + let x = bounds["X"], let y = bounds["Y"], + let width = bounds["Width"], let height = bounds["Height"] + else { return nil } + + let cgWindow = CGRect(x: x, y: y, width: width, height: height) + for screen in NSScreen.screens { + guard let number = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID else { continue } + let cgDisplay = CGDisplayBounds(number) + if cgDisplay.intersects(cgWindow) { + let convertedY = screen.frame.maxY - (cgWindow.minY - cgDisplay.minY) - cgWindow.height + return NSRect(x: cgWindow.minX, y: convertedY, width: cgWindow.width, height: cgWindow.height) + } + } + let primaryTop = NSScreen.screens.first?.frame.maxY ?? 0 + return NSRect(x: x, y: primaryTop - y - height, width: width, height: height) + } +} + +private struct OrbitPermissionGuideView: View { + @ObservedObject var coordinator: OrbitPermissionCoordinator + @ObservedObject var model: OrbitPermissionGuideModel + let appURL: URL + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + var body: some View { + HStack(spacing: 12) { + OrbitDraggablePermissionTile(appURL: appURL, isDragging: $model.isDragging) + .frame(width: 44, height: 44) + .accessibilityLabel("Orbit app tile") + .accessibilityHint("Drag Orbit into the \(model.permission.title) list in System Settings.") + + VStack(alignment: .leading, spacing: 3) { + Text("Add Orbit to \(model.permission.title)") + .font(.system(size: 13, weight: .semibold)) + .foregroundStyle(DS.Colors.textPrimary) + Text(guideMessage) + .font(.system(size: 11)) + .foregroundStyle(DS.Colors.textSecondary) + .lineLimit(2) + } + Spacer(minLength: 8) + Button("Reveal Orbit") { coordinator.revealOrbitInFinder() } + .accessibilityLabel("Reveal Orbit in Finder") + Button("Open Settings") { coordinator.openSettings(for: model.permission) } + } + .buttonStyle(.borderless) + .padding(.horizontal, 14) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .background( + RoundedRectangle(cornerRadius: model.isAttachedToSettings ? 12 : 18, style: .continuous) + .fill(Color(nsColor: .windowBackgroundColor).opacity(0.97)) + .overlay( + RoundedRectangle(cornerRadius: model.isAttachedToSettings ? 12 : 18, style: .continuous) + .stroke(DS.Colors.accent.opacity(0.35), lineWidth: 1) + ) + ) + .opacity(model.isDragging && !reduceMotion ? 0.82 : 1) + .animation(reduceMotion ? nil : .easeOut(duration: 0.16), value: model.isDragging) + } + + private var guideMessage: String { + if model.permission == .screenRecording, + coordinator.statusByPermission[.screenRecording] == .restartRequired + { + return "Screen access was enabled. Quit and reopen Orbit, then retry the live capture check." + } + return "Drag only the Orbit tile. This guide stays attached to System Settings." + } +} + +private struct OrbitDraggablePermissionTile: NSViewRepresentable { + let appURL: URL + @Binding var isDragging: Bool + + func makeNSView(context: Context) -> OrbitDraggablePermissionTileView { + let view = OrbitDraggablePermissionTileView(appURL: appURL) + view.onDraggingChanged = { isDragging = $0 } + return view + } + + func updateNSView(_ nsView: OrbitDraggablePermissionTileView, context: Context) { + nsView.appURL = appURL + nsView.onDraggingChanged = { isDragging = $0 } + } +} + +private final class OrbitDraggablePermissionTileView: NSView, NSDraggingSource { + var appURL: URL { didSet { needsDisplay = true } } + var onDraggingChanged: ((Bool) -> Void)? + private var mouseDownEvent: NSEvent? + + init(appURL: URL) { + self.appURL = appURL + super.init(frame: .zero) + toolTip = "Drag Orbit into System Settings" + setAccessibilityElement(true) + setAccessibilityRole(.button) + setAccessibilityLabel("Orbit app tile") + } + + required init?(coder: NSCoder) { nil } + + override func mouseDown(with event: NSEvent) { mouseDownEvent = event } + + override func mouseDragged(with event: NSEvent) { + guard let mouseDownEvent else { return } + onDraggingChanged?(true) + let item = NSDraggingItem(pasteboardWriter: appURL as NSURL) + let image = NSWorkspace.shared.icon(forFile: appURL.path) + image.size = NSSize(width: 48, height: 48) + item.setDraggingFrame(bounds, contents: image) + beginDraggingSession(with: [item], event: mouseDownEvent, source: self) + self.mouseDownEvent = nil + } + + override func draw(_ dirtyRect: NSRect) { + NSColor.controlAccentColor.withAlphaComponent(0.16).setFill() + NSBezierPath(roundedRect: bounds.insetBy(dx: 1, dy: 1), xRadius: 11, yRadius: 11).fill() + let image = NSWorkspace.shared.icon(forFile: appURL.path) + image.draw(in: bounds.insetBy(dx: 6, dy: 6)) + } + + func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation { .copy } + func draggingSession(_ session: NSDraggingSession, endedAt screenPoint: NSPoint, operation: NSDragOperation) { onDraggingChanged?(false) } + func ignoreModifierKeys(for session: NSDraggingSession) -> Bool { true } +} diff --git a/Orbit/OrbitScreenCaptureUtility.swift b/Orbit/OrbitScreenCaptureUtility.swift index 13ee8f1..154b2b2 100644 --- a/Orbit/OrbitScreenCaptureUtility.swift +++ b/Orbit/OrbitScreenCaptureUtility.swift @@ -48,8 +48,9 @@ enum OrbitScreenCaptureUtility { let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) guard !content.displays.isEmpty else { - throw NSError(domain: "OrbitScreenCapture", code: -1, - userInfo: [NSLocalizedDescriptionKey: "No display available for capture"]) + throw NSError( + domain: "OrbitScreenCapture", code: -1, + userInfo: [NSLocalizedDescriptionKey: "No display available for capture"]) } let mouseLocation = NSEvent.mouseLocation @@ -90,9 +91,11 @@ enum OrbitScreenCaptureUtility { // Use NSScreen.frame (AppKit coordinates, bottom-left origin) so // displayFrame is in the same coordinate system as NSEvent.mouseLocation // and the overlay window's screenFrame in OrbitCursorOverlayView. - let displayFrame = nsScreenByDisplayID[display.displayID]?.frame - ?? CGRect(x: display.frame.origin.x, y: display.frame.origin.y, - width: CGFloat(display.width), height: CGFloat(display.height)) + let displayFrame = + nsScreenByDisplayID[display.displayID]?.frame + ?? CGRect( + x: display.frame.origin.x, y: display.frame.origin.y, + width: CGFloat(display.width), height: CGFloat(display.height)) let isCursorScreen = displayFrame.contains(mouseLocation) let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) @@ -113,8 +116,10 @@ enum OrbitScreenCaptureUtility { configuration: configuration ) - guard let jpegData = NSBitmapImageRep(cgImage: cgImage) - .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) else { + guard + let jpegData = NSBitmapImageRep(cgImage: cgImage) + .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) + else { continue } @@ -127,22 +132,24 @@ enum OrbitScreenCaptureUtility { screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — secondary screen" } - capturedScreens.append(OrbitScreenCapture( - imageData: jpegData, - label: screenLabel, - isCursorScreen: isCursorScreen, - screenNumber: displayIndex + 1, - displayWidthInPoints: Int(displayFrame.width), - displayHeightInPoints: Int(displayFrame.height), - displayFrame: displayFrame, - screenshotWidthInPixels: configuration.width, - screenshotHeightInPixels: configuration.height - )) + capturedScreens.append( + OrbitScreenCapture( + imageData: jpegData, + label: screenLabel, + isCursorScreen: isCursorScreen, + screenNumber: displayIndex + 1, + displayWidthInPoints: Int(displayFrame.width), + displayHeightInPoints: Int(displayFrame.height), + displayFrame: displayFrame, + screenshotWidthInPixels: configuration.width, + screenshotHeightInPixels: configuration.height + )) } guard !capturedScreens.isEmpty else { - throw NSError(domain: "OrbitScreenCapture", code: -2, - userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"]) + throw NSError( + domain: "OrbitScreenCapture", code: -2, + userInfo: [NSLocalizedDescriptionKey: "Failed to capture any screen"]) } return capturedScreens diff --git a/Orbit/OrbitSettings.swift b/Orbit/OrbitSettings.swift index 226d714..558363d 100644 --- a/Orbit/OrbitSettings.swift +++ b/Orbit/OrbitSettings.swift @@ -17,53 +17,86 @@ enum OrbitVoicePreset: String, CaseIterable, Identifiable { } } -enum OrbitCodexReasoningEffort: String, CaseIterable, Identifiable { - case low - case medium - case high - case xhigh +struct OrbitCodexReasoningEffort: RawRepresentable, Hashable, Identifiable, Sendable { + let rawValue: String + + init(rawValue: String) { + self.rawValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + } + + static let none = Self(rawValue: "none") + static let low = Self(rawValue: "low") + static let medium = Self(rawValue: "medium") + static let high = Self(rawValue: "high") + static let xhigh = Self(rawValue: "xhigh") + static let max = Self(rawValue: "max") + static let allCases: [Self] = [.none, .low, .medium, .high, .xhigh, .max] var id: String { rawValue } var displayName: String { - switch self { - case .low: + switch rawValue { + case Self.none.rawValue: + return "None" + case Self.low.rawValue: return "Low" - case .medium: + case Self.medium.rawValue: return "Medium" - case .high: + case Self.high.rawValue: return "High" - case .xhigh: + case Self.xhigh.rawValue: return "X-High" + case Self.max.rawValue: + return "Max" + default: + return rawValue.replacingOccurrences(of: "_", with: " ").capitalized } } var level: Int { - switch self { - case .low: + switch rawValue { + case Self.none.rawValue: + return 0 + case Self.low.rawValue: return 1 - case .medium: + case Self.medium.rawValue: return 2 - case .high: + case Self.high.rawValue: return 3 - case .xhigh: + case Self.xhigh.rawValue: return 4 + case Self.max.rawValue: + return 5 + default: + return 2 } } } -enum OrbitCodexServiceTier: String, CaseIterable, Identifiable { - case standard - case fast +struct OrbitCodexServiceTier: RawRepresentable, Hashable, Identifiable, Sendable { + let rawValue: String + + init(rawValue: String) { + self.rawValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + } + + static let serverDefault = Self(rawValue: "") + static let standard = Self(rawValue: "standard") + static let fast = Self(rawValue: "fast") + static let allCases: [Self] = [.serverDefault, .standard, .fast] var id: String { rawValue } var displayName: String { - switch self { - case .standard: + switch rawValue { + case Self.serverDefault.rawValue: + return "Default" + case Self.standard.rawValue: return "Standard" - case .fast: + case Self.fast.rawValue: return "Fast" + default: + return rawValue.replacingOccurrences(of: "_", with: " ").capitalized } } } @@ -76,31 +109,38 @@ struct OrbitCodexModelOption: Identifiable, Equatable { let defaultEffort: OrbitCodexReasoningEffort? let inputModalities: [String] let isDefault: Bool + let supportedServiceTiers: [OrbitCodexServiceTier] + let upgradeModel: String? + let upgradeMessage: String? var id: String { model } - static let fallbackPickerModels: [OrbitCodexModelOption] = [ - OrbitCodexModelOption( - model: "gpt-5.4", - displayName: "GPT-5.4", - shortDisplayName: "5.4", - supportedEfforts: OrbitCodexReasoningEffort.allCases, - defaultEffort: .medium, - inputModalities: ["text", "image"], - isDefault: true - ), - OrbitCodexModelOption( - model: "gpt-5.4-mini", - displayName: "GPT-5.4 Mini", - shortDisplayName: "5.4 Mini", - supportedEfforts: OrbitCodexReasoningEffort.allCases, - defaultEffort: .medium, - inputModalities: ["text", "image"], - isDefault: false - ) - ] - - static let fallbackDefaultModel = "gpt-5.4" + init( + model: String, + displayName: String, + shortDisplayName: String, + supportedEfforts: [OrbitCodexReasoningEffort], + defaultEffort: OrbitCodexReasoningEffort?, + inputModalities: [String], + isDefault: Bool, + supportedServiceTiers: [OrbitCodexServiceTier] = [], + upgradeModel: String? = nil, + upgradeMessage: String? = nil + ) { + self.model = model + self.displayName = displayName + self.shortDisplayName = shortDisplayName + self.supportedEfforts = supportedEfforts + self.defaultEffort = defaultEffort + self.inputModalities = inputModalities + self.isDefault = isDefault + self.supportedServiceTiers = supportedServiceTiers + self.upgradeModel = upgradeModel + self.upgradeMessage = upgradeMessage + } + + static let fallbackPickerModels: [OrbitCodexModelOption] = [] + static let fallbackDefaultModel = "" static func fallbackOption(for model: String) -> OrbitCodexModelOption? { fallbackPickerModels.first(where: { $0.model == model }) @@ -130,10 +170,33 @@ final class OrbitSettings: ObservableObject { @Published var codexActionModel: String { didSet { let normalized = codexActionModel.trimmingCharacters(in: .whitespacesAndNewlines) - UserDefaults.standard.set( - normalized.isEmpty ? OrbitCodexModelOption.fallbackDefaultModel : normalized, - forKey: Keys.codexActionModel - ) + if normalized.isEmpty { + UserDefaults.standard.removeObject(forKey: Keys.codexActionModel) + } else { + UserDefaults.standard.set(normalized, forKey: Keys.codexActionModel) + } + } + } + + @Published var codexAgentFolder: String { + didSet { + let normalized = codexAgentFolder.trimmingCharacters(in: .whitespacesAndNewlines) + if normalized.isEmpty { + UserDefaults.standard.removeObject(forKey: Keys.codexAgentFolder) + } else { + UserDefaults.standard.set(normalized, forKey: Keys.codexAgentFolder) + } + } + } + + @Published var microphoneDeviceUID: String { + didSet { + let normalized = microphoneDeviceUID.trimmingCharacters(in: .whitespacesAndNewlines) + if normalized.isEmpty { + UserDefaults.standard.removeObject(forKey: Keys.microphoneDeviceUID) + } else { + UserDefaults.standard.set(normalized, forKey: Keys.microphoneDeviceUID) + } } } @@ -154,50 +217,53 @@ final class OrbitSettings: ObservableObject { static let codexReasoningEffort = "orbit.codexReasoningEffort" static let codexServiceTier = "orbit.codexServiceTier" static let codexActionModel = "orbit.codexActionModel" + static let codexAgentFolder = "orbit.codexAgentFolder" static let appleTTSVoiceIdentifier = "orbit.appleTTSVoiceIdentifier" + static let microphoneDeviceUID = "orbit.microphoneDeviceUID" } - private static let currentProductDefaultsGeneration = 2 + private static let currentProductDefaultsGeneration = 3 private init() { Self.applyProductDefaultResetIfNeeded() - voicePreset = OrbitVoicePreset( - rawValue: UserDefaults.standard.string(forKey: Keys.voicePreset) ?? "" - ) ?? .localVoice - showCursor = UserDefaults.standard.object(forKey: Keys.showCursor) == nil + voicePreset = + OrbitVoicePreset( + rawValue: UserDefaults.standard.string(forKey: Keys.voicePreset) ?? "" + ) ?? .localVoice + showCursor = + UserDefaults.standard.object(forKey: Keys.showCursor) == nil ? true : UserDefaults.standard.bool(forKey: Keys.showCursor) - codexReasoningEffort = OrbitCodexReasoningEffort( - rawValue: UserDefaults.standard.string(forKey: Keys.codexReasoningEffort) ?? "" - ) ?? .medium - codexServiceTier = OrbitCodexServiceTier( - rawValue: UserDefaults.standard.string(forKey: Keys.codexServiceTier) ?? "" - ) ?? { - let bundledDefault = AppBundleConfiguration.stringValue(forKey: "CodexActionServiceTier") ?? "" - return OrbitCodexServiceTier(rawValue: bundledDefault) ?? .fast - }() + let storedEffort = UserDefaults.standard.string(forKey: Keys.codexReasoningEffort) ?? "" + codexReasoningEffort = storedEffort.isEmpty ? .medium : OrbitCodexReasoningEffort(rawValue: storedEffort) + let storedTier = UserDefaults.standard.string(forKey: Keys.codexServiceTier) ?? "" + codexServiceTier = + storedTier.isEmpty + ? { + let bundledDefault = AppBundleConfiguration.stringValue(forKey: "CodexActionServiceTier") ?? "" + return bundledDefault.isEmpty ? .serverDefault : OrbitCodexServiceTier(rawValue: bundledDefault) + }() : OrbitCodexServiceTier(rawValue: storedTier) codexActionModel = { - let stored = UserDefaults.standard.string(forKey: Keys.codexActionModel) + let stored = + UserDefaults.standard.string(forKey: Keys.codexActionModel) ?? AppBundleConfiguration.stringValue(forKey: "CodexActionModel") ?? "" let normalized = stored.trimmingCharacters(in: .whitespacesAndNewlines) - return normalized.isEmpty ? OrbitCodexModelOption.fallbackDefaultModel : normalized + return normalized }() - appleTTSVoiceIdentifier = UserDefaults.standard.string(forKey: Keys.appleTTSVoiceIdentifier) + codexAgentFolder = UserDefaults.standard.string(forKey: Keys.codexAgentFolder) ?? "" + appleTTSVoiceIdentifier = + UserDefaults.standard.string(forKey: Keys.appleTTSVoiceIdentifier) ?? AppBundleConfiguration.stringValue(forKey: "AppleTTSVoiceIdentifier") ?? "" + microphoneDeviceUID = UserDefaults.standard.string(forKey: Keys.microphoneDeviceUID) ?? "" } private static func applyProductDefaultResetIfNeeded() { let storedGeneration = UserDefaults.standard.integer(forKey: Keys.productDefaultsGeneration) guard storedGeneration < Self.currentProductDefaultsGeneration else { return } - UserDefaults.standard.removeObject(forKey: Keys.voicePreset) - UserDefaults.standard.removeObject(forKey: Keys.codexReasoningEffort) - UserDefaults.standard.removeObject(forKey: Keys.codexServiceTier) - UserDefaults.standard.removeObject(forKey: Keys.codexActionModel) - UserDefaults.standard.removeObject(forKey: Keys.appleTTSVoiceIdentifier) UserDefaults.standard.set(Self.currentProductDefaultsGeneration, forKey: Keys.productDefaultsGeneration) } } diff --git a/Orbit/OrbitTemporaryCaptureLease.swift b/Orbit/OrbitTemporaryCaptureLease.swift new file mode 100644 index 0000000..a60d2b0 --- /dev/null +++ b/Orbit/OrbitTemporaryCaptureLease.swift @@ -0,0 +1,97 @@ +import Foundation + +/// Owns one per-turn screen capture. The file exists only while Codex can still +/// consume it and is removed when the turn reaches any terminal state. +nonisolated final class OrbitTemporaryCaptureLease: @unchecked Sendable { + static let directoryName = "OrbitTemporaryCaptures" + static let filePrefix = "capture-" + static let staleAge: TimeInterval = 60 * 60 + private static let cleanupQueue = DispatchQueue(label: "com.orbit.capture-cleanup", qos: .utility) + + let turnID: UUID + let fileURL: URL + + private let lock = NSLock() + private var hasReleased = false + + private init(turnID: UUID, fileURL: URL) { + self.turnID = turnID + self.fileURL = fileURL + } + + static func create( + data: Data, + turnID: UUID = UUID(), + temporaryDirectory: URL = FileManager.default.temporaryDirectory + ) async throws -> OrbitTemporaryCaptureLease { + try await Task.detached(priority: .userInitiated) { + let fileManager = FileManager.default + let directory = temporaryDirectory.appendingPathComponent(directoryName, isDirectory: true) + try fileManager.createDirectory( + at: directory, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + try fileManager.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) + + let fileURL = + directory + .appendingPathComponent("\(filePrefix)\(turnID.uuidString)") + .appendingPathExtension("jpg") + guard + fileManager.createFile( + atPath: fileURL.path, + contents: data, + attributes: [.posixPermissions: 0o600] + ) + else { + throw CocoaError(.fileWriteUnknown) + } + try fileManager.setAttributes([.posixPermissions: 0o600], ofItemAtPath: fileURL.path) + return OrbitTemporaryCaptureLease(turnID: turnID, fileURL: fileURL) + }.value + } + + func release() { + lock.lock() + guard !hasReleased else { + lock.unlock() + return + } + hasReleased = true + lock.unlock() + Self.cleanupQueue.sync { + try? FileManager.default.removeItem(at: fileURL) + } + } + + static func sweepStaleCaptures( + now: Date = Date(), + temporaryDirectory: URL = FileManager.default.temporaryDirectory + ) async { + await Task.detached(priority: .utility) { + let fileManager = FileManager.default + let directory = temporaryDirectory.appendingPathComponent(directoryName, isDirectory: true) + guard + let entries = try? fileManager.contentsOfDirectory( + at: directory, + includingPropertiesForKeys: [.contentModificationDateKey, .isRegularFileKey], + options: [.skipsHiddenFiles] + ) + else { return } + + for entry in entries where entry.lastPathComponent.hasPrefix(filePrefix) { + guard let values = try? entry.resourceValues(forKeys: [.contentModificationDateKey, .isRegularFileKey]), + values.isRegularFile == true, + let modifiedAt = values.contentModificationDate, + now.timeIntervalSince(modifiedAt) >= staleAge + else { continue } + try? fileManager.removeItem(at: entry) + } + }.value + } + + deinit { + release() + } +} diff --git a/Orbit/OverlayWindow.swift b/Orbit/OverlayWindow.swift index 3f76aba..fc96e50 100644 --- a/Orbit/OverlayWindow.swift +++ b/Orbit/OverlayWindow.swift @@ -144,7 +144,7 @@ struct OrbitCursorOverlayView: View { "over here!", "click this!", "here it is!", - "found it!" + "found it!", ] var body: some View { @@ -340,13 +340,16 @@ struct OrbitCursorOverlayView: View { // When a UI element location is detected, navigate the Orbit cursor to // that position so it points at the element. guard let screenLocation = newLocation, - let displayFrame = orbitManager.detectedElementDisplayFrame else { + let displayFrame = orbitManager.detectedElementDisplayFrame + else { return } // Only navigate if the target is on THIS screen - guard screenFrame.contains(CGPoint(x: displayFrame.midX, y: displayFrame.midY)) - || displayFrame == screenFrame else { + guard + screenFrame.contains(CGPoint(x: displayFrame.midX, y: displayFrame.midY)) + || displayFrame == screenFrame + else { return } @@ -622,21 +625,25 @@ struct OrbitCursorOverlayView: View { // Quadratic bezier: B(t) = (1-t)²·P0 + 2(1-t)t·P1 + t²·P2 let oneMinusT = 1.0 - t - let bezierX = oneMinusT * oneMinusT * startPosition.x - + 2.0 * oneMinusT * t * controlPoint.x - + t * t * endPosition.x - let bezierY = oneMinusT * oneMinusT * startPosition.y - + 2.0 * oneMinusT * t * controlPoint.y - + t * t * endPosition.y + let bezierX = + oneMinusT * oneMinusT * startPosition.x + + 2.0 * oneMinusT * t * controlPoint.x + + t * t * endPosition.x + let bezierY = + oneMinusT * oneMinusT * startPosition.y + + 2.0 * oneMinusT * t * controlPoint.y + + t * t * endPosition.y self.cursorPosition = CGPoint(x: bezierX, y: bezierY) // Rotation: face the direction of travel by computing the tangent // to the bezier curve. B'(t) = 2(1-t)(P1-P0) + 2t(P2-P1) - let tangentX = 2.0 * oneMinusT * (controlPoint.x - startPosition.x) - + 2.0 * t * (endPosition.x - controlPoint.x) - let tangentY = 2.0 * oneMinusT * (controlPoint.y - startPosition.y) - + 2.0 * t * (endPosition.y - controlPoint.y) + let tangentX = + 2.0 * oneMinusT * (controlPoint.x - startPosition.x) + + 2.0 * t * (endPosition.x - controlPoint.x) + let tangentY = + 2.0 * oneMinusT * (controlPoint.y - startPosition.y) + + 2.0 * t * (endPosition.y - controlPoint.y) self.triangleRotationDegrees = atan2(tangentY, tangentX) * (180.0 / .pi) - OrbitBranding.defaultMarkHeadingDegrees let scalePulse = sin(linearProgress * .pi) @@ -667,7 +674,8 @@ struct OrbitCursorOverlayView: View { // Use custom bubble text from the Orbit manager (e.g. onboarding demo) // if available, otherwise fall back to a random pointer phrase - let pointerPhrase = orbitManager.detectedElementBubbleText + let pointerPhrase = + orbitManager.detectedElementBubbleText ?? navigationPointerPhrases.randomElement() ?? "right here!" @@ -1028,18 +1036,22 @@ class OrbitOverlayWindowManager { let windowsToFade = overlayWindows overlayWindows.removeAll() - NSAnimationContext.runAnimationGroup({ context in - context.duration = duration - context.timingFunction = CAMediaTimingFunction(name: .easeIn) - for window in windowsToFade { - window.animator().alphaValue = 0 - } - }, completionHandler: { - for window in windowsToFade { - window.orderOut(nil) - window.contentView = nil - } - }) + NSAnimationContext.runAnimationGroup( + { context in + context.duration = duration + context.timingFunction = CAMediaTimingFunction(name: .easeIn) + for window in windowsToFade { + window.animator().alphaValue = 0 + } + }, + completionHandler: { + MainActor.assumeIsolated { + for window in windowsToFade { + window.orderOut(nil) + window.contentView = nil + } + } + }) } func isShowingOverlay() -> Bool { diff --git a/Orbit/TextToSpeechProvider.swift b/Orbit/TextToSpeechProvider.swift index 4006ab8..bab2327 100644 --- a/Orbit/TextToSpeechProvider.swift +++ b/Orbit/TextToSpeechProvider.swift @@ -1,4 +1,4 @@ -import AppKit +import AVFoundation import Foundation protocol TextToSpeechProvider: AnyObject { @@ -11,7 +11,7 @@ protocol TextToSpeechProvider: AnyObject { func stopPlayback() } -struct OrbitAppleVoiceOption: Identifiable, Equatable { +struct OrbitAppleVoiceOption: Identifiable, Equatable, Sendable { let identifier: String let name: String let language: String @@ -20,512 +20,90 @@ struct OrbitAppleVoiceOption: Identifiable, Equatable { var id: String { identifier } } +/// Public AVFoundation-only catalog. Orbit intentionally does not inspect Siri +/// preference plists or private frameworks to discover voices. enum OrbitAppleVoiceCatalog { - private static let siriVoiceIdentifierPrefix = "com.apple.speech.synthesis.voice.custom.siri." - private static let assistantVoiceMapPath = "/System/Library/PrivateFrameworks/SiriTTSService.framework/Versions/A/Resources/AssistantVoiceMap.plist" - private static let voiceServicesPrefsPath = "\(NSHomeDirectory())/Library/Preferences/com.apple.voiceservices.plist" - private static let noveltyVoiceNames: Set = [ - "bad news", "bahh", "bells", "boing", "bubbles", "cellos", "fred", - "good news", "jester", "junior", "organ", "superstar", "trinoids", - "whisper", "wobble", "zarvox" - ] - - static func availableVoices(for localeIdentifier: String = Locale.autoupdatingCurrent.identifier) -> [OrbitAppleVoiceOption] { - let subscribedIdentifier = subscribedSiriVoiceIdentifier(for: localeIdentifier) - let candidates = assistantVoiceMapCandidates(for: localeIdentifier) - let resolvedCandidates = candidates.isEmpty - ? fallbackSiriCandidates(for: localeIdentifier) - : candidates - let finalCandidates: [OrbitAppleVoiceCandidate] - if resolvedCandidates.isEmpty, let subscribedIdentifier, - let subscribedCandidate = assistantVoiceCandidate(for: subscribedIdentifier) { - finalCandidates = [subscribedCandidate] - } else { - finalCandidates = resolvedCandidates - } - - return finalCandidates.map { voice in - OrbitAppleVoiceOption( - identifier: voice.identifier, - name: voice.name, - language: voice.language, - displayName: voice.displayName - ) - } + static func availableVoices( + for localeIdentifier: String = Locale.autoupdatingCurrent.identifier + ) -> [OrbitAppleVoiceOption] { + let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") + let baseLanguage = Locale(identifier: localeIdentifier).language.languageCode?.identifier + + return AVSpeechSynthesisVoice.speechVoices() + .map { voice in + OrbitAppleVoiceOption( + identifier: voice.identifier, + name: voice.name, + language: voice.language, + displayName: "\(voice.name) · \(localizedLanguageName(voice.language))" + ) + } + .sorted { lhs, rhs in + let lhsScore = localeScore(lhs.language, exact: normalizedLocale, base: baseLanguage) + let rhsScore = localeScore(rhs.language, exact: normalizedLocale, base: baseLanguage) + if lhsScore != rhsScore { return lhsScore > rhsScore } + if lhs.language != rhs.language { return lhs.language < rhs.language } + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending + } } static func resolvedVoice( preferredIdentifier: String?, localeIdentifier: String = Locale.autoupdatingCurrent.identifier ) -> String? { - if let preferredIdentifier = normalizedPreferredIdentifier(preferredIdentifier), - let resolved = bestReachableIdentifier(for: preferredIdentifier) { - return resolved + let voices = availableVoices(for: localeIdentifier) + if let preferred = normalizedPreferredIdentifier(preferredIdentifier), + voices.contains(where: { $0.identifier == preferred }) + { + return preferred } - if let subscribedIdentifier = subscribedSiriVoiceIdentifier(for: localeIdentifier), - let resolved = bestReachableIdentifier(for: subscribedIdentifier) { - return resolved + let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") + if let exact = voices.first(where: { $0.language.caseInsensitiveCompare(normalizedLocale) == .orderedSame }) { + return exact.identifier } - - if let automaticVoice = availableVoices(for: localeIdentifier).first, - let resolved = bestReachableIdentifier(for: automaticVoice.identifier) { - return resolved + if let base = Locale(identifier: localeIdentifier).language.languageCode?.identifier, + let languageMatch = voices.first(where: { $0.language.lowercased().hasPrefix(base.lowercased()) }) + { + return languageMatch.identifier } - - return nil + return voices.first?.identifier } static func currentSelectionSummary( preferredIdentifier: String?, localeIdentifier: String = Locale.autoupdatingCurrent.identifier ) -> String { - let automatic = normalizedPreferredIdentifier(preferredIdentifier) == nil - let candidates = availableVoices(for: localeIdentifier) - guard let identifier = resolvedVoice(preferredIdentifier: preferredIdentifier, localeIdentifier: localeIdentifier), - let voice = candidates.first(where: { $0.identifier == identifier }) - ?? assistantVoiceOption(for: identifier) else { - return automatic ? "Auto: Unavailable" : "Unavailable" + let isAutomatic = normalizedPreferredIdentifier(preferredIdentifier) == nil + guard + let identifier = resolvedVoice( + preferredIdentifier: preferredIdentifier, + localeIdentifier: localeIdentifier + ), let voice = availableVoices(for: localeIdentifier).first(where: { $0.identifier == identifier }) + else { + return isAutomatic ? "Automatic · unavailable" : "Voice unavailable" } - - return automatic ? "Auto · \(voice.name)" : voice.displayName + return isAutomatic ? "Automatic · \(voice.name)" : voice.displayName } static func normalizedPreferredIdentifier(_ preferredIdentifier: String?) -> String? { - guard let preferredIdentifier else { return nil } - let trimmed = preferredIdentifier.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return nil } - if isReachableSiriVoiceIdentifier(trimmed) { - return trimmed - } - let baseIdentifier = baseSiriIdentifier(from: trimmed) - return isReachableSiriVoiceIdentifier(baseIdentifier) ? baseIdentifier : nil - } - - static func makeSynthesizer(for identifier: String?) -> NSSpeechSynthesizer? { - let trimmedIdentifier = identifier?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - if trimmedIdentifier.isEmpty { - return NSSpeechSynthesizer() - } - - return NSSpeechSynthesizer(voice: NSSpeechSynthesizer.VoiceName(rawValue: trimmedIdentifier)) - } - - private static func allVoiceCandidates() -> [OrbitAppleVoiceCandidate] { - let standardCandidates = NSSpeechSynthesizer.availableVoices.map(\.rawValue) - let extraCandidates = discoveredInstalledVoiceIdentifiers() - let identifiers = Array(Set(standardCandidates + extraCandidates)) - - return identifiers.compactMap { identifier in - makeVoiceCandidate(for: identifier) - } - } - - private static func siriVoiceCandidates() -> [OrbitAppleVoiceCandidate] { - allVoiceCandidates().filter { voice in - voice.identifier.hasPrefix(siriVoiceIdentifierPrefix) - } - } - - private static func fallbackSiriCandidates(for localeIdentifier: String) -> [OrbitAppleVoiceCandidate] { - let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") - let baseLanguageCode = Locale(identifier: localeIdentifier).language.languageCode?.identifier - - let exactMatches = siriVoiceCandidates().filter { voice in - voice.language == normalizedLocale - } - - let baseMatches = siriVoiceCandidates().filter { voice in - guard let baseLanguageCode else { return false } - return voice.language == baseLanguageCode || voice.language.hasPrefix("\(baseLanguageCode)-") - } - - return deduplicatedAndSortedVoices(exactMatches + baseMatches + siriVoiceCandidates()) - } - - private static func deduplicatedAndSortedVoices(_ voices: [OrbitAppleVoiceCandidate]) -> [OrbitAppleVoiceCandidate] { - voices - .reduce(into: [String: OrbitAppleVoiceCandidate]()) { partialResult, voice in - if let existing = partialResult[voice.identifier] { - if voiceScore(voice) > voiceScore(existing) { - partialResult[voice.identifier] = voice - } - } else { - partialResult[voice.identifier] = voice - } - } - .values - .sorted { lhs, rhs in - if let lhsOrder = lhs.order, let rhsOrder = rhs.order, lhsOrder != rhsOrder { - return lhsOrder < rhsOrder - } - let lhsScore = voiceScore(lhs) - let rhsScore = voiceScore(rhs) - if lhsScore != rhsScore { - return lhsScore > rhsScore - } - return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending - } - } - - private static func isReachableSiriVoiceIdentifier(_ identifier: String) -> Bool { - if identifier.hasPrefix(siriVoiceIdentifierPrefix), makeSynthesizer(for: identifier) != nil { - return true - } - let baseIdentifier = baseSiriIdentifier(from: identifier) - return baseIdentifier.hasPrefix(siriVoiceIdentifierPrefix) && makeSynthesizer(for: baseIdentifier) != nil - } - - private static func baseSiriIdentifier(from identifier: String) -> String { - guard identifier.hasPrefix(siriVoiceIdentifierPrefix) else { return identifier } - if identifier.hasSuffix(".premium") { - return String(identifier.dropLast(".premium".count)) - } - return identifier + guard let value = preferredIdentifier?.trimmingCharacters(in: .whitespacesAndNewlines), + !value.isEmpty + else { return nil } + return value } - private static func discoveredInstalledVoiceIdentifiers() -> [String] { - let prefsURL = FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent("Library/Preferences/com.apple.speech.voice.prefs.plist") - - guard let prefs = NSDictionary(contentsOf: prefsURL) as? [String: Any] else { - return [] - } - - var identifiers: Set = [] - - if let installationLog = prefs["SpeechDataInstallationLog"] as? [String: Any] { - for key in installationLog.keys { - if let stripped = key.split(separator: ":", maxSplits: 1).last, key.hasPrefix("VOICEID:") { - identifiers.insert(String(stripped)) - } - } - } - - if let voiceStatistics = prefs["VoiceStatistics"] as? [String: Any], - let perVoiceTable = voiceStatistics["PerVoiceTable"] as? [String: Any] { - for key in perVoiceTable.keys { - identifiers.insert(key) - } - } - - return Array(identifiers) - } - - private static func installedVariants(for identifier: String) -> [String] { - let targetBase = baseSiriIdentifier(from: identifier) - let installed = discoveredInstalledVoiceIdentifiers() - let variants = installed.filter { installedIdentifier in - let normalizedInstalled = baseSiriIdentifier(from: installedIdentifier) - return normalizedInstalled == targetBase - } - if variants.isEmpty, targetBase != identifier { - return [identifier, targetBase].filter { isReachableSiriVoiceIdentifier($0) } - } - return variants - } - - private static func bestReachableIdentifier(for identifier: String) -> String? { - let candidates = Array(Set(installedVariants(for: identifier) + [identifier, baseSiriIdentifier(from: identifier)])) - .filter { isReachableSiriVoiceIdentifier($0) } - guard !candidates.isEmpty else { return nil } - return candidates.max { lhs, rhs in - siriIdentifierScore(lhs) < siriIdentifierScore(rhs) - } + private static func localeScore(_ language: String, exact: String, base: String?) -> Int { + if language.caseInsensitiveCompare(exact) == .orderedSame { return 2 } + if let base, language.lowercased().hasPrefix(base.lowercased()) { return 1 } + return 0 } - private static func siriIdentifierScore(_ identifier: String) -> Int { - let lowered = identifier.lowercased() - var score = 100 - if lowered.contains(".premiumhigh") { - score += 400 - } else if lowered.contains(".premium") { - score += 300 - } - if lowered.contains(".neuralax.") { - score += 120 - } else if lowered.contains(".neural.") { - score += 100 - } else if lowered.contains(".natural.") { - score += 80 - } else if lowered.contains(".gryphon.") { - score += 60 - } - return score - } - - private static func assistantVoiceMapCandidates(for localeIdentifier: String) -> [OrbitAppleVoiceCandidate] { - guard let voiceMap = NSDictionary(contentsOfFile: assistantVoiceMapPath) as? [String: Any] else { - return [] - } - - let requestedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") - let requestedBaseLanguage = Locale(identifier: localeIdentifier).language.languageCode?.identifier - - let exactLocale = voiceMap.keys.first(where: { $0.caseInsensitiveCompare(requestedLocale) == .orderedSame }) - let baseMatches = voiceMap.keys - .filter { locale in - guard let requestedBaseLanguage else { return false } - return locale.hasPrefix("\(requestedBaseLanguage)-") - } - .sorted() - let matchedLocale = exactLocale - ?? preferredLocaleMatch(from: baseMatches, requestedLocale: requestedLocale) - - guard let matchedLocale, - let entries = voiceMap[matchedLocale] as? [[String: Any]] else { - return [] - } - - let candidates = entries.compactMap { entry -> OrbitAppleVoiceCandidate? in - guard let identifier = entry["identifier"] as? String, - isReachableSiriVoiceIdentifier(identifier) else { - return nil - } - - let order = entry["order"] as? Int - let rawName = (entry["name"] as? String) ?? parsedCustomSiriName(from: identifier) ?? "Siri" - let displayName = order.map { "Voice \($0) · \(formattedVoiceName(rawName))" } ?? formattedVoiceName(rawName) - - return OrbitAppleVoiceCandidate( - identifier: bestReachableIdentifier(for: identifier) ?? identifier, - name: formattedVoiceName(rawName), - language: matchedLocale, - displayName: premiumAwareDisplayName(displayName, identifier: bestReachableIdentifier(for: identifier) ?? identifier), - order: order - ) - } - - return deduplicatedAndSortedVoices(candidates) - } - - private static func preferredLocaleMatch(from candidates: [String], requestedLocale: String) -> String? { - guard !candidates.isEmpty else { return nil } - if candidates.contains("en-US") { - return "en-US" - } - let normalizedRequested = requestedLocale.replacingOccurrences(of: "_", with: "-") - if let regionMatch = candidates.first(where: { $0.caseInsensitiveCompare(normalizedRequested) == .orderedSame }) { - return regionMatch - } - return candidates.first - } - - private static func subscribedSiriVoiceIdentifier(for localeIdentifier: String) -> String? { - guard let prefs = NSDictionary(contentsOfFile: voiceServicesPrefsPath) as? [String: Any], - let subscribedAssets = prefs["subscribedAssets"] as? [String: Any] else { - return nil - } - - let requestedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") - let requestedBaseLanguage = Locale(identifier: localeIdentifier).language.languageCode?.identifier - - for serviceGroup in subscribedAssets.values { - guard let serviceMap = serviceGroup as? [String: Any] else { continue } - for assets in serviceMap.values { - guard let entries = assets as? [[String: Any]] else { continue } - for entry in entries { - let languages = (entry["Languages"] as? [String])?.map { $0.replacingOccurrences(of: "_", with: "-") } ?? [] - let matchesLocale = languages.contains { $0.caseInsensitiveCompare(requestedLocale) == .orderedSame } - || languages.contains { language in - guard let requestedBaseLanguage else { return false } - return language.hasPrefix("\(requestedBaseLanguage)-") - } - guard matchesLocale, let name = entry["Name"] as? String else { continue } - let identifier = "\(siriVoiceIdentifierPrefix)\(name.lowercased())" - if isReachableSiriVoiceIdentifier(identifier) { - return identifier - } - } - } - } - - return nil - } - - private static func assistantVoiceCandidate(for identifier: String) -> OrbitAppleVoiceCandidate? { - guard let voiceMap = NSDictionary(contentsOfFile: assistantVoiceMapPath) as? [String: Any] else { - return nil - } - - let targetIdentifier = baseSiriIdentifier(from: identifier) - for (locale, rawEntries) in voiceMap { - guard let entries = rawEntries as? [[String: Any]] else { continue } - for entry in entries { - guard let entryIdentifier = entry["identifier"] as? String, - baseSiriIdentifier(from: entryIdentifier) == targetIdentifier, - isReachableSiriVoiceIdentifier(entryIdentifier) else { - continue - } - - let order = entry["order"] as? Int - let rawName = (entry["name"] as? String) ?? parsedCustomSiriName(from: targetIdentifier) ?? "Siri" - let displayName = order.map { "Voice \($0) · \(formattedVoiceName(rawName))" } ?? formattedVoiceName(rawName) - return OrbitAppleVoiceCandidate( - identifier: bestReachableIdentifier(for: targetIdentifier) ?? targetIdentifier, - name: formattedVoiceName(rawName), - language: locale, - displayName: premiumAwareDisplayName(displayName, identifier: bestReachableIdentifier(for: targetIdentifier) ?? targetIdentifier), - order: order - ) - } - } - - return nil - } - - private static func assistantVoiceOption(for identifier: String) -> OrbitAppleVoiceOption? { - guard let candidate = assistantVoiceCandidate(for: identifier) else { return nil } - return OrbitAppleVoiceOption( - identifier: candidate.identifier, - name: candidate.name, - language: candidate.language, - displayName: candidate.displayName - ) - } - - private static func makeVoiceCandidate(for identifier: String) -> OrbitAppleVoiceCandidate? { - guard makeSynthesizer(for: identifier) != nil else { - return nil - } - - let attributes = NSSpeechSynthesizer.attributes( - forVoice: NSSpeechSynthesizer.VoiceName(rawValue: identifier) - ) - - let localeKey = NSSpeechSynthesizer.VoiceAttributeKey(rawValue: "VoiceLocaleIdentifier") - let nameKey = NSSpeechSynthesizer.VoiceAttributeKey(rawValue: "VoiceName") - - let language = (attributes[localeKey] as? String)?.replacingOccurrences(of: "_", with: "-") - ?? Locale.autoupdatingCurrent.identifier.replacingOccurrences(of: "_", with: "-") - let reportedName = attributes[nameKey] as? String - - let displayName = displayName(for: identifier, reportedName: reportedName) - let name = compactName(for: identifier, reportedName: reportedName) - - return OrbitAppleVoiceCandidate( - identifier: identifier, - name: name, - language: language, - displayName: displayName, - order: nil - ) - } - - private static func isPreferredNaturalVoice(_ voice: OrbitAppleVoiceCandidate) -> Bool { - let identifier = voice.identifier.lowercased() - let name = voice.name.lowercased() - - if identifier.contains("eloquence") { - return false - } - - if noveltyVoiceNames.contains(name) { - return false - } - - return true - } - - private static func voiceScore(_ voice: OrbitAppleVoiceCandidate) -> Int { - var score = 100 - let identifier = voice.identifier.lowercased() - let name = voice.name.lowercased() - - if identifier.contains("custom.siri") && identifier.contains("premium") { - score += 300 - } else if identifier.contains("custom.siri") { - score += 220 - } else if identifier.contains("alex") { - score += 180 - } else if identifier.contains("enhanced") { - score += 120 - } else if identifier.contains("super-compact") { - score -= 30 - } else if identifier.contains("compact") { - score -= 15 - } - - if name == "samantha" { - score += 5 - } - - return score - } - - private static func compactName(for identifier: String, reportedName: String?) -> String { - if identifier.contains("custom.siri") { - let parsedName = parsedCustomSiriName(from: identifier) ?? (reportedName ?? "Siri") - let name = formattedVoiceName(parsedName) - return premiumAwareDisplayName(name, identifier: identifier) - } - - if identifier.localizedCaseInsensitiveContains("Alex") { - return "Alex" - } - - return reportedName ?? identifier - } - - private static func displayName(for identifier: String, reportedName: String?) -> String { - if identifier.contains("custom.siri") { - let parsedName = parsedCustomSiriName(from: identifier) ?? "Siri" - let name = formattedVoiceName(parsedName) - return premiumAwareDisplayName(name, identifier: identifier) - } - - if identifier.localizedCaseInsensitiveContains("Alex") { - return "Alex" - } - - return reportedName ?? identifier - } - - private static func parsedCustomSiriName(from identifier: String) -> String? { - let components = identifier.split(separator: ".") - guard let siriIndex = components.firstIndex(of: "siri"), - components.count > siriIndex + 1 else { - return nil - } - - let rawName = String(components[siriIndex + 1]) - guard !rawName.isEmpty else { - return nil - } - - return rawName.prefix(1).uppercased() + rawName.dropFirst() - } - - private static func formattedVoiceName(_ rawName: String) -> String { - rawName - .split(separator: "-", omittingEmptySubsequences: true) - .map { part in - let value = String(part) - return value.prefix(1).uppercased() + value.dropFirst() - } - .joined(separator: "-") - } - - private static func premiumAwareDisplayName(_ name: String, identifier: String) -> String { - let lowered = identifier.lowercased() - if lowered.contains(".premiumhigh") { - return "\(name) Premium+" - } - if lowered.contains(".premium") { - return "\(name) Premium" - } - return name + private static func localizedLanguageName(_ identifier: String) -> String { + Locale.autoupdatingCurrent.localizedString(forIdentifier: identifier) ?? identifier } } -private struct OrbitAppleVoiceCandidate: Hashable { - let identifier: String - let name: String - let language: String - let displayName: String - let order: Int? -} - enum OrbitTTSProviderFactory { @MainActor static func makePrimaryProvider(for voicePreset: OrbitVoicePreset) -> any TextToSpeechProvider { @@ -544,105 +122,94 @@ enum OrbitTTSProviderFactory { } @MainActor -final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, NSSpeechSynthesizerDelegate { - let displayName = "Apple Speech" - let isConfigured = true - let unavailableExplanation: String? = nil - - private var synthesizer: NSSpeechSynthesizer? +final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, AVSpeechSynthesizerDelegate { + let displayName = "Apple Local Speech" + private let synthesizer = AVSpeechSynthesizer() private var currentSpeakContinuation: CheckedContinuation? private var lastLoggedVoiceIdentifier: String? override init() { super.init() + synthesizer.delegate = self + } + + var isConfigured: Bool { + !AVSpeechSynthesisVoice.speechVoices().isEmpty + } + + var unavailableExplanation: String? { + isConfigured ? nil : "No installed Apple speech voice is available. Install one in System Settings > Accessibility > Spoken Content." } var isPlaying: Bool { - synthesizer?.isSpeaking ?? false + synthesizer.isSpeaking } func speakText(_ text: String) async throws { + let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty else { return } + guard isConfigured else { + throw NSError( + domain: "AppleSystemTTSProvider", + code: -1, + userInfo: [NSLocalizedDescriptionKey: unavailableExplanation ?? "Apple local speech is unavailable."] + ) + } + stopPlayback() + let utterance = AVSpeechUtterance(string: normalized) + if let identifier = OrbitAppleVoiceCatalog.resolvedVoice( + preferredIdentifier: OrbitSettings.shared.appleTTSVoiceIdentifier + ) { + utterance.voice = AVSpeechSynthesisVoice(identifier: identifier) + logVoiceSelectionIfNeeded(identifier) + } + utterance.rate = AVSpeechUtteranceDefaultSpeechRate try await withCheckedThrowingContinuation { continuation in currentSpeakContinuation = continuation - let synthesizer = NSSpeechSynthesizer() - - self.synthesizer = synthesizer - synthesizer.delegate = self - synthesizer.usesFeedbackWindow = false - logVoiceSelectionIfNeeded(synthesizer) - - if !synthesizer.startSpeaking(text) { - currentSpeakContinuation = nil - self.synthesizer = nil - continuation.resume( - throwing: NSError( - domain: "AppleSystemTTSProvider", - code: -4, - userInfo: [NSLocalizedDescriptionKey: "Apple system speech could not start."] - ) - ) - } + synthesizer.speak(utterance) } } func stopPlayback() { - if synthesizer?.isSpeaking == true { - synthesizer?.stopSpeaking() + if synthesizer.isSpeaking || synthesizer.isPaused { + synthesizer.stopSpeaking(at: .immediate) } - synthesizer = nil - if let continuation = currentSpeakContinuation { currentSpeakContinuation = nil continuation.resume() } } - nonisolated func speechSynthesizer(_ sender: NSSpeechSynthesizer, didFinishSpeaking finishedSpeaking: Bool) { + nonisolated func speechSynthesizer( + _ synthesizer: AVSpeechSynthesizer, + didFinish utterance: AVSpeechUtterance + ) { Task { @MainActor [weak self] in - if finishedSpeaking { - self?.finishCurrentSpeech() - } else { - self?.failCurrentSpeech() - } + self?.finishCurrentSpeech() } } - private func finishCurrentSpeech() { - guard let continuation = currentSpeakContinuation else { return } - synthesizer = nil - currentSpeakContinuation = nil - continuation.resume() + nonisolated func speechSynthesizer( + _ synthesizer: AVSpeechSynthesizer, + didCancel utterance: AVSpeechUtterance + ) { + Task { @MainActor [weak self] in + self?.finishCurrentSpeech() + } } - private func failCurrentSpeech() { + private func finishCurrentSpeech() { guard let continuation = currentSpeakContinuation else { return } - synthesizer = nil currentSpeakContinuation = nil - continuation.resume( - throwing: NSError( - domain: "AppleSystemTTSProvider", - code: -2, - userInfo: [NSLocalizedDescriptionKey: "Apple system speech was interrupted."] - ) - ) + continuation.resume() } - private func logVoiceSelectionIfNeeded(_ synthesizer: NSSpeechSynthesizer) { - let voiceIdentifier = synthesizer.voice()?.rawValue ?? "system-default" - guard lastLoggedVoiceIdentifier != voiceIdentifier else { return } - lastLoggedVoiceIdentifier = voiceIdentifier - - var label = "System Default" - if let voiceName = synthesizer.voice() { - let attributes = NSSpeechSynthesizer.attributes(forVoice: voiceName) - let nameKey = NSSpeechSynthesizer.VoiceAttributeKey(rawValue: "VoiceName") - if let systemName = attributes[nameKey] as? String, !systemName.isEmpty { - label = systemName - } - } - - print("🗣️ Apple Speech voice: \(label) [\(voiceIdentifier)]") + private func logVoiceSelectionIfNeeded(_ identifier: String) { + guard lastLoggedVoiceIdentifier != identifier else { return } + lastLoggedVoiceIdentifier = identifier + let label = AVSpeechSynthesisVoice(identifier: identifier)?.name ?? "System voice" + OrbitSupportLog.append("voice", "local speech voice selected: \(label)") } } diff --git a/Orbit/WindowPositionManager.swift b/Orbit/WindowPositionManager.swift index 2a8afd4..37d2c75 100644 --- a/Orbit/WindowPositionManager.swift +++ b/Orbit/WindowPositionManager.swift @@ -51,7 +51,7 @@ class WindowPositionManager { return .alreadyGranted case .systemPrompt: hasAttemptedAccessibilitySystemPromptDuringCurrentLaunch = true - let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue(): true] as CFDictionary + let options = ["AXTrustedCheckOptionPrompt" as CFString: true] as CFDictionary _ = AXIsProcessTrustedWithOptions(options) case .systemSettings: openAccessibilitySettings() @@ -169,7 +169,8 @@ class WindowPositionManager { // the window is currently on, or finally the main screen. let targetScreen: NSScreen if let displayID, - let matchingScreen = NSScreen.screens.first(where: { $0.displayID == displayID }) { + let matchingScreen = NSScreen.screens.first(where: { $0.displayID == displayID }) + { targetScreen = matchingScreen } else if let currentScreen = mainWindow.screen { targetScreen = currentScreen @@ -205,7 +206,8 @@ class WindowPositionManager { // Get the frontmost application that isn't us guard let frontApp = NSWorkspace.shared.frontmostApplication, - frontApp.processIdentifier != ProcessInfo.processInfo.processIdentifier else { + frontApp.processIdentifier != ProcessInfo.processInfo.processIdentifier + else { return } @@ -220,14 +222,16 @@ class WindowPositionManager { var positionValue: AnyObject? var sizeValue: AnyObject? guard AXUIElementCopyAttributeValue(focusedWindow as! AXUIElement, kAXPositionAttribute as CFString, &positionValue) == .success, - AXUIElementCopyAttributeValue(focusedWindow as! AXUIElement, kAXSizeAttribute as CFString, &sizeValue) == .success else { + AXUIElementCopyAttributeValue(focusedWindow as! AXUIElement, kAXSizeAttribute as CFString, &sizeValue) == .success + else { return } var otherPosition = CGPoint.zero var otherSize = CGSize.zero guard AXValueGetValue(positionValue as! AXValue, .cgPoint, &otherPosition), - AXValueGetValue(sizeValue as! AXValue, .cgSize, &otherSize) else { + AXValueGetValue(sizeValue as! AXValue, .cgSize, &otherSize) + else { return } @@ -250,7 +254,7 @@ class WindowPositionManager { // If the other window's right edge extends past our window's left edge, shrink it. if otherRight > ourLeft { let newWidth = ourLeft - otherPosition.x - guard newWidth > 200 else { return } // Don't shrink too small + guard newWidth > 200 else { return } // Don't shrink too small var newSize = CGSize(width: newWidth, height: otherSize.height) guard let newSizeValue = AXValueCreate(.cgSize, &newSize) else { return } diff --git a/OrbitTests/OrbitTests.swift b/OrbitTests/OrbitTests.swift index e6f7e7c..b53f70e 100644 --- a/OrbitTests/OrbitTests.swift +++ b/OrbitTests/OrbitTests.swift @@ -1,5 +1,6 @@ import Foundation import Testing + @testable import Orbit @MainActor @@ -48,6 +49,50 @@ struct OrbitTests { #expect(shouldTreatPermissionAsGranted) } + @Test func temporaryCaptureLeaseUsesPrivatePermissionsAndDeletesOnRelease() async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + let lease = try await OrbitTemporaryCaptureLease.create( + data: Data("screen".utf8), + temporaryDirectory: root + ) + let attributes = try FileManager.default.attributesOfItem(atPath: lease.fileURL.path) + let permissions = try #require(attributes[.posixPermissions] as? NSNumber) + + #expect(permissions.intValue & 0o777 == 0o600) + #expect(FileManager.default.fileExists(atPath: lease.fileURL.path)) + lease.release() + #expect(!FileManager.default.fileExists(atPath: lease.fileURL.path)) + } + + @Test func startupSweepDeletesOnlyStaleOrbitOwnedCaptures() async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let captureDirectory = root.appendingPathComponent(OrbitTemporaryCaptureLease.directoryName, isDirectory: true) + try FileManager.default.createDirectory(at: captureDirectory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + + let staleCapture = captureDirectory.appendingPathComponent("\(OrbitTemporaryCaptureLease.filePrefix)stale.jpg") + let freshCapture = captureDirectory.appendingPathComponent("\(OrbitTemporaryCaptureLease.filePrefix)fresh.jpg") + let unrelated = captureDirectory.appendingPathComponent("other-app.tmp") + FileManager.default.createFile(atPath: staleCapture.path, contents: Data(), attributes: nil) + FileManager.default.createFile(atPath: freshCapture.path, contents: Data(), attributes: nil) + FileManager.default.createFile(atPath: unrelated.path, contents: Data(), attributes: nil) + try FileManager.default.setAttributes( + [.modificationDate: Date(timeIntervalSinceNow: -(OrbitTemporaryCaptureLease.staleAge + 60))], + ofItemAtPath: staleCapture.path + ) + + await OrbitTemporaryCaptureLease.sweepStaleCaptures(temporaryDirectory: root) + + #expect(!FileManager.default.fileExists(atPath: staleCapture.path)) + #expect(FileManager.default.fileExists(atPath: freshCapture.path)) + #expect(FileManager.default.fileExists(atPath: unrelated.path)) + } + @Test func modelInstructionsStayTopLevelInGeneratedCodexConfig() async throws { let config = OrbitCodexEnvironment.makeConfigContents( logDirectory: URL(fileURLWithPath: "/tmp/orbit-log"), @@ -78,6 +123,36 @@ struct OrbitTests { #expect(config.contains("model = \"gpt-5.4\"")) #expect(config.contains("model_reasoning_effort = \"medium\"")) #expect(config.contains("service_tier = \"fast\"")) + #expect(!config.contains("multi_agent = false")) + #expect(config.contains("approval_policy = \"never\"")) + #expect(config.contains("sandbox_mode = \"danger-full-access\"")) + } + + @Test func generatedCodexConfigDefersToServerDefaultWhenModelIsEmpty() async throws { + let config = OrbitCodexEnvironment.makeConfigContents( + logDirectory: URL(fileURLWithPath: "/tmp/orbit-log"), + sqliteDirectory: URL(fileURLWithPath: "/tmp/orbit-sqlite"), + configuredSkillPaths: [:], + model: "" + ) + + #expect(!config.contains("\nmodel = ")) + #expect(!config.hasPrefix("model = ")) + #expect(!config.contains("service_tier =")) + } + + @Test func generatedCodexConfigIncludesOnlyConfiguredBundledSkills() async throws { + let docsPath = URL(fileURLWithPath: "/tmp/orbit-skills/openai-docs/SKILL.md") + let config = OrbitCodexEnvironment.makeConfigContents( + logDirectory: URL(fileURLWithPath: "/tmp/orbit-log"), + sqliteDirectory: URL(fileURLWithPath: "/tmp/orbit-sqlite"), + configuredSkillPaths: ["openai-docs": docsPath], + modelInstructionsPath: "/tmp/OrbitModelInstructions.md" + ) + + #expect(config.contains("[[skills.config]]")) + #expect(config.contains("path = \"/tmp/orbit-skills/openai-docs/SKILL.md\"")) + #expect(config.contains("enabled = true")) } @Test func writableSupportDirectoryPassesPreflight() async throws { @@ -107,6 +182,24 @@ struct OrbitTests { } } + @Test func supportPathOccupiedByAFileFailsClearly() async throws { + try withTemporaryDirectory { temporaryRoot in + let appSupportDirectory = temporaryRoot.appendingPathComponent("Application Support", isDirectory: true) + try FileManager.default.createDirectory(at: appSupportDirectory, withIntermediateDirectories: true) + let orbitSupportPath = appSupportDirectory.appendingPathComponent("Orbit") + try Data("not a directory".utf8).write(to: orbitSupportPath) + + do { + _ = try OrbitCodexEnvironment.validateSupportRootDirectory( + applicationSupportDirectory: appSupportDirectory + ) + Issue.record("Expected support-directory preflight to reject a regular file.") + } catch { + #expect(error.localizedDescription.contains("found a file instead")) + } + } + } + @Test func unwritableSupportDirectoryReturnsRecoveryInstructions() async throws { try withTemporaryDirectory { temporaryRoot in let appSupportDirectory = temporaryRoot.appendingPathComponent("Application Support", isDirectory: true) @@ -138,7 +231,7 @@ struct OrbitTests { } @Test func modelCatalogParsingSupportsSnakeCasePayload() async throws { - let parsed = CodexAppServerActionProvider.parseModelCatalog(from: [ + let parsed = OrbitCodexModelCatalog.parse(from: [ "data": [ [ "slug": "gpt-5.4", @@ -148,10 +241,10 @@ struct OrbitTests { "supported_reasoning_levels": [ ["effort": "low"], ["effort": "medium"], - ["effort": "high"] + ["effort": "high"], ], "default_reasoning_level": "medium", - "priority": 1 + "priority": 1, ], [ "slug": "gpt-5.4-mini", @@ -160,11 +253,11 @@ struct OrbitTests { "input_modalities": ["text", "image"], "supported_reasoning_levels": [ ["effort": "low"], - ["effort": "medium"] + ["effort": "medium"], ], "default_reasoning_level": "medium", - "priority": 2 - ] + "priority": 2, + ], ] ]) @@ -176,7 +269,7 @@ struct OrbitTests { } @Test func modelCatalogParsingSupportsLegacyCamelCasePayload() async throws { - let parsed = CodexAppServerActionProvider.parseModelCatalog(from: [ + let parsed = OrbitCodexModelCatalog.parse(from: [ "data": [ [ "model": "gpt-5.4", @@ -185,10 +278,10 @@ struct OrbitTests { "inputModalities": ["text", "image"], "supportedReasoningEfforts": [ ["reasoningEffort": "medium"], - ["reasoningEffort": "high"] + ["reasoningEffort": "high"], ], "defaultReasoningEffort": "medium", - "isDefault": true + "isDefault": true, ] ] ]) @@ -199,6 +292,35 @@ struct OrbitTests { #expect(parsed.first?.defaultEffort == .medium) } + @Test func modelCatalogPreservesFutureEffortsTiersAndUpgradeMetadata() async throws { + let parsed = OrbitCodexModelCatalog.parse(from: [ + "data": [ + [ + "model": "gpt-future", + "displayName": "GPT Future", + "inputModalities": ["text", "image", "audio"], + "supportedReasoningEfforts": [ + ["reasoningEffort": "none"], + ["reasoningEffort": "max"], + ["reasoningEffort": "orbital"], + ], + "defaultReasoningEffort": "orbital", + "supportedServiceTiers": ["standard", "priority-plus"], + "upgradeModel": "gpt-future-2", + "upgradeMessage": "A newer model is available.", + "isDefault": true, + ] + ] + ]) + + let model = try #require(parsed.first) + #expect(model.supportedEfforts.map(\.rawValue) == ["none", "max", "orbital"]) + #expect(model.defaultEffort?.rawValue == "orbital") + #expect(model.supportedServiceTiers.map(\.rawValue) == ["standard", "priority-plus"]) + #expect(model.upgradeModel == "gpt-future-2") + #expect(model.inputModalities == ["text", "image", "audio"]) + } + @Test func commentaryBufferMergeAvoidsOverlappingStreamDuplication() async throws { let merged = CodexAppServerActionProvider.mergedCommentaryBuffer( existing: "Using the pdf skill to make a polished", @@ -208,6 +330,135 @@ struct OrbitTests { #expect(merged == "Using the pdf skill to make a polished illustrated PDF") } + @Test func boundedBufferRetainsOnlyNewestBytes() async throws { + var buffer = OrbitBoundedDataBuffer(capacity: 5) + buffer.append(Data("abc".utf8)) + buffer.append(Data("def".utf8)) + #expect(String(data: buffer.data, encoding: .utf8) == "bcdef") + } + + @Test func boundedBufferFramesCompleteLines() async throws { + var buffer = OrbitBoundedDataBuffer(capacity: 32) + buffer.append(Data("first\nsecond".utf8)) + let firstData = buffer.popLine() + let firstLine = try #require(firstData) + #expect(String(data: firstLine, encoding: .utf8) == "first") + #expect(buffer.popLine() == nil) + buffer.append(Data("\n".utf8)) + let secondData = buffer.popLine() + let secondLine = try #require(secondData) + #expect(String(data: secondLine, encoding: .utf8) == "second") + } + + @Test func transportActorFramesOutputAndBoundsErrorSnapshots() async throws { + let transport = OrbitCodexTransportActor() + let firstLines = await transport.ingestStandardOutput(Data("{\"id\":1}\npartial".utf8)) + #expect(firstLines.count == 1) + #expect(String(data: firstLines[0], encoding: .utf8) == "{\"id\":1}") + let secondLines = await transport.ingestStandardOutput(Data("-line\n".utf8)) + #expect(String(data: try #require(secondLines.first), encoding: .utf8) == "partial-line") + + let snapshot = await transport.ingestStandardError(Data(repeating: 0x78, count: 1_048_600)) + #expect(snapshot.count == 1_048_576) + await transport.reset() + } + + @Test func commentaryBufferIsBounded() async throws { + let merged = CodexAppServerActionProvider.mergedCommentaryBuffer( + existing: String(repeating: "a", count: 65_530), + incomingDelta: String(repeating: "b", count: 100) + ) + #expect(merged.count == 65_536) + #expect(merged.hasSuffix(String(repeating: "b", count: 100))) + } + + @Test func supportLogRedactsSensitiveContext() async throws { + let raw = + "Authorization: Bearer secret-token prompt=show my password \(NSHomeDirectory())/Desktop OrbitTemporaryCaptures/capture-private.jpg command=[rm, secret]" + let sanitized = OrbitSupportLog.sanitize(raw) + #expect(!sanitized.contains("secret-token")) + #expect(!sanitized.contains(NSHomeDirectory())) + #expect(!sanitized.contains("show my password")) + #expect(!sanitized.contains("capture-private.jpg")) + } + + @Test func bundledSkillRoutingUsesTokenBoundaries() async throws { + #expect(OrbitBundledSkills.matchesAnyKeyword(in: "Open the Codex docs", keywords: ["codex"])) + #expect(OrbitBundledSkills.matchesAnyKeyword(in: "Please make a PDF.", keywords: ["pdf"])) + #expect(!OrbitBundledSkills.matchesAnyKeyword(in: "codexterity", keywords: ["codex"])) + #expect(!OrbitBundledSkills.matchesAnyKeyword(in: "spreadsheetish", keywords: ["sheet"])) + } + + @Test func approvalRouterAutoAcceptsEveryAppServerApprovalShape() async throws { + #expect(OrbitCodexApprovalRouter.route(method: "mcpServer/elicitation/request", params: [:]) == .acceptMCP) + #expect(OrbitCodexApprovalRouter.route(method: "item/commandExecution/requestApproval", params: [:]) == .acceptForSession) + #expect(OrbitCodexApprovalRouter.route(method: "item/fileChange/requestApproval", params: [:]) == .acceptForSession) + #expect( + OrbitCodexApprovalRouter.route( + method: "item/permissions/requestApproval", + params: ["permissions": ["network": true, "fileSystem": true]] + ) == .grantPermissions(network: true, fileSystem: true) + ) + } + + @Test func sessionCoordinatorPreservesAutomationAndServerDefaults() async throws { + let coordinator = OrbitCodexSessionCoordinator( + model: "", + effort: .max, + serviceTier: .serverDefault, + workingDirectory: "/tmp/orbit-agent" + ) + let thread = coordinator.threadStartParameters(sandbox: "danger-full-access") + #expect(thread["approvalPolicy"] as? String == "never") + #expect(thread["sandbox"] as? String == "danger-full-access") + #expect(thread["model"] == nil) + #expect(thread["serviceTier"] == nil) + + let turn = coordinator.turnStartParameters(threadID: "thread-1", input: []) + #expect(turn["effort"] as? String == "max") + #expect(turn["model"] == nil) + #expect(turn["serviceTier"] == nil) + } + + @Test func subagentActivityParsesGeneratedProtocolFixtures() async throws { + let direct = CodexAppServerActionProvider.parseSubagentActivities(from: [ + "type": "subAgentActivity", + "id": "activity-1", + "agentThreadId": "thread-child", + "agentPath": "/root/review", + "kind": "started", + ]) + #expect(direct.first?.agentPath == "/root/review") + #expect(direct.first?.status == "started") + + let collab = CodexAppServerActionProvider.parseSubagentActivities(from: [ + "type": "collabAgentToolCall", + "id": "tool-1", + "model": "server-model", + "agentsStates": [ + "thread-child": ["status": "completed", "message": "review finished"] + ], + ]) + #expect(collab.first?.threadID == "thread-child") + #expect(collab.first?.status == "completed") + #expect(collab.first?.model == "server-model") + + let reduced = OrbitCodexActivityReducer.reducing( + existing: direct, + item: [ + "type": "subAgentActivity", + "id": "activity-2", + "agentThreadId": "thread-child", + "agentPath": "/root/review", + "kind": "completed", + "message": "Review complete", + ] + ) + #expect(reduced.count == 1) + #expect(reduced.first?.status == "completed") + #expect(reduced.first?.message == "Review complete") + } + @Test func earlyCommentarySpeechWaitsForStableChunk() async throws { let tooEarly = CodexAppServerActionProvider.speakableCommentarySnippet( from: "Using the pdf skill" diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 0000000..0d41403 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,37 @@ +# Product + +## Register + +product + +## Users + +Orbit is for macOS users who want a capable Codex assistant available from any app without moving their work into a terminal or chat window. They are usually mid-task, often using voice, and need the interface to explain current state without becoming another workspace to manage. + +## Product Purpose + +Orbit gives every request current-screen context, routes it through a persistent local Codex session, and returns useful spoken, visual, or desktop guidance. Success means the user can ask once, understand what Orbit is doing, and receive a verified result without repeated permission dialogs or setup friction. + +## Brand Personality + +Quiet, capable, candid. Orbit should feel like a precise macOS instrument: calm at rest, immediately legible during work, and honest about its unusually broad local access. + +## Anti-references + +- A Clicky skin outside the proven permission-onboarding mechanics. +- Generic AI dashboards built from nested glass cards, decorative gradients, and duplicated status labels. +- Persistent screen-share indicators or approval theater that imply the user must supervise every local action. +- Playful labels such as “Magic Drag” where a direct instruction is clearer. +- Dense developer configuration exposed without progressive disclosure. + +## Design Principles + +- **One task, one hierarchy.** The current request and its state always outrank settings and decoration. +- **Transparency without friction.** Explain screen capture and unrestricted automation clearly once, then stay out of the way. +- **Platform-native first.** Use familiar macOS controls, keyboard behavior, terminology, and accessibility semantics. +- **State must earn motion.** Animation communicates listening, progress, dragging, success, or interruption only. +- **Reliable recovery is part of the interface.** Every denied, unavailable, disconnected, or stale state offers a specific next action. + +## Accessibility & Inclusion + +Meet WCAG 2.2 AA contrast where applicable. Every control must work with keyboard navigation and VoiceOver, every state must be understandable without color alone, and every animation must respect Reduce Motion. Copy must remain readable at larger accessibility text sizes without clipping or hiding actions. diff --git a/README.md b/README.md index b6dac1f..0b4882f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Orbit is a direct-download macOS assistant that brings a live Codex session onto your desktop in a way that feels native, visual, and immediate. -Instead of opening a separate coding tool or chat tab, you talk to Orbit from anywhere on your Mac. It can capture your current screen, narrate what it is doing, move its cursor overlay to the right place, and help you learn or complete tasks in context. +Instead of opening a separate coding tool or chat tab, you talk to Orbit from anywhere on your Mac. Every submitted request gets one fresh screen capture, so Codex sees the context you meant to share. Orbit then deletes that temporary image when the turn succeeds, fails, is interrupted, or is cancelled. Orbit is an independent open-source project. It is built on Codex by OpenAI, but it is not an official OpenAI product. @@ -57,18 +57,21 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ ## Quick start -1. Grant the requested desktop permissions. +1. Grant Microphone, Accessibility, and Screen Recording access. The guided permission rail lets you drag only the Orbit app tile into the relevant System Settings list. 2. Orbit checks whether its Codex runtime is already authenticated. 3. If authentication is required, Orbit opens the managed ChatGPT browser flow for you. 4. Choose `Local` or `Cloud` voice. 5. If you choose `Cloud`, add your OpenAI API key once. Orbit validates it and stores it in your macOS Keychain. -6. Hold the Orbit push-to-talk shortcut and ask for help from any screen. +6. Review the one-time unrestricted automation disclosure, then hold the Orbit push-to-talk shortcut and ask for help from any screen. ## How Orbit works - Orbit runs as a menu bar app with a compact panel and overlay HUD. - Each app run keeps one live Codex app-server session active in the background. -- Orbit can attach your current screen to requests so Codex has visual context. +- Orbit requires and attaches one fresh current-screen capture to every submitted request. It does not continuously record the screen and shows no persistent capture indicator. +- If capture fails, Orbit blocks the request and offers a retry instead of silently sending without visual context. +- Codex runs with `danger-full-access`, `approval_policy = "never"`, and automatically accepted app-server approvals. Orbit can run commands and edit files without asking for each operation. +- Team-up remains explicit: child agents appear only when you ask for them or applicable instructions require delegation. - Final responses can include pointing tags, which Orbit turns into cursor guidance on screen. - Voice, auth, and action settings stay lightweight and local to the machine. @@ -76,8 +79,9 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ ### Local -- Apple Speech transcription -- Apple system speech +- Apple on-device speech recognition (Orbit refuses silent network fallback) +- Apple `AVSpeechSynthesizer` voices, synthesized on-device +- installed voice and microphone selection with local preview/level test - no extra API key required ### Cloud @@ -92,7 +96,9 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ - Orbit has **no hosted backend** of its own. - Cloud voice uses the user-supplied OpenAI API key stored in Keychain. - Orbit keeps its Codex runtime state in `~/Library/Application Support/Orbit/CodexHome`. -- Orbit is a direct-download macOS utility and requires full desktop permissions to work as designed. +- Temporary captures use mode `0600` in an Orbit-owned temporary directory and are swept after terminal turn states or crash recovery. +- Support logs are private, bounded, rotated, allowlisted, and redact credentials, prompts, capture paths, home paths, and full command arguments. +- Orbit intentionally grants Codex unrestricted filesystem and command access. The optional Agent Folder changes starting context, not the security boundary. Read [SECURITY.md](SECURITY.md) before reporting vulnerabilities. @@ -102,7 +108,8 @@ Requirements: - macOS 14.2 or later - Xcode -- a local or bundled Codex runtime +- Codex CLI `0.144.0` for release packaging +- `chrome-devtools-mcp` `1.5.0` in the bundled browser runtime Build locally: @@ -126,7 +133,8 @@ Orbit is distributed as a signed direct download. The release pipeline: - bundles a Codex runtime into the final app - exports a signed Developer ID build - creates a PKG installer and DMG fallback -- notarizes and staples the public artifacts when credentials are configured +- refuses to publish unless the app, PKG, and DMG are signed, notarized, stapled, and pass Gatekeeper validation +- emits a runtime manifest and SBOM, then smoke-tests the bundled Codex, Node, and browser MCP executables Release automation lives in [scripts/release.sh](scripts/release.sh). More notes are in [scripts/README.md](scripts/README.md). @@ -143,6 +151,6 @@ Release automation lives in [scripts/release.sh](scripts/release.sh). More notes - Follow updates: [x.com/4xiom_](https://x.com/4xiom_) - Launch site: [orbitcodex.org](https://orbitcodex.org) -## Acknowledgements +## Design provenance -Orbit’s codebase is original, but the project was meaningfully inspired by the interface ideas explored in Clicky. Thanks to that project for helping make desktop-native AI interaction feel possible. +Orbit’s product UI, visual system, components, and brand assets are original. A prior internal permission-onboarding experiment informed only the mechanics of anchoring a draggable app tile beside System Settings; no external product chrome or design tokens are copied. diff --git a/SECURITY.md b/SECURITY.md index 0ebe78c..f6abcb8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -38,5 +38,11 @@ We will aim to: - Orbit has no hosted backend of its own. - Cloud voice API keys are stored in the macOS Keychain. - Orbit is a direct-download macOS app that requires full desktop permissions to function. +- Orbit launches Codex with `danger-full-access` and `approval_policy = "never"`, and automatically accepts app-server approval requests. This is an intentional product contract: Codex can run commands and edit files without per-operation prompts. +- Every submitted request requires one fresh screen capture. Orbit does not continuously record; if capture fails, the request is not sent. +- Each temporary capture is owned by its turn, stored with mode `0600`, deleted on every terminal/cancellation path, and eligible for Orbit-only crash-recovery cleanup on the next launch. +- Local voice uses public Apple on-device speech APIs. Cloud speech is optional, explicitly selected, disclosed as AI-generated voice, and Keychain-backed. +- Support logs are mode `0600`, capped at 5 MiB with three rotations, and redact credentials, authorization headers, prompt text, capture paths, home paths, and full command arguments. +- Public release tooling fails closed unless the app, PKG, and DMG pass signing, notarization, stapling, and Gatekeeper checks. Because Orbit intentionally works with elevated desktop access, reports involving unauthorized screen capture, credential exposure, command execution, auth/session leakage, or installer tampering should be sent privately first. diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md new file mode 100644 index 0000000..d615d0e --- /dev/null +++ b/docs/PRIVACY.md @@ -0,0 +1,9 @@ +# Orbit privacy model + +Orbit captures the current screen once for every submitted request. It does not continuously record and does not keep a persistent capture indicator. + +The request is blocked if the capture cannot be created. A successful capture is written to an Orbit-owned temporary directory with mode `0600`, associated with that turn, and removed after success, failure, interruption, cancellation, launch failure, or app termination. On launch, Orbit sweeps only stale files that match its own capture naming contract; unrelated temporary files are never touched. + +Local speech recognition requires Apple on-device recognition. Orbit does not silently fall back to network recognition. Local speech output uses public AVFoundation voices on-device. OpenAI cloud speech is optional, must be explicitly selected, and stores its API key in macOS Keychain. + +Orbit has no hosted backend of its own. Codex and explicitly selected cloud providers still receive the content needed to perform the request. Because Codex runs unrestricted with automatic approvals, it can execute commands and read or edit files available to the signed-in macOS user. diff --git a/docs/SETUP.md b/docs/SETUP.md new file mode 100644 index 0000000..586eb18 --- /dev/null +++ b/docs/SETUP.md @@ -0,0 +1,13 @@ +# Orbit setup and permissions + +Orbit asks for three macOS permissions in order: + +1. **Microphone** for push-to-talk. macOS presents the native prompt. +2. **Accessibility** for the global shortcut and guided desktop interaction. +3. **Screen Recording** for the single current-screen capture attached to each request. The live Screen Content probe is part of this step. + +For Accessibility and Screen Recording, Orbit opens the correct System Settings pane and anchors a small guide to that window. Only the Orbit app tile is draggable. The guide follows Settings across displays, closes when Settings closes or you press Escape, respects Reduce Motion, and dismisses after permission succeeds. “Reveal Orbit in Finder” and “Open Settings” remain available for keyboard and VoiceOver users. + +After permissions, Orbit explains its unrestricted automation contract once. Codex runs with full filesystem/command access and automatic approvals. There is deliberately no approval toggle or restricted mode in v1.1.0. + +The optional Agent Folder selects the starting working context. It does not restrict which files Codex can access. diff --git a/release-manifest.json b/release-manifest.json new file mode 100644 index 0000000..9ebb40c --- /dev/null +++ b/release-manifest.json @@ -0,0 +1,8 @@ +{ + "version": "1.0.7", + "downloadURL": "https://github.com/4xiomdev/orbit/releases/download/v1.0.7/Orbit-1.0.7.pkg", + "sha256": "bf377f3014632bf7e4a829404d7e0ea71f5a857282ebd4c30f29524ae7ee6c25", + "minimumMacOS": "14.2", + "codexRuntimeVersion": "0.118.0", + "browserMCPVersion": "0.21.0" +} diff --git a/scripts/bundle_codex_runtime.sh b/scripts/bundle_codex_runtime.sh index cf12a0c..4b94849 100755 --- a/scripts/bundle_codex_runtime.sh +++ b/scripts/bundle_codex_runtime.sh @@ -7,6 +7,8 @@ PROJECT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" BROWSER_RUNTIME_SOURCE_DIR="${PROJECT_DIR}/BundledResources/browser-runtime" BUNDLED_SKILLS_SOURCE_DIR="${PROJECT_DIR}/BundledResources/skills" MODEL_INSTRUCTIONS_SOURCE_PATH="${PROJECT_DIR}/BundledResources/orbit-model-instructions.md" +EXPECTED_CODEX_VERSION="0.144.0" +EXPECTED_BROWSER_MCP_VERSION="1.5.0" if [[ -z "${APP_BUNDLE_PATH}" ]]; then echo "usage: $0 /absolute/path/to/Orbit.app" >&2 @@ -46,8 +48,20 @@ if [[ -z "${CODEX_SOURCE}" ]]; then exit 1 fi +CODEX_VERSION_OUTPUT="$("${CODEX_SOURCE}" --version 2>/dev/null || true)" +CODEX_VERSION="${CODEX_VERSION_OUTPUT##* }" +if [[ "${CODEX_VERSION}" != "${EXPECTED_CODEX_VERSION}" ]]; then + if [[ "${ORBIT_ALLOW_UNPINNED_RUNTIME:-0}" == "1" ]]; then + echo "WARNING: bundling unpinned Codex ${CODEX_VERSION}; expected ${EXPECTED_CODEX_VERSION}. Local development only." >&2 + else + echo "Codex runtime mismatch: found ${CODEX_VERSION_OUTPUT:-unknown}; expected codex-cli ${EXPECTED_CODEX_VERSION}." >&2 + exit 1 + fi +fi + CODEX_JS="$(resolve_realpath "${CODEX_SOURCE}")" PACKAGE_ROOT="$(cd "$(dirname "${CODEX_JS}")/.." && pwd)" +PACKAGE_SCOPE_ROOT="$(dirname "${PACKAGE_ROOT}")" NODE_BIN="$(cd "$(dirname "${CODEX_SOURCE}")" && pwd)/node" if [[ ! -x "${NODE_BIN}" ]]; then @@ -69,9 +83,9 @@ find_vendor_root() { return 1 } -VENDOR_ROOT="$(find_vendor_root "${PACKAGE_ROOT}" || true)" +VENDOR_ROOT="$(find_vendor_root "${PACKAGE_SCOPE_ROOT}" || true)" if [[ -z "${VENDOR_ROOT}" ]]; then - echo "Could not find Codex vendor runtime under ${PACKAGE_ROOT}" >&2 + echo "Could not find Codex vendor runtime under ${PACKAGE_SCOPE_ROOT}" >&2 exit 1 fi @@ -121,6 +135,12 @@ cp "${BROWSER_RUNTIME_SOURCE_DIR}/package-lock.json" "${TMP_BROWSER_RUNTIME}/pac PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm ci --omit=dev --ignore-scripts --no-audit --no-fund ) +BROWSER_MCP_VERSION="$("${NODE_BIN}" -p "require('${TMP_BROWSER_RUNTIME}/node_modules/chrome-devtools-mcp/package.json').version")" +if [[ "${BROWSER_MCP_VERSION}" != "${EXPECTED_BROWSER_MCP_VERSION}" ]]; then + echo "Browser MCP runtime mismatch: found ${BROWSER_MCP_VERSION}; expected ${EXPECTED_BROWSER_MCP_VERSION}." >&2 + exit 1 +fi + mkdir -p "${BROWSER_TOOLS_ROOT}" ditto "${TMP_BROWSER_RUNTIME}/node_modules" "${BROWSER_TOOLS_ROOT}/node_modules" @@ -155,4 +175,62 @@ else exit 1 fi +PATH="${BIN_ROOT}:${PATH}" "${BIN_ROOT}/codex" --version | grep -F "${EXPECTED_CODEX_VERSION}" >/dev/null +"${BIN_ROOT}/node" --version >/dev/null +"${BIN_ROOT}/chrome-devtools-mcp" --help >/dev/null +"${BIN_ROOT}/playwright-mcp" --help >/dev/null +python3 "${SCRIPT_DIR}/smoke_browser_mcp.py" "${BIN_ROOT}/chrome-devtools-mcp" + +NATIVE_CODEX_PATH="$(find "${VENDOR_DEST}" -type f -name codex -perm -111 | sort | head -n 1)" +if [[ -z "${NATIVE_CODEX_PATH}" ]]; then + echo "Bundled native Codex executable was not found." >&2 + exit 1 +fi +CODEX_SHA256="$(shasum -a 256 "${NATIVE_CODEX_PATH}" | awk '{print $1}')" +NODE_SHA256="$(shasum -a 256 "${BIN_ROOT}/node" | awk '{print $1}')" +SCHEMA_TEMP="$(mktemp -d)" +PATH="${BIN_ROOT}:${PATH}" "${BIN_ROOT}/codex" app-server generate-json-schema --experimental --out "${SCHEMA_TEMP}" +SCHEMA_SOURCE="${SCHEMA_TEMP}/codex_app_server_protocol.v2.schemas.json" +if [[ ! -f "${SCHEMA_SOURCE}" ]]; then + echo "Pinned Codex did not generate the expected app-server v2 schema." >&2 + exit 1 +fi +cp "${SCHEMA_SOURCE}" "${RUNTIME_ROOT}/CodexAppServerProtocol.v2.json" +SCHEMA_SHA256="$(shasum -a 256 "${RUNTIME_ROOT}/CodexAppServerProtocol.v2.json" | awk '{print $1}')" +rm -rf "${SCHEMA_TEMP}" + +TYPE_BINDINGS_TEMP="$(mktemp -d)" +PATH="${BIN_ROOT}:${PATH}" "${BIN_ROOT}/codex" app-server generate-ts --experimental --out "${TYPE_BINDINGS_TEMP}" +if ! find "${TYPE_BINDINGS_TEMP}" -type f -name '*.ts' -print -quit | grep -q .; then + echo "Pinned Codex did not generate app-server TypeScript bindings." >&2 + exit 1 +fi +TYPE_BINDINGS_DEST="${RUNTIME_ROOT}/CodexAppServerProtocolTypes" +ditto "${TYPE_BINDINGS_TEMP}" "${TYPE_BINDINGS_DEST}" +TYPE_BINDINGS_SHA256="$( + cd "${TYPE_BINDINGS_DEST}" + while IFS= read -r binding; do shasum -a 256 "${binding}"; done < <(find . -type f -name '*.ts' | sort) | shasum -a 256 | awk '{print $1}' +)" +rm -rf "${TYPE_BINDINGS_TEMP}" + +cat > "${RUNTIME_ROOT}/OrbitRuntimeManifest.json" < "${RUNTIME_ROOT}/OrbitRuntimeSBOM.cdx.json" +) +chmod 644 "${RUNTIME_ROOT}/OrbitRuntimeSBOM.cdx.json" + echo "Bundled Codex runtime into ${RUNTIME_ROOT}" diff --git a/scripts/check_changed_line_coverage.py b/scripts/check_changed_line_coverage.py new file mode 100755 index 0000000..d5ad085 --- /dev/null +++ b/scripts/check_changed_line_coverage.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Enforce coverage for changed executable lines in Orbit's non-view core modules.""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + + +THRESHOLD = 0.80 +CORE_FILES = { + "Orbit/OrbitBoundedDataBuffer.swift", + "Orbit/OrbitCodexActivityReducer.swift", + "Orbit/OrbitCodexContracts.swift", + "Orbit/OrbitCodexEnvironment.swift", + "Orbit/OrbitCodexModelCatalog.swift", + "Orbit/OrbitCodexTransport.swift", + "Orbit/OrbitTemporaryCaptureLease.swift", +} +HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +def changed_lines(base_ref: str, path: str) -> set[int]: + diff = subprocess.check_output( + ["git", "diff", "--unified=0", base_ref, "--", path], + text=True, + ) + lines: set[int] = set() + for raw_line in diff.splitlines(): + match = HUNK.match(raw_line) + if not match: + continue + start = int(match.group(1)) + count = int(match.group(2) or "1") + lines.update(range(start, start + count)) + return lines + + +def main() -> int: + if len(sys.argv) not in (2, 3): + print(f"usage: {sys.argv[0]} /path/to/result.xcresult [base-ref]", file=sys.stderr) + return 2 + + result_bundle = Path(sys.argv[1]).resolve() + base_ref = sys.argv[2] if len(sys.argv) == 3 else "origin/main" + archive = subprocess.check_output( + ["xcrun", "xccov", "view", "--archive", "--json", str(result_bundle)], + text=True, + ) + coverage = json.loads(archive) + repository = Path.cwd().resolve() + by_relative_path = { + str(Path(path).resolve().relative_to(repository)): entries + for path, entries in coverage.items() + if Path(path).resolve().is_relative_to(repository) + } + + executable = 0 + covered = 0 + details: list[str] = [] + for path in sorted(CORE_FILES): + changed = changed_lines(base_ref, path) + entries = by_relative_path.get(path, []) + relevant = [entry for entry in entries if entry.get("isExecutable") and entry["line"] in changed] + file_covered = sum(1 for entry in relevant if entry.get("executionCount", 0) > 0) + executable += len(relevant) + covered += file_covered + if relevant: + details.append(f"{path}: {file_covered}/{len(relevant)}") + + if executable == 0: + print("No changed executable lines found in coverage-enforced core modules.") + return 0 + + ratio = covered / executable + print("Changed-line coverage: " + ", ".join(details)) + print(f"Total: {covered}/{executable} ({ratio:.1%}); required: {THRESHOLD:.0%}") + return 0 if ratio >= THRESHOLD else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release.sh b/scripts/release.sh index 7864442..0c7c66d 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -15,45 +15,57 @@ EXPORT_DIR="${BUILD_DIR}/export" DMG_BACKGROUND="${PROJECT_DIR}/dmg-background.png" GITHUB_REPO="${GITHUB_REPO:-}" ORBIT_SKIP_GITHUB_RELEASE="${ORBIT_SKIP_GITHUB_RELEASE:-0}" +ORBIT_LOCAL_UNSIGNED_BUILD="${ORBIT_LOCAL_UNSIGNED_BUILD:-0}" BUNDLE_RUNTIME_SCRIPT="${PROJECT_DIR}/scripts/bundle_codex_runtime.sh" GENERATE_BRAND_ASSETS_SCRIPT="${PROJECT_DIR}/scripts/generate_brand_assets.swift" DEVELOPMENT_TEAM_ID="${ORBIT_DEVELOPMENT_TEAM:-}" DEVELOPER_ID_IDENTITY="${ORBIT_DEVELOPER_ID_IDENTITY:-}" DEVELOPER_ID_INSTALLER_IDENTITY="${ORBIT_DEVELOPER_ID_INSTALLER_IDENTITY:-}" -if [[ -z "${DEVELOPMENT_TEAM_ID}" ]]; then - DEVELOPMENT_TEAM_ID="$(defaults read com.apple.dt.Xcode IDEProvisioningTeamManagerLastSelectedTeamID 2>/dev/null || true)" -fi - -if [[ -z "${DEVELOPMENT_TEAM_ID}" ]]; then - IDENTITY_OUTPUT="$(security find-identity -v -p codesigning 2>/dev/null || true)" - DEVELOPMENT_TEAM_ID="$(printf '%s\n' "${IDENTITY_OUTPUT}" | grep -Eo '\([A-Z0-9]{10}\)' | tr -d '()' | head -n 1)" -fi - -if [[ -z "${DEVELOPMENT_TEAM_ID}" ]]; then - echo "❌ No local Apple signing team was detected. Set ORBIT_DEVELOPMENT_TEAM and try again." - exit 1 -fi - -if [[ -z "${DEVELOPER_ID_IDENTITY}" ]]; then - DEVELOPER_ID_IDENTITY="$( - security find-identity -v -p codesigning 2>/dev/null | - sed -nE "s/^[[:space:]]*[0-9]+\) [A-F0-9]+ \"(Developer ID Application: .+ \\(${DEVELOPMENT_TEAM_ID}\\))\"$/\\1/p" | - head -n 1 - )" -fi +if [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" == "1" ]]; then + ORBIT_SKIP_GITHUB_RELEASE=1 + export ORBIT_ALLOW_UNPINNED_RUNTIME="${ORBIT_ALLOW_UNPINNED_RUNTIME:-1}" + echo "WARNING: local unsigned-development mode is enabled; publishing is forcibly disabled." >&2 +else + if [[ -z "${DEVELOPMENT_TEAM_ID}" ]]; then + DEVELOPMENT_TEAM_ID="$(defaults read com.apple.dt.Xcode IDEProvisioningTeamManagerLastSelectedTeamID 2>/dev/null || true)" + fi + if [[ -z "${DEVELOPMENT_TEAM_ID}" ]]; then + IDENTITY_OUTPUT="$(security find-identity -v -p codesigning 2>/dev/null || true)" + DEVELOPMENT_TEAM_ID="$(printf '%s\n' "${IDENTITY_OUTPUT}" | grep -Eo '\([A-Z0-9]{10}\)' | tr -d '()' | head -n 1)" + fi + if [[ -z "${DEVELOPMENT_TEAM_ID}" ]]; then + echo "❌ No local Apple signing team was detected. Set ORBIT_DEVELOPMENT_TEAM and try again." + exit 1 + fi -if [[ -z "${DEVELOPER_ID_IDENTITY}" ]]; then - echo "❌ No Developer ID Application identity was detected for team ${DEVELOPMENT_TEAM_ID}." - exit 1 -fi + if [[ -z "${DEVELOPER_ID_IDENTITY}" ]]; then + DEVELOPER_ID_IDENTITY="$( + security find-identity -v -p codesigning 2>/dev/null | + sed -nE "s/^[[:space:]]*[0-9]+\) [A-F0-9]+ \"(Developer ID Application: .+ \\(${DEVELOPMENT_TEAM_ID}\\))\"$/\\1/p" | + head -n 1 + )" + fi + if [[ -z "${DEVELOPER_ID_IDENTITY}" ]]; then + echo "❌ No Developer ID Application identity was detected for team ${DEVELOPMENT_TEAM_ID}." + exit 1 + fi -if [[ -z "${DEVELOPER_ID_INSTALLER_IDENTITY}" ]]; then - DEVELOPER_ID_INSTALLER_IDENTITY="$( - security find-identity -v -p basic 2>/dev/null | - sed -nE "s/^[[:space:]]*[0-9]+\) [A-F0-9]+ \"(Developer ID Installer: .+ \\(${DEVELOPMENT_TEAM_ID}\\))\"$/\\1/p" | - head -n 1 - )" + if [[ -z "${DEVELOPER_ID_INSTALLER_IDENTITY}" ]]; then + DEVELOPER_ID_INSTALLER_IDENTITY="$( + security find-identity -v -p basic 2>/dev/null | + sed -nE "s/^[[:space:]]*[0-9]+\) [A-F0-9]+ \"(Developer ID Installer: .+ \\(${DEVELOPMENT_TEAM_ID}\\))\"$/\\1/p" | + head -n 1 + )" + fi + if [[ -z "${DEVELOPER_ID_INSTALLER_IDENTITY}" ]]; then + echo "❌ A Developer ID Installer identity is required for a publishable release." + exit 1 + fi + if ! xcrun notarytool history --keychain-profile "AC_PASSWORD" >/dev/null 2>&1; then + echo "❌ Apple notarization is unavailable. Verify AC_PASSWORD and accept all required Apple Developer agreements." + exit 1 + fi fi if [[ -z "${GITHUB_REPO}" ]] && command -v git >/dev/null 2>&1; then @@ -73,11 +85,12 @@ fi DEFAULT_MARKETING_VERSION="$( sed -nE 's/^[[:space:]]*MARKETING_VERSION = ([^;]+);$/\1/p' "${PROJECT_DIR}/Orbit.xcodeproj/project.pbxproj" | head -n 1 )" -MARKETING_VERSION="${1:-${DEFAULT_MARKETING_VERSION:-1.0.4}}" +RELEASE_VERSION="${1:-${DEFAULT_MARKETING_VERSION:-1.0.4}}" +MARKETING_VERSION="${RELEASE_VERSION%%-*}" BUILD_NUMBER="${2:-$(date +%Y%m%d%H%M)}" -TAG="v${MARKETING_VERSION}" -DMG_PATH="${BUILD_DIR}/${APP_NAME}-${MARKETING_VERSION}.dmg" -PKG_PATH="${BUILD_DIR}/${APP_NAME}-${MARKETING_VERSION}.pkg" +TAG="v${RELEASE_VERSION}" +DMG_PATH="${BUILD_DIR}/${APP_NAME}-${RELEASE_VERSION}.dmg" +PKG_PATH="${BUILD_DIR}/${APP_NAME}-${RELEASE_VERSION}.pkg" PKG_ROOT="${BUILD_DIR}/pkg-root" PKG_SCRIPTS_DIR="${PROJECT_DIR}/scripts/installer" PKG_COMPONENT_PLIST="${BUILD_DIR}/components.plist" @@ -89,7 +102,11 @@ if [[ -n "${GITHUB_REPO}" ]]; then else echo " Repo: (not configured)" fi -echo " Team: ${DEVELOPMENT_TEAM_ID}" +if [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" == "1" ]]; then + echo " Signing: local unsigned-development mode" +else + echo " Team: ${DEVELOPMENT_TEAM_ID}" +fi echo "" rm -rf "${BUILD_DIR}" @@ -98,7 +115,8 @@ mkdir -p "${BUILD_DIR}" "${EXPORT_DIR}" echo "🎨 Generating branded app assets..." swift "${GENERATE_BRAND_ASSETS_SCRIPT}" -cat > "${EXPORT_OPTIONS}" < "${EXPORT_OPTIONS}" < @@ -112,24 +130,37 @@ cat > "${EXPORT_OPTIONS}" < PLIST +fi echo "📦 Archiving..." -xcodebuild archive \ - -project "${PROJECT_DIR}/Orbit.xcodeproj" \ - -scheme "${SCHEME}" \ - -archivePath "${ARCHIVE_PATH}" \ - MARKETING_VERSION="${MARKETING_VERSION}" \ - CURRENT_PROJECT_VERSION="${BUILD_NUMBER}" \ - DEVELOPMENT_TEAM="${DEVELOPMENT_TEAM_ID}" +ARCHIVE_ARGS=( + archive + -project "${PROJECT_DIR}/Orbit.xcodeproj" + -scheme "${SCHEME}" + -archivePath "${ARCHIVE_PATH}" + "MARKETING_VERSION=${MARKETING_VERSION}" + "CURRENT_PROJECT_VERSION=${BUILD_NUMBER}" +) +if [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" == "1" ]]; then + ARCHIVE_ARGS+=(CODE_SIGNING_ALLOWED=NO CODE_SIGNING_REQUIRED=NO) +else + ARCHIVE_ARGS+=("DEVELOPMENT_TEAM=${DEVELOPMENT_TEAM_ID}") +fi +xcodebuild "${ARCHIVE_ARGS[@]}" echo "📎 Bundling Codex runtime..." "${BUNDLE_RUNTIME_SCRIPT}" "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" -echo "📤 Exporting signed app..." -xcodebuild -exportArchive \ - -archivePath "${ARCHIVE_PATH}" \ - -exportPath "${EXPORT_DIR}" \ - -exportOptionsPlist "${EXPORT_OPTIONS}" +if [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" == "1" ]]; then + echo "📤 Copying unsigned development app..." + cp -R "${ARCHIVE_PATH}/Products/Applications/${APP_NAME}.app" "${EXPORT_DIR}/${APP_NAME}.app" +else + echo "📤 Exporting signed app..." + xcodebuild -exportArchive \ + -archivePath "${ARCHIVE_PATH}" \ + -exportPath "${EXPORT_DIR}" \ + -exportOptionsPlist "${EXPORT_OPTIONS}" +fi EXPORT_APP_PATH="${EXPORT_DIR}/${APP_NAME}.app" NODE_ENTITLEMENTS_PATH="${PROJECT_DIR}/Orbit/CodexRuntimeNode.entitlements" @@ -138,26 +169,37 @@ if [[ -f "${EXPORT_APP_PATH}/Contents/Resources/LocalSecrets.plist" ]]; then rm -f "${EXPORT_APP_PATH}/Contents/Resources/LocalSecrets.plist" fi -echo "🔏 Re-signing bundled runtime executables..." -while IFS= read -r executable_path; do - if [[ "$(basename "${executable_path}")" == "node" ]]; then - codesign --force \ - --sign "${DEVELOPER_ID_IDENTITY}" \ - --options runtime \ - --timestamp \ - --entitlements "${NODE_ENTITLEMENTS_PATH}" \ - "${executable_path}" - else - codesign --force --sign "${DEVELOPER_ID_IDENTITY}" --options runtime --timestamp "${executable_path}" - fi -done < <(find "${EXPORT_APP_PATH}/Contents/Resources/CodexRuntime" -type f -perm -111 | sort) +if [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" != "1" ]]; then + echo "🔏 Re-signing bundled runtime executables..." + while IFS= read -r executable_path; do + if [[ "$(basename "${executable_path}")" == "node" ]]; then + codesign --force \ + --sign "${DEVELOPER_ID_IDENTITY}" \ + --options runtime \ + --timestamp \ + --entitlements "${NODE_ENTITLEMENTS_PATH}" \ + "${executable_path}" + else + codesign --force --sign "${DEVELOPER_ID_IDENTITY}" --options runtime --timestamp "${executable_path}" + fi + done < <(find "${EXPORT_APP_PATH}/Contents/Resources/CodexRuntime" -type f -perm -111 | sort) -codesign --force \ - --sign "${DEVELOPER_ID_IDENTITY}" \ - --options runtime \ - --timestamp \ - --entitlements "${PROJECT_DIR}/Orbit/Orbit.entitlements" \ - "${EXPORT_APP_PATH}" + codesign --force \ + --sign "${DEVELOPER_ID_IDENTITY}" \ + --options runtime \ + --timestamp \ + --entitlements "${PROJECT_DIR}/Orbit/Orbit.entitlements" \ + "${EXPORT_APP_PATH}" + codesign --verify --deep --strict --verbose=2 "${EXPORT_APP_PATH}" + + APP_NOTARY_ZIP="${BUILD_DIR}/${APP_NAME}-${RELEASE_VERSION}-notary.zip" + ditto -c -k --keepParent "${EXPORT_APP_PATH}" "${APP_NOTARY_ZIP}" + echo "🔏 Notarizing app..." + xcrun notarytool submit "${APP_NOTARY_ZIP}" --keychain-profile "AC_PASSWORD" --wait + xcrun stapler staple "${EXPORT_APP_PATH}" + xcrun stapler validate "${EXPORT_APP_PATH}" + spctl --assess --type execute --verbose=2 "${EXPORT_APP_PATH}" +fi echo "📦 Building installer package..." rm -rf "${PKG_ROOT}" @@ -198,8 +240,11 @@ PKGBUILD_ARGS=( if [[ -n "${DEVELOPER_ID_INSTALLER_IDENTITY}" ]]; then PKGBUILD_ARGS+=(--sign "${DEVELOPER_ID_INSTALLER_IDENTITY}") -else +elif [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" == "1" ]]; then echo "⚠️ No Developer ID Installer identity detected for team ${DEVELOPMENT_TEAM_ID}. Building an unsigned PKG." +else + echo "❌ Refusing to build an unsigned publishable PKG." + exit 1 fi pkgbuild "${PKGBUILD_ARGS[@]}" "${PKG_PATH}" @@ -216,19 +261,24 @@ create-dmg \ "${DMG_PATH}" \ "${EXPORT_APP_PATH}" -if xcrun notarytool history --keychain-profile "AC_PASSWORD" >/dev/null 2>&1; then +if [[ "${ORBIT_LOCAL_UNSIGNED_BUILD}" != "1" ]]; then + codesign --force --sign "${DEVELOPER_ID_IDENTITY}" --timestamp "${DMG_PATH}" + codesign --verify --verbose=2 "${DMG_PATH}" + pkgutil --check-signature "${PKG_PATH}" + echo "🔏 Notarizing DMG..." xcrun notarytool submit "${DMG_PATH}" --keychain-profile "AC_PASSWORD" --wait xcrun stapler staple "${DMG_PATH}" - if [[ -n "${DEVELOPER_ID_INSTALLER_IDENTITY}" ]]; then - echo "🔏 Notarizing PKG..." - xcrun notarytool submit "${PKG_PATH}" --keychain-profile "AC_PASSWORD" --wait - xcrun stapler staple "${PKG_PATH}" - else - echo "⚠️ Skipping PKG notarization because no Developer ID Installer identity is configured." - fi + xcrun stapler validate "${DMG_PATH}" + spctl --assess --type open --context context:primary-signature --verbose=2 "${DMG_PATH}" + + echo "🔏 Notarizing PKG..." + xcrun notarytool submit "${PKG_PATH}" --keychain-profile "AC_PASSWORD" --wait + xcrun stapler staple "${PKG_PATH}" + xcrun stapler validate "${PKG_PATH}" + spctl --assess --type install --verbose=2 "${PKG_PATH}" else - echo "⚠️ Skipping notarization because AC_PASSWORD credentials are not configured in Keychain." + echo "⚠️ Local development build: notarization, stapling, and Gatekeeper publication checks skipped." fi if [[ "${ORBIT_SKIP_GITHUB_RELEASE}" == "1" ]]; then @@ -239,10 +289,15 @@ elif [[ -n "${GITHUB_REPO}" ]] && command -v gh >/dev/null 2>&1; then if [[ -f "${PKG_PATH}" ]]; then RELEASE_ASSETS+=("${PKG_PATH}") fi + RELEASE_FLAGS=() + if [[ "${TAG}" == *"-rc."* ]]; then + RELEASE_FLAGS+=(--prerelease) + fi gh release create "${TAG}" "${RELEASE_ASSETS[@]}" \ --repo "${GITHUB_REPO}" \ --title "${TAG}" \ - --notes "Orbit ${TAG}" + --notes "Orbit ${TAG}" \ + "${RELEASE_FLAGS[@]}" else echo "⚠️ Skipping GitHub release creation because repo is not configured or GitHub CLI is unavailable." fi diff --git a/scripts/smoke_browser_mcp.py b/scripts/smoke_browser_mcp.py new file mode 100755 index 0000000..36f7148 --- /dev/null +++ b/scripts/smoke_browser_mcp.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Initialize the bundled Chrome MCP and perform one read-only discovery call.""" + +from __future__ import annotations + +import json +import select +import subprocess +import sys +from typing import Any + + +def main() -> int: + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} /path/to/chrome-devtools-mcp", file=sys.stderr) + return 2 + + process = subprocess.Popen( + [sys.argv[1], "--headless", "--isolated", "--no-usage-statistics", "--no-performance-crux"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + bufsize=1, + ) + + def send(payload: dict[str, Any]) -> None: + assert process.stdin is not None + process.stdin.write(json.dumps(payload) + "\n") + process.stdin.flush() + + def receive(timeout: float = 20) -> dict[str, Any]: + assert process.stdout is not None + readable, _, _ = select.select([process.stdout], [], [], timeout) + if not readable: + raise TimeoutError("timed out waiting for Chrome MCP response") + line = process.stdout.readline() + if not line: + raise RuntimeError("Chrome MCP exited before responding") + return json.loads(line) + + try: + send( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "orbit-release-smoke", "version": "1.1.0"}, + }, + } + ) + initialized = receive() + version = initialized.get("result", {}).get("serverInfo", {}).get("version") + if version != "1.5.0": + raise RuntimeError(f"unexpected Chrome MCP version: {version!r}") + + send({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}) + send({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + tools = receive().get("result", {}).get("tools", []) + list_pages = next((tool for tool in tools if tool.get("name") == "list_pages"), None) + if not list_pages or not list_pages.get("annotations", {}).get("readOnlyHint"): + raise RuntimeError("Chrome MCP did not expose list_pages as a read-only tool") + + send( + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "list_pages", "arguments": {}}, + } + ) + content = receive().get("result", {}).get("content", []) + if not any("Pages" in item.get("text", "") for item in content): + raise RuntimeError("Chrome MCP browser discovery returned no page list") + print("Chrome MCP 1.5.0 initialized and list_pages completed in an isolated browser.") + return 0 + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_release_manifest.py b/scripts/validate_release_manifest.py new file mode 100644 index 0000000..c21bc35 --- /dev/null +++ b/scripts/validate_release_manifest.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json +import re +import sys +from pathlib import Path +from urllib.parse import urlparse + + +def main() -> int: + path = Path(sys.argv[1] if len(sys.argv) > 1 else "release-manifest.json") + data = json.loads(path.read_text(encoding="utf-8")) + required = { + "version", + "downloadURL", + "sha256", + "minimumMacOS", + "codexRuntimeVersion", + "browserMCPVersion", + } + missing = sorted(required - data.keys()) + if missing: + raise ValueError(f"missing manifest fields: {', '.join(missing)}") + if any(not isinstance(data[key], str) or not data[key].strip() for key in required): + raise ValueError("all release manifest fields must be non-empty strings") + if not re.fullmatch(r"[0-9a-f]{64}", data["sha256"]): + raise ValueError("sha256 must be 64 lowercase hexadecimal characters") + parsed = urlparse(data["downloadURL"]) + if parsed.scheme != "https" or not parsed.netloc: + raise ValueError("downloadURL must be an absolute HTTPS URL") + version_pattern = r"\d+(?:\.\d+){1,2}(?:-[0-9A-Za-z.-]+)?" + for key in ("version", "codexRuntimeVersion", "browserMCPVersion"): + if not re.fullmatch(version_pattern, data[key]): + raise ValueError(f"{key} is not a supported version string") + if not re.fullmatch(r"\d+\.\d+(?:\.\d+)?", data["minimumMacOS"]): + raise ValueError("minimumMacOS is not a supported macOS version") + print(f"validated {path}: Orbit {data['version']}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From d787a4b51608791531a33849e80306ab48167302 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Thu, 9 Jul 2026 16:36:10 -0400 Subject: [PATCH 2/8] fix: enforce runtime pins in CI --- .github/workflows/ci.yml | 2 +- BundledResources/browser-runtime/package-lock.json | 2 +- BundledResources/browser-runtime/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e9f09b..91a19d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: macos-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/BundledResources/browser-runtime/package-lock.json b/BundledResources/browser-runtime/package-lock.json index 4cd27c5..f4854b2 100644 --- a/BundledResources/browser-runtime/package-lock.json +++ b/BundledResources/browser-runtime/package-lock.json @@ -10,7 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@playwright/mcp": "0.0.70", - "chrome-devtools-mcp": "^1.5.0" + "chrome-devtools-mcp": "1.5.0" } }, "node_modules/@playwright/mcp": { diff --git a/BundledResources/browser-runtime/package.json b/BundledResources/browser-runtime/package.json index 20fb035..661fad4 100644 --- a/BundledResources/browser-runtime/package.json +++ b/BundledResources/browser-runtime/package.json @@ -6,6 +6,6 @@ "license": "UNLICENSED", "dependencies": { "@playwright/mcp": "0.0.70", - "chrome-devtools-mcp": "^1.5.0" + "chrome-devtools-mcp": "1.5.0" } } From f435c641cb0afb8863ccb779a6498e70a183060f Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Thu, 9 Jul 2026 16:40:31 -0400 Subject: [PATCH 3/8] fix: pin Swift 6 CI toolchain --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91a19d7..ae84b58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,9 @@ on: jobs: build-and-test: - runs-on: macos-latest + runs-on: macos-26 + env: + DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer steps: - name: Check out repository uses: actions/checkout@v7 @@ -17,6 +19,7 @@ jobs: - name: Validate release manifest and pinned browser runtime run: | + xcodebuild -version python3 scripts/validate_release_manifest.py test "$(node -p "require('./BundledResources/browser-runtime/package.json').dependencies['chrome-devtools-mcp']")" = "1.5.0" From ced1a17ceb18f5f33d772fa7491fbd72a0653890 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Thu, 9 Jul 2026 23:35:52 -0400 Subject: [PATCH 4/8] fix: prefer high-quality Apple voices --- Orbit/OrbitManager.swift | 3 + Orbit/OrbitPanelView.swift | 90 ++++++++++-------- Orbit/TextToSpeechProvider.swift | 156 ++++++++++++++++++++++++++++--- OrbitTests/OrbitTests.swift | 69 ++++++++++++++ 4 files changed, 268 insertions(+), 50 deletions(-) diff --git a/Orbit/OrbitManager.swift b/Orbit/OrbitManager.swift index 68a578a..e713e40 100644 --- a/Orbit/OrbitManager.swift +++ b/Orbit/OrbitManager.swift @@ -47,6 +47,7 @@ final class OrbitManager: ObservableObject { @Published private(set) var textToSpeechProviderDisplayName: String = "" @Published private(set) var availableAppleVoices: [OrbitAppleVoiceOption] = [] @Published private(set) var selectedAppleVoiceSummary: String = "Auto" + @Published private(set) var appleVoiceQualityNotice: String? @Published private(set) var isPreviewingAppleVoice = false @Published private(set) var availableMicrophones: [OrbitAudioInputDevice] = [] @Published private(set) var microphoneTestLevel: CGFloat = 0 @@ -240,6 +241,7 @@ final class OrbitManager: ObservableObject { self.selectedAppleVoiceSummary = OrbitAppleVoiceCatalog.currentSelectionSummary( preferredIdentifier: OrbitSettings.shared.appleTTSVoiceIdentifier ) + self.appleVoiceQualityNotice = OrbitAppleVoiceCatalog.qualityNotice() self.availableMicrophones = OrbitAudioInputCatalog.devices() self.codexSessionSummary = actionProvider.sessionStatusSummary self.codexConfigurationSummary = actionProvider.configurationSummary @@ -611,6 +613,7 @@ final class OrbitManager: ObservableObject { selectedAppleVoiceSummary = OrbitAppleVoiceCatalog.currentSelectionSummary( preferredIdentifier: settings.appleTTSVoiceIdentifier ) + appleVoiceQualityNotice = OrbitAppleVoiceCatalog.qualityNotice() orbitDictationManager.refreshConfiguredProviders() } diff --git a/Orbit/OrbitPanelView.swift b/Orbit/OrbitPanelView.swift index 4bf9487..dabde16 100644 --- a/Orbit/OrbitPanelView.swift +++ b/Orbit/OrbitPanelView.swift @@ -881,51 +881,63 @@ struct OrbitPanelView: View { } private var speechOutputRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel( - icon: "speaker.wave.2.fill", - title: "Voice", - subtitle: orbitSettings.voicePreset == .localVoice ? "System speech" : "OpenAI speech" - ) + VStack(alignment: .leading, spacing: 6) { + HStack(alignment: .center, spacing: 12) { + rowLabel( + icon: "speaker.wave.2.fill", + title: "Voice", + subtitle: orbitSettings.voicePreset == .localVoice ? "On-device Apple speech" : "OpenAI speech" + ) - Spacer(minLength: 8) + Spacer(minLength: 8) - if orbitSettings.voicePreset == .localVoice { - VStack(alignment: .trailing, spacing: 5) { - Menu { - Button("Automatic") { orbitManager.selectAppleVoice("") } - Divider() - ForEach(orbitManager.availableAppleVoices) { voice in - Button(voice.displayName) { orbitManager.selectAppleVoice(voice.identifier) } + if orbitSettings.voicePreset == .localVoice { + VStack(alignment: .trailing, spacing: 5) { + Menu { + Button("Automatic (recommended)") { orbitManager.selectAppleVoice("") } + Divider() + ForEach(orbitManager.availableAppleVoices) { voice in + Button(voice.displayName) { orbitManager.selectAppleVoice(voice.identifier) } + } + } label: { + Text(orbitManager.selectedAppleVoiceSummary) + .font(.system(size: 11, weight: .medium)) + .lineLimit(1) + .frame(maxWidth: 152, alignment: .trailing) } - } label: { - Text(orbitManager.selectedAppleVoiceSummary) - .font(.system(size: 11, weight: .medium)) - .lineLimit(1) - .frame(maxWidth: 112, alignment: .trailing) - } - .menuStyle(.borderlessButton) + .menuStyle(.borderlessButton) - Button(orbitManager.isPreviewingAppleVoice ? "Stop" : "Preview") { - orbitManager.toggleAppleVoicePreview() + Button(orbitManager.isPreviewingAppleVoice ? "Stop" : "Preview") { + orbitManager.toggleAppleVoicePreview() + } + .buttonStyle(.borderless) + .font(.system(size: 10, weight: .semibold)) } - .buttonStyle(.borderless) - .font(.system(size: 10, weight: .semibold)) + } else { + Text(panelTextToSpeechLabel) + .font(.system(size: 11.5, weight: .medium)) + .foregroundColor(DS.Colors.textSecondary) + .padding(.horizontal, 10) + .padding(.vertical, 7) + .orbitGlassCard( + shape: Capsule(style: .continuous), + fillOpacity: 0.22, + borderOpacity: 0.12, + highlightOpacity: 0.16, + shadowOpacity: 0.08, + glowOpacity: 0.01 + ) } - } else { - Text(panelTextToSpeechLabel) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.22, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) + } + + if orbitSettings.voicePreset == .localVoice, + let notice = orbitManager.appleVoiceQualityNotice + { + Text(notice) + .font(.system(size: 9.5, weight: .medium)) + .foregroundColor(DS.Colors.warningText.opacity(0.88)) + .fixedSize(horizontal: false, vertical: true) + .accessibilityLabel(notice) } } } diff --git a/Orbit/TextToSpeechProvider.swift b/Orbit/TextToSpeechProvider.swift index bab2327..eb525c3 100644 --- a/Orbit/TextToSpeechProvider.swift +++ b/Orbit/TextToSpeechProvider.swift @@ -11,13 +11,72 @@ protocol TextToSpeechProvider: AnyObject { func stopPlayback() } +enum OrbitAppleVoiceQuality: Int, Equatable, Sendable { + case standard = 1 + case enhanced = 2 + case premium = 3 + + init(_ quality: AVSpeechSynthesisVoiceQuality) { + switch quality { + case .premium: + self = .premium + case .enhanced: + self = .enhanced + default: + self = .standard + } + } + + var displayName: String { + switch self { + case .standard: return "Standard" + case .enhanced: return "Enhanced" + case .premium: return "Premium" + } + } +} + +enum OrbitAppleVoiceCategory: Int, Equatable, Sendable { + case novelty + case legacy + case natural +} + struct OrbitAppleVoiceOption: Identifiable, Equatable, Sendable { let identifier: String let name: String let language: String + let quality: OrbitAppleVoiceQuality + let category: OrbitAppleVoiceCategory + let isSystemDefault: Bool let displayName: String var id: String { identifier } + + var automaticPriority: Int { + switch quality { + case .premium: + return 600 + case .enhanced: + return 500 + case .standard where isSystemDefault: + return 400 + case .standard where category == .natural: + return 300 + case .standard where category == .legacy: + return 100 + case .standard: + return 0 + } + } + + var qualityDisplayName: String { + switch category { + case .novelty: return "Novelty" + case .legacy: return "Legacy" + case .natural: return quality.displayName + } + } } /// Public AVFoundation-only catalog. Orbit intentionally does not inspect Siri @@ -28,20 +87,35 @@ enum OrbitAppleVoiceCatalog { ) -> [OrbitAppleVoiceOption] { let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") let baseLanguage = Locale(identifier: localeIdentifier).language.languageCode?.identifier + let systemDefaultIdentifier = AVSpeechSynthesisVoice(language: normalizedLocale)?.identifier return AVSpeechSynthesisVoice.speechVoices() .map { voice in - OrbitAppleVoiceOption( + let quality = OrbitAppleVoiceQuality(voice.quality) + let category = category(for: voice.identifier) + let qualityDisplayName: String + switch category { + case .novelty: qualityDisplayName = "Novelty" + case .legacy: qualityDisplayName = "Legacy" + case .natural: qualityDisplayName = quality.displayName + } + return OrbitAppleVoiceOption( identifier: voice.identifier, name: voice.name, language: voice.language, - displayName: "\(voice.name) · \(localizedLanguageName(voice.language))" + quality: quality, + category: category, + isSystemDefault: voice.identifier == systemDefaultIdentifier, + displayName: "\(voice.name) · \(localizedLanguageName(voice.language)) · \(qualityDisplayName)" ) } .sorted { lhs, rhs in let lhsScore = localeScore(lhs.language, exact: normalizedLocale, base: baseLanguage) let rhsScore = localeScore(rhs.language, exact: normalizedLocale, base: baseLanguage) if lhsScore != rhsScore { return lhsScore > rhsScore } + if lhs.automaticPriority != rhs.automaticPriority { + return lhs.automaticPriority > rhs.automaticPriority + } if lhs.language != rhs.language { return lhs.language < rhs.language } return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending } @@ -52,22 +126,47 @@ enum OrbitAppleVoiceCatalog { localeIdentifier: String = Locale.autoupdatingCurrent.identifier ) -> String? { let voices = availableVoices(for: localeIdentifier) + let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") + let baseLanguage = Locale(identifier: localeIdentifier).language.languageCode?.identifier + return resolvedVoiceIdentifier( + preferredIdentifier: preferredIdentifier, + voices: voices, + normalizedLocale: normalizedLocale, + baseLanguage: baseLanguage + ) + } + + static func resolvedVoiceIdentifier( + preferredIdentifier: String?, + voices: [OrbitAppleVoiceOption], + normalizedLocale: String, + baseLanguage: String? + ) -> String? { if let preferred = normalizedPreferredIdentifier(preferredIdentifier), voices.contains(where: { $0.identifier == preferred }) { return preferred } - let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") - if let exact = voices.first(where: { $0.language.caseInsensitiveCompare(normalizedLocale) == .orderedSame }) { - return exact.identifier + let exactMatches = voices.filter { + $0.language.caseInsensitiveCompare(normalizedLocale) == .orderedSame + && $0.automaticPriority > 0 } - if let base = Locale(identifier: localeIdentifier).language.languageCode?.identifier, - let languageMatch = voices.first(where: { $0.language.lowercased().hasPrefix(base.lowercased()) }) - { - return languageMatch.identifier + if let recommended = exactMatches.max(by: automaticPreferenceAscending) { + return recommended.identifier + } + + if let baseLanguage { + let languageMatches = voices.filter { + $0.language.lowercased().hasPrefix(baseLanguage.lowercased()) + && $0.automaticPriority > 0 + } + if let recommended = languageMatches.max(by: automaticPreferenceAscending) { + return recommended.identifier + } } - return voices.first?.identifier + + return voices.max(by: automaticPreferenceAscending)?.identifier } static func currentSelectionSummary( @@ -83,7 +182,22 @@ enum OrbitAppleVoiceCatalog { else { return isAutomatic ? "Automatic · unavailable" : "Voice unavailable" } - return isAutomatic ? "Automatic · \(voice.name)" : voice.displayName + return isAutomatic + ? "Automatic · \(voice.name) · \(voice.qualityDisplayName)" + : voice.displayName + } + + static func qualityNotice( + for localeIdentifier: String = Locale.autoupdatingCurrent.identifier + ) -> String? { + let normalizedLocale = localeIdentifier.replacingOccurrences(of: "_", with: "-") + let baseLanguage = Locale(identifier: localeIdentifier).language.languageCode?.identifier + let relevantVoices = availableVoices(for: localeIdentifier).filter { voice in + voice.language.caseInsensitiveCompare(normalizedLocale) == .orderedSame + || baseLanguage.map { voice.language.lowercased().hasPrefix($0.lowercased()) } == true + } + guard !relevantVoices.contains(where: { $0.quality != .standard }) else { return nil } + return "Only Standard Apple voices are installed. Add an Enhanced or Premium voice in System Settings → Accessibility → Read & Speak." } static func normalizedPreferredIdentifier(_ preferredIdentifier: String?) -> String? { @@ -102,6 +216,26 @@ enum OrbitAppleVoiceCatalog { private static func localizedLanguageName(_ identifier: String) -> String { Locale.autoupdatingCurrent.localizedString(forIdentifier: identifier) ?? identifier } + + private static func category(for identifier: String) -> OrbitAppleVoiceCategory { + if identifier.contains(".speech.synthesis.voice.") { + return .novelty + } + if identifier.contains(".eloquence.") { + return .legacy + } + return .natural + } + + private static func automaticPreferenceAscending( + _ lhs: OrbitAppleVoiceOption, + _ rhs: OrbitAppleVoiceOption + ) -> Bool { + if lhs.automaticPriority != rhs.automaticPriority { + return lhs.automaticPriority < rhs.automaticPriority + } + return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedDescending + } } enum OrbitTTSProviderFactory { diff --git a/OrbitTests/OrbitTests.swift b/OrbitTests/OrbitTests.swift index b53f70e..68fc993 100644 --- a/OrbitTests/OrbitTests.swift +++ b/OrbitTests/OrbitTests.swift @@ -22,6 +22,75 @@ struct OrbitTests { return try body(directoryURL) } + private func appleVoice( + _ identifier: String, + name: String, + quality: OrbitAppleVoiceQuality = .standard, + category: OrbitAppleVoiceCategory = .natural, + isSystemDefault: Bool = false + ) -> OrbitAppleVoiceOption { + OrbitAppleVoiceOption( + identifier: identifier, + name: name, + language: "en-US", + quality: quality, + category: category, + isSystemDefault: isSystemDefault, + displayName: name + ) + } + + @Test func automaticAppleVoicePrefersPremiumOverLegacyAndSystemDefaultVoices() { + let voices = [ + appleVoice("albert", name: "Albert", category: .novelty), + appleVoice("samantha", name: "Samantha", isSystemDefault: true), + appleVoice("ava-enhanced", name: "Ava", quality: .enhanced), + appleVoice("ava-premium", name: "Ava", quality: .premium), + ] + + let selected = OrbitAppleVoiceCatalog.resolvedVoiceIdentifier( + preferredIdentifier: nil, + voices: voices, + normalizedLocale: "en-US", + baseLanguage: "en" + ) + + #expect(selected == "ava-premium") + } + + @Test func automaticAppleVoiceUsesAppleLocaleDefaultWhenNoHighQualityVoiceIsInstalled() { + let voices = [ + appleVoice("albert", name: "Albert", category: .novelty), + appleVoice("eddy", name: "Eddy", category: .legacy), + appleVoice("samantha", name: "Samantha", isSystemDefault: true), + ] + + let selected = OrbitAppleVoiceCatalog.resolvedVoiceIdentifier( + preferredIdentifier: nil, + voices: voices, + normalizedLocale: "en-US", + baseLanguage: "en" + ) + + #expect(selected == "samantha") + } + + @Test func explicitAppleVoiceChoiceRemainsAuthoritative() { + let voices = [ + appleVoice("albert", name: "Albert", category: .novelty), + appleVoice("samantha", name: "Samantha", isSystemDefault: true), + ] + + let selected = OrbitAppleVoiceCatalog.resolvedVoiceIdentifier( + preferredIdentifier: "albert", + voices: voices, + normalizedLocale: "en-US", + baseLanguage: "en" + ) + + #expect(selected == "albert") + } + @Test func firstPermissionRequestUsesSystemPromptOnly() async throws { let presentationDestination = WindowPositionManager.permissionRequestPresentationDestination( hasPermissionNow: false, From ae02193c2fa6e64d392c1cea592b984eae7f6eb7 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Fri, 10 Jul 2026 00:07:23 -0400 Subject: [PATCH 5/8] fix: isolate realtime dictation callbacks --- Orbit/OrbitDictationManager.swift | 103 +++++++++++++++++--------- Orbit/OrbitSpeechToTextProvider.swift | 2 +- OrbitTests/OrbitTests.swift | 46 ++++++++++++ 3 files changed, 116 insertions(+), 35 deletions(-) diff --git a/Orbit/OrbitDictationManager.swift b/Orbit/OrbitDictationManager.swift index 2680c17..2c60f7a 100644 --- a/Orbit/OrbitDictationManager.swift +++ b/Orbit/OrbitDictationManager.swift @@ -569,10 +569,20 @@ final class OrbitDictationManager: NSObject, ObservableObject { let inputFormat = inputNode.outputFormat(forBus: 0) inputNode.removeTap(onBus: 0) - inputNode.installTap(onBus: 0, bufferSize: 1024, format: inputFormat) { [weak self] buffer, _ in - self?.activeTranscriptionSession?.appendAudioBuffer(buffer) - self?.updateAudioPowerLevel(from: buffer) - } + let audioTapBlock = Self.makeAudioTapBlock( + transcriptionSession: activeTranscriptionSession, + reportPowerLevel: { [weak self] boostedLevel in + Task { @MainActor [weak self] in + self?.updateAudioPowerLevel(boostedLevel) + } + } + ) + inputNode.installTap( + onBus: 0, + bufferSize: 1024, + format: inputFormat, + block: audioTapBlock + ) audioEngine.prepare() try audioEngine.start() @@ -708,12 +718,25 @@ final class OrbitDictationManager: NSObject, ObservableObject { return orderedKeyterms } - private func updateAudioPowerLevel(from audioBuffer: AVAudioPCMBuffer) { - guard let channelData = audioBuffer.floatChannelData else { return } + /// AVAudioEngine invokes tap blocks on a real-time audio queue. Build the + /// block outside MainActor isolation so Swift does not insert an invalid + /// main-executor precondition around the callback. + nonisolated static func makeAudioTapBlock( + transcriptionSession: any SpeechToTextStreamingSession, + reportPowerLevel: @escaping @Sendable (Float) -> Void + ) -> AVAudioNodeTapBlock { + { audioBuffer, _ in + transcriptionSession.appendAudioBuffer(audioBuffer) + reportPowerLevel(audioPowerLevel(from: audioBuffer)) + } + } + + nonisolated private static func audioPowerLevel(from audioBuffer: AVAudioPCMBuffer) -> Float { + guard let channelData = audioBuffer.floatChannelData else { return 0 } let channelSamples = channelData[0] let frameCount = Int(audioBuffer.frameLength) - guard frameCount > 0 else { return } + guard frameCount > 0 else { return 0 } var summedSquares: Float = 0 for sampleIndex in 0..= Self.recordedAudioPowerHistorySampleIntervalSeconds + { + lastRecordedAudioPowerSampleDate = now + appendRecordedAudioPowerSample( + max(CGFloat(boostedLevel), Self.recordedAudioPowerHistoryBaselineLevel) ) - self.currentAudioPowerLevel = smoothedAudioPowerLevel - - let now = Date() - if now.timeIntervalSince(self.lastRecordedAudioPowerSampleDate) - >= Self.recordedAudioPowerHistorySampleIntervalSeconds - { - self.lastRecordedAudioPowerSampleDate = now - self.appendRecordedAudioPowerSample( - max(CGFloat(boostedLevel), Self.recordedAudioPowerHistoryBaselineLevel) - ) - } } } @@ -819,11 +840,7 @@ final class OrbitDictationManager: NSObject, ObservableObject { currentPermissionProblem = nil return true case .notDetermined: - let isGranted = await withCheckedContinuation { continuation in - AVCaptureDevice.requestAccess(for: .audio) { isGranted in - continuation.resume(returning: isGranted) - } - } + let isGranted = await Self.requestMicrophoneAccess() currentPermissionProblem = isGranted ? nil : .microphoneAccessDenied return isGranted case .denied, .restricted: @@ -841,11 +858,8 @@ final class OrbitDictationManager: NSObject, ObservableObject { currentPermissionProblem = nil return true case .notDetermined: - let isGranted = await withCheckedContinuation { continuation in - SFSpeechRecognizer.requestAuthorization { authorizationStatus in - continuation.resume(returning: authorizationStatus == .authorized) - } - } + let authorizationStatus = await Self.requestSpeechRecognitionAuthorization() + let isGranted = authorizationStatus == .authorized currentPermissionProblem = isGranted ? nil : .speechRecognitionDenied return isGranted case .denied, .restricted: @@ -857,6 +871,27 @@ final class OrbitDictationManager: NSObject, ObservableObject { } } + /// Framework permission completions are delivered on framework-owned + /// queues. Keeping these bridges nonisolated prevents callbacks from + /// inheriting OrbitDictationManager's MainActor executor requirement. + nonisolated private static func requestMicrophoneAccess() async -> Bool { + await withCheckedContinuation { continuation in + AVCaptureDevice.requestAccess(for: .audio) { isGranted in + continuation.resume(returning: isGranted) + } + } + } + + nonisolated private static func requestSpeechRecognitionAuthorization() async + -> SFSpeechRecognizerAuthorizationStatus + { + await withCheckedContinuation { continuation in + SFSpeechRecognizer.requestAuthorization { authorizationStatus in + continuation.resume(returning: authorizationStatus) + } + } + } + func openRelevantPrivacySettings() { let settingsURLString: String diff --git a/Orbit/OrbitSpeechToTextProvider.swift b/Orbit/OrbitSpeechToTextProvider.swift index 98271fe..1c9e6b0 100644 --- a/Orbit/OrbitSpeechToTextProvider.swift +++ b/Orbit/OrbitSpeechToTextProvider.swift @@ -8,7 +8,7 @@ import AVFoundation import Foundation -protocol SpeechToTextStreamingSession: AnyObject { +nonisolated protocol SpeechToTextStreamingSession: AnyObject, Sendable { var finalTranscriptFallbackDelaySeconds: TimeInterval { get } func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) func requestFinalTranscript() diff --git a/OrbitTests/OrbitTests.swift b/OrbitTests/OrbitTests.swift index 68fc993..eaa6a94 100644 --- a/OrbitTests/OrbitTests.swift +++ b/OrbitTests/OrbitTests.swift @@ -1,8 +1,54 @@ +import AVFoundation import Foundation import Testing @testable import Orbit +private final class OrbitAudioTapTestSession: SpeechToTextStreamingSession, @unchecked Sendable { + let finalTranscriptFallbackDelaySeconds: TimeInterval = 0 + + private let lock = NSLock() + private var appendedBufferCount = 0 + + var bufferCount: Int { + lock.withLock { appendedBufferCount } + } + + func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) { + lock.withLock { + appendedBufferCount += 1 + } + } + + func requestFinalTranscript() {} + func cancel() {} +} + +struct OrbitAudioTapIsolationTests { + @Test func audioTapBlockRunsOutsideMainActorWithoutExecutorTrap() throws { + let format = try #require( + AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 1) + ) + let buffer = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 4)) + buffer.frameLength = 4 + let samples = try #require(buffer.floatChannelData?[0]) + samples[0] = 0.02 + samples[1] = -0.02 + samples[2] = 0.04 + samples[3] = -0.04 + + let session = OrbitAudioTapTestSession() + let tapBlock = OrbitDictationManager.makeAudioTapBlock( + transcriptionSession: session, + reportPowerLevel: { _ in } + ) + + tapBlock(buffer, AVAudioTime(sampleTime: 0, atRate: format.sampleRate)) + + #expect(session.bufferCount == 1) + } +} + @MainActor struct OrbitTests { From 04d92e109fbb97e8a3c8a1920fa840b47bccba02 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Fri, 10 Jul 2026 00:32:09 -0400 Subject: [PATCH 6/8] feat: harden Orbit voice and Codex runtime --- .github/workflows/ci.yml | 3 +- AGENTS.md | 15 +- DESIGN.md | 12 + Orbit/AGENTS.md | 18 +- Orbit/ActionProvider.swift | 10 + Orbit/AppBundleConfiguration.swift | 5 +- Orbit/AppleSpeechTranscriptionProvider.swift | 71 ++-- Orbit/CodexAppServerActionProvider.swift | 242 ++++++++--- Orbit/DesignSystem.swift | 54 +-- Orbit/OpenAITTSProvider.swift | 83 +++- Orbit/OpenAITranscriptionProvider.swift | 29 +- Orbit/OrbitAudioInput.swift | 31 +- Orbit/OrbitCodexContracts.swift | 20 + Orbit/OrbitCodexModelCatalog.swift | 71 +++- Orbit/OrbitManager.swift | 290 ++++++++++--- Orbit/OrbitPanelView.swift | 259 ++++++++---- Orbit/OrbitScreenCaptureUtility.swift | 238 +++++++---- Orbit/OrbitSettings.swift | 73 +++- Orbit/OrbitTemporaryCaptureLease.swift | 32 +- Orbit/OrbitVoiceCoordinator.swift | 184 ++++++++ Orbit/OverlayWindow.swift | 85 ++-- Orbit/SayNoraTTSProvider.swift | 393 ++++++++++++++++++ Orbit/TextToSpeechProvider.swift | 58 ++- Orbit/WindowPositionManager.swift | 11 +- OrbitTests/OrbitCodexLifecycleTests.swift | 62 +++ OrbitTests/OrbitCodexModelCatalogTests.swift | 137 ++++++ OrbitTests/OrbitTests.swift | 56 ++- OrbitTests/OrbitVoiceTests.swift | 198 +++++++++ PRODUCT.md | 11 +- README.md | 14 +- SECURITY.md | 5 +- SUPPORT.md | 10 +- docs/PRIVACY.md | 8 +- docs/SETUP.md | 13 +- release-manifest.json | 11 +- scripts/release.sh | 77 +++- .../tests/test_validate_release_manifest.py | 83 ++++ scripts/validate_release_manifest.py | 188 ++++++++- 38 files changed, 2662 insertions(+), 498 deletions(-) create mode 100644 Orbit/OrbitVoiceCoordinator.swift create mode 100644 Orbit/SayNoraTTSProvider.swift create mode 100644 OrbitTests/OrbitCodexLifecycleTests.swift create mode 100644 OrbitTests/OrbitCodexModelCatalogTests.swift create mode 100644 OrbitTests/OrbitVoiceTests.swift create mode 100644 scripts/tests/test_validate_release_manifest.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae84b58..afb1385 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,8 @@ jobs: - name: Validate release manifest and pinned browser runtime run: | xcodebuild -version - python3 scripts/validate_release_manifest.py + python3 scripts/validate_release_manifest.py --project-root . + python3 -m unittest discover -s scripts/tests -p 'test_*.py' test "$(node -p "require('./BundledResources/browser-runtime/package.json').dependencies['chrome-devtools-mcp']")" = "1.5.0" - name: Build Orbit diff --git a/AGENTS.md b/AGENTS.md index 9432140..61e5604 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,14 +5,16 @@ Orbit is a Codex-native macOS menu bar assistant. ## Current architecture - UI shell: SwiftUI + AppKit panel/overlay -- STT: OpenAI `gpt-4o-mini-transcribe` with Apple Speech fallback -- Brain and actions: Codex app-server -- TTS: OpenAI `gpt-4o-mini-tts` with Apple system speech fallback +- STT: Apple on-device recognition in Local mode; OpenAI `gpt-4o-mini-transcribe` only in explicit Cloud mode +- Brain and actions: one warm Codex app-server thread with a live account model catalog +- TTS: Nora Premium through `/usr/bin/say` in Local mode, public AVFoundation fallback, and OpenAI `gpt-4o-mini-tts` in Cloud mode ## Important files - [Orbit/OrbitManager.swift](Orbit/OrbitManager.swift) — unified Codex routing, screenshots, overlay updates, action summaries - [Orbit/OrbitDictationManager.swift](Orbit/OrbitDictationManager.swift) — push-to-talk capture and STT session management +- [Orbit/OrbitVoiceCoordinator.swift](Orbit/OrbitVoiceCoordinator.swift) — provider-neutral narration formatting, interruption, fallback, and duplicate suppression +- [Orbit/SayNoraTTSProvider.swift](Orbit/SayNoraTTSProvider.swift) — Nora availability probing, secure temporary AIFF rendering, playback, and cleanup - [Orbit/OrbitSettings.swift](Orbit/OrbitSettings.swift) — persisted voice mode, Codex effort, and overlay settings - [Orbit/CodexAppServerActionProvider.swift](Orbit/CodexAppServerActionProvider.swift) — persistent Codex session and event streaming - [Orbit/OverlayWindow.swift](Orbit/OverlayWindow.swift) — Orbit cursor, HUD, and pointing animations @@ -20,5 +22,8 @@ Orbit is a Codex-native macOS menu bar assistant. ## Notes - Keep the point-tag flow in v1 instead of introducing a second coordinate detection architecture. -- Orbit now uses one persistent Codex session for both answers and actions. -- Menu bar settings should stay compact; prefer clear model/effort controls over sprawling configuration UI. +- Orbit uses one warm Codex thread for both answers and actions. Model, effort, and service tier changes preserve it; Agent Folder starts fresh context. +- Treat app-server `model/list` as authoritative. Never hard-code account availability or discard unknown future effort and tier strings. +- Keep model, effort, and service tier visually connected. Keep Local and Cloud privacy copy separate and accurate. +- Nora narration files belong only in Orbit's mode `0700` local-speech directory, use mode `0600`, and must be removed after every terminal playback path. Never invoke `/usr/bin/say` through a shell. +- Menu bar settings should stay compact; prefer connected, adaptive Codex controls over sprawling configuration UI. diff --git a/DESIGN.md b/DESIGN.md index c76f699..29adfc8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -163,6 +163,18 @@ Orbit is flat by default and separates layers through Night, Graphite, and Raise Settings use compact rows and focused drill-down pages. Back, Escape, and keyboard focus follow standard macOS expectations. +### Connected Codex Configuration + +Model, reasoning effort, and service tier form one tonal group because their available values come from the same live Codex catalog. The model remains visually primary. Effort and tier update to the selected model's supported values; dynamic or long catalogs use menus instead of compressed glyph-only segments. Unknown future values keep their server-provided text. + +Agent Folder is adjacent context, not a security control. Its copy must state that changing it begins fresh working context while unrestricted filesystem access remains unchanged. + +### Voice and Privacy States + +Local and Cloud are distinct product states, not decorative presets. Local identifies Nora Premium when available and names the Apple fallback when it is not. Cloud states plainly that speech audio is sent to OpenAI. Never show local-only privacy copy while Cloud is selected. + +Voice controls expose provider availability, preview or stop state, microphone choice, and input level without duplicating the same status in the header. Recovery copy belongs beside the unavailable provider. + ### Permission Coach The coach is a non-activating detached surface anchored to System Settings. Only the Orbit tile moves during drag. It disappears automatically after success and supplies Finder and keyboard fallbacks without persistent chrome. diff --git a/Orbit/AGENTS.md b/Orbit/AGENTS.md index 58a2d5e..1e2fd4a 100644 --- a/Orbit/AGENTS.md +++ b/Orbit/AGENTS.md @@ -7,22 +7,23 @@ - `OrbitSpeechToTextProvider.swift` defines the STT abstraction and selects providers from the active Orbit preset. - `AppleSpeechTranscriptionProvider.swift` is the default on-device macOS STT provider and never silently falls back to the network. - `OpenAITranscriptionProvider.swift` is the explicitly selected cloud STT provider using `gpt-4o-mini-transcribe`. -- `TextToSpeechProvider.swift` defines the TTS abstraction plus provider factory logic. +- `OrbitVoiceCoordinator.swift` owns narration formatting, interruption, duplicate suppression, and provider fallback. +- `SayNoraTTSProvider.swift` verifies Nora Premium availability, renders through `/usr/bin/say` to a private temporary AIFF, plays it locally, and removes it on every terminal path. +- `TextToSpeechProvider.swift` defines the TTS abstraction and public AVFoundation local fallback. - `OpenAITTSProvider.swift` is the default cloud TTS provider using `gpt-4o-mini-tts`. -- `TextToSpeechProvider.swift` also includes `AppleSystemTTSProvider` as the local speech fallback. - `OrbitOpenAIVoiceConfiguration.swift` stores the Cloud voice API key in Keychain and validates it. - `OrbitCodexEnvironment.swift` prepares Orbit's isolated Codex home, bundled browser MCP config, and bundled skill inventory. - `OrbitBundledSkills.swift` selects the Orbit-owned and curated skills to inject into turns when the request clearly matches them. ### Actions and state - `ActionProvider.swift` defines Orbit's unified Codex request contract. -- `CodexAppServerActionProvider.swift` streams action progress from `codex app-server`. +- `CodexAppServerActionProvider.swift` keeps one warm Codex thread, treats `model/list` as authoritative, and streams action progress from `codex app-server`. - `OrbitSettings.swift` stores persisted menu bar settings in `UserDefaults`, including voice mode, Codex effort, and cursor visibility. - `OrbitManager.swift` orchestrates Codex turns, current-screen screenshot capture, cursor overlay, and spoken summaries. ### UI shell - `OrbitApp.swift` boots the menu bar app and startup services. -- `OrbitPanelView.swift` renders the compact panel, including Orbit preset controls and action status. +- `OrbitPanelView.swift` renders separate activity and settings states, a connected model/effort/tier group, and accurate Local/Cloud privacy states. - `MenuBarPanelManager.swift` and `OverlayWindow.swift` own the menu bar shell and cursor-adjacent overlay behavior. - `OrbitScreenCaptureUtility.swift` captures current-screen context for Codex turns. @@ -32,8 +33,15 @@ - Codex model default: the current app-server `model/list` default - Codex effort default: `medium` - Codex service tier default: the app-server default; optional tiers appear only when returned -- TTS default: Apple on-device `AVSpeechSynthesizer` +- TTS default: Nora Premium through local `/usr/bin/say`, with public AVFoundation fallback - Cloud voice option: OpenAI `gpt-4o-mini-transcribe` and `gpt-4o-mini-tts` - Unified assistant path: Codex app-server - Bundled browser tools: `chrome-devtools-mcp`, `@playwright/mcp` - Bundled skills: `doc`, `pdf`, `slides`, `spreadsheet`, `screenshot`, `transcribe`, `speech`, `openai-docs` + +## Invariants + +- Model, reasoning effort, and service tier use the live account catalog and preserve the warm Codex thread. +- Agent Folder changes working context by starting a fresh thread after any active turn; it never narrows filesystem access. +- Local narration uses only Orbit-owned temporary AIFF files in a mode `0700` directory, sets each file to mode `0600`, and deletes files after playback, failure, interruption, cancellation, or stale-file recovery. +- Cloud voice copy must state that speech audio is sent to OpenAI. Local copy must not imply cloud transfer. diff --git a/Orbit/ActionProvider.swift b/Orbit/ActionProvider.swift index 05047eb..a8ea6ef 100644 --- a/Orbit/ActionProvider.swift +++ b/Orbit/ActionProvider.swift @@ -150,6 +150,15 @@ enum OrbitActionEvent { case interrupted(String) case completed(String) case failed(String) + + var isTerminal: Bool { + switch self { + case .interrupted, .completed, .failed: + return true + case .phase, .commentary, .liveUpdate, .toolPrompt, .subagentActivity: + return false + } + } } struct OrbitSubagentActivity: Identifiable, Equatable, Sendable { @@ -170,6 +179,7 @@ struct OrbitToolPrompt: Equatable { } struct OrbitActionRequest { + let id: UUID let transcript: String let screenshotPath: String? let screenshotLabel: String? diff --git a/Orbit/AppBundleConfiguration.swift b/Orbit/AppBundleConfiguration.swift index a2dc90e..4f1ff13 100644 --- a/Orbit/AppBundleConfiguration.swift +++ b/Orbit/AppBundleConfiguration.swift @@ -141,9 +141,12 @@ enum OrbitSupportLog { let patterns: [(String, String)] = [ (#"(?i)(authorization\s*[:=]\s*)(?:bearer\s+)?[^\s,;]+"#, "$1[redacted]"), (#"(?i)\b(?:sk|sess|token|key)-[A-Za-z0-9._-]{8,}\b"#, "[credential-redacted]"), + (#"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"#, "[jwt-redacted]"), + (#"(?i)([?&](?:access_token|api[_-]?key|token|secret|key)=)[^&\s]+"#, "$1[redacted]"), + (#"(?i)\b(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|GOOGLE_API_KEY|GITHUB_TOKEN|NPM_TOKEN)\s*=\s*[^\s,;]+"#, "credential=[redacted]"), (#"(?i)(?:prompt|transcript)\s*[:=].*$"#, "prompt=[redacted]"), (#"/[^\s]+/OrbitTemporaryCaptures/capture-[^\s]+\.jpg"#, "[temporary-capture]"), - (#"(?i)(?:arguments|argv|command)\s*[:=]\s*\[[^\]]*\]"#, "arguments=[redacted]"), + (#"(?i)(?:arguments|argv|command)\s*[:=].*$"#, "arguments=[redacted]"), ] for (pattern, replacement) in patterns { value = value.replacingOccurrences(of: pattern, with: replacement, options: .regularExpression) diff --git a/Orbit/AppleSpeechTranscriptionProvider.swift b/Orbit/AppleSpeechTranscriptionProvider.swift index 022732a..4d8d4a8 100644 --- a/Orbit/AppleSpeechTranscriptionProvider.swift +++ b/Orbit/AppleSpeechTranscriptionProvider.swift @@ -71,6 +71,7 @@ nonisolated private final class AppleSpeechTranscriptionSession: NSObject, Speec private let onTranscriptUpdate: (String) -> Void private let onFinalTranscriptReady: (String) -> Void private let onError: (Error) -> Void + private let stateLock = NSLock() private var latestRecognizedText = "" private var hasRequestedFinalTranscript = false @@ -106,51 +107,67 @@ nonisolated private final class AppleSpeechTranscriptionSession: NSObject, Speec } func appendAudioBuffer(_ audioBuffer: AVAudioPCMBuffer) { - guard !hasRequestedFinalTranscript else { return } - recognitionRequest.append(audioBuffer) + stateLock.withLock { + guard !hasRequestedFinalTranscript else { return } + recognitionRequest.append(audioBuffer) + } } func requestFinalTranscript() { - guard !hasRequestedFinalTranscript else { return } - hasRequestedFinalTranscript = true - recognitionRequest.endAudio() + stateLock.withLock { + guard !hasRequestedFinalTranscript else { return } + hasRequestedFinalTranscript = true + recognitionRequest.endAudio() + } } func cancel() { - recognitionTask?.cancel() - recognitionTask = nil + let task = stateLock.withLock { () -> SFSpeechRecognitionTask? in + hasRequestedFinalTranscript = true + let task = recognitionTask + recognitionTask = nil + return task + } + task?.cancel() } private func handleRecognitionEvent( result: SFSpeechRecognitionResult?, error: Error? ) { - if let result { - latestRecognizedText = result.bestTranscription.formattedString - onTranscriptUpdate(latestRecognizedText) - - if result.isFinal { - deliverFinalTranscriptIfNeeded(latestRecognizedText) - return + var transcriptUpdate: String? + var finalTranscript: String? + var reportedError: Error? + + stateLock.withLock { + if let result { + latestRecognizedText = result.bestTranscription.formattedString + transcriptUpdate = latestRecognizedText + if result.isFinal, !hasDeliveredFinalTranscript { + hasDeliveredFinalTranscript = true + finalTranscript = latestRecognizedText + } } - } - - guard let error else { return } - if hasRequestedFinalTranscript && !latestRecognizedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - deliverFinalTranscriptIfNeeded(latestRecognizedText) - } else { - onError(error) + if let error, finalTranscript == nil { + if hasRequestedFinalTranscript, + !latestRecognizedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + !hasDeliveredFinalTranscript + { + hasDeliveredFinalTranscript = true + finalTranscript = latestRecognizedText + } else if !hasDeliveredFinalTranscript { + reportedError = error + } + } } - } - private func deliverFinalTranscriptIfNeeded(_ transcriptText: String) { - guard !hasDeliveredFinalTranscript else { return } - hasDeliveredFinalTranscript = true - onFinalTranscriptReady(transcriptText) + if let transcriptUpdate { onTranscriptUpdate(transcriptUpdate) } + if let finalTranscript { onFinalTranscriptReady(finalTranscript) } + if let reportedError { onError(reportedError) } } deinit { - recognitionTask?.cancel() + cancel() } } diff --git a/Orbit/CodexAppServerActionProvider.swift b/Orbit/CodexAppServerActionProvider.swift index 57a79af..eef184d 100644 --- a/Orbit/CodexAppServerActionProvider.swift +++ b/Orbit/CodexAppServerActionProvider.swift @@ -1,4 +1,5 @@ import AppKit +import Darwin import Foundation private enum OrbitMcpStartupState: Equatable { @@ -36,7 +37,7 @@ final class CodexAppServerActionProvider: ActionProvider { private var stdinHandle: FileHandle? private var stdoutHandle: FileHandle? private var stderrHandle: FileHandle? - private let transport = OrbitCodexTransportActor() + private var transport = OrbitCodexTransportActor() private var stderrSnapshot = Data() private var nextRequestID = 99 private var activeThreadID: String? @@ -55,6 +56,8 @@ final class CodexAppServerActionProvider: ActionProvider { private var lastEmittedProgress: OrbitActionProgress? private var intentionalShutdownInProgress = false private var prewarmTask: Task? + private var activePrewarmIsFresh = false + private var freshPrewarmRequested = false private var streamedCommentaryBuffer = "" private var hasEmittedEarlyCommentary = false private var availableModelOptions: [OrbitCodexModelOption] = OrbitCodexModelOption.fallbackPickerModels @@ -68,6 +71,10 @@ final class CodexAppServerActionProvider: ActionProvider { private var lastLoginID: String? private var loginRequestTimeoutTask: Task? private var pendingTurnStartRetryTask: Task? + private var interruptTimeoutTask: Task? + private var turnStartRequestInFlight = false + private var connectionGeneration: UInt64 = 0 + private var activeConnectionGeneration: UInt64? private var lastEmittedLiveCommentary: String? private var preparedCodexHome: OrbitPreparedCodexHome? private var mcpStartupStates: [String: OrbitMcpStartupState] = CodexAppServerActionProvider.makeInitialMcpStartupStates() @@ -208,10 +215,14 @@ final class CodexAppServerActionProvider: ActionProvider { status = .failed(warmupError) authState = .runtimeUnavailable(warmupError) notifyStateChanged() - onEvent(.failed(warmupError)) + terminalizePendingAction(.failed(warmupError)) return } + // A process failure can terminalize the request while prewarm is + // suspended. Never continue by sending a ghost turn afterward. + guard eventHandler != nil else { return } + guard isAuthenticatedForTurns else { let message: String switch authState { @@ -228,7 +239,7 @@ final class CodexAppServerActionProvider: ActionProvider { } status = .failed(message) notifyStateChanged() - onEvent(.failed(message)) + terminalizePendingAction(.failed(message)) return } @@ -282,12 +293,31 @@ final class CodexAppServerActionProvider: ActionProvider { func prewarmSession(forceFreshSession: Bool = false) async -> String? { if let prewarmTask { - return await prewarmTask.value + if forceFreshSession, !activePrewarmIsFresh { + freshPrewarmRequested = true + } + let existingResult = await prewarmTask.value + if freshPrewarmRequested { + freshPrewarmRequested = false + return await launchPrewarmSession(forceFreshSession: true) + } + if let currentPrewarmTask = self.prewarmTask { + return await currentPrewarmTask.value + } + return existingResult } + return await launchPrewarmSession(forceFreshSession: forceFreshSession) + } + + private func launchPrewarmSession(forceFreshSession: Bool) async -> String? { + activePrewarmIsFresh = forceFreshSession let task = Task { @MainActor [weak self] in guard let self else { return "Orbit could not connect to Codex app-server." } - defer { self.prewarmTask = nil } + defer { + self.prewarmTask = nil + self.activePrewarmIsFresh = false + } return await self.performPrewarmSession(forceFreshSession: forceFreshSession) } prewarmTask = task @@ -349,6 +379,9 @@ final class CodexAppServerActionProvider: ActionProvider { private func performPrewarmSession(forceFreshSession: Bool = false) async -> String? { if forceFreshSession { + terminalizePendingAction( + .interrupted("the codex session was restarted before the current request finished.") + ) restartServerForRecovery() } else if let process, !process.isRunning { teardownProcess() @@ -363,6 +396,9 @@ final class CodexAppServerActionProvider: ActionProvider { var lastFailureMessage: String? for attempt in 0.. String? in - guard case .failed(let errorMessage) = mcpStartupStates[serverName] else { return nil } - let detail = errorMessage?.trimmingCharacters(in: .whitespacesAndNewlines) - if let detail, !detail.isEmpty { - return "- \(serverName) browser tools are unavailable in this Orbit session: \(detail)" - } + guard case .failed = mcpStartupStates[serverName] else { return nil } return "- \(serverName) browser tools are unavailable in this Orbit session" } @@ -1088,6 +1146,8 @@ final class CodexAppServerActionProvider: ActionProvider { "-> turn/start model=\(currentActionModel) effort=\(resolvedEffortForCurrentModel.rawValue) tier=\(resolvedServiceTier.rawValue) inputItems=\(inputItems.count)" ) + turnStartRequestInFlight = true + self.pendingPrompt = nil sendJSON([ "method": "turn/start", "id": 2, @@ -1153,6 +1213,7 @@ final class CodexAppServerActionProvider: ActionProvider { let requestID = nextClientRequestID() appendDebugEvent("-> turn/steer #\(requestID) turn=\(String(activeTurnID.suffix(6))) inputItems=\(inputItems.count)") + self.pendingPrompt = nil sendJSON([ "method": "turn/steer", "id": requestID, @@ -1204,21 +1265,52 @@ final class CodexAppServerActionProvider: ActionProvider { try? await Task.sleep(nanoseconds: 8_000_000_000) await MainActor.run { guard let self, !self.hasReceivedInitializeResponse else { return } - let stderrText = String(data: self.stderrSnapshot, encoding: .utf8)? + let stderrText = String(data: self.stderrSnapshot, encoding: .utf8) + .map(Self.safeDiagnostic)? .trimmingCharacters(in: .whitespacesAndNewlines) let message = stderrText?.isEmpty == false ? "Orbit could not connect to Codex app-server. \(stderrText!)" : "Orbit could not connect to Codex app-server." self.status = .failed(message) - self.eventHandler?(.failed(message)) + self.terminalizePendingAction(.failed(message)) self.teardownProcess() } } } - private func handleProcessTermination(_ terminatedProcess: Process) { - guard process === terminatedProcess else { return } + private func beginInterruptTimeoutWatch(threadID: String, turnID: String) { + interruptTimeoutTask?.cancel() + interruptTimeoutTask = Task { @MainActor [weak self] in + try? await Task.sleep(nanoseconds: 5_000_000_000) + guard let self, + self.activeThreadID == threadID, + self.activeTurnID == turnID, + self.isAwaitingTurnCompletion + else { return } + + let message = "the codex turn did not stop in time, so Orbit restarted the session." + self.appendDebugEvent("turn/interrupt timed out; restarting session") + self.status = .interrupted(message) + self.terminalizePendingAction(.interrupted(message)) + self.restartServerForRecovery() + Task { @MainActor [weak self] in + _ = await self?.prewarmSession() + } + } + } + + private func terminalizePendingAction(_ event: OrbitActionEvent) { + guard let eventHandler else { return } + self.eventHandler = nil + pendingPrompt = nil + latestRequest = nil + turnStartRequestInFlight = false + eventHandler(event) + } + + private func handleProcessTermination(_ terminatedProcess: Process, generation: UInt64) { + guard activeConnectionGeneration == generation, process === terminatedProcess else { return } let wasHandlingSession = hasSentInitialize || hasSentThreadStart @@ -1237,16 +1329,15 @@ final class CodexAppServerActionProvider: ActionProvider { return } - let stderrText = String(data: stderrSnapshot, encoding: .utf8)? + let stderrText = String(data: stderrSnapshot, encoding: .utf8) + .map(Self.safeDiagnostic)? .trimmingCharacters(in: .whitespacesAndNewlines) let message = stderrText?.isEmpty == false ? "Codex action process exited: \(stderrText!)" : "Codex action process exited unexpectedly." status = .failed(message) - if isAwaitingTurnCompletion { - eventHandler?(.failed(message)) - } + terminalizePendingAction(.failed(message)) teardownProcess() } @@ -1397,7 +1488,9 @@ final class CodexAppServerActionProvider: ActionProvider { } if Self.browserToolServerNames.contains(serverName) { - let errorMessage = (params["error"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) + let errorMessage = (params["error"] as? String) + .map(Self.safeDiagnostic)? + .trimmingCharacters(in: .whitespacesAndNewlines) switch status { case "starting": mcpStartupStates[serverName] = .starting @@ -1405,6 +1498,8 @@ final class CodexAppServerActionProvider: ActionProvider { mcpStartupStates[serverName] = .ready case "failed": mcpStartupStates[serverName] = .failed(errorMessage) + case "cancelled": + mcpStartupStates[serverName] = .failed("startup cancelled") default: break } @@ -1415,15 +1510,7 @@ final class CodexAppServerActionProvider: ActionProvider { } else if status == "ready" && (serverName == "playwright" || serverName == "chrome-devtools") { emitPhase(.thinking, rawSource: "\(serverName) tools are ready") } else if status == "failed" && (serverName == "playwright" || serverName == "chrome-devtools") { - let detail = { - let trimmed = (params["error"] as? String)? - .trimmingCharacters(in: .whitespacesAndNewlines) - if let trimmed, !trimmed.isEmpty { - return trimmed - } - return "browser tools failed to start." - }() - appendDebugEvent("\(serverName) tools failed: \(detail)") + appendDebugEvent("\(serverName) tools failed to start") } if pendingPrompt != nil, activeThreadID != nil, !isAwaitingTurnCompletion { @@ -1527,7 +1614,7 @@ final class CodexAppServerActionProvider: ActionProvider { emitPhase(progressForTool(server: server, tool: tool), rawSource: "using \(server) \(tool)") case "commandExecution": if let command = item["command"] as? [String], !command.isEmpty { - emitPhase(progressForCommand(command), rawSource: "running \(command.joined(separator: " "))") + emitPhase(progressForCommand(command), rawSource: "running a command") } case "fileChange": emitPhase(.editingFiles, detail: "editing files in the current session.", rawSource: "preparing changes") @@ -1660,7 +1747,7 @@ final class CodexAppServerActionProvider: ActionProvider { if let progressText { let cleaned = progressText.trimmingCharacters(in: .whitespacesAndNewlines) if !cleaned.isEmpty { - appendDebugEvent("<- tool/progress \(cleaned)") + appendDebugEvent("<- tool/progress bytes=\(cleaned.utf8.count)") } } } @@ -1753,7 +1840,7 @@ final class CodexAppServerActionProvider: ActionProvider { return OrbitActionProgress(phase: .openingBrowser) } - return OrbitActionProgress(phase: .runningCommand, detail: "running \(command.joined(separator: " ")).") + return OrbitActionProgress(phase: .runningCommand, detail: "running a command in the current session.") } private func visualContextMessage(for request: OrbitActionRequest?) -> String? { @@ -2076,9 +2163,13 @@ final class CodexAppServerActionProvider: ActionProvider { private static func startupFailureMessage(for error: Error) -> String { let nsError = error as NSError if nsError.domain == "OrbitCodexEnvironment" { - return nsError.localizedDescription + return safeDiagnostic(nsError.localizedDescription) } - return "Orbit could not start Codex app-server: \(nsError.localizedDescription)" + return "Orbit could not start Codex app-server: \(safeDiagnostic(nsError.localizedDescription))" + } + + private static func safeDiagnostic(_ value: String) -> String { + OrbitSupportLog.sanitize(value) } private func resolveCodexExecutable() throws -> String { @@ -2161,6 +2252,10 @@ final class CodexAppServerActionProvider: ActionProvider { } private func teardownProcess() { + let retiredTransport = transport + if let process, process.isRunning { + gracefullyStopProcess(process) + } intentionalShutdownInProgress = false loginRequestTimeoutTask?.cancel() loginRequestTimeoutTask = nil @@ -2168,6 +2263,8 @@ final class CodexAppServerActionProvider: ActionProvider { startupTimeoutTask = nil pendingTurnStartRetryTask?.cancel() pendingTurnStartRetryTask = nil + interruptTimeoutTask?.cancel() + interruptTimeoutTask = nil stdoutHandle?.readabilityHandler = nil stderrHandle?.readabilityHandler = nil try? stdinHandle?.close() @@ -2177,8 +2274,9 @@ final class CodexAppServerActionProvider: ActionProvider { stderrHandle = nil stdinHandle = nil process = nil + activeConnectionGeneration = nil stderrSnapshot.removeAll(keepingCapacity: false) - Task { await transport.reset() } + Task { await retiredTransport.reset() } activeThreadID = nil activeTurnID = nil pendingPrompt = nil @@ -2187,6 +2285,7 @@ final class CodexAppServerActionProvider: ActionProvider { hasReceivedInitializeResponse = false hasSentInitialize = false hasSentThreadStart = false + turnStartRequestInFlight = false isAwaitingTurnCompletion = false hasOpenedBrowserInCurrentTurn = false streamedCommentaryBuffer = "" @@ -2206,16 +2305,25 @@ final class CodexAppServerActionProvider: ActionProvider { notifyStateChanged() } + private func gracefullyStopProcess(_ process: Process) { + process.terminationHandler = nil + try? stdinHandle?.close() + guard process.isRunning else { return } + + let processIdentifier = process.processIdentifier + process.terminate() + Task.detached(priority: .utility) { + try? await Task.sleep(nanoseconds: 1_500_000_000) + var childStatus: Int32 = 0 + if Darwin.waitpid(processIdentifier, &childStatus, WNOHANG) == 0 { + _ = Darwin.kill(processIdentifier, SIGKILL) + } + } + } + private func restartServerForRecovery() { startupTimeoutTask?.cancel() startupTimeoutTask = nil - - if let process, process.isRunning { - intentionalShutdownInProgress = true - process.terminationHandler = nil - process.terminate() - } - teardownProcess() } } diff --git a/Orbit/DesignSystem.swift b/Orbit/DesignSystem.swift index 5cb21e1..3c15922 100644 --- a/Orbit/DesignSystem.swift +++ b/Orbit/DesignSystem.swift @@ -93,11 +93,11 @@ enum DS { /// Accent fill — used for solid button backgrounds. /// #2563eb → ~5.1:1 contrast with white text (WCAG AA). - static let accent = Color(hex: "#667076") + static let accent = blue600 /// Accent hover — slightly darker blue for hover state. /// #1d4ed8 → ~6.5:1 contrast with white text (WCAG AA+). - static let accentHover = Color(hex: "#79848B") + static let accentHover = blue700 /// Accent text — bright blue used for accent-colored text and icons /// on dark backgrounds (links, active nav items, highlighted labels). @@ -250,32 +250,20 @@ struct DSGlassCardModifier: ViewModifier { func body(content: Content) -> some View { content .background { - ZStack { - shape - .fill(.ultraThinMaterial) - - shape - .fill(Color.black.opacity(fillOpacity)) - - shape - .stroke(Color.white.opacity(borderOpacity), lineWidth: 0.8) - - shape - .stroke( - LinearGradient( - colors: [ - Color.white.opacity(highlightOpacity), - Color.white.opacity(0.03), - ], - startPoint: .topLeading, - endPoint: .bottomTrailing - ), - lineWidth: 1 + shape + .fill(DS.Colors.surface1.opacity(max(0.88, min(0.98, 1 - (fillOpacity * 0.12))))) + .overlay( + shape.stroke( + DS.Colors.borderSubtle.opacity(max(0.55, borderOpacity * 4)), + lineWidth: 0.75 ) - .blur(radius: 0.2) - } - .shadow(color: Color.black.opacity(shadowOpacity), radius: 24, x: 0, y: 18) - .shadow(color: Color.white.opacity(glowOpacity), radius: 16, x: 0, y: 0) + ) + .shadow( + color: Color.black.opacity(shadowOpacity >= 0.2 ? 0.28 : 0), + radius: shadowOpacity >= 0.2 ? 14 : 0, + x: 0, + y: shadowOpacity >= 0.2 ? 8 : 0 + ) } } } @@ -293,7 +281,6 @@ struct DSQuietStatusChip: View { Text(title) .font(.system(size: 10, weight: .semibold)) .foregroundColor(DS.Colors.textSecondary) - .textCase(.uppercase) } .padding(.horizontal, 10) .padding(.vertical, 6) @@ -316,6 +303,7 @@ struct DSQuietStatusChip: View { struct DSPrimaryButtonStyle: ButtonStyle { var isFullWidth: Bool = true + @Environment(\.accessibilityReduceMotion) private var reduceMotion @State private var isHovered = false // Separate state for the scale expansion so it animates on a slower, @@ -352,8 +340,8 @@ struct DSPrimaryButtonStyle: ButtonStyle { radius: isHoverGlowActive ? (isGlowBreathingIn ? 16 : 10) : 0 ) // Hover: gradually expand to 1.03. Press: snap down to 0.97. - .scaleEffect(configuration.isPressed ? 0.97 : (isHoverScaleExpanded ? 1.03 : 1.0)) - .animation(.easeOut(duration: 0.1), value: configuration.isPressed) + .scaleEffect(reduceMotion ? 1 : (configuration.isPressed ? 0.97 : (isHoverScaleExpanded ? 1.03 : 1.0))) + .animation(reduceMotion ? nil : .easeOut(duration: 0.1), value: configuration.isPressed) .onHover { hovering in // Background color — fast snap so the button feels responsive withAnimation(.easeOut(duration: 0.15)) { @@ -361,18 +349,18 @@ struct DSPrimaryButtonStyle: ButtonStyle { } // Scale — slow, gradual expansion (like the button is swelling) - withAnimation(.easeInOut(duration: hovering ? 0.6 : 0.3)) { + withAnimation(reduceMotion ? nil : .easeInOut(duration: hovering ? 0.6 : 0.3)) { isHoverScaleExpanded = hovering } // Glow — builds up gradually on entry, fades faster on exit - withAnimation(.easeInOut(duration: hovering ? 0.6 : 0.3)) { + withAnimation(reduceMotion ? nil : .easeInOut(duration: hovering ? 0.6 : 0.3)) { isHoverGlowActive = hovering } // Breathing glow loop — gentle pulse while hovered. // The 2.5s cycle keeps it feeling organic, not mechanical. - if hovering { + if hovering && !reduceMotion { withAnimation( .easeInOut(duration: 2.5) .repeatForever(autoreverses: true) diff --git a/Orbit/OpenAITTSProvider.swift b/Orbit/OpenAITTSProvider.swift index b452fe1..3252dad 100644 --- a/Orbit/OpenAITTSProvider.swift +++ b/Orbit/OpenAITTSProvider.swift @@ -11,7 +11,10 @@ final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDele private let session: URLSession private let endpointURL = URL(string: "https://api.openai.com/v1/audio/speech")! private var audioPlayer: AVAudioPlayer? + private var currentPlayerIdentifier: ObjectIdentifier? private var currentSpeakContinuation: CheckedContinuation? + private var currentNetworkTask: Task<(Data, URLResponse), Error>? + private var generation = 0 init(voicePreset: OrbitVoicePreset) { self.voicePreset = voicePreset @@ -41,6 +44,8 @@ final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDele func speakText(_ text: String) async throws { stopPlayback() + generation &+= 1 + let requestGeneration = generation guard let resolvedAPIKey else { throw NSError( @@ -64,7 +69,20 @@ final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDele ] request.httpBody = try JSONSerialization.data(withJSONObject: body) - let (data, response) = try await session.data(for: request) + let networkTask = Task { try await session.data(for: request) } + currentNetworkTask = networkTask + let (data, response) = try await withTaskCancellationHandler { + try await networkTask.value + } onCancel: { + networkTask.cancel() + Task { @MainActor [weak self] in + self?.cancelIfCurrent(requestGeneration) + } + } + if generation == requestGeneration { + currentNetworkTask = nil + } + guard generation == requestGeneration else { throw CancellationError() } guard let httpResponse = response as? HTTPURLResponse else { throw NSError( @@ -83,48 +101,68 @@ final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDele ) } - try await withCheckedThrowingContinuation { continuation in - do { - let player = try AVAudioPlayer(data: data) - player.delegate = self - audioPlayer = player - currentSpeakContinuation = continuation - if !player.play() { + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + do { + guard generation == requestGeneration else { + continuation.resume(throwing: CancellationError()) + return + } + let player = try AVAudioPlayer(data: data) + player.delegate = self + audioPlayer = player + currentPlayerIdentifier = ObjectIdentifier(player) + currentSpeakContinuation = continuation + if !player.play() { + audioPlayer = nil + currentPlayerIdentifier = nil + currentSpeakContinuation = nil + continuation.resume( + throwing: NSError( + domain: "OpenAITTSProvider", + code: -3, + userInfo: [NSLocalizedDescriptionKey: "OpenAI voice could not start playback."] + ) + ) + } + } catch { audioPlayer = nil + currentPlayerIdentifier = nil currentSpeakContinuation = nil - continuation.resume( - throwing: NSError( - domain: "OpenAITTSProvider", - code: -3, - userInfo: [NSLocalizedDescriptionKey: "OpenAI voice could not start playback."] - ) - ) + continuation.resume(throwing: error) } - } catch { - audioPlayer = nil - currentSpeakContinuation = nil - continuation.resume(throwing: error) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.cancelIfCurrent(requestGeneration) } } } func stopPlayback() { + generation &+= 1 + currentNetworkTask?.cancel() + currentNetworkTask = nil if audioPlayer?.isPlaying == true { audioPlayer?.stop() } audioPlayer = nil + currentPlayerIdentifier = nil if let currentSpeakContinuation { self.currentSpeakContinuation = nil - currentSpeakContinuation.resume() + currentSpeakContinuation.resume(throwing: CancellationError()) } } nonisolated func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + let playerIdentifier = ObjectIdentifier(player) Task { @MainActor [weak self] in guard let self else { return } + guard self.currentPlayerIdentifier == playerIdentifier else { return } let continuation = self.currentSpeakContinuation self.currentSpeakContinuation = nil self.audioPlayer = nil + self.currentPlayerIdentifier = nil if flag { continuation?.resume() } else { @@ -139,6 +177,11 @@ final class OpenAITTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDele } } + private func cancelIfCurrent(_ requestGeneration: Int) { + guard generation == requestGeneration else { return } + stopPlayback() + } + private var preferredVoiceName: String { switch voicePreset { case .localVoice, .cloudVoice: diff --git a/Orbit/OpenAITranscriptionProvider.swift b/Orbit/OpenAITranscriptionProvider.swift index 65c5a93..19dbe35 100644 --- a/Orbit/OpenAITranscriptionProvider.swift +++ b/Orbit/OpenAITranscriptionProvider.swift @@ -57,6 +57,8 @@ nonisolated private final class OpenAITranscriptionSession: SpeechToTextStreamin private static let transcriptionURL = URL(string: "https://api.openai.com/v1/audio/transcriptions")! private static let targetSampleRate = 16_000 + private static let maximumRecordingDurationSeconds = 120 + private static let maximumPCM16ByteCount = targetSampleRate * 2 * maximumRecordingDurationSeconds private let apiKey: String private let modelName: String @@ -72,6 +74,7 @@ nonisolated private final class OpenAITranscriptionSession: SpeechToTextStreamin private var hasRequestedFinalTranscript = false private var hasDeliveredFinalTranscript = false private var isCancelled = false + private var hasReportedRecordingLimit = false private var transcriptionUploadTask: Task? init( @@ -105,6 +108,17 @@ nonisolated private final class OpenAITranscriptionSession: SpeechToTextStreamin stateQueue.async { guard !self.hasRequestedFinalTranscript, !self.isCancelled else { return } + guard self.bufferedPCM16AudioData.count + pcmData.count <= Self.maximumPCM16ByteCount else { + self.hasRequestedFinalTranscript = true + guard !self.hasReportedRecordingLimit else { return } + self.hasReportedRecordingLimit = true + self.onError( + OpenAITranscriptionProviderError( + message: "Cloud dictation is limited to two minutes. Release the shortcut and try a shorter request." + ) + ) + return + } self.bufferedPCM16AudioData.append(pcmData) } } @@ -125,10 +139,10 @@ nonisolated private final class OpenAITranscriptionSession: SpeechToTextStreamin stateQueue.async { self.isCancelled = true self.bufferedPCM16AudioData.removeAll(keepingCapacity: false) + self.transcriptionUploadTask?.cancel() + self.transcriptionUploadTask = nil + self.urlSession.invalidateAndCancel() } - - transcriptionUploadTask?.cancel() - urlSession.invalidateAndCancel() } private func transcribeBufferedAudio(_ bufferedPCM16AudioData: Data) async { @@ -230,8 +244,13 @@ nonisolated private final class OpenAITranscriptionSession: SpeechToTextStreamin } private func deliverFinalTranscript(_ transcriptText: String) { - guard !hasDeliveredFinalTranscript else { return } - hasDeliveredFinalTranscript = true + let shouldDeliver = stateQueue.sync { () -> Bool in + guard !hasDeliveredFinalTranscript, !isCancelled else { return false } + hasDeliveredFinalTranscript = true + transcriptionUploadTask = nil + return true + } + guard shouldDeliver else { return } onFinalTranscriptReady(transcriptText) } } diff --git a/Orbit/OrbitAudioInput.swift b/Orbit/OrbitAudioInput.swift index c07c31b..ef39302 100644 --- a/Orbit/OrbitAudioInput.swift +++ b/Orbit/OrbitAudioInput.swift @@ -94,19 +94,19 @@ enum OrbitAudioInputCatalog { final class OrbitMicrophoneLevelMonitor { private var engine: AVAudioEngine? - func start(deviceUID: String, onLevel: @escaping @MainActor (CGFloat) -> Void) throws { + func start( + deviceUID: String, + onLevel: @escaping @MainActor @Sendable (CGFloat) -> Void + ) throws { stop() let engine = AVAudioEngine() let input = engine.inputNode try OrbitAudioInputCatalog.applySelectedDevice(uid: deviceUID, to: input) let format = input.outputFormat(forBus: 0) - input.installTap(onBus: 0, bufferSize: 1_024, format: format) { buffer, _ in - guard let samples = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return } - var squares: Float = 0 - for index in 0.. Void + ) -> AVAudioNodeTapBlock { + { buffer, _ in + reportLevel(audioLevel(from: buffer)) + } + } + + nonisolated private static func audioLevel(from buffer: AVAudioPCMBuffer) -> Float { + guard let samples = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return 0 } + var squares: Float = 0 + for index in 0.. Bool { + guard !isModelCatalogPending, !isAccountReadPending else { return false } + + switch authState { + case .authenticated: + return hasReadySession + case .authRequired, .loginInProgress, .authFailed, .runtimeUnavailable: + return true + case .unknown, .checking: + return false + } + } +} diff --git a/Orbit/OrbitCodexModelCatalog.swift b/Orbit/OrbitCodexModelCatalog.swift index 24cecc7..fa41dc2 100644 --- a/Orbit/OrbitCodexModelCatalog.swift +++ b/Orbit/OrbitCodexModelCatalog.swift @@ -23,15 +23,43 @@ enum OrbitCodexModelCatalog { (item["supportedReasoningEfforts"] as? [Any]) ?? (item["supported_reasoning_levels"] as? [Any]) ?? [] - let efforts = effortValues.compactMap(reasoningEffort(from:)) + let efforts = deduplicated(effortValues.compactMap(reasoningEffort(from:))) let defaultEffortValue = firstString(item, keys: ["defaultReasoningEffort", "default_reasoning_level"]) - let defaultEffort = defaultEffortValue.isEmpty ? nil : OrbitCodexReasoningEffort(rawValue: defaultEffortValue) + let parsedDefaultEffort = + defaultEffortValue.isEmpty ? nil : OrbitCodexReasoningEffort(rawValue: defaultEffortValue) + let defaultEffort = parsedDefaultEffort.flatMap { parsedDefault in + efforts.first(where: { $0 == parsedDefault }) ?? parsedDefault + } let tierValues = - (item["supportedServiceTiers"] as? [Any]) + (item["serviceTiers"] as? [Any]) + ?? (item["supportedServiceTiers"] as? [Any]) ?? (item["supported_service_tiers"] as? [Any]) + ?? (item["additionalSpeedTiers"] as? [Any]) ?? [] - let serviceTiers = tierValues.compactMap(serviceTier(from:)) + var serviceTiers = deduplicated(tierValues.compactMap(serviceTier(from:))) + if let deprecatedTierValues = item["additionalSpeedTiers"] as? [Any] { + serviceTiers = deduplicated(serviceTiers + deprecatedTierValues.compactMap(serviceTier(from:))) + } + let defaultServiceTierValue = firstString( + item, + keys: ["defaultServiceTier", "default_service_tier"] + ) + let parsedDefaultServiceTier = + defaultServiceTierValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + ? nil : OrbitCodexServiceTier(rawValue: defaultServiceTierValue) + let defaultServiceTier = parsedDefaultServiceTier.flatMap { parsedDefault in + serviceTiers.first(where: { $0 == parsedDefault }) ?? parsedDefault + } + + let upgradeInfo = item["upgradeInfo"] as? [String: Any] + let availabilityNux = item["availabilityNux"] as? [String: Any] + let upgradeModel = + optionalString(upgradeInfo ?? [:], keys: ["model"]) + ?? optionalString(item, keys: ["upgrade", "upgradeModel", "upgrade_model"]) + let upgradeMessage = + optionalString(upgradeInfo ?? [:], keys: ["upgradeCopy"]) + ?? optionalString(item, keys: ["upgradeMessage", "upgrade_message"]) let displayNameValue = firstString(item, keys: ["displayName", "display_name"]) .trimmingCharacters(in: .whitespacesAndNewlines) @@ -39,19 +67,23 @@ enum OrbitCodexModelCatalog { displayNameValue.isEmpty || displayNameValue.caseInsensitiveCompare(identifier) == .orderedSame ? humanizedIdentifier(identifier) : displayNameValue - let resolvedEfforts = efforts.isEmpty ? (defaultEffort.map { [$0] } ?? OrbitCodexReasoningEffort.allCases) : efforts return ( OrbitCodexModelOption( model: identifier, displayName: displayName, shortDisplayName: shortName(displayName), - supportedEfforts: resolvedEfforts, + supportedEfforts: efforts, defaultEffort: defaultEffort, inputModalities: modalities, isDefault: (item["isDefault"] as? Bool) ?? (item["is_default"] as? Bool) ?? false, supportedServiceTiers: serviceTiers, - upgradeModel: optionalString(item, keys: ["upgradeModel", "upgrade_model"]), - upgradeMessage: optionalString(item, keys: ["upgradeMessage", "upgrade_message"]) + defaultServiceTier: defaultServiceTier, + modelDescription: optionalString(item, keys: ["description"]), + availabilityMessage: optionalString(availabilityNux ?? [:], keys: ["message"]), + upgradeModel: upgradeModel, + upgradeMessage: upgradeMessage, + upgradeModelLink: optionalString(upgradeInfo ?? [:], keys: ["modelLink"]), + upgradeMigrationMarkdown: optionalString(upgradeInfo ?? [:], keys: ["migrationMarkdown"]) ), item["priority"] as? Int ?? Int.max ) @@ -75,19 +107,38 @@ enum OrbitCodexModelCatalog { } else { return nil } - return rawValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : OrbitCodexReasoningEffort(rawValue: rawValue) + guard !rawValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } + let detailText = (value as? [String: Any]).flatMap { optionalString($0, keys: ["description"]) } + return OrbitCodexReasoningEffort(rawValue: rawValue, detailText: detailText) } private static func serviceTier(from value: Any) -> OrbitCodexServiceTier? { let rawValue: String + let advertisedName: String? + let detailText: String? if let string = value as? String { rawValue = string + advertisedName = nil + detailText = nil } else if let dictionary = value as? [String: Any] { rawValue = firstString(dictionary, keys: ["serviceTier", "tier", "value", "id"]) + advertisedName = optionalString(dictionary, keys: ["name"]) + detailText = optionalString(dictionary, keys: ["description"]) } else { return nil } - return rawValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : OrbitCodexServiceTier(rawValue: rawValue) + return rawValue.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + ? nil + : OrbitCodexServiceTier( + rawValue: rawValue, + advertisedName: advertisedName, + detailText: detailText + ) + } + + private static func deduplicated(_ values: [Value]) -> [Value] { + var seen = Set() + return values.filter { seen.insert($0).inserted } } private static func firstString(_ dictionary: [String: Any], keys: [String]) -> String { diff --git a/Orbit/OrbitManager.swift b/Orbit/OrbitManager.swift index e713e40..0c7dfea 100644 --- a/Orbit/OrbitManager.swift +++ b/Orbit/OrbitManager.swift @@ -48,6 +48,8 @@ final class OrbitManager: ObservableObject { @Published private(set) var availableAppleVoices: [OrbitAppleVoiceOption] = [] @Published private(set) var selectedAppleVoiceSummary: String = "Auto" @Published private(set) var appleVoiceQualityNotice: String? + @Published private(set) var isNoraVoiceAvailable = false + @Published private(set) var isCheckingNoraVoice = false @Published private(set) var isPreviewingAppleVoice = false @Published private(set) var availableMicrophones: [OrbitAudioInputDevice] = [] @Published private(set) var microphoneTestLevel: CGFloat = 0 @@ -91,16 +93,19 @@ final class OrbitManager: ObservableObject { private let fallbackTextToSpeechProvider: any TextToSpeechProvider private let actionProvider = CodexAppServerActionProvider() private var lastCodexScreenCapture: OrbitScreenCapture? - private var activeCaptureLease: OrbitTemporaryCaptureLease? + private var captureLeasesByRequestID: [UUID: OrbitTemporaryCaptureLease] = [:] private let microphoneLevelMonitor = OrbitMicrophoneLevelMonitor() private var currentResponseTask: Task? + private var currentResponseRequestID: UUID? private var shortcutTransitionCancellable: AnyCancellable? private var voiceStateCancellable: AnyCancellable? private var audioPowerCancellable: AnyCancellable? private var accessibilityCheckTimer: Timer? private var applicationActivationCancellable: AnyCancellable? + private var screenAccessProbeTask: Task? + private var noraAvailabilityTask: Task? private var pendingKeyboardShortcutStartTask: Task? private var voicePresetCancellable: AnyCancellable? private var appleVoiceCancellable: AnyCancellable? @@ -108,13 +113,16 @@ final class OrbitManager: ObservableObject { private var codexReasoningEffortCancellable: AnyCancellable? private var codexServiceTierCancellable: AnyCancellable? private var codexModelCancellable: AnyCancellable? + private var codexAgentFolderCancellable: AnyCancellable? private var codexOverlayDismissTask: Task? private var transientHideTask: Task? private var onboardingTask: Task? private var onboardingLaunchTask: Task? private var codexSessionWarmupTask: Task? private var actionAcknowledgementTask: Task? + private var completionNarrationTask: Task? private var codexWarmupGeneration: Int = 0 + private var pendingCodexContextRestart = false private var hasSpokenActionAcknowledgement = false private var escapeKeyMonitor: Any? private var lastObservedSetupStage: OrbitSetupStage? @@ -265,7 +273,14 @@ final class OrbitManager: ObservableObject { } func start() { + if settings.voicePreset == .localVoice, + !settings.appleTTSVoiceIdentifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + { + settings.appleTTSVoiceIdentifier = "" + } refreshAllPermissions() + refreshLiveScreenContentAccessIfPermitted() + refreshNoraVoiceAvailability() print( "🪐 Orbit start — accessibility: \(hasAccessibilityPermission), screen: \(hasScreenRecordingPermission), mic: \(hasMicrophonePermission), screenContent: \(hasScreenContentPermission), onboarded: \(hasCompletedOnboarding)" ) @@ -310,22 +325,27 @@ final class OrbitManager: ObservableObject { onboardingTask?.cancel() codexSessionWarmupTask?.cancel() actionAcknowledgementTask?.cancel() + completionNarrationTask?.cancel() currentResponseTask?.cancel() currentResponseTask = nil + currentResponseRequestID = nil actionProvider.cancelCurrentAction() permissionCoordinator.dismissGuide() microphoneLevelMonitor.stop() isTestingMicrophone = false - releaseTemporaryCapture() + releaseAllTemporaryCaptures() shortcutTransitionCancellable?.cancel() voiceStateCancellable?.cancel() audioPowerCancellable?.cancel() applicationActivationCancellable?.cancel() + screenAccessProbeTask?.cancel() + noraAvailabilityTask?.cancel() voicePresetCancellable?.cancel() appleVoiceCancellable?.cancel() showCursorCancellable?.cancel() codexModelCancellable?.cancel() + codexAgentFolderCancellable?.cancel() accessibilityCheckTimer?.invalidate() accessibilityCheckTimer = nil if let escapeKeyMonitor { @@ -374,15 +394,9 @@ final class OrbitManager: ObservableObject { if !previouslyHadMicrophone && hasMicrophonePermission { OrbitAnalytics.trackPermissionGranted(permission: "microphone") } - // Screen content permission is persisted after a successful real capture. - // Treat that as usable screen access for setup even if CGPreflight lags - // behind on a fresh build. - if !hasScreenContentPermission { - hasScreenContentPermission = UserDefaults.standard.bool(forKey: "hasScreenContentPermission") - } - - if hasScreenContentPermission && !hasScreenRecordingPermission { - hasScreenRecordingPermission = true + if !hasScreenRecordingPermission { + hasScreenContentPermission = false + UserDefaults.standard.removeObject(forKey: "hasScreenContentPermission") } updateScreenAccessDiagnostic() @@ -441,17 +455,6 @@ final class OrbitManager: ObservableObject { // MARK: - Private - /// Triggers the system microphone prompt if the user has never been asked. - /// Once granted/denied the status sticks and polling picks it up. - private func promptForMicrophoneIfNotDetermined() { - guard AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined else { return } - AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in - Task { @MainActor [weak self] in - self?.hasMicrophonePermission = granted - } - } - } - /// Polls all permissions frequently so the UI updates live after the /// user grants them in System Settings. Screen Recording is the exception — /// macOS requires an app restart for that one to take effect. @@ -484,6 +487,8 @@ final class OrbitManager: ObservableObject { .sink { [weak self] _ in guard let self else { return } self.refreshAllPermissions() + self.refreshLiveScreenContentAccessIfPermitted() + self.refreshNoraVoiceAvailability(forceRefresh: true) if self.setupStage == .permissions && !self.hasCompletedOnboarding { self.startPermissionPolling() } else { @@ -493,6 +498,28 @@ final class OrbitManager: ObservableObject { } } + private func refreshLiveScreenContentAccessIfPermitted() { + screenAccessProbeTask?.cancel() + guard hasScreenRecordingPermission else { + hasScreenContentPermission = false + return + } + + screenAccessProbeTask = Task { @MainActor [weak self] in + let canAccess = await OrbitScreenCaptureUtility.canAccessScreenContent() + guard let self, !Task.isCancelled else { return } + self.hasScreenContentPermission = canAccess + if canAccess { + WindowPositionManager.recordConfirmedScreenRecordingPermission() + UserDefaults.standard.set(true, forKey: "hasScreenContentPermission") + } else { + WindowPositionManager.clearPreviouslyConfirmedScreenRecordingPermission() + UserDefaults.standard.removeObject(forKey: "hasScreenContentPermission") + } + self.synchronizeSetupState() + } + } + private func bindInterruptShortcut() { if let escapeKeyMonitor { NSEvent.removeMonitor(escapeKeyMonitor) @@ -590,23 +617,34 @@ final class OrbitManager: ObservableObject { codexServiceTierCancellable = settings.$codexServiceTier .receive(on: DispatchQueue.main) .sink { [weak self] _ in - guard let self else { return } - self.refreshActionProviderPresentation() - self.ensureCodexSessionReady(forceFreshSession: true) + self?.refreshActionProviderPresentation() } codexModelCancellable = settings.$codexActionModel + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in + self?.refreshActionProviderPresentation() + } + + codexAgentFolderCancellable = settings.$codexAgentFolder + .dropFirst() .receive(on: DispatchQueue.main) .sink { [weak self] _ in guard let self else { return } - self.refreshActionProviderPresentation() - self.ensureCodexSessionReady(forceFreshSession: true) + if self.actionProvider.canInterruptCurrentAction { + self.pendingCodexContextRestart = true + self.appendActionUpdate("working folder will apply after this turn") + } else { + self.ensureCodexSessionReady(forceFreshSession: true) + } } } private func refreshTextToSpeechProvider() { textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() + completionNarrationTask?.cancel() + completionNarrationTask = nil textToSpeechProvider = OrbitTTSProviderFactory.makePrimaryProvider(for: settings.voicePreset) textToSpeechProviderDisplayName = textToSpeechProvider.displayName availableAppleVoices = OrbitAppleVoiceCatalog.availableVoices() @@ -614,9 +652,31 @@ final class OrbitManager: ObservableObject { preferredIdentifier: settings.appleTTSVoiceIdentifier ) appleVoiceQualityNotice = OrbitAppleVoiceCatalog.qualityNotice() + refreshNoraVoiceAvailability() orbitDictationManager.refreshConfiguredProviders() } + func refreshNoraVoiceAvailability(forceRefresh: Bool = false) { + noraAvailabilityTask?.cancel() + guard settings.voicePreset == .localVoice else { + isCheckingNoraVoice = false + return + } + if !forceRefresh, let cachedValue = OrbitNoraVoiceAvailability.cachedValue { + isNoraVoiceAvailable = cachedValue + isCheckingNoraVoice = false + return + } + + isCheckingNoraVoice = true + noraAvailabilityTask = Task { @MainActor [weak self] in + let isAvailable = await OrbitNoraVoiceAvailability.probe(forceRefresh: forceRefresh) + guard let self, !Task.isCancelled else { return } + self.isNoraVoiceAvailable = isAvailable + self.isCheckingNoraVoice = false + } + } + func selectAppleVoice(_ identifier: String) { settings.appleTTSVoiceIdentifier = identifier } @@ -632,7 +692,10 @@ final class OrbitManager: ObservableObject { Task { [weak self] in guard let self else { return } defer { isPreviewingAppleVoice = false } - try? await textToSpeechProvider.speakText("Orbit is ready. This voice stays on your Mac.") + try? await speakNarration( + "Orbit is ready. This voice stays on your Mac.", + source: .preview + ) } } @@ -760,10 +823,15 @@ final class OrbitManager: ObservableObject { } private func submitTranscriptToActionProvider(transcript: String) { + let previousRequestID = currentResponseRequestID currentResponseTask?.cancel() - releaseTemporaryCapture() + if let previousRequestID { + releaseTemporaryCapture(for: previousRequestID) + } textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() + completionNarrationTask?.cancel() + completionNarrationTask = nil isRunningOnboardingTour = false pendingToolPrompt = nil let shouldResetPresentation = !actionProvider.canInterruptCurrentAction @@ -783,16 +851,32 @@ final class OrbitManager: ObservableObject { showCodexActivityOverlayCard() scheduleActionAcknowledgementFallback() - currentResponseTask = Task { - defer { self.currentResponseTask = nil } + let requestID = UUID() + currentResponseRequestID = requestID + currentResponseTask = Task { [weak self] in + guard let self else { return } + defer { + if currentResponseRequestID == requestID { + currentResponseTask = nil + } + } let request: OrbitActionRequest do { - request = try await prepareUnifiedCodexRequest(transcript: transcript) + request = try await prepareUnifiedCodexRequest( + transcript: transcript, + requestID: requestID + ) + try Task.checkCancellation() + guard currentResponseRequestID == requestID else { + releaseTemporaryCapture(for: requestID) + return + } } catch is CancellationError { - releaseTemporaryCapture() + releaseTemporaryCapture(for: requestID) return } catch { - releaseTemporaryCapture() + releaseTemporaryCapture(for: requestID) + guard currentResponseRequestID == requestID else { return } hasScreenContentPermission = false UserDefaults.standard.removeObject(forKey: "hasScreenContentPermission") WindowPositionManager.clearPreviouslyConfirmedScreenRecordingPermission() @@ -805,6 +889,7 @@ final class OrbitManager: ObservableObject { voiceState = .idle showCodexActivityOverlayCard() scheduleCodexActivityOverlayDismiss() + currentResponseRequestID = nil return } @@ -812,13 +897,19 @@ final class OrbitManager: ObservableObject { request ) { [weak self] event in Task { @MainActor [weak self] in - self?.handleActionEvent(event) + self?.handleActionEvent(event, requestID: requestID) } } } } - private func handleActionEvent(_ event: OrbitActionEvent) { + private func handleActionEvent(_ event: OrbitActionEvent, requestID: UUID) { + guard currentResponseRequestID == requestID else { + if event.isTerminal { + releaseTemporaryCapture(for: requestID) + } + return + } switch event { case .phase(let progress): pendingToolPrompt = nil @@ -842,7 +933,7 @@ final class OrbitManager: ObservableObject { activeActionDetailLine = activities.last.map { "Team-up: \($0.agentPath) · \($0.status)" } showCodexActivityOverlayCard() case .interrupted(let summary): - releaseTemporaryCapture() + releaseTemporaryCapture(for: requestID) let spokenSummary = conciseDetailLine(from: summary.isEmpty ? "stopped." : summary) cancelActionAcknowledgementFlow() textToSpeechProvider.stopPlayback() @@ -862,12 +953,12 @@ final class OrbitManager: ObservableObject { scheduleCodexActivityOverlayDismiss() scheduleTransientHideIfNeeded() case .completed(let summary): - releaseTemporaryCapture() + releaseTemporaryCapture(for: requestID) cancelActionAcknowledgementFlow() pendingToolPrompt = nil - handleCompletedCodexSummary(summary) + handleCompletedCodexSummary(summary, requestID: requestID) case .failed(let errorMessage): - releaseTemporaryCapture() + releaseTemporaryCapture(for: requestID) let shortDetail = conciseDetailLine(from: errorMessage) cancelActionAcknowledgementFlow() textToSpeechProvider.stopPlayback() @@ -884,15 +975,30 @@ final class OrbitManager: ObservableObject { appendActionUpdate(OrbitActionPhase.failed.summaryText) showCodexActivityOverlayCard() scheduleCodexActivityOverlayDismiss() - Task { - await self.speakCompletionText(nil, fallback: "i could not finish that action.") + completionNarrationTask?.cancel() + completionNarrationTask = Task { [weak self] in + await self?.speakCompletionText( + nil, + fallback: "i could not finish that action.", + source: .failure, + turnIdentifier: requestID.uuidString + ) + } + } + + if event.isTerminal, currentResponseRequestID == requestID { + currentResponseRequestID = nil + currentResponseTask = nil + if pendingCodexContextRestart { + pendingCodexContextRestart = false + ensureCodexSessionReady(forceFreshSession: true) } } refreshActionProviderPresentation() } - private func handleCompletedCodexSummary(_ summary: String) { + private func handleCompletedCodexSummary(_ summary: String, requestID: UUID) { let responseParse = Self.parseOrbitResponse(from: summary) let spokenSummary = responseParse.spokenText.isEmpty ? "done." : responseParse.spokenText @@ -911,8 +1017,14 @@ final class OrbitManager: ObservableObject { } showCodexActivityOverlayCard() scheduleCodexActivityOverlayDismiss() - Task { - await self.speakCompletionText(spokenSummary, fallback: nil) + completionNarrationTask?.cancel() + completionNarrationTask = Task { [weak self] in + await self?.speakCompletionText( + spokenSummary, + fallback: nil, + source: .completion, + turnIdentifier: requestID.uuidString + ) } } @@ -932,16 +1044,34 @@ final class OrbitManager: ObservableObject { return (data: capture.imageData, label: label) } - private func prepareUnifiedCodexRequest(transcript: String) async throws -> OrbitActionRequest { + private func prepareUnifiedCodexRequest( + transcript: String, + requestID: UUID + ) async throws -> OrbitActionRequest { do { let activeScreenCapture = try await OrbitScreenCaptureUtility.captureCurrentScreenAsJPEG() + try Task.checkCancellation() - lastCodexScreenCapture = activeScreenCapture let labeledCapture = buildPrimaryScreenLabel(for: activeScreenCapture) let cursorPoint = currentCursorPointInScreenshotPixels(for: activeScreenCapture) - let lease = try await OrbitTemporaryCaptureLease.create(data: labeledCapture.data) - activeCaptureLease = lease + let lease = try await OrbitTemporaryCaptureLease.create( + data: labeledCapture.data, + turnID: requestID + ) + do { + try Task.checkCancellation() + guard currentResponseRequestID == requestID else { + lease.release() + throw CancellationError() + } + } catch { + lease.release() + throw error + } + lastCodexScreenCapture = activeScreenCapture + captureLeasesByRequestID[requestID] = lease return OrbitActionRequest( + id: requestID, transcript: transcript, screenshotPath: lease.fileURL.path, screenshotLabel: labeledCapture.label, @@ -960,15 +1090,24 @@ final class OrbitManager: ObservableObject { } } - private func releaseTemporaryCapture() { - let lease = activeCaptureLease - activeCaptureLease = nil + private func releaseTemporaryCapture(for requestID: UUID) { + let lease = captureLeasesByRequestID.removeValue(forKey: requestID) guard let lease else { return } Task.detached(priority: .utility) { lease.release() } } + private func releaseAllTemporaryCaptures() { + let leases = Array(captureLeasesByRequestID.values) + captureLeasesByRequestID.removeAll(keepingCapacity: false) + for lease in leases { + Task.detached(priority: .utility) { + lease.release() + } + } + } + private func applyCodexPointDirective(_ parseResult: PointingParseResult) { guard let coordinate = parseResult.coordinate, let resolvedTarget = resolveGlobalScreenTarget( @@ -1136,32 +1275,65 @@ final class OrbitManager: ObservableObject { actionAcknowledgementTask = Task { @MainActor [weak self] in guard let self else { return } do { - try await textToSpeechProvider.speakText(acknowledgement) + try await speakNarration( + acknowledgement, + source: .earlyCommentary, + turnIdentifier: currentResponseRequestID?.uuidString + ) } catch { OrbitSupportLog.append("voice", "failed early commentary speech: \(error.localizedDescription)") } } } - private func speakCompletionText(_ primary: String?, fallback: String?) async { + private func speakCompletionText( + _ primary: String?, + fallback: String?, + source: OrbitNarrationSource, + turnIdentifier: String? + ) async { let trimmedPrimary = primary?.trimmingCharacters(in: .whitespacesAndNewlines) let trimmedFallback = fallback?.trimmingCharacters(in: .whitespacesAndNewlines) let finalText = (trimmedPrimary?.isEmpty == false ? trimmedPrimary : trimmedFallback) ?? "done." await waitForCurrentSpeechToSettle(maximumWait: 2.4) + guard !Task.isCancelled else { return } voiceState = .responding do { - try await textToSpeechProvider.speakText(finalText) + try await speakNarration( + finalText, + source: source, + turnIdentifier: turnIdentifier + ) } catch { let visibleError = trimmedFallback?.isEmpty == false ? trimmedFallback! : error.localizedDescription OrbitSupportLog.append("voice", "speech failed: \(visibleError)") } + guard !Task.isCancelled else { return } voiceState = .idle scheduleTransientHideIfNeeded() } + private func speakNarration( + _ text: String, + source: OrbitNarrationSource, + turnIdentifier: String? = nil + ) async throws { + if let coordinator = textToSpeechProvider as? OrbitVoiceCoordinator { + try await coordinator.speak( + OrbitNarrationRequest( + text: text, + source: source, + turnIdentifier: turnIdentifier + ) + ) + } else { + try await textToSpeechProvider.speakText(text) + } + } + private func waitForCurrentSpeechToSettle(maximumWait: TimeInterval) async { let deadline = Date().addingTimeInterval(maximumWait) while textToSpeechProvider.isPlaying || fallbackTextToSpeechProvider.isPlaying, @@ -1363,13 +1535,17 @@ final class OrbitManager: ObservableObject { } func interruptCurrentAction() { + let interruptedRequestID = currentResponseRequestID currentResponseTask?.cancel() currentResponseTask = nil + currentResponseRequestID = nil orbitDictationManager.cancelCurrentDictation() textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() actionProvider.cancelCurrentAction() - releaseTemporaryCapture() + if let interruptedRequestID { + releaseTemporaryCapture(for: interruptedRequestID) + } voiceState = .idle activeActionProgress = OrbitActionProgress( phase: .interrupted, @@ -1689,7 +1865,7 @@ final class OrbitManager: ObservableObject { private func speakOnboardingMessage(_ message: String) async { do { - try await textToSpeechProvider.speakText(message) + try await speakNarration(message, source: .onboarding) } catch { OrbitSupportLog.append("voice", "failed onboarding speech: \(error.localizedDescription)") } diff --git a/Orbit/OrbitPanelView.swift b/Orbit/OrbitPanelView.swift index dabde16..72347cd 100644 --- a/Orbit/OrbitPanelView.swift +++ b/Orbit/OrbitPanelView.swift @@ -57,6 +57,10 @@ struct OrbitPanelView: View { .frame(width: 312) .frame(maxHeight: 720) .background(panelBackground) + .onExitCommand { + NotificationCenter.default.post(name: .orbitDismissPanel, object: nil) + } + .onAppear { resetLegacyAppleVoiceSelection() } } private var header: some View { @@ -87,7 +91,10 @@ struct OrbitPanelView: View { Spacer(minLength: 8) - DSQuietStatusChip(title: statusText, tint: statusDotColor) + if shouldShowHeaderStatus { + DSQuietStatusChip(title: statusText, tint: statusDotColor) + .accessibilityLabel("Orbit status: \(statusText)") + } Button { NotificationCenter.default.post(name: .orbitDismissPanel, object: nil) @@ -107,6 +114,8 @@ struct OrbitPanelView: View { } .buttonStyle(.plain) .pointerCursor() + .accessibilityLabel("Close Orbit panel") + .accessibilityHint("Returns focus to the current application") } .padding(.horizontal, 2) .padding(.top, 1) @@ -177,7 +186,7 @@ struct OrbitPanelView: View { voicePresetRow rowDivider - codexModelRow + codexConfigurationGroup if let notice = orbitManager.codexModelMigrationNotice { Text(notice) .font(.system(size: 10.5, weight: .medium)) @@ -188,12 +197,6 @@ struct OrbitPanelView: View { rowDivider agentFolderRow rowDivider - if availableServiceTiers.count > 1 { - codexServiceTierRow - rowDivider - } - codexReasoningEffortRow - rowDivider providerRow(icon: "waveform.badge.mic", title: "Speech to Text", value: panelSpeechToTextLabel) rowDivider microphoneInputRow @@ -611,7 +614,7 @@ struct OrbitPanelView: View { .font(.system(size: 13, weight: .semibold)) .foregroundColor(DS.Colors.textPrimary) - Text("Live local session") + Text("Persistent Codex session") .font(.system(size: 10.5, weight: .medium)) .foregroundColor(DS.Colors.textTertiary) .lineLimit(1) @@ -640,6 +643,7 @@ struct OrbitPanelView: View { .buttonStyle(.plain) .pointerCursor() .help("Interrupt Codex") + .accessibilityLabel("Interrupt current Codex task") } Button { @@ -661,9 +665,13 @@ struct OrbitPanelView: View { .buttonStyle(.plain) .pointerCursor() .help("Reconnect Codex") + .accessibilityLabel("Reconnect Codex session") } - DSQuietStatusChip(title: actionStatusLabel, tint: codexStatusColor) + if shouldShowCodexStatusChip { + DSQuietStatusChip(title: actionStatusLabel, tint: codexStatusColor) + .accessibilityLabel("Codex status: \(actionStatusLabel)") + } } Text(codexSummaryLine) @@ -728,6 +736,37 @@ struct OrbitPanelView: View { .padding(.vertical, 2) } + private var codexConfigurationGroup: some View { + VStack(alignment: .leading, spacing: 9) { + HStack { + Label("Codex configuration", systemImage: "cpu") + .font(.system(size: 11, weight: .semibold)) + .foregroundStyle(DS.Colors.textSecondary) + Spacer(minLength: 8) + Text("Account catalog") + .font(.system(size: 9.5, weight: .medium)) + .foregroundStyle(DS.Colors.textTertiary) + } + + codexModelRow + rowDivider.padding(.leading, -26) + + if availableServiceTiers.count > 1 { + codexServiceTierRow + rowDivider.padding(.leading, -26) + } + + codexReasoningEffortRow + } + .padding(10) + .background( + RoundedRectangle(cornerRadius: 10, style: .continuous) + .fill(DS.Colors.surface2) + ) + .accessibilityElement(children: .contain) + .accessibilityLabel("Codex configuration") + } + private var codexModelRow: some View { HStack(alignment: .center, spacing: 12) { rowLabel( @@ -740,24 +779,53 @@ struct OrbitPanelView: View { modelSelectorMenu } + .accessibilityElement(children: .contain) + .accessibilityLabel("Codex model") + .accessibilityValue(selectedModelShortLabel) } private var codexReasoningEffortRow: some View { HStack(alignment: .center, spacing: 12) { - rowLabel(icon: "dial.medium", title: "Codex Effort", subtitle: nil) + rowLabel(icon: "dial.medium", title: "Effort", subtitle: "Options supported by the selected model") Spacer(minLength: 8) + codexEffortSelector + } + } + + @ViewBuilder + private var codexEffortSelector: some View { + if orbitManager.availableCodexEfforts.count <= 4 { segmentedControl(spacing: 2) { ForEach(orbitManager.availableCodexEfforts) { effort in - effortOptionButton( - effort: effort, + settingOptionButton( + label: effort.displayName, isSelected: orbitSettings.codexReasoningEffort == effort ) { orbitSettings.codexReasoningEffort = effort } } } + } else { + Menu { + ForEach(orbitManager.availableCodexEfforts) { effort in + Button { + orbitSettings.codexReasoningEffort = effort + } label: { + if orbitSettings.codexReasoningEffort == effort { + Label(effort.displayName, systemImage: "checkmark") + } else { + Text(effort.displayName) + } + } + } + } label: { + selectorLabel(orbitSettings.codexReasoningEffort.displayName) + } + .menuStyle(.borderlessButton) + .accessibilityLabel("Codex effort") + .accessibilityValue(orbitSettings.codexReasoningEffort.displayName) } } @@ -767,19 +835,46 @@ struct OrbitPanelView: View { Spacer(minLength: 8) - segmentedControl(spacing: 2) { - ForEach(availableServiceTiers) { tier in - settingOptionButton( - label: tier.displayName, - isSelected: orbitSettings.codexServiceTier == tier - ) { - orbitSettings.codexServiceTier = tier + if availableServiceTiers.count <= 3 { + segmentedControl(spacing: 2) { + ForEach(availableServiceTiers) { tier in + settingOptionButton( + label: tier.displayName, + isSelected: orbitSettings.codexServiceTier == tier + ) { + orbitSettings.codexServiceTier = tier + } + } + } + } else { + Menu { + ForEach(availableServiceTiers) { tier in + Button(tier.displayName) { orbitSettings.codexServiceTier = tier } } + } label: { + selectorLabel(orbitSettings.codexServiceTier.displayName) } + .menuStyle(.borderlessButton) + .accessibilityLabel("Codex service tier") + .accessibilityValue(orbitSettings.codexServiceTier.displayName) } } } + private func selectorLabel(_ title: String) -> some View { + HStack(spacing: 5) { + Text(title) + .lineLimit(1) + Image(systemName: "chevron.up.chevron.down") + .font(.system(size: 8.5, weight: .semibold)) + } + .font(.system(size: 10.5, weight: .semibold)) + .foregroundStyle(DS.Colors.textSecondary) + .padding(.horizontal, 9) + .padding(.vertical, 6) + .background(Capsule().fill(DS.Colors.surface3)) + } + private var availableServiceTiers: [OrbitCodexServiceTier] { let serverTiers = orbitManager.availableCodexModels @@ -878,6 +973,8 @@ struct OrbitPanelView: View { } .menuStyle(.borderlessButton) .pointerCursor() + .accessibilityLabel("Codex model") + .accessibilityValue(selectedModelShortLabel) } private var speechOutputRow: some View { @@ -886,32 +983,30 @@ struct OrbitPanelView: View { rowLabel( icon: "speaker.wave.2.fill", title: "Voice", - subtitle: orbitSettings.voicePreset == .localVoice ? "On-device Apple speech" : "OpenAI speech" + subtitle: orbitSettings.voicePreset == .localVoice ? "Nora Premium · on device" : "OpenAI speech" ) Spacer(minLength: 8) if orbitSettings.voicePreset == .localVoice { VStack(alignment: .trailing, spacing: 5) { - Menu { - Button("Automatic (recommended)") { orbitManager.selectAppleVoice("") } - Divider() - ForEach(orbitManager.availableAppleVoices) { voice in - Button(voice.displayName) { orbitManager.selectAppleVoice(voice.identifier) } - } - } label: { - Text(orbitManager.selectedAppleVoiceSummary) - .font(.system(size: 11, weight: .medium)) - .lineLimit(1) - .frame(maxWidth: 152, alignment: .trailing) - } - .menuStyle(.borderlessButton) + Text(noraVoiceStatusLabel) + .font(.system(size: 11, weight: .medium)) + .foregroundColor( + orbitManager.isNoraVoiceAvailable + ? DS.Colors.textSecondary + : DS.Colors.warningText + ) + .lineLimit(1) + .accessibilityLabel("Local narrator") + .accessibilityValue(noraVoiceStatusLabel) Button(orbitManager.isPreviewingAppleVoice ? "Stop" : "Preview") { orbitManager.toggleAppleVoicePreview() } .buttonStyle(.borderless) .font(.system(size: 10, weight: .semibold)) + .accessibilityLabel(orbitManager.isPreviewingAppleVoice ? "Stop voice preview" : "Preview selected voice") } } else { Text(panelTextToSpeechLabel) @@ -930,25 +1025,38 @@ struct OrbitPanelView: View { } } - if orbitSettings.voicePreset == .localVoice, - let notice = orbitManager.appleVoiceQualityNotice + if orbitSettings.voicePreset == .localVoice, !orbitManager.isNoraVoiceAvailable, + !orbitManager.isCheckingNoraVoice { - Text(notice) + Text("Nora Premium is unavailable to Orbit. Local narration will use the best compatible Apple fallback voice.") .font(.system(size: 9.5, weight: .medium)) .foregroundColor(DS.Colors.warningText.opacity(0.88)) .fixedSize(horizontal: false, vertical: true) - .accessibilityLabel(notice) + .accessibilityLabel( + "Nora Premium is unavailable. Orbit will use the best compatible Apple fallback voice." + ) } } } + private var noraVoiceStatusLabel: String { + if orbitManager.isCheckingNoraVoice { return "Checking Nora…" } + return orbitManager.isNoraVoiceAvailable ? "Nora Premium" : "Fallback voice" + } + + private func resetLegacyAppleVoiceSelection() { + let selected = orbitSettings.appleTTSVoiceIdentifier.trimmingCharacters(in: .whitespacesAndNewlines) + guard !selected.isEmpty else { return } + orbitManager.selectAppleVoice("") + } + private var microphoneInputRow: some View { VStack(alignment: .leading, spacing: 6) { HStack(spacing: 10) { rowLabel( icon: "mic", title: "Microphone", - subtitle: orbitManager.microphoneDeviceNotice ?? "Local input · no audio leaves the Mac" + subtitle: orbitManager.microphoneDeviceNotice ?? microphonePrivacyLabel ) Spacer(minLength: 6) Menu { @@ -968,16 +1076,28 @@ struct OrbitPanelView: View { HStack(spacing: 8) { ProgressView(value: orbitManager.microphoneTestLevel) .progressViewStyle(.linear) + .accessibilityLabel("Microphone input level") + .accessibilityValue("\(Int(orbitManager.microphoneTestLevel * 100)) percent") Button(orbitManager.isTestingMicrophone ? "Stop" : "Test") { orbitManager.toggleMicrophoneTest() } .buttonStyle(.borderless) .font(.system(size: 10, weight: .semibold)) + .accessibilityLabel(orbitManager.isTestingMicrophone ? "Stop microphone test" : "Test microphone") } } .onAppear { orbitManager.refreshMicrophones() } } + private var microphonePrivacyLabel: String { + switch orbitSettings.voicePreset { + case .localVoice: + return "On-device recognition · audio stays on this Mac" + case .cloudVoice: + return "Cloud recognition · speech audio is sent to OpenAI" + } + } + private var selectedMicrophoneLabel: String { let selectedUID = orbitSettings.microphoneDeviceUID.trimmingCharacters(in: .whitespacesAndNewlines) guard !selectedUID.isEmpty else { return "System" } @@ -1071,7 +1191,7 @@ struct OrbitPanelView: View { Spacer(minLength: 8) Toggle( - "", + "Show Orbit cursor", isOn: Binding( get: { orbitManager.isOrbitCursorEnabled }, set: { orbitManager.setOrbitCursorEnabled($0) } @@ -1081,6 +1201,8 @@ struct OrbitPanelView: View { .labelsHidden() .tint(Color.white.opacity(0.8)) .scaleEffect(0.8) + .accessibilityLabel("Show Orbit cursor") + .accessibilityValue(orbitManager.isOrbitCursorEnabled ? "On" : "Off") } } @@ -1283,25 +1405,6 @@ struct OrbitPanelView: View { ?? orbitSettings.codexActionModel } - private func effortOptionButton( - effort: OrbitCodexReasoningEffort, - isSelected: Bool, - action: @escaping () -> Void - ) -> some View { - Button(action: action) { - EffortGlyph(level: effort.level, isSelected: isSelected) - .padding(.horizontal, 6) - .padding(.vertical, 7) - .background( - Capsule(style: .continuous) - .fill(isSelected ? Color.white.opacity(0.10) : Color.clear) - ) - } - .buttonStyle(.plain) - .pointerCursor() - .accessibilityLabel(Text(effort.displayName)) - } - private func settingOptionButton( label: String, isSelected: Bool, @@ -1323,6 +1426,9 @@ struct OrbitPanelView: View { } .buttonStyle(.plain) .pointerCursor() + .accessibilityLabel(label) + .accessibilityValue(isSelected ? "Selected" : "Not selected") + .accessibilityAddTraits(isSelected ? .isSelected : []) } private func primaryButton(_ title: String, action: @escaping () -> Void) -> some View { @@ -1437,6 +1543,19 @@ struct OrbitPanelView: View { } } + private var shouldShowHeaderStatus: Bool { + orbitManager.setupStage != .ready || panelSection != .activity + } + + private var shouldShowCodexStatusChip: Bool { + switch orbitManager.activeActionStatus { + case .idle: + return false + case .running, .waitingForApproval, .completed, .interrupted, .failed: + return true + } + } + private var statusDotColor: Color { if orbitManager.isRunningOnboardingTour { return Color.white.opacity(0.88) @@ -1814,25 +1933,3 @@ struct OrbitPanelView: View { } } } - -private struct EffortGlyph: View { - let level: Int - let isSelected: Bool - - var body: some View { - HStack(alignment: .bottom, spacing: 2.5) { - ForEach(0..<5, id: \.self) { index in - RoundedRectangle(cornerRadius: 1.5, style: .continuous) - .fill(barColor(for: index)) - .frame(width: 3, height: CGFloat(5 + (index * 3))) - } - } - } - - private func barColor(for index: Int) -> Color { - if index < level { - return isSelected ? DS.Colors.textPrimary : DS.Colors.textSecondary - } - return Color.white.opacity(isSelected ? 0.18 : 0.10) - } -} diff --git a/Orbit/OrbitScreenCaptureUtility.swift b/Orbit/OrbitScreenCaptureUtility.swift index 154b2b2..69e0dbe 100644 --- a/Orbit/OrbitScreenCaptureUtility.swift +++ b/Orbit/OrbitScreenCaptureUtility.swift @@ -25,20 +25,71 @@ struct OrbitScreenCapture { @MainActor enum OrbitScreenCaptureUtility { + /// Checks whether ScreenCaptureKit can enumerate a display without taking + /// an image. Permission refreshes can call this without creating an extra + /// capture outside the once-per-request contract. + static func canAccessScreenContent() async -> Bool { + do { + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + return !content.displays.isEmpty + } catch { + return false + } + } + static func captureCurrentScreenAsJPEG() async throws -> OrbitScreenCapture { - let captures = try await captureAllScreensAsJPEG() - if let cursorCapture = captures.first(where: { $0.isCursorScreen }) { - return cursorCapture + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + guard !content.displays.isEmpty else { + throw NSError( + domain: "OrbitScreenCapture", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "No display available for capture."] + ) } - guard let firstCapture = captures.first else { + let mouseLocation = NSEvent.mouseLocation + let screensByDisplayID = nsScreensByDisplayID() + let cursorDisplayID = selectedDisplayID( + cursorLocation: mouseLocation, + screenFramesByDisplayID: screensByDisplayID.mapValues(\.frame), + availableDisplayIDs: content.displays.map(\.displayID) + ) + guard + let selectedDisplay = content.displays.first(where: { + $0.displayID == cursorDisplayID + }) ?? content.displays.first + else { throw NSError( domain: "OrbitScreenCapture", code: -3, + userInfo: [NSLocalizedDescriptionKey: "Failed to resolve the current screen."] + ) + } + + let ownWindows = windowsOwnedByOrbit(in: content) + guard + let capture = try await capture( + display: selectedDisplay, + screenNumber: 1, + displayCount: 1, + mouseLocation: mouseLocation, + nsScreen: screensByDisplayID[selectedDisplay.displayID], + excludingWindows: ownWindows + ) + else { + throw NSError( + domain: "OrbitScreenCapture", + code: -2, userInfo: [NSLocalizedDescriptionKey: "Failed to capture the current screen."] ) } - return firstCapture + return capture } /// Captures all connected displays as JPEG data, labeling each with @@ -57,10 +108,7 @@ enum OrbitScreenCaptureUtility { // Exclude all windows belonging to this app so the AI sees // only the user's content, not our overlays or panels. - let ownBundleIdentifier = Bundle.main.bundleIdentifier - let ownAppWindows = content.windows.filter { window in - window.owningApplication?.bundleIdentifier == ownBundleIdentifier - } + let ownAppWindows = windowsOwnedByOrbit(in: content) // Build a lookup from display ID to NSScreen so we can use AppKit-coordinate // frames instead of CG-coordinate frames. NSEvent.mouseLocation and NSScreen.frame @@ -68,12 +116,7 @@ enum OrbitScreenCaptureUtility { // Core Graphics coordinates (top-left origin). On multi-display setups, the Y // origins differ for secondary displays, which breaks cursor-contains checks // and downstream coordinate conversions. - var nsScreenByDisplayID: [CGDirectDisplayID: NSScreen] = [:] - for screen in NSScreen.screens { - if let screenNumber = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID { - nsScreenByDisplayID[screenNumber] = screen - } - } + let nsScreenByDisplayID = nsScreensByDisplayID() // Sort displays so the cursor screen is always first let sortedDisplays = content.displays.sorted { displayA, displayB in @@ -88,62 +131,16 @@ enum OrbitScreenCaptureUtility { var capturedScreens: [OrbitScreenCapture] = [] for (displayIndex, display) in sortedDisplays.enumerated() { - // Use NSScreen.frame (AppKit coordinates, bottom-left origin) so - // displayFrame is in the same coordinate system as NSEvent.mouseLocation - // and the overlay window's screenFrame in OrbitCursorOverlayView. - let displayFrame = - nsScreenByDisplayID[display.displayID]?.frame - ?? CGRect( - x: display.frame.origin.x, y: display.frame.origin.y, - width: CGFloat(display.width), height: CGFloat(display.height)) - let isCursorScreen = displayFrame.contains(mouseLocation) - - let filter = SCContentFilter(display: display, excludingWindows: ownAppWindows) - - let configuration = SCStreamConfiguration() - let maxDimension = 1280 - let aspectRatio = CGFloat(display.width) / CGFloat(display.height) - if display.width >= display.height { - configuration.width = maxDimension - configuration.height = Int(CGFloat(maxDimension) / aspectRatio) - } else { - configuration.height = maxDimension - configuration.width = Int(CGFloat(maxDimension) * aspectRatio) - } - - let cgImage = try await SCScreenshotManager.captureImage( - contentFilter: filter, - configuration: configuration - ) - - guard - let jpegData = NSBitmapImageRep(cgImage: cgImage) - .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) - else { - continue - } - - let screenLabel: String - if sortedDisplays.count == 1 { - screenLabel = "user's screen (cursor is here)" - } else if isCursorScreen { - screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — cursor is on this screen (primary focus)" - } else { - screenLabel = "screen \(displayIndex + 1) of \(sortedDisplays.count) — secondary screen" + if let capture = try await capture( + display: display, + screenNumber: displayIndex + 1, + displayCount: sortedDisplays.count, + mouseLocation: mouseLocation, + nsScreen: nsScreenByDisplayID[display.displayID], + excludingWindows: ownAppWindows + ) { + capturedScreens.append(capture) } - - capturedScreens.append( - OrbitScreenCapture( - imageData: jpegData, - label: screenLabel, - isCursorScreen: isCursorScreen, - screenNumber: displayIndex + 1, - displayWidthInPoints: Int(displayFrame.width), - displayHeightInPoints: Int(displayFrame.height), - displayFrame: displayFrame, - screenshotWidthInPixels: configuration.width, - screenshotHeightInPixels: configuration.height - )) } guard !capturedScreens.isEmpty else { @@ -154,4 +151,105 @@ enum OrbitScreenCaptureUtility { return capturedScreens } + + /// Resolves the cursor display without touching ScreenCaptureKit objects so + /// multi-display selection can be regression tested deterministically. + static func selectedDisplayID( + cursorLocation: CGPoint, + screenFramesByDisplayID: [CGDirectDisplayID: CGRect], + availableDisplayIDs: [CGDirectDisplayID] + ) -> CGDirectDisplayID? { + availableDisplayIDs.first(where: { + screenFramesByDisplayID[$0]?.contains(cursorLocation) == true + }) ?? availableDisplayIDs.first + } + + private static func nsScreensByDisplayID() -> [CGDirectDisplayID: NSScreen] { + Dictionary( + uniqueKeysWithValues: NSScreen.screens.compactMap { screen in + guard + let displayID = screen.deviceDescription[ + NSDeviceDescriptionKey("NSScreenNumber") + ] as? CGDirectDisplayID + else { return nil } + return (displayID, screen) + } + ) + } + + private static func windowsOwnedByOrbit(in content: SCShareableContent) -> [SCWindow] { + let ownBundleIdentifier = Bundle.main.bundleIdentifier + return content.windows.filter { + $0.owningApplication?.bundleIdentifier == ownBundleIdentifier + } + } + + private static func capture( + display: SCDisplay, + screenNumber: Int, + displayCount: Int, + mouseLocation: CGPoint, + nsScreen: NSScreen?, + excludingWindows: [SCWindow] + ) async throws -> OrbitScreenCapture? { + // AppKit cursor coordinates and NSScreen frames share a bottom-left + // origin. SCDisplay frames use Core Graphics coordinates on secondary + // displays, so prefer the matching NSScreen whenever it is available. + let displayFrame = + nsScreen?.frame + ?? CGRect( + x: display.frame.origin.x, + y: display.frame.origin.y, + width: CGFloat(display.width), + height: CGFloat(display.height) + ) + let isCursorScreen = displayFrame.contains(mouseLocation) + let filter = SCContentFilter(display: display, excludingWindows: excludingWindows) + let configuration = captureConfiguration(for: display) + let cgImage = try await SCScreenshotManager.captureImage( + contentFilter: filter, + configuration: configuration + ) + guard + let jpegData = NSBitmapImageRep(cgImage: cgImage) + .representation(using: .jpeg, properties: [.compressionFactor: 0.8]) + else { return nil } + + let screenLabel: String + if displayCount == 1, isCursorScreen { + screenLabel = "user's current screen (cursor is here)" + } else if displayCount == 1 { + screenLabel = "user's available screen" + } else if isCursorScreen { + screenLabel = "screen \(screenNumber) of \(displayCount) — cursor is on this screen (primary focus)" + } else { + screenLabel = "screen \(screenNumber) of \(displayCount) — secondary screen" + } + + return OrbitScreenCapture( + imageData: jpegData, + label: screenLabel, + isCursorScreen: isCursorScreen, + screenNumber: screenNumber, + displayWidthInPoints: Int(displayFrame.width), + displayHeightInPoints: Int(displayFrame.height), + displayFrame: displayFrame, + screenshotWidthInPixels: configuration.width, + screenshotHeightInPixels: configuration.height + ) + } + + private static func captureConfiguration(for display: SCDisplay) -> SCStreamConfiguration { + let configuration = SCStreamConfiguration() + let maxDimension = 1280 + let aspectRatio = CGFloat(display.width) / CGFloat(display.height) + if display.width >= display.height { + configuration.width = maxDimension + configuration.height = Int(CGFloat(maxDimension) / aspectRatio) + } else { + configuration.height = maxDimension + configuration.width = Int(CGFloat(maxDimension) * aspectRatio) + } + return configuration + } } diff --git a/Orbit/OrbitSettings.swift b/Orbit/OrbitSettings.swift index 558363d..57ac8dd 100644 --- a/Orbit/OrbitSettings.swift +++ b/Orbit/OrbitSettings.swift @@ -19,9 +19,15 @@ enum OrbitVoicePreset: String, CaseIterable, Identifiable { struct OrbitCodexReasoningEffort: RawRepresentable, Hashable, Identifiable, Sendable { let rawValue: String + let detailText: String? init(rawValue: String) { - self.rawValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + self.init(rawValue: rawValue, detailText: nil) + } + + init(rawValue: String, detailText: String?) { + self.rawValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines) + self.detailText = detailText?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty } static let none = Self(rawValue: "none") @@ -32,10 +38,20 @@ struct OrbitCodexReasoningEffort: RawRepresentable, Hashable, Identifiable, Send static let max = Self(rawValue: "max") static let allCases: [Self] = [.none, .low, .medium, .high, .xhigh, .max] - var id: String { rawValue } + var id: String { normalizedIdentifier } + + var normalizedIdentifier: String { rawValue.lowercased() } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.normalizedIdentifier == rhs.normalizedIdentifier + } + + func hash(into hasher: inout Hasher) { + hasher.combine(normalizedIdentifier) + } var displayName: String { - switch rawValue { + switch normalizedIdentifier { case Self.none.rawValue: return "None" case Self.low.rawValue: @@ -54,7 +70,7 @@ struct OrbitCodexReasoningEffort: RawRepresentable, Hashable, Identifiable, Send } var level: Int { - switch rawValue { + switch normalizedIdentifier { case Self.none.rawValue: return 0 case Self.low.rawValue: @@ -75,9 +91,17 @@ struct OrbitCodexReasoningEffort: RawRepresentable, Hashable, Identifiable, Send struct OrbitCodexServiceTier: RawRepresentable, Hashable, Identifiable, Sendable { let rawValue: String + let advertisedName: String? + let detailText: String? init(rawValue: String) { - self.rawValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + self.init(rawValue: rawValue, advertisedName: nil, detailText: nil) + } + + init(rawValue: String, advertisedName: String?, detailText: String?) { + self.rawValue = rawValue.trimmingCharacters(in: .whitespacesAndNewlines) + self.advertisedName = advertisedName?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty + self.detailText = detailText?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty } static let serverDefault = Self(rawValue: "") @@ -85,10 +109,24 @@ struct OrbitCodexServiceTier: RawRepresentable, Hashable, Identifiable, Sendable static let fast = Self(rawValue: "fast") static let allCases: [Self] = [.serverDefault, .standard, .fast] - var id: String { rawValue } + var id: String { normalizedIdentifier } + + var normalizedIdentifier: String { rawValue.lowercased() } + + static func == (lhs: Self, rhs: Self) -> Bool { + lhs.normalizedIdentifier == rhs.normalizedIdentifier + } + + func hash(into hasher: inout Hasher) { + hasher.combine(normalizedIdentifier) + } var displayName: String { - switch rawValue { + if let advertisedName { + return advertisedName + } + + switch normalizedIdentifier { case Self.serverDefault.rawValue: return "Default" case Self.standard.rawValue: @@ -110,8 +148,13 @@ struct OrbitCodexModelOption: Identifiable, Equatable { let inputModalities: [String] let isDefault: Bool let supportedServiceTiers: [OrbitCodexServiceTier] + let defaultServiceTier: OrbitCodexServiceTier? + let modelDescription: String? + let availabilityMessage: String? let upgradeModel: String? let upgradeMessage: String? + let upgradeModelLink: String? + let upgradeMigrationMarkdown: String? var id: String { model } @@ -124,8 +167,13 @@ struct OrbitCodexModelOption: Identifiable, Equatable { inputModalities: [String], isDefault: Bool, supportedServiceTiers: [OrbitCodexServiceTier] = [], + defaultServiceTier: OrbitCodexServiceTier? = nil, + modelDescription: String? = nil, + availabilityMessage: String? = nil, upgradeModel: String? = nil, - upgradeMessage: String? = nil + upgradeMessage: String? = nil, + upgradeModelLink: String? = nil, + upgradeMigrationMarkdown: String? = nil ) { self.model = model self.displayName = displayName @@ -135,8 +183,13 @@ struct OrbitCodexModelOption: Identifiable, Equatable { self.inputModalities = inputModalities self.isDefault = isDefault self.supportedServiceTiers = supportedServiceTiers + self.defaultServiceTier = defaultServiceTier + self.modelDescription = modelDescription + self.availabilityMessage = availabilityMessage self.upgradeModel = upgradeModel self.upgradeMessage = upgradeMessage + self.upgradeModelLink = upgradeModelLink + self.upgradeMigrationMarkdown = upgradeMigrationMarkdown } static let fallbackPickerModels: [OrbitCodexModelOption] = [] @@ -147,6 +200,10 @@ struct OrbitCodexModelOption: Identifiable, Equatable { } } +extension String { + fileprivate var nilIfEmpty: String? { isEmpty ? nil : self } +} + @MainActor final class OrbitSettings: ObservableObject { static let shared = OrbitSettings() diff --git a/Orbit/OrbitTemporaryCaptureLease.swift b/Orbit/OrbitTemporaryCaptureLease.swift index a60d2b0..f32773d 100644 --- a/Orbit/OrbitTemporaryCaptureLease.swift +++ b/Orbit/OrbitTemporaryCaptureLease.swift @@ -5,7 +5,6 @@ import Foundation nonisolated final class OrbitTemporaryCaptureLease: @unchecked Sendable { static let directoryName = "OrbitTemporaryCaptures" static let filePrefix = "capture-" - static let staleAge: TimeInterval = 60 * 60 private static let cleanupQueue = DispatchQueue(label: "com.orbit.capture-cleanup", qos: .utility) let turnID: UUID @@ -66,7 +65,13 @@ nonisolated final class OrbitTemporaryCaptureLease: @unchecked Sendable { } static func sweepStaleCaptures( - now: Date = Date(), + now _: Date = Date(), + temporaryDirectory: URL = FileManager.default.temporaryDirectory + ) async { + await sweepOrphanedCaptures(temporaryDirectory: temporaryDirectory) + } + + static func sweepOrphanedCaptures( temporaryDirectory: URL = FileManager.default.temporaryDirectory ) async { await Task.detached(priority: .utility) { @@ -75,22 +80,35 @@ nonisolated final class OrbitTemporaryCaptureLease: @unchecked Sendable { guard let entries = try? fileManager.contentsOfDirectory( at: directory, - includingPropertiesForKeys: [.contentModificationDateKey, .isRegularFileKey], + includingPropertiesForKeys: [.isRegularFileKey, .isSymbolicLinkKey], options: [.skipsHiddenFiles] ) else { return } - for entry in entries where entry.lastPathComponent.hasPrefix(filePrefix) { - guard let values = try? entry.resourceValues(forKeys: [.contentModificationDateKey, .isRegularFileKey]), + // No live lease can survive a process restart. Delete every strictly + // named Orbit capture on launch, including a fresh file left by a + // crash, while preserving unrelated and look-alike temporary files. + for entry in entries where isOwnedCapture(entry) { + guard + let values = try? entry.resourceValues( + forKeys: [.isRegularFileKey, .isSymbolicLinkKey] + ), values.isRegularFile == true, - let modifiedAt = values.contentModificationDate, - now.timeIntervalSince(modifiedAt) >= staleAge + values.isSymbolicLink != true else { continue } try? fileManager.removeItem(at: entry) } }.value } + static func isOwnedCapture(_ fileURL: URL) -> Bool { + guard fileURL.pathExtension.lowercased() == "jpg" else { return false } + let stem = fileURL.deletingPathExtension().lastPathComponent + guard stem.hasPrefix(filePrefix) else { return false } + let identifier = String(stem.dropFirst(filePrefix.count)) + return UUID(uuidString: identifier) != nil + } + deinit { release() } diff --git a/Orbit/OrbitVoiceCoordinator.swift b/Orbit/OrbitVoiceCoordinator.swift new file mode 100644 index 0000000..b05e785 --- /dev/null +++ b/Orbit/OrbitVoiceCoordinator.swift @@ -0,0 +1,184 @@ +import Foundation + +enum OrbitNarrationSource: Sendable { + case preview + case onboarding + case earlyCommentary + case completion + case failure +} + +struct OrbitNarrationRequest: Sendable { + let text: String + let source: OrbitNarrationSource + let turnIdentifier: String? + + init( + text: String, + source: OrbitNarrationSource = .completion, + turnIdentifier: String? = nil + ) { + self.text = text + self.source = source + self.turnIdentifier = turnIdentifier + } +} + +enum OrbitNarrationFormatter { + static func spokenText(from rawText: String, maximumLength: Int = 560) -> String { + guard maximumLength > 0 else { return "" } + + var text = rawText + text = text.replacingOccurrences( + of: #"\[POINT:(?:none|[^\]]+)\]"#, + with: "", + options: .regularExpression + ) + text = text.replacingOccurrences( + of: #"```(?:[A-Za-z0-9_+-]+)?\s*([\s\S]*?)```"#, + with: "$1", + options: .regularExpression + ) + text = text.replacingOccurrences( + of: #"!?\[([^\]]+)\]\((?:[^()]|\([^)]*\))+\)"#, + with: "$1", + options: .regularExpression + ) + text = text.replacingOccurrences(of: #"https?://\S+"#, with: "", options: .regularExpression) + text = text.replacingOccurrences( + of: #"(?m)^\s{0,3}(?:#{1,6}\s+|[-*+]\s+|\d+[.)]\s+)"#, + with: "", + options: .regularExpression + ) + text = text.replacingOccurrences(of: #"[*_~`]"#, with: "", options: .regularExpression) + text = text.replacingOccurrences(of: #"\s+"#, with: " ", options: .regularExpression) + text = text.trimmingCharacters(in: .whitespacesAndNewlines) + + guard text.count > maximumLength else { return text } + let prefix = String(text.prefix(maximumLength)) + let wordBoundary = prefix.range(of: #"\s+\S*$"#, options: .regularExpression)?.lowerBound + let clipped = wordBoundary.map { String(prefix[..<$0]) } ?? prefix + return clipped.trimmingCharacters(in: .whitespacesAndNewlines) + "…" + } + + static func comparisonKey(for rawText: String) -> String { + spokenText(from: rawText, maximumLength: 2_000) + .folding(options: [.caseInsensitive, .diacriticInsensitive], locale: .current) + .replacingOccurrences(of: #"[^\p{L}\p{N}]+"#, with: " ", options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + } +} + +struct OrbitNarrationDeduplicator { + private struct Entry { + let key: String + let turnIdentifier: String? + let timestamp: Date + } + + private var recentEntries: [Entry] = [] + private let retentionInterval: TimeInterval + + init(retentionInterval: TimeInterval = 20) { + self.retentionInterval = retentionInterval + } + + mutating func shouldSpeak( + _ text: String, + turnIdentifier: String?, + now: Date = Date() + ) -> Bool { + let key = OrbitNarrationFormatter.comparisonKey(for: text) + guard !key.isEmpty else { return false } + + recentEntries.removeAll { now.timeIntervalSince($0.timestamp) > retentionInterval } + let isDuplicate = recentEntries.contains { entry in + guard entry.turnIdentifier == turnIdentifier else { return false } + return entry.key == key || entry.key.hasPrefix(key) || key.hasPrefix(entry.key) + } + guard !isDuplicate else { return false } + + recentEntries.append(Entry(key: key, turnIdentifier: turnIdentifier, timestamp: now)) + return true + } + + mutating func reset() { + recentEntries.removeAll(keepingCapacity: false) + } +} + +@MainActor +final class OrbitVoiceCoordinator: TextToSpeechProvider { + private let primaryProvider: any TextToSpeechProvider + private let fallbackProvider: (any TextToSpeechProvider)? + private var generation = 0 + private var deduplicator = OrbitNarrationDeduplicator() + + init( + primary: any TextToSpeechProvider, + fallback: (any TextToSpeechProvider)? = nil + ) { + self.primaryProvider = primary + self.fallbackProvider = fallback + } + + var displayName: String { primaryProvider.displayName } + + var isConfigured: Bool { + primaryProvider.isConfigured || fallbackProvider?.isConfigured == true + } + + var unavailableExplanation: String? { + if isConfigured { return nil } + return primaryProvider.unavailableExplanation ?? fallbackProvider?.unavailableExplanation + } + + var isPlaying: Bool { + primaryProvider.isPlaying || fallbackProvider?.isPlaying == true + } + + func speakText(_ text: String) async throws { + try await speak( + OrbitNarrationRequest( + text: text, + turnIdentifier: UUID().uuidString + ) + ) + } + + func speak(_ request: OrbitNarrationRequest) async throws { + let formattedText = OrbitNarrationFormatter.spokenText(from: request.text) + guard + deduplicator.shouldSpeak( + formattedText, + turnIdentifier: request.turnIdentifier + ) + else { return } + + stopPlayback() + let requestGeneration = generation + + do { + try await primaryProvider.speakText(formattedText) + } catch is CancellationError { + throw CancellationError() + } catch { + guard requestGeneration == generation, let fallbackProvider, fallbackProvider.isConfigured else { + throw error + } + try await fallbackProvider.speakText(formattedText) + } + + guard requestGeneration == generation else { throw CancellationError() } + } + + func stopPlayback() { + generation &+= 1 + primaryProvider.stopPlayback() + fallbackProvider?.stopPlayback() + } + + func resetNarrationHistory() { + deduplicator.reset() + } +} diff --git a/Orbit/OverlayWindow.swift b/Orbit/OverlayWindow.swift index fc96e50..0fe21c2 100644 --- a/Orbit/OverlayWindow.swift +++ b/Orbit/OverlayWindow.swift @@ -76,6 +76,7 @@ struct OrbitCursorOverlayView: View { let isFirstAppearance: Bool let showWelcomeOnFirstAppearance: Bool @ObservedObject var orbitManager: OrbitManager + @Environment(\.accessibilityReduceMotion) private var reduceMotion @State private var cursorPosition: CGPoint @State private var isCursorOnThisScreen: Bool @@ -158,8 +159,8 @@ struct OrbitCursorOverlayView: View { .padding(.trailing, 18) .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topTrailing) .transition(.move(edge: .trailing).combined(with: .opacity)) - .animation(.spring(response: 0.28, dampingFraction: 0.82), value: orbitManager.showCodexActivityOverlay) - .animation(.easeInOut(duration: 0.18), value: orbitManager.activeActionDetailLine) + .animation(reduceMotion ? nil : .spring(response: 0.28, dampingFraction: 0.82), value: orbitManager.showCodexActivityOverlay) + .animation(reduceMotion ? nil : .easeInOut(duration: 0.18), value: orbitManager.activeActionDetailLine) .allowsHitTesting(false) } @@ -191,8 +192,8 @@ struct OrbitCursorOverlayView: View { ) .opacity(bubbleOpacity) .position(x: cursorPosition.x + 10 + (bubbleSize.width / 2), y: cursorPosition.y + 18) - .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) - .animation(.easeOut(duration: 0.5), value: bubbleOpacity) + .animation(reduceMotion ? nil : .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(reduceMotion ? nil : .easeOut(duration: 0.5), value: bubbleOpacity) .onPreferenceChange(SizePreferenceKey.self) { newSize in bubbleSize = newSize } @@ -226,8 +227,8 @@ struct OrbitCursorOverlayView: View { ) .opacity(orbitManager.onboardingPromptOpacity) .position(x: cursorPosition.x + 14 + (onboardingBubbleSize.width / 2), y: cursorPosition.y + 22) - .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) - .animation(.easeOut(duration: 0.4), value: orbitManager.onboardingPromptOpacity) + .animation(reduceMotion ? nil : .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(reduceMotion ? nil : .easeOut(duration: 0.4), value: orbitManager.onboardingPromptOpacity) .onPreferenceChange(OnboardingBubbleSizePreferenceKey.self) { newSize in onboardingBubbleSize = newSize } @@ -264,9 +265,9 @@ struct OrbitCursorOverlayView: View { .scaleEffect(navigationBubbleScale) .opacity(navigationBubbleOpacity) .position(x: cursorPosition.x + 10 + (navigationBubbleSize.width / 2), y: cursorPosition.y + 18) - .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) - .animation(.spring(response: 0.4, dampingFraction: 0.6), value: navigationBubbleScale) - .animation(.easeOut(duration: 0.5), value: navigationBubbleOpacity) + .animation(reduceMotion ? nil : .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(reduceMotion ? nil : .spring(response: 0.4, dampingFraction: 0.6), value: navigationBubbleScale) + .animation(reduceMotion ? nil : .easeOut(duration: 0.5), value: navigationBubbleOpacity) .onPreferenceChange(NavigationBubbleSizePreferenceKey.self) { newSize in navigationBubbleSize = newSize } @@ -279,28 +280,28 @@ struct OrbitCursorOverlayView: View { .opacity(orbitCursorIsVisibleOnThisScreen && (orbitManager.voiceState == .idle || orbitManager.voiceState == .responding) ? cursorOpacity : 0) .position(cursorPosition) .animation( - orbitNavigationMode == .followingCursor + !reduceMotion && orbitNavigationMode == .followingCursor ? .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0) : nil, value: cursorPosition ) - .animation(.easeIn(duration: 0.25), value: orbitManager.voiceState) + .animation(reduceMotion ? nil : .easeIn(duration: 0.25), value: orbitManager.voiceState) .animation( - orbitNavigationMode == .navigatingToTarget ? nil : .easeInOut(duration: 0.3), + reduceMotion || orbitNavigationMode == .navigatingToTarget ? nil : .easeInOut(duration: 0.3), value: triangleRotationDegrees ) OrbitListeningCursorView(audioPowerLevel: orbitManager.currentAudioPowerLevel) .opacity(orbitCursorIsVisibleOnThisScreen && orbitManager.voiceState == .listening ? cursorOpacity : 0) .position(cursorPosition) - .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) - .animation(.easeIn(duration: 0.15), value: orbitManager.voiceState) + .animation(reduceMotion ? nil : .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(reduceMotion ? nil : .easeIn(duration: 0.15), value: orbitManager.voiceState) OrbitProcessingCursorView() .opacity(orbitCursorIsVisibleOnThisScreen && orbitManager.voiceState == .processing ? cursorOpacity : 0) .position(cursorPosition) - .animation(.spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) - .animation(.easeIn(duration: 0.15), value: orbitManager.voiceState) + .animation(reduceMotion ? nil : .spring(response: 0.2, dampingFraction: 0.6, blendDuration: 0), value: cursorPosition) + .animation(reduceMotion ? nil : .easeIn(duration: 0.15), value: orbitManager.voiceState) } .frame(width: screenFrame.width, height: screenFrame.height) @@ -319,7 +320,7 @@ struct OrbitCursorOverlayView: View { // and only if the cursor starts on this screen if showWelcomeOnFirstAppearance && isFirstAppearance && isCursorOnThisScreen { self.welcomeText = self.fullWelcomeMessage - withAnimation(.easeIn(duration: 2.0)) { + withAnimation(reduceMotion ? nil : .easeIn(duration: 2.0)) { self.cursorOpacity = 1.0 } DispatchQueue.main.asyncAfter(deadline: .now() + 0.8) { @@ -421,9 +422,9 @@ struct OrbitCursorOverlayView: View { .lineLimit(2) } - if !orbitManager.recentActionUpdates.isEmpty { + if !visibleRecentActionUpdates.isEmpty { VStack(alignment: .leading, spacing: 5) { - ForEach(Array(orbitManager.recentActionUpdates.suffix(4).enumerated()), id: \.offset) { _, update in + ForEach(Array(visibleRecentActionUpdates.enumerated()), id: \.offset) { _, update in HStack(alignment: .top, spacing: 6) { Circle() .fill(Color.white.opacity(0.55)) @@ -438,12 +439,6 @@ struct OrbitCursorOverlayView: View { } } - if !orbitManager.codexConfigurationSummary.isEmpty { - Text(orbitManager.codexConfigurationSummary) - .font(.system(size: 10, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .lineLimit(1) - } } .padding(.horizontal, 14) .padding(.vertical, 12) @@ -500,6 +495,16 @@ struct OrbitCursorOverlayView: View { orbitManager.activeActionDetailLine } + private var visibleRecentActionUpdates: [String] { + let activeDetail = orbitManager.activeActionDetailLine? + .trimmingCharacters(in: .whitespacesAndNewlines) + let filtered = orbitManager.recentActionUpdates.filter { update in + guard let activeDetail, !activeDetail.isEmpty else { return true } + return update.trimmingCharacters(in: .whitespacesAndNewlines) != activeDetail + } + return Array(filtered.suffix(3)) + } + // MARK: - Cursor Tracking private func startTrackingCursor() { @@ -591,6 +596,13 @@ struct OrbitCursorOverlayView: View { ) { navigationAnimationTask?.cancel() + if reduceMotion { + cursorPosition = destination + orbitFlightScale = 1 + onComplete() + return + } + let startPosition = cursorPosition let endPosition = destination @@ -679,6 +691,16 @@ struct OrbitCursorOverlayView: View { ?? navigationPointerPhrases.randomElement() ?? "right here!" + if reduceMotion { + navigationBubbleScale = 1 + navigationBubbleText = pointerPhrase + DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { + guard self.orbitNavigationMode == .pointingAtTarget else { return } + self.startFlyingBackToCursor() + } + return + } + streamNavigationBubbleCharacter(phrase: pointerPhrase, characterIndex: 0) { // All characters streamed — hold for 3 seconds, then fly back DispatchQueue.main.asyncAfter(deadline: .now() + 3.0) { @@ -825,10 +847,11 @@ private struct OrbitDashedMarkStroke: View { private struct OrbitCursorGlyphView: View { var isReturning: Bool = false + @Environment(\.accessibilityReduceMotion) private var reduceMotion var body: some View { TimelineView(.animation) { timeline in - let cycleProgress = cycleProgress(for: timeline.date, duration: 4.0) + let cycleProgress = reduceMotion ? 0 : cycleProgress(for: timeline.date, duration: 4.0) let auraOpacity = 0.75 - (0.25 * CGFloat(cos(Double(cycleProgress) * .pi * 2))) ZStack { @@ -858,10 +881,11 @@ private struct OrbitCursorGlyphView: View { private struct OrbitListeningCursorView: View { let audioPowerLevel: CGFloat + @Environment(\.accessibilityReduceMotion) private var reduceMotion var body: some View { TimelineView(.animation) { timeline in - let cycle = cycleProgress(for: timeline.date, duration: 1.2) + let cycle = reduceMotion ? 0 : cycleProgress(for: timeline.date, duration: 1.2) let resonancePhase = cubicBezierProgress( cycle, c1x: 0.2, @@ -887,9 +911,11 @@ private struct OrbitListeningCursorView: View { } private struct OrbitProcessingCursorView: View { + @Environment(\.accessibilityReduceMotion) private var reduceMotion + var body: some View { TimelineView(.animation) { timeline in - let cycle = cycleProgress(for: timeline.date, duration: 0.9) + let cycle = reduceMotion ? 0 : cycleProgress(for: timeline.date, duration: 0.9) let circumference = CGFloat.pi * OrbitCursorMetrics.railDiameter let dashOn = circumference * 0.22 let dashOff = max(circumference - dashOn, 0.01) @@ -920,10 +946,11 @@ private struct OrbitProcessingCursorView: View { private struct OrbitMiniSpinner: View { var tint: Color = Color.white.opacity(0.96) + @Environment(\.accessibilityReduceMotion) private var reduceMotion var body: some View { TimelineView(.animation) { timeline in - let cycle = cycleProgress(for: timeline.date, duration: 1.2) + let cycle = reduceMotion ? 0 : cycleProgress(for: timeline.date, duration: 1.2) ZStack { OrbitMarkShape() diff --git a/Orbit/SayNoraTTSProvider.swift b/Orbit/SayNoraTTSProvider.swift new file mode 100644 index 0000000..ac6d020 --- /dev/null +++ b/Orbit/SayNoraTTSProvider.swift @@ -0,0 +1,393 @@ +import AVFoundation +import Foundation + +struct OrbitNoraVoiceUnavailableError: LocalizedError { + var errorDescription: String? { + "Nora Premium is not available through the local Apple speech service." + } +} + +nonisolated final class OrbitSayProcessHandle: @unchecked Sendable { + private let lock = NSLock() + private var process: Process? + private var isCancelled = false + + func install(_ process: Process) { + let shouldTerminate = lock.withLock { + self.process = process + return isCancelled + } + if shouldTerminate, process.isRunning { + process.terminate() + } + } + + func finish() { + lock.withLock { + process = nil + } + } + + func cancel() { + let runningProcess = lock.withLock { () -> Process? in + isCancelled = true + return process + } + if runningProcess?.isRunning == true { + runningProcess?.terminate() + } + } + + var cancelled: Bool { + lock.withLock { isCancelled } + } +} + +enum OrbitSayAudioRenderer { + nonisolated static let executableURL = URL(fileURLWithPath: "/usr/bin/say") + + nonisolated static var temporaryDirectoryURL: URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("com.orbit.codex", isDirectory: true) + .appendingPathComponent("local-speech", isDirectory: true) + } + + static func makeSecureTemporaryAudioURL(prefix: String) throws -> URL { + let directoryURL = temporaryDirectoryURL + try FileManager.default.createDirectory( + at: directoryURL, + withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700] + ) + try FileManager.default.setAttributes( + [.posixPermissions: 0o700], + ofItemAtPath: directoryURL.path + ) + + let outputURL = directoryURL.appendingPathComponent( + "\(prefix)-\(UUID().uuidString).aiff", + isDirectory: false + ) + guard + FileManager.default.createFile( + atPath: outputURL.path, + contents: Data(), + attributes: [.posixPermissions: 0o600] + ) + else { + throw CocoaError(.fileWriteUnknown) + } + return outputURL + } + + nonisolated static func sweepStaleTemporaryAudio( + olderThan interval: TimeInterval = 60 * 60, + now: Date = Date() + ) { + guard + let items = try? FileManager.default.contentsOfDirectory( + at: temporaryDirectoryURL, + includingPropertiesForKeys: [.contentModificationDateKey, .isRegularFileKey], + options: [.skipsHiddenFiles] + ) + else { return } + + for item in items where item.pathExtension == "aiff" { + guard + let values = try? item.resourceValues(forKeys: [.contentModificationDateKey, .isRegularFileKey]), + values.isRegularFile == true, + let modifiedAt = values.contentModificationDate, + now.timeIntervalSince(modifiedAt) >= interval + else { continue } + try? FileManager.default.removeItem(at: item) + } + } + + static func render( + text: String, + voiceName: String, + outputURL: URL, + processHandle: OrbitSayProcessHandle + ) async throws { + try await withTaskCancellationHandler { + try await Task.detached(priority: .userInitiated) { + guard !processHandle.cancelled else { throw CancellationError() } + let process = Process() + process.executableURL = executableURL + process.arguments = ["-v", voiceName, "-o", outputURL.path, text] + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + processHandle.install(process) + defer { processHandle.finish() } + + try process.run() + process.waitUntilExit() + guard !processHandle.cancelled, !Task.isCancelled else { + throw CancellationError() + } + guard process.terminationReason == .exit, process.terminationStatus == 0 else { + throw NSError( + domain: "OrbitSayAudioRenderer", + code: Int(process.terminationStatus), + userInfo: [NSLocalizedDescriptionKey: "Apple local speech could not synthesize audio."] + ) + } + + let attributes = try FileManager.default.attributesOfItem(atPath: outputURL.path) + guard (attributes[.size] as? NSNumber)?.intValue ?? 0 > 64 else { + throw NSError( + domain: "OrbitSayAudioRenderer", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Apple local speech returned empty audio."] + ) + } + try FileManager.default.setAttributes( + [.posixPermissions: 0o600], + ofItemAtPath: outputURL.path + ) + }.value + } onCancel: { + processHandle.cancel() + } + } + + static func rendersDistinctVoice( + candidateAudio: Data, + fallbackAudio: Data + ) -> Bool { + candidateAudio.count > 64 && fallbackAudio.count > 64 && candidateAudio != fallbackAudio + } +} + +nonisolated private final class OrbitNoraAvailabilityCache: @unchecked Sendable { + private let lock = NSLock() + private var storedValue: Bool? + + var value: Bool? { + lock.withLock { storedValue } + } + + func store(_ value: Bool) { + lock.withLock { + storedValue = value + } + } +} + +enum OrbitNoraVoiceAvailability { + private static let cache = OrbitNoraAvailabilityCache() + private static let fallbackProbeVoice = "__orbit_missing_voice_\(UUID().uuidString)__" + + static var cachedValue: Bool? { cache.value } + + static func probe(forceRefresh: Bool = false) async -> Bool { + if !forceRefresh, let cachedValue { return cachedValue } + guard FileManager.default.isExecutableFile(atPath: OrbitSayAudioRenderer.executableURL.path) else { + cache.store(false) + return false + } + + var candidateURL: URL? + var fallbackURL: URL? + defer { + if let candidateURL { try? FileManager.default.removeItem(at: candidateURL) } + if let fallbackURL { try? FileManager.default.removeItem(at: fallbackURL) } + } + + do { + candidateURL = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "nora-probe") + fallbackURL = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "default-probe") + let probeText = "Orbit local voice availability probe." + try await OrbitSayAudioRenderer.render( + text: probeText, + voiceName: "Nora", + outputURL: candidateURL!, + processHandle: OrbitSayProcessHandle() + ) + try await OrbitSayAudioRenderer.render( + text: probeText, + voiceName: fallbackProbeVoice, + outputURL: fallbackURL!, + processHandle: OrbitSayProcessHandle() + ) + let candidateData = try Data(contentsOf: candidateURL!, options: .mappedIfSafe) + let fallbackData = try Data(contentsOf: fallbackURL!, options: .mappedIfSafe) + let isAvailable = OrbitSayAudioRenderer.rendersDistinctVoice( + candidateAudio: candidateData, + fallbackAudio: fallbackData + ) + cache.store(isAvailable) + return isAvailable + } catch { + if error is CancellationError { return false } + cache.store(false) + return false + } + } +} + +@MainActor +final class SayNoraTTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDelegate { + let displayName = "Nora Premium · Apple Local" + + private var audioPlayer: AVAudioPlayer? + private var currentPlayerIdentifier: ObjectIdentifier? + private var availabilityProbeTask: Task? + private var renderProcessHandle: OrbitSayProcessHandle? + private var currentOutputURL: URL? + private var currentSpeakContinuation: CheckedContinuation? + private var generation = 0 + + override init() { + super.init() + Task.detached(priority: .utility) { + OrbitSayAudioRenderer.sweepStaleTemporaryAudio() + } + } + + var isConfigured: Bool { + FileManager.default.isExecutableFile(atPath: OrbitSayAudioRenderer.executableURL.path) + && OrbitNoraVoiceAvailability.cachedValue != false + } + + var unavailableExplanation: String? { + isConfigured + ? nil + : "Nora Premium is not available. Orbit can fall back to an AVFoundation voice." + } + + var isPlaying: Bool { + audioPlayer?.isPlaying == true || renderProcessHandle != nil || availabilityProbeTask != nil + } + + func speakText(_ text: String) async throws { + let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty else { return } + + stopPlayback() + generation &+= 1 + let requestGeneration = generation + + try await withTaskCancellationHandler { + let probeTask = Task { await OrbitNoraVoiceAvailability.probe() } + availabilityProbeTask = probeTask + let isNoraAvailable = await probeTask.value + if generation == requestGeneration { + availabilityProbeTask = nil + } + guard isNoraAvailable else { + throw OrbitNoraVoiceUnavailableError() + } + guard generation == requestGeneration else { throw CancellationError() } + + let outputURL = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "nora") + currentOutputURL = outputURL + let processHandle = OrbitSayProcessHandle() + renderProcessHandle = processHandle + + do { + try await OrbitSayAudioRenderer.render( + text: normalized, + voiceName: "Nora", + outputURL: outputURL, + processHandle: processHandle + ) + } catch { + if generation == requestGeneration { + renderProcessHandle = nil + cleanupOutputFile() + } + throw error + } + + guard generation == requestGeneration else { throw CancellationError() } + renderProcessHandle = nil + let player = try AVAudioPlayer(contentsOf: outputURL) + player.delegate = self + audioPlayer = player + currentPlayerIdentifier = ObjectIdentifier(player) + + try await withCheckedThrowingContinuation { continuation in + currentSpeakContinuation = continuation + guard player.play() else { + currentSpeakContinuation = nil + audioPlayer = nil + currentPlayerIdentifier = nil + cleanupOutputFile() + continuation.resume( + throwing: NSError( + domain: "SayNoraTTSProvider", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Nora audio could not start playback."] + ) + ) + return + } + } + } onCancel: { + Task { @MainActor [weak self] in + self?.cancelIfCurrent(requestGeneration) + } + } + } + + func stopPlayback() { + generation &+= 1 + availabilityProbeTask?.cancel() + availabilityProbeTask = nil + renderProcessHandle?.cancel() + renderProcessHandle = nil + audioPlayer?.stop() + audioPlayer = nil + currentPlayerIdentifier = nil + let continuation = currentSpeakContinuation + currentSpeakContinuation = nil + cleanupOutputFile() + continuation?.resume(throwing: CancellationError()) + } + + nonisolated func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { + let playerIdentifier = ObjectIdentifier(player) + Task { @MainActor [weak self] in + self?.finishPlayback(playerIdentifier: playerIdentifier, successfully: flag) + } + } + + private func cancelIfCurrent(_ requestGeneration: Int) { + guard generation == requestGeneration else { return } + stopPlayback() + } + + private func finishPlayback(playerIdentifier: ObjectIdentifier, successfully: Bool) { + guard currentPlayerIdentifier == playerIdentifier else { return } + audioPlayer = nil + currentPlayerIdentifier = nil + let continuation = currentSpeakContinuation + currentSpeakContinuation = nil + cleanupOutputFile() + if successfully { + continuation?.resume() + } else { + continuation?.resume( + throwing: NSError( + domain: "SayNoraTTSProvider", + code: -2, + userInfo: [NSLocalizedDescriptionKey: "Nora audio playback was interrupted."] + ) + ) + } + } + + private func cleanupOutputFile() { + guard let currentOutputURL else { return } + self.currentOutputURL = nil + try? FileManager.default.removeItem(at: currentOutputURL) + } + + deinit { + renderProcessHandle?.cancel() + if let currentOutputURL { + try? FileManager.default.removeItem(at: currentOutputURL) + } + } +} diff --git a/Orbit/TextToSpeechProvider.swift b/Orbit/TextToSpeechProvider.swift index eb525c3..94a217a 100644 --- a/Orbit/TextToSpeechProvider.swift +++ b/Orbit/TextToSpeechProvider.swift @@ -243,9 +243,15 @@ enum OrbitTTSProviderFactory { static func makePrimaryProvider(for voicePreset: OrbitVoicePreset) -> any TextToSpeechProvider { switch voicePreset { case .localVoice: - return AppleSystemTTSProvider() + return OrbitVoiceCoordinator( + primary: SayNoraTTSProvider(), + fallback: AppleSystemTTSProvider() + ) case .cloudVoice: - return OpenAITTSProvider(voicePreset: voicePreset) + return OrbitVoiceCoordinator( + primary: OpenAITTSProvider(voicePreset: voicePreset), + fallback: AppleSystemTTSProvider() + ) } } @@ -260,7 +266,9 @@ final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, AVSpeechSynt let displayName = "Apple Local Speech" private let synthesizer = AVSpeechSynthesizer() private var currentSpeakContinuation: CheckedContinuation? + private var currentUtteranceIdentifier: ObjectIdentifier? private var lastLoggedVoiceIdentifier: String? + private var generation = 0 override init() { super.init() @@ -291,6 +299,8 @@ final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, AVSpeechSynt } stopPlayback() + generation &+= 1 + let requestGeneration = generation let utterance = AVSpeechUtterance(string: normalized) if let identifier = OrbitAppleVoiceCatalog.resolvedVoice( preferredIdentifier: OrbitSettings.shared.appleTTSVoiceIdentifier @@ -300,19 +310,32 @@ final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, AVSpeechSynt } utterance.rate = AVSpeechUtteranceDefaultSpeechRate - try await withCheckedThrowingContinuation { continuation in - currentSpeakContinuation = continuation - synthesizer.speak(utterance) + try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + guard generation == requestGeneration else { + continuation.resume(throwing: CancellationError()) + return + } + currentUtteranceIdentifier = ObjectIdentifier(utterance) + currentSpeakContinuation = continuation + synthesizer.speak(utterance) + } + } onCancel: { + Task { @MainActor [weak self] in + self?.cancelIfCurrent(requestGeneration) + } } } func stopPlayback() { + generation &+= 1 + currentUtteranceIdentifier = nil if synthesizer.isSpeaking || synthesizer.isPaused { synthesizer.stopSpeaking(at: .immediate) } if let continuation = currentSpeakContinuation { currentSpeakContinuation = nil - continuation.resume() + continuation.resume(throwing: CancellationError()) } } @@ -320,8 +343,9 @@ final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, AVSpeechSynt _ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance ) { + let utteranceIdentifier = ObjectIdentifier(utterance) Task { @MainActor [weak self] in - self?.finishCurrentSpeech() + self?.finishCurrentSpeech(for: utteranceIdentifier) } } @@ -329,17 +353,33 @@ final class AppleSystemTTSProvider: NSObject, TextToSpeechProvider, AVSpeechSynt _ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance ) { + let utteranceIdentifier = ObjectIdentifier(utterance) Task { @MainActor [weak self] in - self?.finishCurrentSpeech() + self?.cancelCurrentSpeech(for: utteranceIdentifier) } } - private func finishCurrentSpeech() { + private func cancelIfCurrent(_ requestGeneration: Int) { + guard generation == requestGeneration else { return } + stopPlayback() + } + + private func finishCurrentSpeech(for utteranceIdentifier: ObjectIdentifier) { + guard currentUtteranceIdentifier == utteranceIdentifier else { return } + currentUtteranceIdentifier = nil guard let continuation = currentSpeakContinuation else { return } currentSpeakContinuation = nil continuation.resume() } + private func cancelCurrentSpeech(for utteranceIdentifier: ObjectIdentifier) { + guard currentUtteranceIdentifier == utteranceIdentifier else { return } + currentUtteranceIdentifier = nil + guard let continuation = currentSpeakContinuation else { return } + currentSpeakContinuation = nil + continuation.resume(throwing: CancellationError()) + } + private func logVoiceSelectionIfNeeded(_ identifier: String) { guard lastLoggedVoiceIdentifier != identifier else { return } lastLoggedVoiceIdentifier = identifier diff --git a/Orbit/WindowPositionManager.swift b/Orbit/WindowPositionManager.swift index 37d2c75..a1bc5fb 100644 --- a/Orbit/WindowPositionManager.swift +++ b/Orbit/WindowPositionManager.swift @@ -84,10 +84,9 @@ class WindowPositionManager { return hasScreenRecordingPermissionNow } - /// Returns true when the app should proceed with session launch without showing - /// the permission gate again. This intentionally falls back to the last known - /// granted state because CGPreflightScreenCaptureAccess() can sometimes return a - /// false negative even though the user has already approved the app. + /// Returns true only when the current process can verify Screen Recording. + /// Historical grants are useful for restart guidance, but must never be treated + /// as live authorization after the user revokes TCC access. static func shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch() -> Bool { shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( hasScreenRecordingPermissionNow: hasScreenRecordingPermission(), @@ -97,9 +96,9 @@ class WindowPositionManager { static func shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( hasScreenRecordingPermissionNow: Bool, - hasPreviouslyConfirmedScreenRecordingPermission: Bool + hasPreviouslyConfirmedScreenRecordingPermission _: Bool ) -> Bool { - hasScreenRecordingPermissionNow || hasPreviouslyConfirmedScreenRecordingPermission + hasScreenRecordingPermissionNow } static func clearPreviouslyConfirmedScreenRecordingPermission() { diff --git a/OrbitTests/OrbitCodexLifecycleTests.swift b/OrbitTests/OrbitCodexLifecycleTests.swift new file mode 100644 index 0000000..ecf2176 --- /dev/null +++ b/OrbitTests/OrbitCodexLifecycleTests.swift @@ -0,0 +1,62 @@ +import Testing + +@testable import Orbit + +@MainActor +struct OrbitCodexLifecycleTests { + @Test func authenticatedStartupRequiresAReadyThread() { + let unresolved = OrbitCodexStartupReadiness.isResolved( + authState: .authenticated(email: nil, plan: nil), + hasReadySession: false, + isModelCatalogPending: false, + isAccountReadPending: false + ) + let resolved = OrbitCodexStartupReadiness.isResolved( + authState: .authenticated(email: nil, plan: nil), + hasReadySession: true, + isModelCatalogPending: false, + isAccountReadPending: false + ) + + #expect(!unresolved) + #expect(resolved) + } + + @Test func startupWaitsForAccountAndModelCatalog() { + #expect( + !OrbitCodexStartupReadiness.isResolved( + authState: .authenticated(email: nil, plan: nil), + hasReadySession: true, + isModelCatalogPending: true, + isAccountReadPending: false + ) + ) + #expect( + !OrbitCodexStartupReadiness.isResolved( + authState: .authenticated(email: nil, plan: nil), + hasReadySession: true, + isModelCatalogPending: false, + isAccountReadPending: true + ) + ) + } + + @Test func authenticationRequiredCanResolveWithoutAThread() { + #expect( + OrbitCodexStartupReadiness.isResolved( + authState: .authRequired, + hasReadySession: false, + isModelCatalogPending: false, + isAccountReadPending: false + ) + ) + } + + @Test func supportDiagnosticsAreRedactedAndBounded() { + let diagnostic = "Authorization: Bearer secret-token " + String(repeating: "x", count: 8_192) + let sanitized = OrbitSupportLog.sanitize(diagnostic) + + #expect(sanitized.count <= 4_096) + #expect(!sanitized.contains("secret-token")) + } +} diff --git a/OrbitTests/OrbitCodexModelCatalogTests.swift b/OrbitTests/OrbitCodexModelCatalogTests.swift new file mode 100644 index 0000000..b9f9d6c --- /dev/null +++ b/OrbitTests/OrbitCodexModelCatalogTests.swift @@ -0,0 +1,137 @@ +import Foundation +import Testing + +@testable import Orbit + +@MainActor +struct OrbitCodexModelCatalogTests { + @Test func parsesPinnedCodex0144ModelContractWithoutLosingMetadata() throws { + let models = OrbitCodexModelCatalog.parse(from: [ + "data": [ + [ + "id": "gpt-5.6-terra", + "model": "gpt-5.6-terra", + "displayName": "GPT-5.6 Terra", + "description": "A careful coding model.", + "hidden": false, + "isDefault": true, + "inputModalities": ["text", "image"], + "defaultReasoningEffort": "Orbital-X", + "supportedReasoningEfforts": [ + [ + "reasoningEffort": "none", + "description": "No additional reasoning.", + ], + [ + "reasoningEffort": "Orbital-X", + "description": "Account-specific future effort.", + ], + ], + "defaultServiceTier": "Priority-Plus", + "serviceTiers": [ + [ + "id": "standard", + "name": "Standard", + "description": "Normal latency.", + ], + [ + "id": "Priority-Plus", + "name": "Priority Plus", + "description": "Fastest available tier.", + ], + ], + "availabilityNux": [ + "message": "Available through your workspace preview." + ], + "upgrade": "gpt-5.6-terra-legacy-target", + "upgradeInfo": [ + "model": "gpt-5.7-terra", + "upgradeCopy": "Move to GPT-5.7 Terra.", + "migrationMarkdown": "Review the migration notes.", + "modelLink": "https://developers.openai.com/models/gpt-5.7-terra", + ], + ] + ] + ]) + + let model = try #require(models.first) + #expect(model.model == "gpt-5.6-terra") + #expect(model.modelDescription == "A careful coding model.") + #expect(model.availabilityMessage == "Available through your workspace preview.") + #expect(model.supportedEfforts.map(\.rawValue) == ["none", "Orbital-X"]) + #expect(model.supportedEfforts.last?.detailText == "Account-specific future effort.") + #expect(model.defaultEffort?.rawValue == "Orbital-X") + #expect(model.defaultEffort?.detailText == "Account-specific future effort.") + #expect(model.supportedServiceTiers.map(\.rawValue) == ["standard", "Priority-Plus"]) + #expect(model.supportedServiceTiers.last?.displayName == "Priority Plus") + #expect(model.supportedServiceTiers.last?.detailText == "Fastest available tier.") + #expect(model.defaultServiceTier?.rawValue == "Priority-Plus") + #expect(model.defaultServiceTier?.displayName == "Priority Plus") + #expect(model.upgradeModel == "gpt-5.7-terra") + #expect(model.upgradeMessage == "Move to GPT-5.7 Terra.") + #expect(model.upgradeMigrationMarkdown == "Review the migration notes.") + #expect(model.upgradeModelLink == "https://developers.openai.com/models/gpt-5.7-terra") + } + + @Test func preservesProtocolIdentifiersWhileMatchingThemCaseInsensitively() throws { + let effort = OrbitCodexReasoningEffort(rawValue: " Future-Effort ") + let persistedEffort = OrbitCodexReasoningEffort(rawValue: "future-effort") + let tier = OrbitCodexServiceTier( + rawValue: " Priority-Plus ", + advertisedName: "Priority Plus", + detailText: "Fast path" + ) + let persistedTier = OrbitCodexServiceTier(rawValue: "priority-plus") + + #expect(effort.rawValue == "Future-Effort") + #expect(effort == persistedEffort) + #expect(tier.rawValue == "Priority-Plus") + #expect(tier == persistedTier) + #expect(tier.displayName == "Priority Plus") + } + + @Test func doesNotInventReasoningEffortsWhenCatalogAdvertisesNone() throws { + let models = OrbitCodexModelCatalog.parse(from: [ + "data": [ + [ + "id": "non-reasoning-model", + "model": "non-reasoning-model", + "displayName": "Non-reasoning model", + "description": "No selectable reasoning effort.", + "hidden": false, + "isDefault": false, + "inputModalities": ["text", "image"], + "supportedReasoningEfforts": [], + ] + ] + ]) + + let model = try #require(models.first) + #expect(model.supportedEfforts.isEmpty) + #expect(model.defaultEffort == nil) + } + + @Test func mergesDeprecatedAdditionalSpeedTiersWithoutDuplicates() throws { + let models = OrbitCodexModelCatalog.parse(from: [ + "data": [ + [ + "model": "gpt-compatible", + "displayName": "GPT Compatible", + "hidden": false, + "isDefault": false, + "inputModalities": ["text", "image"], + "defaultReasoningEffort": "medium", + "supportedReasoningEfforts": [["reasoningEffort": "medium"]], + "serviceTiers": [ + ["id": "Fast", "name": "Fast lane", "description": "Fast"] + ], + "additionalSpeedTiers": ["fast", "burst"], + ] + ] + ]) + + let model = try #require(models.first) + #expect(model.supportedServiceTiers.map(\.rawValue) == ["Fast", "burst"]) + #expect(model.supportedServiceTiers.first?.displayName == "Fast lane") + } +} diff --git a/OrbitTests/OrbitTests.swift b/OrbitTests/OrbitTests.swift index eaa6a94..5d92a64 100644 --- a/OrbitTests/OrbitTests.swift +++ b/OrbitTests/OrbitTests.swift @@ -155,13 +155,13 @@ struct OrbitTests { #expect(presentationDestination == .systemSettings) } - @Test func knownGrantedScreenRecordingPermissionSkipsTheGate() async throws { + @Test func historicalScreenRecordingPermissionDoesNotBypassTheLiveGate() async throws { let shouldTreatPermissionAsGranted = WindowPositionManager.shouldTreatScreenRecordingPermissionAsGrantedForSessionLaunch( hasScreenRecordingPermissionNow: false, hasPreviouslyConfirmedScreenRecordingPermission: true ) - #expect(shouldTreatPermissionAsGranted) + #expect(!shouldTreatPermissionAsGranted) } @Test func temporaryCaptureLeaseUsesPrivatePermissionsAndDeletesOnRelease() async throws { @@ -183,29 +183,57 @@ struct OrbitTests { #expect(!FileManager.default.fileExists(atPath: lease.fileURL.path)) } - @Test func startupSweepDeletesOnlyStaleOrbitOwnedCaptures() async throws { + @Test func startupSweepDeletesFreshOrphanedOrbitCapturesOnly() async throws { let root = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString, isDirectory: true) let captureDirectory = root.appendingPathComponent(OrbitTemporaryCaptureLease.directoryName, isDirectory: true) try FileManager.default.createDirectory(at: captureDirectory, withIntermediateDirectories: true) defer { try? FileManager.default.removeItem(at: root) } - let staleCapture = captureDirectory.appendingPathComponent("\(OrbitTemporaryCaptureLease.filePrefix)stale.jpg") - let freshCapture = captureDirectory.appendingPathComponent("\(OrbitTemporaryCaptureLease.filePrefix)fresh.jpg") + let orphanedCapture = captureDirectory.appendingPathComponent( + "\(OrbitTemporaryCaptureLease.filePrefix)\(UUID().uuidString).jpg" + ) + let lookalike = captureDirectory.appendingPathComponent( + "\(OrbitTemporaryCaptureLease.filePrefix)not-a-uuid.jpg" + ) let unrelated = captureDirectory.appendingPathComponent("other-app.tmp") - FileManager.default.createFile(atPath: staleCapture.path, contents: Data(), attributes: nil) - FileManager.default.createFile(atPath: freshCapture.path, contents: Data(), attributes: nil) - FileManager.default.createFile(atPath: unrelated.path, contents: Data(), attributes: nil) - try FileManager.default.setAttributes( - [.modificationDate: Date(timeIntervalSinceNow: -(OrbitTemporaryCaptureLease.staleAge + 60))], - ofItemAtPath: staleCapture.path + let symlink = captureDirectory.appendingPathComponent( + "\(OrbitTemporaryCaptureLease.filePrefix)\(UUID().uuidString).jpg" ) + FileManager.default.createFile(atPath: orphanedCapture.path, contents: Data(), attributes: nil) + FileManager.default.createFile(atPath: lookalike.path, contents: Data(), attributes: nil) + FileManager.default.createFile(atPath: unrelated.path, contents: Data(), attributes: nil) + try FileManager.default.createSymbolicLink(at: symlink, withDestinationURL: unrelated) - await OrbitTemporaryCaptureLease.sweepStaleCaptures(temporaryDirectory: root) + await OrbitTemporaryCaptureLease.sweepOrphanedCaptures(temporaryDirectory: root) - #expect(!FileManager.default.fileExists(atPath: staleCapture.path)) - #expect(FileManager.default.fileExists(atPath: freshCapture.path)) + #expect(!FileManager.default.fileExists(atPath: orphanedCapture.path)) + #expect(FileManager.default.fileExists(atPath: lookalike.path)) #expect(FileManager.default.fileExists(atPath: unrelated.path)) + #expect(try symlink.resourceValues(forKeys: [.isSymbolicLinkKey]).isSymbolicLink == true) + } + + @Test func currentScreenSelectionUsesTheDisplayContainingTheCursor() { + let selected = OrbitScreenCaptureUtility.selectedDisplayID( + cursorLocation: CGPoint(x: 1_250, y: 400), + screenFramesByDisplayID: [ + 10: CGRect(x: 0, y: 0, width: 1_000, height: 800), + 20: CGRect(x: 1_000, y: 0, width: 1_000, height: 800), + ], + availableDisplayIDs: [10, 20] + ) + + #expect(selected == 20) + } + + @Test func currentScreenSelectionFallsBackDeterministically() { + let selected = OrbitScreenCaptureUtility.selectedDisplayID( + cursorLocation: CGPoint(x: -5_000, y: -5_000), + screenFramesByDisplayID: [10: CGRect(x: 0, y: 0, width: 1_000, height: 800)], + availableDisplayIDs: [20, 10] + ) + + #expect(selected == 20) } @Test func modelInstructionsStayTopLevelInGeneratedCodexConfig() async throws { diff --git a/OrbitTests/OrbitVoiceTests.swift b/OrbitTests/OrbitVoiceTests.swift new file mode 100644 index 0000000..bcc72ec --- /dev/null +++ b/OrbitTests/OrbitVoiceTests.swift @@ -0,0 +1,198 @@ +import AVFoundation +import Foundation +import Testing + +@testable import Orbit + +@MainActor +private final class OrbitVoiceTestProvider: TextToSpeechProvider { + let displayName: String + let isConfigured: Bool + let unavailableExplanation: String? + private(set) var spokenTexts: [String] = [] + private(set) var stopCount = 0 + var error: Error? + + init( + displayName: String, + isConfigured: Bool = true, + error: Error? = nil + ) { + self.displayName = displayName + self.isConfigured = isConfigured + self.unavailableExplanation = isConfigured ? nil : "Unavailable" + self.error = error + } + + var isPlaying: Bool { false } + + func speakText(_ text: String) async throws { + if let error { throw error } + spokenTexts.append(text) + } + + func stopPlayback() { + stopCount += 1 + } +} + +private struct OrbitVoiceTestError: Error {} + +nonisolated private final class OrbitVoiceTestLockedLevel: @unchecked Sendable { + private let lock = NSLock() + private var value: Float? + + func store(_ value: Float) { + lock.withLock { self.value = value } + } + + func load() -> Float? { + lock.withLock { value } + } +} + +nonisolated private final class OrbitVoiceTestAudioBuffer: @unchecked Sendable { + let value: AVAudioPCMBuffer + + init(_ value: AVAudioPCMBuffer) { + self.value = value + } +} + +@MainActor +struct OrbitNarrationPrimitiveTests { + @Test func formatterRemovesVisualMarkupAndBoundsSpeech() { + let formatted = OrbitNarrationFormatter.spokenText( + from: """ + ## Finished + - Open [the report](https://example.com/report). + - Value is `42`. [POINT:none] + """, + maximumLength: 60 + ) + + #expect(formatted == "Finished Open the report. Value is 42.") + } + + @Test func deduplicatorSuppressesOverlappingSpeechOnlyWithinSameTurn() { + var deduplicator = OrbitNarrationDeduplicator(retentionInterval: 60) + let now = Date() + + let firstResult = deduplicator.shouldSpeak( + "Opening the report.", + turnIdentifier: "turn-a", + now: now + ) + let duplicateResult = deduplicator.shouldSpeak( + "Opening the report. I will check every page.", + turnIdentifier: "turn-a", + now: now + ) + let otherTurnResult = deduplicator.shouldSpeak( + "Opening the report.", + turnIdentifier: "turn-b", + now: now + ) + + #expect(firstResult) + #expect(!duplicateResult) + #expect(otherTurnResult) + } + + @Test func noraProbeRejectsDefaultFallbackAudio() { + let candidate = Data([1, 2, 3] + Array(repeating: 4, count: 80)) + let different = Data([1, 2, 3] + Array(repeating: 5, count: 80)) + + #expect( + !OrbitSayAudioRenderer.rendersDistinctVoice( + candidateAudio: candidate, + fallbackAudio: candidate + ) + ) + #expect( + OrbitSayAudioRenderer.rendersDistinctVoice( + candidateAudio: candidate, + fallbackAudio: different + ) + ) + } + + @Test func installedNoraProducesDistinctLocalAudioWhenPresent() async { + let isAvailable = await OrbitNoraVoiceAvailability.probe(forceRefresh: true) + print("Nora Premium distinct local render available: \(isAvailable)") + guard isAvailable else { return } + #expect(isAvailable) + } + + @Test func secureSpeechFilesUseOwnerOnlyPermissions() throws { + let url = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "permission-test") + defer { try? FileManager.default.removeItem(at: url) } + + let attributes = try FileManager.default.attributesOfItem(atPath: url.path) + #expect((attributes[.posixPermissions] as? NSNumber)?.intValue == 0o600) + } + + @Test func microphoneTestTapRunsOutsideMainActor() async throws { + let format = try #require(AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 1)) + let buffer = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 4)) + buffer.frameLength = 4 + let samples = try #require(buffer.floatChannelData?[0]) + samples[0] = 0.01 + samples[1] = -0.02 + samples[2] = 0.03 + samples[3] = -0.04 + + let reportedLevel = OrbitVoiceTestLockedLevel() + let tap = OrbitMicrophoneLevelMonitor.makeLevelTapBlock { level in + reportedLevel.store(level) + } + let sendableBuffer = OrbitVoiceTestAudioBuffer(buffer) + + await Task.detached { + tap( + sendableBuffer.value, + AVAudioTime(sampleTime: 0, atRate: 48_000) + ) + }.value + + #expect((reportedLevel.load() ?? 0) > 0) + } +} + +@MainActor +struct OrbitVoiceCoordinatorTests { + @Test func fallsBackWhenPrimaryProviderFails() async throws { + let primary = OrbitVoiceTestProvider( + displayName: "Primary", + error: OrbitVoiceTestError() + ) + let fallback = OrbitVoiceTestProvider(displayName: "Fallback") + let coordinator = OrbitVoiceCoordinator(primary: primary, fallback: fallback) + + try await coordinator.speak( + OrbitNarrationRequest( + text: "**Done.** [POINT:none]", + turnIdentifier: "turn-a" + ) + ) + + #expect(fallback.spokenTexts == ["Done."]) + } + + @Test func suppressesDuplicateNarrationWithinTurn() async throws { + let provider = OrbitVoiceTestProvider(displayName: "Primary") + let coordinator = OrbitVoiceCoordinator(primary: provider) + + try await coordinator.speak( + OrbitNarrationRequest(text: "Opening the report.", turnIdentifier: "turn-a") + ) + try await coordinator.speak( + OrbitNarrationRequest( + text: "Opening the report. I will inspect it now.", + turnIdentifier: "turn-a" + ) + ) + + #expect(provider.spokenTexts == ["Opening the report."]) + } +} diff --git a/PRODUCT.md b/PRODUCT.md index 0d41403..098fddf 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -10,7 +10,15 @@ Orbit is for macOS users who want a capable Codex assistant available from any a ## Product Purpose -Orbit gives every request current-screen context, routes it through a persistent local Codex session, and returns useful spoken, visual, or desktop guidance. Success means the user can ask once, understand what Orbit is doing, and receive a verified result without repeated permission dialogs or setup friction. +Orbit gives every request current-screen context, routes it through one warm Codex thread, and returns useful spoken, visual, or desktop guidance. Success means the user can ask once, understand what Orbit is doing, and receive a verified result without repeated permission dialogs or setup friction. + +## Product Contracts + +- **One warm thread.** Model, reasoning effort, and service tier are connected controls backed by the signed-in account's authoritative Codex catalog. Changing them does not discard the current thread. +- **Explicit context reset.** Agent Folder sets the working directory and begins fresh Codex context. It does not restrict filesystem access. +- **Local means local.** Local recognition requires Apple's on-device path. Local narration prefers an available Nora Premium voice through macOS speech, deletes its temporary AIFF after use, and falls back to public AVFoundation speech when Nora is unavailable. +- **Cloud is explicit.** Cloud recognition and narration are selected together, disclose that audio is sent to OpenAI, and require a Keychain-backed API key. +- **One capture per request.** Orbit never sends a request without its fresh screen capture and never presents capture as continuous recording. ## Brand Personality @@ -31,6 +39,7 @@ Quiet, capable, candid. Orbit should feel like a precise macOS instrument: calm - **Platform-native first.** Use familiar macOS controls, keyboard behavior, terminology, and accessibility semantics. - **State must earn motion.** Animation communicates listening, progress, dragging, success, or interruption only. - **Reliable recovery is part of the interface.** Every denied, unavailable, disconnected, or stale state offers a specific next action. +- **Connected controls explain connected behavior.** Model, effort, and tier appear as one configuration; input and narration privacy follow the selected Local or Cloud mode. ## Accessibility & Inclusion diff --git a/README.md b/README.md index 0b4882f..443827f 100644 --- a/README.md +++ b/README.md @@ -67,21 +67,24 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ ## How Orbit works - Orbit runs as a menu bar app with a compact panel and overlay HUD. -- Each app run keeps one live Codex app-server session active in the background. +- Each app run keeps one Codex app-server process and one warm Codex thread active in the background. Model, reasoning effort, and service tier changes apply within that thread; changing Agent Folder starts fresh context in the selected folder. +- The model picker is populated from the signed-in account's live `model/list` response. Orbit does not invent preview models or assume fixed reasoning and service-tier options. - Orbit requires and attaches one fresh current-screen capture to every submitted request. It does not continuously record the screen and shows no persistent capture indicator. - If capture fails, Orbit blocks the request and offers a retry instead of silently sending without visual context. - Codex runs with `danger-full-access`, `approval_policy = "never"`, and automatically accepted app-server approvals. Orbit can run commands and edit files without asking for each operation. - Team-up remains explicit: child agents appear only when you ask for them or applicable instructions require delegation. - Final responses can include pointing tags, which Orbit turns into cursor guidance on screen. -- Voice, auth, and action settings stay lightweight and local to the machine. +- The connected Codex settings group keeps model, supported effort, and service tier together. Local and Cloud voice states use separate, explicit privacy copy. ## Voice modes ### Local - Apple on-device speech recognition (Orbit refuses silent network fallback) -- Apple `AVSpeechSynthesizer` voices, synthesized on-device -- installed voice and microphone selection with local preview/level test +- Nora Premium narration through macOS `/usr/bin/say` when that installed voice is available +- private temporary AIFF rendering (`0700` directory, `0600` files), local playback, and deletion after playback, failure, interruption, or cancellation +- public AVFoundation speech as the automatic local fallback when Nora is unavailable +- microphone selection, local preview, and live level test - no extra API key required ### Cloud @@ -97,8 +100,9 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ - Cloud voice uses the user-supplied OpenAI API key stored in Keychain. - Orbit keeps its Codex runtime state in `~/Library/Application Support/Orbit/CodexHome`. - Temporary captures use mode `0600` in an Orbit-owned temporary directory and are swept after terminal turn states or crash recovery. +- Nora narration files are separate Orbit-owned temporary AIFF files. They are mode `0600`, removed after use, and limited to stale-file cleanup inside Orbit's local-speech directory. - Support logs are private, bounded, rotated, allowlisted, and redact credentials, prompts, capture paths, home paths, and full command arguments. -- Orbit intentionally grants Codex unrestricted filesystem and command access. The optional Agent Folder changes starting context, not the security boundary. +- Orbit intentionally grants Codex unrestricted filesystem and command access. The optional Agent Folder starts fresh working context in that folder; it does not change the security boundary. Read [SECURITY.md](SECURITY.md) before reporting vulnerabilities. diff --git a/SECURITY.md b/SECURITY.md index f6abcb8..a7f32a1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -41,7 +41,10 @@ We will aim to: - Orbit launches Codex with `danger-full-access` and `approval_policy = "never"`, and automatically accepts app-server approval requests. This is an intentional product contract: Codex can run commands and edit files without per-operation prompts. - Every submitted request requires one fresh screen capture. Orbit does not continuously record; if capture fails, the request is not sent. - Each temporary capture is owned by its turn, stored with mode `0600`, deleted on every terminal/cancellation path, and eligible for Orbit-only crash-recovery cleanup on the next launch. -- Local voice uses public Apple on-device speech APIs. Cloud speech is optional, explicitly selected, disclosed as AI-generated voice, and Keychain-backed. +- Local recognition requires Apple's on-device speech path and never silently falls back to a network recognizer. +- Local narration prefers an available Nora Premium voice through macOS `/usr/bin/say`. Orbit renders speech into its own mode `0700` temporary directory as a mode `0600` AIFF, plays it locally, and removes it after playback, failure, interruption, or cancellation. Startup cleanup is limited to stale AIFF files inside that Orbit-owned local-speech directory. If Nora is unavailable, Orbit falls back to public AVFoundation speech synthesis. +- Cloud speech is optional and explicitly selected. Speech audio is sent to OpenAI, AI-generated narration is disclosed, and the user-supplied API key is stored in Keychain. +- Orbit keeps one warm Codex thread while model, reasoning effort, or service tier changes. Agent Folder deliberately starts fresh context but does not reduce `danger-full-access`. - Support logs are mode `0600`, capped at 5 MiB with three rotations, and redact credentials, authorization headers, prompt text, capture paths, home paths, and full command arguments. - Public release tooling fails closed unless the app, PKG, and DMG pass signing, notarization, stapling, and Gatekeeper checks. diff --git a/SUPPORT.md b/SUPPORT.md index 6e16eb8..183dfbc 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -12,9 +12,17 @@ - your Orbit version or commit - macOS version - whether you installed via PKG or DMG -- whether the issue is about permissions, Codex auth, cloud voice, or overlay behavior +- whether the issue is about permissions, Codex auth, Local or Cloud voice, model availability, Agent Folder, or overlay behavior +- for voice issues, whether Orbit reports Nora Premium or the Apple fallback and which microphone is selected +- for Codex configuration issues, the model, reasoning effort, service tier, and whether Agent Folder was changed - screenshots when relevant +## Expected behavior + +- Model choices come from the current account's Codex catalog and can differ between accounts. +- Effort and service-tier choices follow the selected model. Changing them keeps the current Codex thread; changing Agent Folder starts fresh context. +- Local mode keeps recognition on device and uses Nora Premium when available, with an Apple AVFoundation narration fallback. Cloud mode sends speech audio to OpenAI and requires a Keychain-backed API key. + ## What Orbit does not provide - a hosted backend diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md index d615d0e..917839d 100644 --- a/docs/PRIVACY.md +++ b/docs/PRIVACY.md @@ -4,6 +4,10 @@ Orbit captures the current screen once for every submitted request. It does not The request is blocked if the capture cannot be created. A successful capture is written to an Orbit-owned temporary directory with mode `0600`, associated with that turn, and removed after success, failure, interruption, cancellation, launch failure, or app termination. On launch, Orbit sweeps only stale files that match its own capture naming contract; unrelated temporary files are never touched. -Local speech recognition requires Apple on-device recognition. Orbit does not silently fall back to network recognition. Local speech output uses public AVFoundation voices on-device. OpenAI cloud speech is optional, must be explicitly selected, and stores its API key in macOS Keychain. +Local speech recognition requires Apple on-device recognition. Orbit does not silently fall back to network recognition. -Orbit has no hosted backend of its own. Codex and explicitly selected cloud providers still receive the content needed to perform the request. Because Codex runs unrestricted with automatic approvals, it can execute commands and read or edit files available to the signed-in macOS user. +Local narration first checks whether Nora Premium is available through the Mac's local speech service. When available, Orbit invokes `/usr/bin/say` without a shell, writes a temporary AIFF into `com.orbit.codex/local-speech`, plays it locally, and deletes it after playback, failure, interruption, or cancellation. The directory is mode `0700`; each AIFF is mode `0600`. A launch-time sweep removes only stale `.aiff` files from that Orbit-owned directory. When Nora is unavailable, Orbit uses its public AVFoundation fallback. + +OpenAI Cloud voice is optional and must be explicitly selected. In Cloud mode, speech audio is sent to OpenAI for transcription and narration is AI-generated. The API key is stored in macOS Keychain. Orbit's interface does not describe Cloud input as staying on the Mac. + +Orbit has no hosted backend of its own. Codex and explicitly selected cloud providers still receive the content needed to perform the request. Because Codex runs unrestricted with automatic approvals, it can execute commands and read or edit files available to the signed-in macOS user. Agent Folder starts new working context in the chosen directory; it does not limit that access. diff --git a/docs/SETUP.md b/docs/SETUP.md index 586eb18..697987e 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -10,4 +10,15 @@ For Accessibility and Screen Recording, Orbit opens the correct System Settings After permissions, Orbit explains its unrestricted automation contract once. Codex runs with full filesystem/command access and automatic approvals. There is deliberately no approval toggle or restricted mode in v1.1.0. -The optional Agent Folder selects the starting working context. It does not restrict which files Codex can access. +## Voice setup + +- **Local** keeps recognition on device. Narration uses Nora Premium through the Mac's local speech service when Orbit verifies that voice is available, with public AVFoundation speech as the fallback. No OpenAI voice key is required. +- **Cloud** sends speech audio to OpenAI for transcription and uses AI-generated OpenAI narration. It is enabled only after explicit selection and a successful Keychain-backed API-key check. + +Orbit keeps local narration audio only long enough to play it. Nora output is a private temporary AIFF and is deleted after completion, failure, interruption, or cancellation. + +## Codex context + +Orbit prewarms one Codex app-server thread. The Model menu comes from the signed-in account's live `model/list` response; reasoning effort and service tier show only values supported by the selected model. Changing those three connected controls keeps the current thread warm. + +The optional Agent Folder changes the working directory and starts fresh Codex context. If a turn is active, Orbit applies the folder after that turn. Agent Folder does not restrict which files Codex can access. diff --git a/release-manifest.json b/release-manifest.json index 9ebb40c..d0736cc 100644 --- a/release-manifest.json +++ b/release-manifest.json @@ -1,8 +1,9 @@ { - "version": "1.0.7", - "downloadURL": "https://github.com/4xiomdev/orbit/releases/download/v1.0.7/Orbit-1.0.7.pkg", - "sha256": "bf377f3014632bf7e4a829404d7e0ea71f5a857282ebd4c30f29524ae7ee6c25", + "releaseStatus": "unpublished", + "version": "1.1.0", + "downloadURL": "", + "sha256": "", "minimumMacOS": "14.2", - "codexRuntimeVersion": "0.118.0", - "browserMCPVersion": "0.21.0" + "codexRuntimeVersion": "0.144.0", + "browserMCPVersion": "1.5.0" } diff --git a/scripts/release.sh b/scripts/release.sh index 0c7c66d..edfc857 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -17,6 +17,7 @@ GITHUB_REPO="${GITHUB_REPO:-}" ORBIT_SKIP_GITHUB_RELEASE="${ORBIT_SKIP_GITHUB_RELEASE:-0}" ORBIT_LOCAL_UNSIGNED_BUILD="${ORBIT_LOCAL_UNSIGNED_BUILD:-0}" BUNDLE_RUNTIME_SCRIPT="${PROJECT_DIR}/scripts/bundle_codex_runtime.sh" +VALIDATE_RELEASE_MANIFEST_SCRIPT="${PROJECT_DIR}/scripts/validate_release_manifest.py" GENERATE_BRAND_ASSETS_SCRIPT="${PROJECT_DIR}/scripts/generate_brand_assets.swift" DEVELOPMENT_TEAM_ID="${ORBIT_DEVELOPMENT_TEAM:-}" DEVELOPER_ID_IDENTITY="${ORBIT_DEVELOPER_ID_IDENTITY:-}" @@ -95,6 +96,27 @@ PKG_ROOT="${BUILD_DIR}/pkg-root" PKG_SCRIPTS_DIR="${PROJECT_DIR}/scripts/installer" PKG_COMPONENT_PLIST="${BUILD_DIR}/components.plist" EXPORT_OPTIONS="${BUILD_DIR}/ExportOptions.plist" +SOURCE_RELEASE_MANIFEST="${PROJECT_DIR}/release-manifest.json" +GENERATED_RELEASE_MANIFEST="${BUILD_DIR}/release-manifest.json" + +python3 "${VALIDATE_RELEASE_MANIFEST_SCRIPT}" \ + "${SOURCE_RELEASE_MANIFEST}" \ + --project-root "${PROJECT_DIR}" +if [[ "${MARKETING_VERSION}" != "${DEFAULT_MARKETING_VERSION}" ]]; then + echo "❌ Release ${RELEASE_VERSION} does not match project marketing version ${DEFAULT_MARKETING_VERSION}." + exit 1 +fi + +if [[ "${ORBIT_SKIP_GITHUB_RELEASE}" != "1" ]]; then + if [[ -z "${GITHUB_REPO}" ]]; then + echo "❌ GITHUB_REPO is required for publication. Set ORBIT_SKIP_GITHUB_RELEASE=1 for a local artifact-only run." + exit 1 + fi + if ! command -v gh >/dev/null 2>&1; then + echo "❌ GitHub CLI is required for publication. Set ORBIT_SKIP_GITHUB_RELEASE=1 for a local artifact-only run." + exit 1 + fi +fi echo "🚀 Releasing ${APP_NAME} ${TAG} (build ${BUILD_NUMBER})" if [[ -n "${GITHUB_REPO}" ]]; then @@ -164,6 +186,12 @@ fi EXPORT_APP_PATH="${EXPORT_DIR}/${APP_NAME}.app" NODE_ENTITLEMENTS_PATH="${PROJECT_DIR}/Orbit/CodexRuntimeNode.entitlements" +RUNTIME_MANIFEST_PATH="${EXPORT_APP_PATH}/Contents/Resources/CodexRuntime/OrbitRuntimeManifest.json" +RUNTIME_SBOM_PATH="${EXPORT_APP_PATH}/Contents/Resources/CodexRuntime/OrbitRuntimeSBOM.cdx.json" +if [[ ! -f "${RUNTIME_MANIFEST_PATH}" || ! -f "${RUNTIME_SBOM_PATH}" ]]; then + echo "❌ Bundled runtime manifest or SBOM is missing from the exported app." + exit 1 +fi if [[ -f "${EXPORT_APP_PATH}/Contents/Resources/LocalSecrets.plist" ]]; then echo "🧼 Removing bundled LocalSecrets from release app..." rm -f "${EXPORT_APP_PATH}/Contents/Resources/LocalSecrets.plist" @@ -283,12 +311,49 @@ fi if [[ "${ORBIT_SKIP_GITHUB_RELEASE}" == "1" ]]; then echo "⚠️ Skipping GitHub release creation because ORBIT_SKIP_GITHUB_RELEASE=1." -elif [[ -n "${GITHUB_REPO}" ]] && command -v gh >/dev/null 2>&1; then +else + PUBLISHED_DOWNLOAD_URL="https://github.com/${GITHUB_REPO}/releases/download/${TAG}/${APP_NAME}-${RELEASE_VERSION}.pkg" + PKG_SHA256="$(shasum -a 256 "${PKG_PATH}" | awk '{print $1}')" + python3 - \ + "${GENERATED_RELEASE_MANIFEST}" \ + "${RELEASE_VERSION}" \ + "${PUBLISHED_DOWNLOAD_URL}" \ + "${PKG_SHA256}" \ + "${SOURCE_RELEASE_MANIFEST}" <<'PY' +import json +import sys +from pathlib import Path + +destination, version, download_url, checksum, source = sys.argv[1:] +manifest = json.loads(Path(source).read_text(encoding="utf-8")) +manifest.update( + { + "releaseStatus": "published", + "version": version, + "downloadURL": download_url, + "sha256": checksum, + } +) +Path(destination).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") +PY + + python3 "${VALIDATE_RELEASE_MANIFEST_SCRIPT}" \ + "${GENERATED_RELEASE_MANIFEST}" \ + --project-root "${PROJECT_DIR}" \ + --require-published \ + --expected-version "${RELEASE_VERSION}" \ + --expected-download-url "${PUBLISHED_DOWNLOAD_URL}" \ + --artifact "${PKG_PATH}" \ + --runtime-manifest "${RUNTIME_MANIFEST_PATH}" + echo "🏷️ Creating GitHub release ${TAG}..." - RELEASE_ASSETS=("${DMG_PATH}") - if [[ -f "${PKG_PATH}" ]]; then - RELEASE_ASSETS+=("${PKG_PATH}") - fi + RELEASE_ASSETS=( + "${DMG_PATH}" + "${PKG_PATH}" + "${GENERATED_RELEASE_MANIFEST}" + "${RUNTIME_MANIFEST_PATH}" + "${RUNTIME_SBOM_PATH}" + ) RELEASE_FLAGS=() if [[ "${TAG}" == *"-rc."* ]]; then RELEASE_FLAGS+=(--prerelease) @@ -298,8 +363,6 @@ elif [[ -n "${GITHUB_REPO}" ]] && command -v gh >/dev/null 2>&1; then --title "${TAG}" \ --notes "Orbit ${TAG}" \ "${RELEASE_FLAGS[@]}" -else - echo "⚠️ Skipping GitHub release creation because repo is not configured or GitHub CLI is unavailable." fi echo "" diff --git a/scripts/tests/test_validate_release_manifest.py b/scripts/tests/test_validate_release_manifest.py new file mode 100644 index 0000000..3c16cc6 --- /dev/null +++ b/scripts/tests/test_validate_release_manifest.py @@ -0,0 +1,83 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "validate_release_manifest", + PROJECT_ROOT / "scripts" / "validate_release_manifest.py", +) +assert SPEC is not None and SPEC.loader is not None +VALIDATOR = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(VALIDATOR) + + +class ReleaseManifestValidationTests(unittest.TestCase): + def test_checked_in_unpublished_manifest_matches_repository(self) -> None: + data = VALIDATOR.validate_manifest( + PROJECT_ROOT / "release-manifest.json", + PROJECT_ROOT, + ) + self.assertEqual(data["version"], "1.1.0") + self.assertEqual(data["codexRuntimeVersion"], "0.144.0") + self.assertEqual(data["browserMCPVersion"], "1.5.0") + + def test_unpublished_manifest_rejects_invented_artifact_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + manifest_path = Path(temporary_directory) / "release-manifest.json" + data = json.loads( + (PROJECT_ROOT / "release-manifest.json").read_text(encoding="utf-8") + ) + data["downloadURL"] = "https://example.com/not-published.pkg" + data["sha256"] = "0" * 64 + manifest_path.write_text(json.dumps(data), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "must leave downloadURL"): + VALIDATOR.validate_manifest(manifest_path, PROJECT_ROOT) + + def test_published_manifest_must_match_artifact_checksum(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + artifact = root / "Orbit-1.1.0.pkg" + artifact.write_bytes(b"signed package fixture") + manifest_path = root / "release-manifest.json" + data = json.loads( + (PROJECT_ROOT / "release-manifest.json").read_text(encoding="utf-8") + ) + data.update( + { + "releaseStatus": "published", + "downloadURL": ( + "https://github.com/4xiomdev/orbit/releases/download/" + "v1.1.0/Orbit-1.1.0.pkg" + ), + "sha256": VALIDATOR.sha256(artifact), + } + ) + manifest_path.write_text(json.dumps(data), encoding="utf-8") + + validated = VALIDATOR.validate_manifest( + manifest_path, + PROJECT_ROOT, + require_published=True, + expected_version="1.1.0", + expected_download_url=data["downloadURL"], + artifact=artifact, + ) + self.assertEqual(validated["sha256"], VALIDATOR.sha256(artifact)) + + artifact.write_bytes(b"tampered package") + with self.assertRaisesRegex(ValueError, "does not match artifact"): + VALIDATOR.validate_manifest( + manifest_path, + PROJECT_ROOT, + require_published=True, + artifact=artifact, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate_release_manifest.py b/scripts/validate_release_manifest.py index c21bc35..1bcf545 100644 --- a/scripts/validate_release_manifest.py +++ b/scripts/validate_release_manifest.py @@ -1,15 +1,90 @@ #!/usr/bin/env python3 +"""Validate Orbit release metadata against the repository and built artifact.""" + +from __future__ import annotations + +import argparse +import hashlib import json import re -import sys from pathlib import Path +from typing import Any from urllib.parse import urlparse -def main() -> int: - path = Path(sys.argv[1] if len(sys.argv) > 1 else "release-manifest.json") +VERSION_PATTERN = re.compile(r"\d+(?:\.\d+){1,2}(?:-[0-9A-Za-z.-]+)?") +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") + + +def single_match(path: Path, pattern: str, label: str) -> str: + matches = sorted(set(re.findall(pattern, path.read_text(encoding="utf-8"), re.MULTILINE))) + if len(matches) != 1: + raise ValueError(f"expected one {label} in {path}, found: {matches}") + return matches[0] + + +def repository_contract(project_root: Path) -> dict[str, str]: + project_file = project_root / "Orbit.xcodeproj" / "project.pbxproj" + bundler = project_root / "scripts" / "bundle_codex_runtime.sh" + browser_package = project_root / "BundledResources" / "browser-runtime" / "package.json" + + project_version = single_match( + project_file, + r"^\s*MARKETING_VERSION = ([^;]+);$", + "MARKETING_VERSION", + ) + minimum_macos = single_match( + project_file, + r"^\s*MACOSX_DEPLOYMENT_TARGET = ([^;]+);$", + "MACOSX_DEPLOYMENT_TARGET", + ) + codex_version = single_match( + bundler, + r'^EXPECTED_CODEX_VERSION="([^"]+)"$', + "EXPECTED_CODEX_VERSION", + ) + browser_version = single_match( + bundler, + r'^EXPECTED_BROWSER_MCP_VERSION="([^"]+)"$', + "EXPECTED_BROWSER_MCP_VERSION", + ) + package_data = json.loads(browser_package.read_text(encoding="utf-8")) + package_browser_version = package_data.get("dependencies", {}).get("chrome-devtools-mcp") + if package_browser_version != browser_version: + raise ValueError( + "browser runtime mismatch: " + f"bundle script expects {browser_version}, package.json pins {package_browser_version}" + ) + + return { + "version": project_version, + "minimumMacOS": minimum_macos, + "codexRuntimeVersion": codex_version, + "browserMCPVersion": browser_version, + } + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_manifest( + path: Path, + project_root: Path, + *, + require_published: bool = False, + expected_version: str | None = None, + expected_download_url: str | None = None, + artifact: Path | None = None, + runtime_manifest: Path | None = None, +) -> dict[str, Any]: data = json.loads(path.read_text(encoding="utf-8")) required = { + "releaseStatus", "version", "downloadURL", "sha256", @@ -20,20 +95,107 @@ def main() -> int: missing = sorted(required - data.keys()) if missing: raise ValueError(f"missing manifest fields: {', '.join(missing)}") - if any(not isinstance(data[key], str) or not data[key].strip() for key in required): - raise ValueError("all release manifest fields must be non-empty strings") - if not re.fullmatch(r"[0-9a-f]{64}", data["sha256"]): - raise ValueError("sha256 must be 64 lowercase hexadecimal characters") - parsed = urlparse(data["downloadURL"]) - if parsed.scheme != "https" or not parsed.netloc: - raise ValueError("downloadURL must be an absolute HTTPS URL") - version_pattern = r"\d+(?:\.\d+){1,2}(?:-[0-9A-Za-z.-]+)?" + + release_status = data["releaseStatus"] + if release_status not in {"unpublished", "published"}: + raise ValueError("releaseStatus must be unpublished or published") + if require_published and release_status != "published": + raise ValueError("a publishable release requires releaseStatus=published") + + for key in ("version", "minimumMacOS", "codexRuntimeVersion", "browserMCPVersion"): + if not isinstance(data[key], str) or not data[key].strip(): + raise ValueError(f"{key} must be a non-empty string") for key in ("version", "codexRuntimeVersion", "browserMCPVersion"): - if not re.fullmatch(version_pattern, data[key]): + if not VERSION_PATTERN.fullmatch(data[key]): raise ValueError(f"{key} is not a supported version string") if not re.fullmatch(r"\d+\.\d+(?:\.\d+)?", data["minimumMacOS"]): raise ValueError("minimumMacOS is not a supported macOS version") - print(f"validated {path}: Orbit {data['version']}") + + contract = repository_contract(project_root) + manifest_base_version = data["version"].split("-", 1)[0] + if manifest_base_version != contract["version"]: + raise ValueError( + f"manifest version {data['version']} does not match project {contract['version']}" + ) + for key in ("minimumMacOS", "codexRuntimeVersion", "browserMCPVersion"): + if data[key] != contract[key]: + raise ValueError( + f"manifest {key}={data[key]} does not match repository {contract[key]}" + ) + if runtime_manifest is not None: + runtime_data = json.loads(runtime_manifest.read_text(encoding="utf-8")) + for key in ("codexRuntimeVersion", "browserMCPVersion"): + if runtime_data.get(key) != data[key]: + raise ValueError( + f"runtime manifest {key}={runtime_data.get(key)} " + f"does not match release manifest {data[key]}" + ) + if expected_version is not None and data["version"] != expected_version: + raise ValueError( + f"manifest version {data['version']} does not match release {expected_version}" + ) + + download_url = data["downloadURL"] + checksum = data["sha256"] + if not isinstance(download_url, str) or not isinstance(checksum, str): + raise ValueError("downloadURL and sha256 must be strings") + + if release_status == "unpublished": + if download_url or checksum: + raise ValueError( + "unpublished manifests must leave downloadURL and sha256 empty" + ) + if artifact is not None or expected_download_url is not None: + raise ValueError("artifact validation requires a published manifest") + else: + parsed = urlparse(download_url) + if parsed.scheme != "https" or not parsed.netloc: + raise ValueError("downloadURL must be an absolute HTTPS URL") + if not SHA256_PATTERN.fullmatch(checksum): + raise ValueError("sha256 must be 64 lowercase hexadecimal characters") + if expected_download_url is not None and download_url != expected_download_url: + raise ValueError( + f"manifest downloadURL {download_url} does not match {expected_download_url}" + ) + if artifact is not None: + actual_checksum = sha256(artifact) + if checksum != actual_checksum: + raise ValueError( + f"manifest sha256 {checksum} does not match artifact {actual_checksum}" + ) + + return data + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("manifest", nargs="?", default="release-manifest.json") + parser.add_argument("--project-root", default=".") + parser.add_argument("--require-published", action="store_true") + parser.add_argument("--expected-version") + parser.add_argument("--expected-download-url") + parser.add_argument("--artifact") + parser.add_argument("--runtime-manifest") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + path = Path(args.manifest).resolve() + data = validate_manifest( + path, + Path(args.project_root).resolve(), + require_published=args.require_published, + expected_version=args.expected_version, + expected_download_url=args.expected_download_url, + artifact=Path(args.artifact).resolve() if args.artifact else None, + runtime_manifest=( + Path(args.runtime_manifest).resolve() if args.runtime_manifest else None + ), + ) + print( + f"validated {path}: Orbit {data['version']} ({data['releaseStatus']})" + ) return 0 From 4f9d4c9bbb465190ba40c13f8b1c33853d37ae55 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Fri, 10 Jul 2026 01:07:25 -0400 Subject: [PATCH 7/8] fix: use exact Shortcuts Voice 4 safely --- AGENTS.md | 6 +- DESIGN.md | 2 +- Orbit.xcodeproj/project.pbxproj | 127 ++++++++++ Orbit/AGENTS.md | 6 +- Orbit/NaturalNoraTTSProvider.swift | 233 +++++++++++++++++ Orbit/OrbitPanelView.swift | 8 +- Orbit/SayNoraTTSProvider.swift | 393 ----------------------------- Orbit/TextToSpeechProvider.swift | 2 +- OrbitNaturalVoiceHelper/main.m | 158 ++++++++++++ OrbitTests/OrbitVoiceTests.swift | 57 +++-- PRODUCT.md | 2 +- README.md | 8 +- SECURITY.md | 2 +- SUPPORT.md | 4 +- docs/PRIVACY.md | 2 +- docs/SETUP.md | 4 +- 16 files changed, 572 insertions(+), 442 deletions(-) create mode 100644 Orbit/NaturalNoraTTSProvider.swift delete mode 100644 Orbit/SayNoraTTSProvider.swift create mode 100644 OrbitNaturalVoiceHelper/main.m diff --git a/AGENTS.md b/AGENTS.md index 61e5604..5fa30cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,14 +7,14 @@ Orbit is a Codex-native macOS menu bar assistant. - UI shell: SwiftUI + AppKit panel/overlay - STT: Apple on-device recognition in Local mode; OpenAI `gpt-4o-mini-transcribe` only in explicit Cloud mode - Brain and actions: one warm Codex app-server thread with a live account model catalog -- TTS: Nora Premium through `/usr/bin/say` in Local mode, public AVFoundation fallback, and OpenAI `gpt-4o-mini-tts` in Cloud mode +- TTS: exact Siri Natural Nora / Shortcuts Voice 4 through the local macOS TextToSpeech runtime, public AVFoundation fallback, and OpenAI `gpt-4o-mini-tts` in Cloud mode ## Important files - [Orbit/OrbitManager.swift](Orbit/OrbitManager.swift) — unified Codex routing, screenshots, overlay updates, action summaries - [Orbit/OrbitDictationManager.swift](Orbit/OrbitDictationManager.swift) — push-to-talk capture and STT session management - [Orbit/OrbitVoiceCoordinator.swift](Orbit/OrbitVoiceCoordinator.swift) — provider-neutral narration formatting, interruption, fallback, and duplicate suppression -- [Orbit/SayNoraTTSProvider.swift](Orbit/SayNoraTTSProvider.swift) — Nora availability probing, secure temporary AIFF rendering, playback, and cleanup +- [Orbit/NaturalNoraTTSProvider.swift](Orbit/NaturalNoraTTSProvider.swift) and [OrbitNaturalVoiceHelper/main.m](OrbitNaturalVoiceHelper/main.m) — exact Voice 4 resolution, crash-isolated local playback, interruption, and runtime fallback - [Orbit/OrbitSettings.swift](Orbit/OrbitSettings.swift) — persisted voice mode, Codex effort, and overlay settings - [Orbit/CodexAppServerActionProvider.swift](Orbit/CodexAppServerActionProvider.swift) — persistent Codex session and event streaming - [Orbit/OverlayWindow.swift](Orbit/OverlayWindow.swift) — Orbit cursor, HUD, and pointing animations @@ -25,5 +25,5 @@ Orbit is a Codex-native macOS menu bar assistant. - Orbit uses one warm Codex thread for both answers and actions. Model, effort, and service tier changes preserve it; Agent Folder starts fresh context. - Treat app-server `model/list` as authoritative. Never hard-code account availability or discard unknown future effort and tier strings. - Keep model, effort, and service tier visually connected. Keep Local and Cloud privacy copy separate and accurate. -- Nora narration files belong only in Orbit's mode `0700` local-speech directory, use mode `0600`, and must be removed after every terminal playback path. Never invoke `/usr/bin/say` through a shell. +- Local narration selects `com.apple.siri.natural.Nora`, never uploads speech, supports interruption, and isolates the dynamically loaded macOS interface in a signed helper so the app can fail safely to public AVFoundation if that interface changes. - Menu bar settings should stay compact; prefer connected, adaptive Codex controls over sprawling configuration UI. diff --git a/DESIGN.md b/DESIGN.md index 29adfc8..f40d636 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -171,7 +171,7 @@ Agent Folder is adjacent context, not a security control. Its copy must state th ### Voice and Privacy States -Local and Cloud are distinct product states, not decorative presets. Local identifies Nora Premium when available and names the Apple fallback when it is not. Cloud states plainly that speech audio is sent to OpenAI. Never show local-only privacy copy while Cloud is selected. +Local and Cloud are distinct product states, not decorative presets. Local identifies Siri Natural Nora / Voice 4 when available and names the Apple fallback when it is not. Cloud states plainly that speech audio is sent to OpenAI. Never show local-only privacy copy while Cloud is selected. Voice controls expose provider availability, preview or stop state, microphone choice, and input level without duplicating the same status in the header. Recovery copy belongs beside the unavailable provider. diff --git a/Orbit.xcodeproj/project.pbxproj b/Orbit.xcodeproj/project.pbxproj index b8dfc56..ba05364 100644 --- a/Orbit.xcodeproj/project.pbxproj +++ b/Orbit.xcodeproj/project.pbxproj @@ -6,6 +6,10 @@ objectVersion = 77; objects = { +/* Begin PBXBuildFile section */ + A1000000000000000000000B /* OrbitNaturalVoiceHelper in Embed Voice Helper */ = {isa = PBXBuildFile; fileRef = A10000000000000000000001 /* OrbitNaturalVoiceHelper */; settings = {ATTRIBUTES = (CodeSignOnCopy, ); }; }; +/* End PBXBuildFile section */ + /* Begin PBXContainerItemProxy section */ 28F22CCD2F56440300A0FC59 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; @@ -14,11 +18,33 @@ remoteGlobalIDString = 28F22CBE2F56440300A0FC59; remoteInfo = "Orbit"; }; + A1000000000000000000000D /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 28F22CB72F56440300A0FC59 /* Project object */; + proxyType = 1; + remoteGlobalIDString = A10000000000000000000006; + remoteInfo = OrbitNaturalVoiceHelper; + }; /* End PBXContainerItemProxy section */ +/* Begin PBXCopyFilesBuildPhase section */ + A1000000000000000000000A /* Embed Voice Helper */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = Contents/Helpers; + dstSubfolderSpec = 1; + files = ( + A1000000000000000000000B /* OrbitNaturalVoiceHelper in Embed Voice Helper */, + ); + name = "Embed Voice Helper"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + /* Begin PBXFileReference section */ 28F22CBF2F56440300A0FC59 /* Orbit.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Orbit.app; sourceTree = BUILT_PRODUCTS_DIR; }; 28F22CCC2F56440300A0FC59 /* OrbitTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "OrbitTests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; + A10000000000000000000001 /* OrbitNaturalVoiceHelper */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = OrbitNaturalVoiceHelper; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -32,6 +58,11 @@ path = "OrbitTests"; sourceTree = ""; }; + A10000000000000000000002 /* OrbitNaturalVoiceHelper */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = OrbitNaturalVoiceHelper; + sourceTree = ""; + }; /* End PBXFileSystemSynchronizedRootGroup section */ /* Begin PBXFrameworksBuildPhase section */ @@ -49,6 +80,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + A10000000000000000000004 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -56,6 +94,7 @@ isa = PBXGroup; children = ( 28F22CC12F56440300A0FC59 /* Orbit */, + A10000000000000000000002 /* OrbitNaturalVoiceHelper */, 28F22CCF2F56440300A0FC59 /* OrbitTests */, 28F22CC02F56440300A0FC59 /* Products */, ); @@ -65,6 +104,7 @@ isa = PBXGroup; children = ( 28F22CBF2F56440300A0FC59 /* Orbit.app */, + A10000000000000000000001 /* OrbitNaturalVoiceHelper */, 28F22CCC2F56440300A0FC59 /* OrbitTests.xctest */, ); name = Products; @@ -80,10 +120,12 @@ 28F22CBB2F56440300A0FC59 /* Sources */, 28F22CBC2F56440300A0FC59 /* Frameworks */, 28F22CBD2F56440300A0FC59 /* Resources */, + A1000000000000000000000A /* Embed Voice Helper */, ); buildRules = ( ); dependencies = ( + A1000000000000000000000C /* PBXTargetDependency */, ); fileSystemSynchronizedGroups = ( 28F22CC12F56440300A0FC59 /* Orbit */, @@ -95,6 +137,28 @@ productReference = 28F22CBF2F56440300A0FC59 /* Orbit.app */; productType = "com.apple.product-type.application"; }; + A10000000000000000000006 /* OrbitNaturalVoiceHelper */ = { + isa = PBXNativeTarget; + buildConfigurationList = A10000000000000000000007 /* Build configuration list for PBXNativeTarget "OrbitNaturalVoiceHelper" */; + buildPhases = ( + A10000000000000000000003 /* Sources */, + A10000000000000000000004 /* Frameworks */, + A10000000000000000000005 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + A10000000000000000000002 /* OrbitNaturalVoiceHelper */, + ); + name = OrbitNaturalVoiceHelper; + packageProductDependencies = ( + ); + productName = OrbitNaturalVoiceHelper; + productReference = A10000000000000000000001 /* OrbitNaturalVoiceHelper */; + productType = "com.apple.product-type.tool"; + }; 28F22CCB2F56440300A0FC59 /* OrbitTests */ = { isa = PBXNativeTarget; buildConfigurationList = 28F22CE32F56440300A0FC59 /* Build configuration list for PBXNativeTarget "OrbitTests" */; @@ -135,6 +199,9 @@ CreatedOnToolsVersion = 26.2; TestTargetID = 28F22CBE2F56440300A0FC59; }; + A10000000000000000000006 = { + CreatedOnToolsVersion = 26.2; + }; }; }; buildConfigurationList = 28F22CBA2F56440300A0FC59 /* Build configuration list for PBXProject "Orbit" */; @@ -154,6 +221,7 @@ projectRoot = ""; targets = ( 28F22CBE2F56440300A0FC59 /* Orbit */, + A10000000000000000000006 /* OrbitNaturalVoiceHelper */, 28F22CCB2F56440300A0FC59 /* OrbitTests */, ); }; @@ -174,6 +242,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + A10000000000000000000005 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -191,6 +266,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + A10000000000000000000003 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -199,6 +281,11 @@ target = 28F22CBE2F56440300A0FC59 /* Orbit */; targetProxy = 28F22CCD2F56440300A0FC59 /* PBXContainerItemProxy */; }; + A1000000000000000000000C /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = A10000000000000000000006 /* OrbitNaturalVoiceHelper */; + targetProxy = A1000000000000000000000D /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ @@ -438,6 +525,37 @@ }; name = Release; }; + A10000000000000000000008 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + MACOSX_DEPLOYMENT_TARGET = 14.2; + ONLY_ACTIVE_ARCH = YES; + OTHER_LDFLAGS = ( + "-framework", + Foundation, + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Debug; + }; + A10000000000000000000009 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + COPY_PHASE_STRIP = YES; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + MACOSX_DEPLOYMENT_TARGET = 14.2; + OTHER_LDFLAGS = ( + "-framework", + Foundation, + ); + PRODUCT_NAME = "$(TARGET_NAME)"; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -468,6 +586,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + A10000000000000000000007 /* Build configuration list for PBXNativeTarget "OrbitNaturalVoiceHelper" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + A10000000000000000000008 /* Debug */, + A10000000000000000000009 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; diff --git a/Orbit/AGENTS.md b/Orbit/AGENTS.md index 1e2fd4a..74f48fb 100644 --- a/Orbit/AGENTS.md +++ b/Orbit/AGENTS.md @@ -8,7 +8,7 @@ - `AppleSpeechTranscriptionProvider.swift` is the default on-device macOS STT provider and never silently falls back to the network. - `OpenAITranscriptionProvider.swift` is the explicitly selected cloud STT provider using `gpt-4o-mini-transcribe`. - `OrbitVoiceCoordinator.swift` owns narration formatting, interruption, duplicate suppression, and provider fallback. -- `SayNoraTTSProvider.swift` verifies Nora Premium availability, renders through `/usr/bin/say` to a private temporary AIFF, plays it locally, and removes it on every terminal path. +- `NaturalNoraTTSProvider.swift` and the separately signed `OrbitNaturalVoiceHelper` resolve the exact Shortcuts Voice 4 identifier through macOS's local TextToSpeech runtime, stream playback, and preserve interruption/fallback behavior without exposing the app process to private-runtime crashes. - `TextToSpeechProvider.swift` defines the TTS abstraction and public AVFoundation local fallback. - `OpenAITTSProvider.swift` is the default cloud TTS provider using `gpt-4o-mini-tts`. - `OrbitOpenAIVoiceConfiguration.swift` stores the Cloud voice API key in Keychain and validates it. @@ -33,7 +33,7 @@ - Codex model default: the current app-server `model/list` default - Codex effort default: `medium` - Codex service tier default: the app-server default; optional tiers appear only when returned -- TTS default: Nora Premium through local `/usr/bin/say`, with public AVFoundation fallback +- TTS default: Siri Natural Nora / Shortcuts Voice 4 through the local macOS speech runtime, with public AVFoundation fallback - Cloud voice option: OpenAI `gpt-4o-mini-transcribe` and `gpt-4o-mini-tts` - Unified assistant path: Codex app-server - Bundled browser tools: `chrome-devtools-mcp`, `@playwright/mcp` @@ -43,5 +43,5 @@ - Model, reasoning effort, and service tier use the live account catalog and preserve the warm Codex thread. - Agent Folder changes working context by starting a fresh thread after any active turn; it never narrows filesystem access. -- Local narration uses only Orbit-owned temporary AIFF files in a mode `0700` directory, sets each file to mode `0600`, and deletes files after playback, failure, interruption, cancellation, or stale-file recovery. +- Local narration must select `com.apple.siri.natural.Nora`, remain on device, support interruption, and keep the dynamically loaded system interface isolated in the disposable helper process so Orbit can fall back safely if it changes. - Cloud voice copy must state that speech audio is sent to OpenAI. Local copy must not imply cloud transfer. diff --git a/Orbit/NaturalNoraTTSProvider.swift b/Orbit/NaturalNoraTTSProvider.swift new file mode 100644 index 0000000..b2dd770 --- /dev/null +++ b/Orbit/NaturalNoraTTSProvider.swift @@ -0,0 +1,233 @@ +import Foundation + +struct OrbitNoraVoiceUnavailableError: LocalizedError { + var errorDescription: String? { + "Siri Natural Nora (Voice 4) is not installed or is unavailable to Orbit." + } +} + +private struct OrbitNoraHelperError: LocalizedError { + let status: Int32 + + var errorDescription: String? { + switch status { + case 2: + "Siri Natural Nora (Voice 4) is not installed." + case 4: + "Siri Natural Nora did not finish before the local playback timeout." + default: + "Siri Natural Nora local playback failed (helper status \(status))." + } + } +} + +nonisolated private final class OrbitNoraAvailabilityCache: @unchecked Sendable { + private let lock = NSLock() + private var storedValue: Bool? + + var value: Bool? { + lock.withLock { storedValue } + } + + func store(_ value: Bool) { + lock.withLock { + storedValue = value + } + } +} + +nonisolated private enum OrbitNaturalVoiceHelperRuntime { + static let executableName = "OrbitNaturalVoiceHelper" + + static var executableURL: URL? { + let candidate = Bundle.main.bundleURL + .appendingPathComponent("Contents/Helpers", isDirectory: true) + .appendingPathComponent(executableName, isDirectory: false) + return FileManager.default.isExecutableFile(atPath: candidate.path) ? candidate : nil + } + + static func probe(timeout: TimeInterval = 3) -> Bool { + guard let executableURL else { return false } + + let process = Process() + process.executableURL = executableURL + process.arguments = ["--probe"] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + + do { + try process.run() + } catch { + return false + } + + let deadline = Date().addingTimeInterval(timeout) + while process.isRunning, Date() < deadline { + Thread.sleep(forTimeInterval: 0.02) + } + if process.isRunning { + process.terminate() + return false + } + return process.terminationStatus == 0 + } +} + +enum OrbitNoraVoiceAvailability { + static let identifier = "com.apple.siri.natural.Nora" + static let displayName = "Siri Natural Nora · Voice 4" + + private static let cache = OrbitNoraAvailabilityCache() + + static var cachedValue: Bool? { cache.value } + + static func probe(forceRefresh: Bool = false) async -> Bool { + if !forceRefresh, let cachedValue { return cachedValue } + + let isAvailable = await Task.detached(priority: .utility) { + OrbitNaturalVoiceHelperRuntime.probe() + }.value + cache.store(isAvailable) + return isAvailable + } +} + +@MainActor +final class NaturalNoraTTSProvider: TextToSpeechProvider { + let displayName = "Siri Natural Nora · Voice 4 · Apple Local" + + private var currentProcess: Process? + private var currentSpeakContinuation: CheckedContinuation? + private var watchdogTask: Task? + private var generation = 0 + + var isConfigured: Bool { + OrbitNoraVoiceAvailability.cachedValue != false + } + + var unavailableExplanation: String? { + isConfigured + ? nil + : "Siri Natural Nora (Voice 4) is unavailable. Orbit can use a compatible AVFoundation fallback." + } + + var isPlaying: Bool { + currentProcess?.isRunning == true || currentSpeakContinuation != nil + } + + func speakText(_ text: String) async throws { + let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !normalized.isEmpty else { return } + + stopPlayback() + generation &+= 1 + let requestGeneration = generation + + try await withTaskCancellationHandler { + guard await OrbitNoraVoiceAvailability.probe() else { + throw OrbitNoraVoiceUnavailableError() + } + guard generation == requestGeneration else { throw CancellationError() } + guard let executableURL = OrbitNaturalVoiceHelperRuntime.executableURL else { + throw OrbitNoraVoiceUnavailableError() + } + + try await withCheckedThrowingContinuation { continuation in + let process = Process() + let inputPipe = Pipe() + process.executableURL = executableURL + process.standardInput = inputPipe + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + process.terminationHandler = { [weak self] terminatedProcess in + let status = terminatedProcess.terminationStatus + Task { @MainActor [weak self] in + self?.finishPlayback( + requestGeneration: requestGeneration, + status: status + ) + } + } + + currentProcess = process + currentSpeakContinuation = continuation + + do { + try process.run() + try inputPipe.fileHandleForWriting.write(contentsOf: Data(normalized.utf8)) + try inputPipe.fileHandleForWriting.close() + startWatchdog( + requestGeneration: requestGeneration, + textLength: normalized.count + ) + } catch { + process.terminationHandler = nil + if process.isRunning { process.terminate() } + currentProcess = nil + currentSpeakContinuation = nil + continuation.resume(throwing: error) + } + } + } onCancel: { + Task { @MainActor [weak self] in + self?.cancelIfCurrent(requestGeneration) + } + } + } + + func stopPlayback() { + generation &+= 1 + watchdogTask?.cancel() + watchdogTask = nil + + let process = currentProcess + currentProcess = nil + process?.terminationHandler = nil + if process?.isRunning == true { process?.terminate() } + + let continuation = currentSpeakContinuation + currentSpeakContinuation = nil + continuation?.resume(throwing: CancellationError()) + } + + private func startWatchdog(requestGeneration: Int, textLength: Int) { + watchdogTask?.cancel() + let seconds = min(185.0, max(20.0, 15.0 + Double(textLength) / 12.0)) + watchdogTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(seconds)) + guard !Task.isCancelled else { return } + self?.timeoutIfCurrent(requestGeneration) + } + } + + private func timeoutIfCurrent(_ requestGeneration: Int) { + guard generation == requestGeneration else { return } + finishPlayback(requestGeneration: requestGeneration, status: 4) + } + + private func cancelIfCurrent(_ requestGeneration: Int) { + guard generation == requestGeneration else { return } + stopPlayback() + } + + private func finishPlayback(requestGeneration: Int, status: Int32) { + guard generation == requestGeneration else { return } + generation &+= 1 + watchdogTask?.cancel() + watchdogTask = nil + + let process = currentProcess + currentProcess = nil + process?.terminationHandler = nil + if process?.isRunning == true { process?.terminate() } + + let continuation = currentSpeakContinuation + currentSpeakContinuation = nil + if status == 0 { + continuation?.resume() + } else { + continuation?.resume(throwing: OrbitNoraHelperError(status: status)) + } + } +} diff --git a/Orbit/OrbitPanelView.swift b/Orbit/OrbitPanelView.swift index 72347cd..8426e93 100644 --- a/Orbit/OrbitPanelView.swift +++ b/Orbit/OrbitPanelView.swift @@ -983,7 +983,7 @@ struct OrbitPanelView: View { rowLabel( icon: "speaker.wave.2.fill", title: "Voice", - subtitle: orbitSettings.voicePreset == .localVoice ? "Nora Premium · on device" : "OpenAI speech" + subtitle: orbitSettings.voicePreset == .localVoice ? "Siri Natural Nora · on device" : "OpenAI speech" ) Spacer(minLength: 8) @@ -1028,12 +1028,12 @@ struct OrbitPanelView: View { if orbitSettings.voicePreset == .localVoice, !orbitManager.isNoraVoiceAvailable, !orbitManager.isCheckingNoraVoice { - Text("Nora Premium is unavailable to Orbit. Local narration will use the best compatible Apple fallback voice.") + Text("Siri Natural Nora (Voice 4) is unavailable to Orbit. Local narration will use the best compatible Apple fallback voice.") .font(.system(size: 9.5, weight: .medium)) .foregroundColor(DS.Colors.warningText.opacity(0.88)) .fixedSize(horizontal: false, vertical: true) .accessibilityLabel( - "Nora Premium is unavailable. Orbit will use the best compatible Apple fallback voice." + "Siri Natural Nora Voice 4 is unavailable. Orbit will use the best compatible Apple fallback voice." ) } } @@ -1041,7 +1041,7 @@ struct OrbitPanelView: View { private var noraVoiceStatusLabel: String { if orbitManager.isCheckingNoraVoice { return "Checking Nora…" } - return orbitManager.isNoraVoiceAvailable ? "Nora Premium" : "Fallback voice" + return orbitManager.isNoraVoiceAvailable ? "Voice 4 · Natural Nora" : "Fallback voice" } private func resetLegacyAppleVoiceSelection() { diff --git a/Orbit/SayNoraTTSProvider.swift b/Orbit/SayNoraTTSProvider.swift deleted file mode 100644 index ac6d020..0000000 --- a/Orbit/SayNoraTTSProvider.swift +++ /dev/null @@ -1,393 +0,0 @@ -import AVFoundation -import Foundation - -struct OrbitNoraVoiceUnavailableError: LocalizedError { - var errorDescription: String? { - "Nora Premium is not available through the local Apple speech service." - } -} - -nonisolated final class OrbitSayProcessHandle: @unchecked Sendable { - private let lock = NSLock() - private var process: Process? - private var isCancelled = false - - func install(_ process: Process) { - let shouldTerminate = lock.withLock { - self.process = process - return isCancelled - } - if shouldTerminate, process.isRunning { - process.terminate() - } - } - - func finish() { - lock.withLock { - process = nil - } - } - - func cancel() { - let runningProcess = lock.withLock { () -> Process? in - isCancelled = true - return process - } - if runningProcess?.isRunning == true { - runningProcess?.terminate() - } - } - - var cancelled: Bool { - lock.withLock { isCancelled } - } -} - -enum OrbitSayAudioRenderer { - nonisolated static let executableURL = URL(fileURLWithPath: "/usr/bin/say") - - nonisolated static var temporaryDirectoryURL: URL { - FileManager.default.temporaryDirectory - .appendingPathComponent("com.orbit.codex", isDirectory: true) - .appendingPathComponent("local-speech", isDirectory: true) - } - - static func makeSecureTemporaryAudioURL(prefix: String) throws -> URL { - let directoryURL = temporaryDirectoryURL - try FileManager.default.createDirectory( - at: directoryURL, - withIntermediateDirectories: true, - attributes: [.posixPermissions: 0o700] - ) - try FileManager.default.setAttributes( - [.posixPermissions: 0o700], - ofItemAtPath: directoryURL.path - ) - - let outputURL = directoryURL.appendingPathComponent( - "\(prefix)-\(UUID().uuidString).aiff", - isDirectory: false - ) - guard - FileManager.default.createFile( - atPath: outputURL.path, - contents: Data(), - attributes: [.posixPermissions: 0o600] - ) - else { - throw CocoaError(.fileWriteUnknown) - } - return outputURL - } - - nonisolated static func sweepStaleTemporaryAudio( - olderThan interval: TimeInterval = 60 * 60, - now: Date = Date() - ) { - guard - let items = try? FileManager.default.contentsOfDirectory( - at: temporaryDirectoryURL, - includingPropertiesForKeys: [.contentModificationDateKey, .isRegularFileKey], - options: [.skipsHiddenFiles] - ) - else { return } - - for item in items where item.pathExtension == "aiff" { - guard - let values = try? item.resourceValues(forKeys: [.contentModificationDateKey, .isRegularFileKey]), - values.isRegularFile == true, - let modifiedAt = values.contentModificationDate, - now.timeIntervalSince(modifiedAt) >= interval - else { continue } - try? FileManager.default.removeItem(at: item) - } - } - - static func render( - text: String, - voiceName: String, - outputURL: URL, - processHandle: OrbitSayProcessHandle - ) async throws { - try await withTaskCancellationHandler { - try await Task.detached(priority: .userInitiated) { - guard !processHandle.cancelled else { throw CancellationError() } - let process = Process() - process.executableURL = executableURL - process.arguments = ["-v", voiceName, "-o", outputURL.path, text] - process.standardOutput = FileHandle.nullDevice - process.standardError = FileHandle.nullDevice - processHandle.install(process) - defer { processHandle.finish() } - - try process.run() - process.waitUntilExit() - guard !processHandle.cancelled, !Task.isCancelled else { - throw CancellationError() - } - guard process.terminationReason == .exit, process.terminationStatus == 0 else { - throw NSError( - domain: "OrbitSayAudioRenderer", - code: Int(process.terminationStatus), - userInfo: [NSLocalizedDescriptionKey: "Apple local speech could not synthesize audio."] - ) - } - - let attributes = try FileManager.default.attributesOfItem(atPath: outputURL.path) - guard (attributes[.size] as? NSNumber)?.intValue ?? 0 > 64 else { - throw NSError( - domain: "OrbitSayAudioRenderer", - code: -1, - userInfo: [NSLocalizedDescriptionKey: "Apple local speech returned empty audio."] - ) - } - try FileManager.default.setAttributes( - [.posixPermissions: 0o600], - ofItemAtPath: outputURL.path - ) - }.value - } onCancel: { - processHandle.cancel() - } - } - - static func rendersDistinctVoice( - candidateAudio: Data, - fallbackAudio: Data - ) -> Bool { - candidateAudio.count > 64 && fallbackAudio.count > 64 && candidateAudio != fallbackAudio - } -} - -nonisolated private final class OrbitNoraAvailabilityCache: @unchecked Sendable { - private let lock = NSLock() - private var storedValue: Bool? - - var value: Bool? { - lock.withLock { storedValue } - } - - func store(_ value: Bool) { - lock.withLock { - storedValue = value - } - } -} - -enum OrbitNoraVoiceAvailability { - private static let cache = OrbitNoraAvailabilityCache() - private static let fallbackProbeVoice = "__orbit_missing_voice_\(UUID().uuidString)__" - - static var cachedValue: Bool? { cache.value } - - static func probe(forceRefresh: Bool = false) async -> Bool { - if !forceRefresh, let cachedValue { return cachedValue } - guard FileManager.default.isExecutableFile(atPath: OrbitSayAudioRenderer.executableURL.path) else { - cache.store(false) - return false - } - - var candidateURL: URL? - var fallbackURL: URL? - defer { - if let candidateURL { try? FileManager.default.removeItem(at: candidateURL) } - if let fallbackURL { try? FileManager.default.removeItem(at: fallbackURL) } - } - - do { - candidateURL = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "nora-probe") - fallbackURL = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "default-probe") - let probeText = "Orbit local voice availability probe." - try await OrbitSayAudioRenderer.render( - text: probeText, - voiceName: "Nora", - outputURL: candidateURL!, - processHandle: OrbitSayProcessHandle() - ) - try await OrbitSayAudioRenderer.render( - text: probeText, - voiceName: fallbackProbeVoice, - outputURL: fallbackURL!, - processHandle: OrbitSayProcessHandle() - ) - let candidateData = try Data(contentsOf: candidateURL!, options: .mappedIfSafe) - let fallbackData = try Data(contentsOf: fallbackURL!, options: .mappedIfSafe) - let isAvailable = OrbitSayAudioRenderer.rendersDistinctVoice( - candidateAudio: candidateData, - fallbackAudio: fallbackData - ) - cache.store(isAvailable) - return isAvailable - } catch { - if error is CancellationError { return false } - cache.store(false) - return false - } - } -} - -@MainActor -final class SayNoraTTSProvider: NSObject, TextToSpeechProvider, AVAudioPlayerDelegate { - let displayName = "Nora Premium · Apple Local" - - private var audioPlayer: AVAudioPlayer? - private var currentPlayerIdentifier: ObjectIdentifier? - private var availabilityProbeTask: Task? - private var renderProcessHandle: OrbitSayProcessHandle? - private var currentOutputURL: URL? - private var currentSpeakContinuation: CheckedContinuation? - private var generation = 0 - - override init() { - super.init() - Task.detached(priority: .utility) { - OrbitSayAudioRenderer.sweepStaleTemporaryAudio() - } - } - - var isConfigured: Bool { - FileManager.default.isExecutableFile(atPath: OrbitSayAudioRenderer.executableURL.path) - && OrbitNoraVoiceAvailability.cachedValue != false - } - - var unavailableExplanation: String? { - isConfigured - ? nil - : "Nora Premium is not available. Orbit can fall back to an AVFoundation voice." - } - - var isPlaying: Bool { - audioPlayer?.isPlaying == true || renderProcessHandle != nil || availabilityProbeTask != nil - } - - func speakText(_ text: String) async throws { - let normalized = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard !normalized.isEmpty else { return } - - stopPlayback() - generation &+= 1 - let requestGeneration = generation - - try await withTaskCancellationHandler { - let probeTask = Task { await OrbitNoraVoiceAvailability.probe() } - availabilityProbeTask = probeTask - let isNoraAvailable = await probeTask.value - if generation == requestGeneration { - availabilityProbeTask = nil - } - guard isNoraAvailable else { - throw OrbitNoraVoiceUnavailableError() - } - guard generation == requestGeneration else { throw CancellationError() } - - let outputURL = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "nora") - currentOutputURL = outputURL - let processHandle = OrbitSayProcessHandle() - renderProcessHandle = processHandle - - do { - try await OrbitSayAudioRenderer.render( - text: normalized, - voiceName: "Nora", - outputURL: outputURL, - processHandle: processHandle - ) - } catch { - if generation == requestGeneration { - renderProcessHandle = nil - cleanupOutputFile() - } - throw error - } - - guard generation == requestGeneration else { throw CancellationError() } - renderProcessHandle = nil - let player = try AVAudioPlayer(contentsOf: outputURL) - player.delegate = self - audioPlayer = player - currentPlayerIdentifier = ObjectIdentifier(player) - - try await withCheckedThrowingContinuation { continuation in - currentSpeakContinuation = continuation - guard player.play() else { - currentSpeakContinuation = nil - audioPlayer = nil - currentPlayerIdentifier = nil - cleanupOutputFile() - continuation.resume( - throwing: NSError( - domain: "SayNoraTTSProvider", - code: -1, - userInfo: [NSLocalizedDescriptionKey: "Nora audio could not start playback."] - ) - ) - return - } - } - } onCancel: { - Task { @MainActor [weak self] in - self?.cancelIfCurrent(requestGeneration) - } - } - } - - func stopPlayback() { - generation &+= 1 - availabilityProbeTask?.cancel() - availabilityProbeTask = nil - renderProcessHandle?.cancel() - renderProcessHandle = nil - audioPlayer?.stop() - audioPlayer = nil - currentPlayerIdentifier = nil - let continuation = currentSpeakContinuation - currentSpeakContinuation = nil - cleanupOutputFile() - continuation?.resume(throwing: CancellationError()) - } - - nonisolated func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully flag: Bool) { - let playerIdentifier = ObjectIdentifier(player) - Task { @MainActor [weak self] in - self?.finishPlayback(playerIdentifier: playerIdentifier, successfully: flag) - } - } - - private func cancelIfCurrent(_ requestGeneration: Int) { - guard generation == requestGeneration else { return } - stopPlayback() - } - - private func finishPlayback(playerIdentifier: ObjectIdentifier, successfully: Bool) { - guard currentPlayerIdentifier == playerIdentifier else { return } - audioPlayer = nil - currentPlayerIdentifier = nil - let continuation = currentSpeakContinuation - currentSpeakContinuation = nil - cleanupOutputFile() - if successfully { - continuation?.resume() - } else { - continuation?.resume( - throwing: NSError( - domain: "SayNoraTTSProvider", - code: -2, - userInfo: [NSLocalizedDescriptionKey: "Nora audio playback was interrupted."] - ) - ) - } - } - - private func cleanupOutputFile() { - guard let currentOutputURL else { return } - self.currentOutputURL = nil - try? FileManager.default.removeItem(at: currentOutputURL) - } - - deinit { - renderProcessHandle?.cancel() - if let currentOutputURL { - try? FileManager.default.removeItem(at: currentOutputURL) - } - } -} diff --git a/Orbit/TextToSpeechProvider.swift b/Orbit/TextToSpeechProvider.swift index 94a217a..7cef6b2 100644 --- a/Orbit/TextToSpeechProvider.swift +++ b/Orbit/TextToSpeechProvider.swift @@ -244,7 +244,7 @@ enum OrbitTTSProviderFactory { switch voicePreset { case .localVoice: return OrbitVoiceCoordinator( - primary: SayNoraTTSProvider(), + primary: NaturalNoraTTSProvider(), fallback: AppleSystemTTSProvider() ) case .cloudVoice: diff --git a/OrbitNaturalVoiceHelper/main.m b/OrbitNaturalVoiceHelper/main.m new file mode 100644 index 0000000..c5beb4f --- /dev/null +++ b/OrbitNaturalVoiceHelper/main.m @@ -0,0 +1,158 @@ +#import + +#import +#import + +static NSString *const OrbitNaturalNoraIdentifier = + @"com.apple.siri.natural.Nora"; + +@interface NSObject (OrbitNaturalVoicePrivateRuntime) ++ (id)voiceForIdentifier:(NSString *)identifier; ++ (id)actionWithString:(NSString *)string shouldQueue:(BOOL)queue; +- (NSString *)identifier; +- (BOOL)isInstalled; +- (void)setVoiceIdentifier:(NSString *)identifier; +- (void)setSpeakingRate:(double)rate; +- (void)setPitch:(double)pitch; +- (void)setVolume:(double)volume; +- (void)setCompletionCallback:(void (^)(void))callback; +- (void)setSpeechEnabled:(BOOL)enabled; +- (void)dispatchSpeechAction:(id)action; +- (void)stopSpeaking; +@end + +static BOOL OrbitClassRespondsToClassSelector(Class candidate, SEL selector) { + return candidate != Nil && class_getClassMethod(candidate, selector) != NULL; +} + +static BOOL OrbitClassRespondsToInstanceSelector(Class candidate, + SEL selector) { + return candidate != Nil && + class_getInstanceMethod(candidate, selector) != NULL; +} + +static BOOL OrbitLoadAndValidateRuntime(void) { + void *handle = dlopen( + "/System/Library/PrivateFrameworks/TextToSpeech.framework/TextToSpeech", + RTLD_NOW | RTLD_LOCAL); + if (handle == NULL) { + return NO; + } + + Class synthesizerClass = NSClassFromString(@"TTSSpeechSynthesizer"); + Class actionClass = NSClassFromString(@"TTSSpeechAction"); + Class managerClass = NSClassFromString(@"TTSSpeechManager"); + + return OrbitClassRespondsToClassSelector(synthesizerClass, + @selector(voiceForIdentifier:)) && + OrbitClassRespondsToClassSelector(actionClass, @selector + (actionWithString:shouldQueue:)) && + OrbitClassRespondsToInstanceSelector(actionClass, + @selector(setVoiceIdentifier:)) && + OrbitClassRespondsToInstanceSelector(actionClass, + @selector(setSpeakingRate:)) && + OrbitClassRespondsToInstanceSelector(actionClass, + @selector(setPitch:)) && + OrbitClassRespondsToInstanceSelector(actionClass, + @selector(setVolume:)) && + OrbitClassRespondsToInstanceSelector(actionClass, @selector + (setCompletionCallback:)) && + OrbitClassRespondsToInstanceSelector(managerClass, + @selector(setSpeechEnabled:)) && + OrbitClassRespondsToInstanceSelector(managerClass, @selector + (dispatchSpeechAction:)) && + OrbitClassRespondsToInstanceSelector(managerClass, + @selector(stopSpeaking)); +} + +static BOOL OrbitExactVoiceIsInstalled(void) { + if (!OrbitLoadAndValidateRuntime()) { + return NO; + } + + @try { + Class synthesizerClass = NSClassFromString(@"TTSSpeechSynthesizer"); + id voice = [synthesizerClass voiceForIdentifier:OrbitNaturalNoraIdentifier]; + return voice != nil && [voice respondsToSelector:@selector(isInstalled)] && + [voice respondsToSelector:@selector(identifier)] && + [voice isInstalled] && + [[voice identifier] isEqualToString:OrbitNaturalNoraIdentifier]; + } @catch (__unused NSException *exception) { + return NO; + } +} + +static int OrbitSpeakText(NSString *text) { + if (text.length == 0 || !OrbitExactVoiceIsInstalled()) { + return 2; + } + + __block BOOL completed = NO; + id manager = nil; + + @try { + Class managerClass = NSClassFromString(@"TTSSpeechManager"); + Class actionClass = NSClassFromString(@"TTSSpeechAction"); + manager = [[managerClass alloc] init]; + if (manager == nil) { + return 3; + } + [manager setSpeechEnabled:YES]; + + id action = [actionClass actionWithString:text shouldQueue:NO]; + if (action == nil) { + return 3; + } + [action setVoiceIdentifier:OrbitNaturalNoraIdentifier]; + [action setSpeakingRate:0.5]; + [action setPitch:1.0]; + [action setVolume:1.0]; + [action setCompletionCallback:^{ + completed = YES; + }]; + [manager dispatchSpeechAction:action]; + + NSTimeInterval timeout = + MIN(180.0, MAX(15.0, 10.0 + (double)text.length / 12.0)); + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:timeout]; + while (!completed && deadline.timeIntervalSinceNow > 0) { + [[NSRunLoop currentRunLoop] + runMode:NSDefaultRunLoopMode + beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.05]]; + } + if (!completed) { + [manager stopSpeaking]; + return 4; + } + return 0; + } @catch (__unused NSException *exception) { + @try { + [manager stopSpeaking]; + } @catch (__unused NSException *stopException) { + } + return 3; + } +} + +int main(int argc, const char *argv[]) { + @autoreleasepool { + if (argc == 2 && strcmp(argv[1], "--probe") == 0) { + return OrbitExactVoiceIsInstalled() ? 0 : 2; + } + if (argc != 1) { + return 64; + } + + NSData *input = + [[NSFileHandle fileHandleWithStandardInput] readDataToEndOfFile]; + if (input.length == 0 || input.length > 1024 * 1024) { + return 65; + } + NSString *text = [[NSString alloc] initWithData:input + encoding:NSUTF8StringEncoding]; + if (text == nil) { + return 65; + } + return OrbitSpeakText(text); + } +} diff --git a/OrbitTests/OrbitVoiceTests.swift b/OrbitTests/OrbitVoiceTests.swift index bcc72ec..2fb9745 100644 --- a/OrbitTests/OrbitVoiceTests.swift +++ b/OrbitTests/OrbitVoiceTests.swift @@ -99,37 +99,42 @@ struct OrbitNarrationPrimitiveTests { #expect(otherTurnResult) } - @Test func noraProbeRejectsDefaultFallbackAudio() { - let candidate = Data([1, 2, 3] + Array(repeating: 4, count: 80)) - let different = Data([1, 2, 3] + Array(repeating: 5, count: 80)) - - #expect( - !OrbitSayAudioRenderer.rendersDistinctVoice( - candidateAudio: candidate, - fallbackAudio: candidate - ) - ) - #expect( - OrbitSayAudioRenderer.rendersDistinctVoice( - candidateAudio: candidate, - fallbackAudio: different - ) - ) + @Test func naturalNoraUsesTheExactShortcutsVoiceIdentifier() { + #expect(OrbitNoraVoiceAvailability.identifier == "com.apple.siri.natural.Nora") + #expect(OrbitNoraVoiceAvailability.displayName == "Siri Natural Nora · Voice 4") } - @Test func installedNoraProducesDistinctLocalAudioWhenPresent() async { + @Test func installedNaturalNoraResolvesThroughTheIsolatedLocalHelper() async throws { + let helperURL = Bundle.main.bundleURL + .appendingPathComponent("Contents/Helpers", isDirectory: true) + .appendingPathComponent("OrbitNaturalVoiceHelper", isDirectory: false) + #expect(FileManager.default.isExecutableFile(atPath: helperURL.path)) + + let process = Process() + process.executableURL = helperURL + process.arguments = ["--probe"] + process.standardInput = FileHandle.nullDevice + process.standardOutput = FileHandle.nullDevice + process.standardError = FileHandle.nullDevice + try process.run() + process.waitUntilExit() + #expect(process.terminationStatus == 0 || process.terminationStatus == 2) + let isAvailable = await OrbitNoraVoiceAvailability.probe(forceRefresh: true) - print("Nora Premium distinct local render available: \(isAvailable)") - guard isAvailable else { return } - #expect(isAvailable) + print("Siri Natural Nora Voice 4 local runtime available: \(isAvailable)") + #expect(isAvailable == (process.terminationStatus == 0)) } - @Test func secureSpeechFilesUseOwnerOnlyPermissions() throws { - let url = try OrbitSayAudioRenderer.makeSecureTemporaryAudioURL(prefix: "permission-test") - defer { try? FileManager.default.removeItem(at: url) } - - let attributes = try FileManager.default.attributesOfItem(atPath: url.path) - #expect((attributes[.posixPermissions] as? NSNumber)?.intValue == 0o600) + @Test func installedNaturalNoraCanSpeakWhenLiveTestingIsEnabled() async throws { + guard ProcessInfo.processInfo.environment["ORBIT_RUN_LOCAL_VOICE_TEST"] == "1" else { + return + } + #expect(await OrbitNoraVoiceAvailability.probe(forceRefresh: true)) + let provider = await NaturalNoraTTSProvider() + try await provider.speakText( + "Orbit is using the same local Siri Natural Nora voice as Shortcuts Voice Four." + ) + #expect(await !provider.isPlaying) } @Test func microphoneTestTapRunsOutsideMainActor() async throws { diff --git a/PRODUCT.md b/PRODUCT.md index 098fddf..ed2a74b 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -16,7 +16,7 @@ Orbit gives every request current-screen context, routes it through one warm Cod - **One warm thread.** Model, reasoning effort, and service tier are connected controls backed by the signed-in account's authoritative Codex catalog. Changing them does not discard the current thread. - **Explicit context reset.** Agent Folder sets the working directory and begins fresh Codex context. It does not restrict filesystem access. -- **Local means local.** Local recognition requires Apple's on-device path. Local narration prefers an available Nora Premium voice through macOS speech, deletes its temporary AIFF after use, and falls back to public AVFoundation speech when Nora is unavailable. +- **Local means local.** Local recognition requires Apple's on-device path. Local narration selects the exact Siri Natural Nora / Voice 4 identifier used by Shortcuts, streams it through the installed macOS speech asset, and falls back to public AVFoundation speech only when that runtime is unavailable. - **Cloud is explicit.** Cloud recognition and narration are selected together, disclose that audio is sent to OpenAI, and require a Keychain-backed API key. - **One capture per request.** Orbit never sends a request without its fresh screen capture and never presents capture as continuous recording. diff --git a/README.md b/README.md index 443827f..90b36d9 100644 --- a/README.md +++ b/README.md @@ -81,9 +81,9 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ ### Local - Apple on-device speech recognition (Orbit refuses silent network fallback) -- Nora Premium narration through macOS `/usr/bin/say` when that installed voice is available -- private temporary AIFF rendering (`0700` directory, `0600` files), local playback, and deletion after playback, failure, interruption, or cancellation -- public AVFoundation speech as the automatic local fallback when Nora is unavailable +- the exact **Siri Natural Nora / Voice 4** used by Shortcuts (`com.apple.siri.natural.Nora`) +- direct on-device playback through macOS's local TextToSpeech runtime in a disposable signed helper, with interruption, crash containment, and no narration file written by Orbit +- public AVFoundation speech as the automatic fallback when Voice 4 or its system runtime is unavailable - microphone selection, local preview, and live level test - no extra API key required @@ -100,7 +100,7 @@ Installers are published through the main [`4xiomdev/orbit`](https://github.com/ - Cloud voice uses the user-supplied OpenAI API key stored in Keychain. - Orbit keeps its Codex runtime state in `~/Library/Application Support/Orbit/CodexHome`. - Temporary captures use mode `0600` in an Orbit-owned temporary directory and are swept after terminal turn states or crash recovery. -- Nora narration files are separate Orbit-owned temporary AIFF files. They are mode `0600`, removed after use, and limited to stale-file cleanup inside Orbit's local-speech directory. +- Local Voice 4 narration is rendered by the installed macOS speech asset and streamed directly to the audio subsystem; Orbit does not upload or persist it. - Support logs are private, bounded, rotated, allowlisted, and redact credentials, prompts, capture paths, home paths, and full command arguments. - Orbit intentionally grants Codex unrestricted filesystem and command access. The optional Agent Folder starts fresh working context in that folder; it does not change the security boundary. diff --git a/SECURITY.md b/SECURITY.md index a7f32a1..65a3ac1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -42,7 +42,7 @@ We will aim to: - Every submitted request requires one fresh screen capture. Orbit does not continuously record; if capture fails, the request is not sent. - Each temporary capture is owned by its turn, stored with mode `0600`, deleted on every terminal/cancellation path, and eligible for Orbit-only crash-recovery cleanup on the next launch. - Local recognition requires Apple's on-device speech path and never silently falls back to a network recognizer. -- Local narration prefers an available Nora Premium voice through macOS `/usr/bin/say`. Orbit renders speech into its own mode `0700` temporary directory as a mode `0600` AIFF, plays it locally, and removes it after playback, failure, interruption, or cancellation. Startup cleanup is limited to stale AIFF files inside that Orbit-owned local-speech directory. If Nora is unavailable, Orbit falls back to public AVFoundation speech synthesis. +- Local narration selects Shortcuts Voice 4 by its exact identifier, `com.apple.siri.natural.Nora`. A separately signed, disposable helper dynamically loads Apple's private local TextToSpeech framework, validates the required selectors, and dispatches the installed speech asset. Orbit does not link that private framework into the app process, upload narration, or write a narration file. The helper is time-bounded and crash-isolated; if Apple removes or changes this interface, Orbit remains alive and falls back to public AVFoundation speech. - Cloud speech is optional and explicitly selected. Speech audio is sent to OpenAI, AI-generated narration is disclosed, and the user-supplied API key is stored in Keychain. - Orbit keeps one warm Codex thread while model, reasoning effort, or service tier changes. Agent Folder deliberately starts fresh context but does not reduce `danger-full-access`. - Support logs are mode `0600`, capped at 5 MiB with three rotations, and redact credentials, authorization headers, prompt text, capture paths, home paths, and full command arguments. diff --git a/SUPPORT.md b/SUPPORT.md index 183dfbc..308eca5 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -13,7 +13,7 @@ - macOS version - whether you installed via PKG or DMG - whether the issue is about permissions, Codex auth, Local or Cloud voice, model availability, Agent Folder, or overlay behavior -- for voice issues, whether Orbit reports Nora Premium or the Apple fallback and which microphone is selected +- for voice issues, whether Orbit reports Siri Natural Nora / Voice 4 or the Apple fallback and which microphone is selected - for Codex configuration issues, the model, reasoning effort, service tier, and whether Agent Folder was changed - screenshots when relevant @@ -21,7 +21,7 @@ - Model choices come from the current account's Codex catalog and can differ between accounts. - Effort and service-tier choices follow the selected model. Changing them keeps the current Codex thread; changing Agent Folder starts fresh context. -- Local mode keeps recognition on device and uses Nora Premium when available, with an Apple AVFoundation narration fallback. Cloud mode sends speech audio to OpenAI and requires a Keychain-backed API key. +- Local mode keeps recognition on device and uses the same Siri Natural Nora / Voice 4 asset as Shortcuts when available, with an Apple AVFoundation narration fallback. Cloud mode sends speech audio to OpenAI and requires a Keychain-backed API key. ## What Orbit does not provide diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md index 917839d..7eb19ee 100644 --- a/docs/PRIVACY.md +++ b/docs/PRIVACY.md @@ -6,7 +6,7 @@ The request is blocked if the capture cannot be created. A successful capture is Local speech recognition requires Apple on-device recognition. Orbit does not silently fall back to network recognition. -Local narration first checks whether Nora Premium is available through the Mac's local speech service. When available, Orbit invokes `/usr/bin/say` without a shell, writes a temporary AIFF into `com.orbit.codex/local-speech`, plays it locally, and deletes it after playback, failure, interruption, or cancellation. The directory is mode `0700`; each AIFF is mode `0600`. A launch-time sweep removes only stale `.aiff` files from that Orbit-owned directory. When Nora is unavailable, Orbit uses its public AVFoundation fallback. +Local narration first checks for the exact Siri Natural Nora voice used by Shortcuts Voice 4 (`com.apple.siri.natural.Nora`). When available, a signed local helper dispatches text through the installed macOS TextToSpeech runtime and streams playback directly to the local audio subsystem. The helper is isolated from the Orbit app process and exits after each utterance. Orbit does not upload narration and does not create a narration file. If this system interface or voice asset is unavailable, Orbit uses its public AVFoundation fallback. OpenAI Cloud voice is optional and must be explicitly selected. In Cloud mode, speech audio is sent to OpenAI for transcription and narration is AI-generated. The API key is stored in macOS Keychain. Orbit's interface does not describe Cloud input as staying on the Mac. diff --git a/docs/SETUP.md b/docs/SETUP.md index 697987e..22bd172 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -12,10 +12,10 @@ After permissions, Orbit explains its unrestricted automation contract once. Cod ## Voice setup -- **Local** keeps recognition on device. Narration uses Nora Premium through the Mac's local speech service when Orbit verifies that voice is available, with public AVFoundation speech as the fallback. No OpenAI voice key is required. +- **Local** keeps recognition on device. Narration uses the exact Siri Natural Nora / Voice 4 asset used by Shortcuts when Orbit verifies that runtime is available, with public AVFoundation speech as the fallback. No OpenAI voice key is required. - **Cloud** sends speech audio to OpenAI for transcription and uses AI-generated OpenAI narration. It is enabled only after explicit selection and a successful Keychain-backed API-key check. -Orbit keeps local narration audio only long enough to play it. Nora output is a private temporary AIFF and is deleted after completion, failure, interruption, or cancellation. +Voice 4 narration is streamed directly through macOS's local audio subsystem. Orbit does not create or retain a narration file. ## Codex context From 31f2a5f587f418a017b2a40b97ca4534ff3ecc89 Mon Sep 17 00:00:00 2001 From: 4xiomLocal <4xiomdev@gmail.com> Date: Mon, 13 Jul 2026 17:35:47 -0400 Subject: [PATCH 8/8] Harden Orbit permissions, sessions, narration, and panel --- Orbit/CodexAppServerActionProvider.swift | 300 +++- Orbit/DesignSystem.swift | 71 +- Orbit/Localizable.xcstrings | 81 + Orbit/MenuBarPanelManager.swift | 23 +- Orbit/OrbitCodexConversationState.swift | 102 ++ Orbit/OrbitManager.swift | 203 +-- Orbit/OrbitPanelView.swift | 1935 --------------------- Orbit/OrbitPanelViewV2.swift | 1101 ++++++++++++ Orbit/OrbitPermissionCoordinator.swift | 810 ++++++--- Orbit/OrbitPermissionModels.swift | 274 +++ Orbit/OrbitVisualQA.swift | 598 +++++++ Orbit/OrbitVoiceCoordinator.swift | 209 ++- OrbitTests/OrbitCodexLifecycleTests.swift | 73 + OrbitTests/OrbitPermissionTests.swift | 172 ++ OrbitTests/OrbitTests.swift | 4 + OrbitTests/OrbitVoiceTests.swift | 118 +- 16 files changed, 3589 insertions(+), 2485 deletions(-) create mode 100644 Orbit/Localizable.xcstrings create mode 100644 Orbit/OrbitCodexConversationState.swift delete mode 100644 Orbit/OrbitPanelView.swift create mode 100644 Orbit/OrbitPanelViewV2.swift create mode 100644 Orbit/OrbitPermissionModels.swift create mode 100644 Orbit/OrbitVisualQA.swift create mode 100644 OrbitTests/OrbitPermissionTests.swift diff --git a/Orbit/CodexAppServerActionProvider.swift b/Orbit/CodexAppServerActionProvider.swift index eef184d..2834077 100644 --- a/Orbit/CodexAppServerActionProvider.swift +++ b/Orbit/CodexAppServerActionProvider.swift @@ -20,6 +20,11 @@ private enum OrbitMcpStartupState: Equatable { @MainActor final class CodexAppServerActionProvider: ActionProvider { + private struct PendingSubmission { + let request: OrbitActionRequest + let handler: @Sendable (OrbitActionEvent) -> Void + } + let displayName = "Codex" private static let browserToolServerNames = ["playwright", "chrome-devtools"] private static func makeInitialMcpStartupStates() -> [String: OrbitMcpStartupState] { @@ -43,6 +48,9 @@ final class CodexAppServerActionProvider: ActionProvider { private var activeThreadID: String? private var pendingPrompt: String? private var eventHandler: (@Sendable (OrbitActionEvent) -> Void)? + private var pendingSubmission: PendingSubmission? + private var pendingSteerRequestID: Int? + private var cancellationPendingTurnID = false private var startupTimeoutTask: Task? private var hasReceivedInitializeResponse = false private var isAwaitingTurnCompletion = false @@ -60,6 +68,7 @@ final class CodexAppServerActionProvider: ActionProvider { private var freshPrewarmRequested = false private var streamedCommentaryBuffer = "" private var hasEmittedEarlyCommentary = false + private var commentaryFlushTask: Task? private var availableModelOptions: [OrbitCodexModelOption] = OrbitCodexModelOption.fallbackPickerModels private var pendingModelCatalogRequestID: Int? private var pendingAccountReadRequestID: Int? @@ -78,6 +87,7 @@ final class CodexAppServerActionProvider: ActionProvider { private var lastEmittedLiveCommentary: String? private var preparedCodexHome: OrbitPreparedCodexHome? private var mcpStartupStates: [String: OrbitMcpStartupState] = CodexAppServerActionProvider.makeInitialMcpStartupStates() + private var conversationState = OrbitCodexConversationState() var stateDidChange: (() -> Void)? var isConfigured: Bool { @@ -201,10 +211,36 @@ final class CodexAppServerActionProvider: ActionProvider { _ request: OrbitActionRequest, onEvent: @escaping @Sendable (OrbitActionEvent) -> Void ) async { - eventHandler = onEvent - if !isAwaitingTurnCompletion { - subagentActivities = [] + if case .waitingForApproval = status { + onEvent( + .failed("Answer the current Codex choice before sending another request.") + ) + appendDebugEvent("submission rejected while waiting on tool choice") + return + } + + if isAwaitingTurnCompletion { + guard pendingSubmission == nil, pendingSteerRequestID == nil else { + onEvent(.failed("Orbit is already adding a follow-up to the current Codex turn.")) + return + } + + pendingSubmission = PendingSubmission(request: request, handler: onEvent) + onEvent( + .phase( + OrbitActionProgress( + phase: .thinking, + detail: "steering the current Codex turn.", + rawSource: "steering the current codex turn" + ) + ) + ) + sendTurnSteer() + return } + + eventHandler = onEvent + subagentActivities = [] latestRequest = request status = .running hasOpenedBrowserInCurrentTurn = false @@ -243,33 +279,13 @@ final class CodexAppServerActionProvider: ActionProvider { return } - if isAwaitingTurnCompletion { - if case .waitingForApproval = status { - emitPhase( - .waitingForChoice, - detail: "answer the current tool question before steering codex.", - rawSource: "orbit is waiting on a tool choice" - ) - appendDebugEvent("turn blocked while waiting on tool choice") - return - } - - pendingPrompt = wrappedPrompt(for: request) - latestRequest = request - latestAgentMessageText = nil - latestFinalAnswerText = nil - streamedCommentaryBuffer = "" - hasEmittedEarlyCommentary = false - lastEmittedLiveCommentary = nil - sendTurnSteer() - return - } - pendingPrompt = wrappedPrompt(for: request) latestAgentMessageText = nil latestFinalAnswerText = nil streamedCommentaryBuffer = "" hasEmittedEarlyCommentary = false + commentaryFlushTask?.cancel() + commentaryFlushTask = nil lastEmittedLiveCommentary = nil attemptPendingTurnStart() } @@ -464,24 +480,31 @@ final class CodexAppServerActionProvider: ActionProvider { } func cancelCurrentAction() { + conversationState.requestCancellation() + if let pendingSubmission { + self.pendingSubmission = nil + pendingSteerRequestID = nil + pendingSubmission.handler(.interrupted("stopped before the follow-up was added.")) + } + if let activeThreadID, let activeTurnID, let process, process.isRunning, isAwaitingTurnCompletion { - appendDebugEvent("-> turn/interrupt turn=\(String(activeTurnID.suffix(6)))") - sendJSON([ - "method": "turn/interrupt", - "id": 3, - "params": [ - "threadId": activeThreadID, - "turnId": activeTurnID, - ], - ]) + sendTurnInterrupt(threadID: activeThreadID, turnID: activeTurnID) + return + } + + if activeThreadID != nil, + process?.isRunning == true, + turnStartRequestInFlight || isAwaitingTurnCompletion + { + cancellationPendingTurnID = true status = .interrupted("stopping the current codex turn.") - emitPhase(.interrupted, detail: "stopping the current codex turn.", rawSource: "interrupting codex") - beginInterruptTimeoutWatch(threadID: activeThreadID, turnID: activeTurnID) + emitPhase(.interrupted, detail: "stopping when Codex confirms the turn.", rawSource: "interrupt pending turn id") + appendDebugEvent("turn cancellation queued until turn id arrives") return } @@ -495,6 +518,22 @@ final class CodexAppServerActionProvider: ActionProvider { status = .idle } + private func sendTurnInterrupt(threadID: String?, turnID: String) { + guard let threadID else { return } + appendDebugEvent("-> turn/interrupt turn=\(String(turnID.suffix(6)))") + sendJSON([ + "method": "turn/interrupt", + "id": 3, + "params": [ + "threadId": threadID, + "turnId": turnID, + ], + ]) + status = .interrupted("stopping the current codex turn.") + emitPhase(.interrupted, detail: "stopping the current codex turn.", rawSource: "interrupting codex") + beginInterruptTimeoutWatch(threadID: threadID, turnID: turnID) + } + private func bootstrapSessionIfNeeded() throws { if let process, !process.isRunning { teardownProcess() @@ -654,6 +693,17 @@ final class CodexAppServerActionProvider: ActionProvider { appendDebugEvent("<- error \(errorMessage)") if let id = message["id"] as? Int { + if id == pendingSteerRequestID { + pendingSteerRequestID = nil + let pending = pendingSubmission + pendingSubmission = nil + conversationState.steerRejected() + pending?.handler(.failed("Orbit could not add that follow-up: \(errorMessage)")) + appendDebugEvent("turn/steer rejected; active turn preserved") + notifyStateChanged() + return + } + if id == pendingCollaborationModeRequestID { pendingCollaborationModeRequestID = nil collaborationModes = [] @@ -721,10 +771,16 @@ final class CodexAppServerActionProvider: ActionProvider { { appendDebugEvent("<- turn/started \(String(turnID.suffix(6)))") activeTurnID = turnID + conversationState.turnStarted(turnID) turnStartRequestInFlight = false isAwaitingTurnCompletion = true + if cancellationPendingTurnID { + cancellationPendingTurnID = false + sendTurnInterrupt(threadID: activeThreadID, turnID: turnID) + return + } emitPhase(.thinking, rawSource: "codex is working on it") - if pendingPrompt != nil { + if pendingSubmission != nil { sendTurnSteer() } } @@ -745,6 +801,7 @@ final class CodexAppServerActionProvider: ActionProvider { if phase == "final_answer" || phase == "finalAnswer" { appendDebugEvent("<- item/completed final_answer") latestFinalAnswerText = text + emitCompletedCommentaryIfNeeded(text: text) } else { appendDebugEvent("<- item/completed agentMessage") latestAgentMessageText = text @@ -757,13 +814,18 @@ final class CodexAppServerActionProvider: ActionProvider { case "item/mcpToolCall/progress": handleMcpToolCallProgress(message) case "turn/completed": + let queuedSubmission = pendingSubmission + pendingSubmission = nil + pendingSteerRequestID = nil let turnStatus = turnStatus(from: message) + _ = conversationState.completeActiveRequest() appendDebugEvent("<- turn/completed status=\(turnStatus ?? "unknown")") let summary = summarizedCompletionMessage(from: message) ?? latestFinalAnswerText ?? latestAgentMessageText ?? "codex finished the action" + emitCompletedCommentaryIfNeeded(text: summary) activeTurnID = nil interruptTimeoutTask?.cancel() interruptTimeoutTask = nil @@ -774,6 +836,8 @@ final class CodexAppServerActionProvider: ActionProvider { latestAgentMessageText = nil streamedCommentaryBuffer = "" hasEmittedEarlyCommentary = false + commentaryFlushTask?.cancel() + commentaryFlushTask = nil hasOpenedBrowserInCurrentTurn = false lastEmittedProgress = nil @@ -787,12 +851,25 @@ final class CodexAppServerActionProvider: ActionProvider { status = .completed(summary) terminalizePendingAction(.completed(summary)) } + + if let queuedSubmission { + beginPendingSubmissionAsFreshTurn(queuedSubmission) + } default: break } return } + if let id = message["id"] as? Int, + id == pendingSteerRequestID, + message["result"] as? [String: Any] != nil + { + pendingSteerRequestID = nil + promotePendingSubmissionAfterSteer() + return + } + if let id = message["id"] as? Int, id == 0, message["result"] as? [String: Any] != nil @@ -901,6 +978,7 @@ final class CodexAppServerActionProvider: ActionProvider { let threadID = thread["id"] as? String { activeThreadID = threadID + conversationState.threadStarted(threadID) hasSentThreadStart = false appendDebugEvent("<- thread/start ok \(String(threadID.suffix(6)))") notifyStateChanged() @@ -922,9 +1000,15 @@ final class CodexAppServerActionProvider: ActionProvider { let turnID = turn["id"] as? String { activeTurnID = turnID + conversationState.turnStarted(turnID) + } + if cancellationPendingTurnID, let activeThreadID, let activeTurnID { + cancellationPendingTurnID = false + sendTurnInterrupt(threadID: activeThreadID, turnID: activeTurnID) + return } emitPhase(.thinking, rawSource: "codex is working on it") - if pendingPrompt != nil, activeTurnID != nil { + if pendingSubmission != nil, activeTurnID != nil { sendTurnSteer() } return @@ -1020,6 +1104,7 @@ final class CodexAppServerActionProvider: ActionProvider { private func sendThreadStart() { guard activeThreadID == nil, !hasSentThreadStart else { return } hasSentThreadStart = true + conversationState.beginNewContext() let params = sessionCoordinator.threadStartParameters( sandbox: AppBundleConfiguration.stringValue(forKey: "CodexActionSandbox") ?? "danger-full-access" ) @@ -1092,6 +1177,9 @@ final class CodexAppServerActionProvider: ActionProvider { private func sendTurnStart() { guard let threadID = activeThreadID, let pendingPrompt else { return } + if let latestRequest { + conversationState.startRequest(latestRequest.id) + } var inputItems: [[String: Any]] = [] @@ -1158,11 +1246,14 @@ final class CodexAppServerActionProvider: ActionProvider { private func sendTurnSteer() { guard let threadID = activeThreadID, let activeTurnID, - let pendingPrompt + let pendingSubmission else { return } + let request = pendingSubmission.request + let prompt = wrappedPrompt(for: request) + var inputItems: [[String: Any]] = [] if let runtimeCapabilityNote { @@ -1172,8 +1263,7 @@ final class CodexAppServerActionProvider: ActionProvider { ]) } - if let latestRequest, - let screenshotPath = latestRequest.screenshotPath, + if let screenshotPath = request.screenshotPath, !screenshotPath.isEmpty { inputItems.append([ @@ -1181,7 +1271,7 @@ final class CodexAppServerActionProvider: ActionProvider { "path": screenshotPath, ]) - if let visualContext = visualContextMessage(for: latestRequest) { + if let visualContext = visualContextMessage(for: request) { inputItems.append([ "type": "text", "text": visualContext, @@ -1189,7 +1279,7 @@ final class CodexAppServerActionProvider: ActionProvider { } } - let activeSkills = activeBundledSkillsForCurrentRequest() + let activeSkills = activeBundledSkills(for: request) if !activeSkills.isEmpty { inputItems.append([ "type": "text", @@ -1208,12 +1298,13 @@ final class CodexAppServerActionProvider: ActionProvider { inputItems.append([ "type": "text", - "text": pendingPrompt, + "text": prompt, ]) let requestID = nextClientRequestID() + pendingSteerRequestID = requestID + conversationState.queueSteer(request.id) appendDebugEvent("-> turn/steer #\(requestID) turn=\(String(activeTurnID.suffix(6))) inputItems=\(inputItems.count)") - self.pendingPrompt = nil sendJSON([ "method": "turn/steer", "id": requestID, @@ -1224,17 +1315,61 @@ final class CodexAppServerActionProvider: ActionProvider { ], ]) - emitPhase( - .thinking, - detail: "steering the current codex turn.", - rawSource: "steering the current codex turn" + } + + private func promotePendingSubmissionAfterSteer() { + guard let pendingSubmission else { return } + self.pendingSubmission = nil + conversationState.steerAccepted() + eventHandler?(.interrupted("steered by a newer Orbit request.")) + eventHandler = pendingSubmission.handler + latestRequest = pendingSubmission.request + latestAgentMessageText = nil + latestFinalAnswerText = nil + streamedCommentaryBuffer = "" + hasEmittedEarlyCommentary = false + commentaryFlushTask?.cancel() + commentaryFlushTask = nil + lastEmittedLiveCommentary = nil + status = .running + emitPhase(.thinking, detail: "working on your follow-up.", rawSource: "turn steer accepted") + appendDebugEvent("<- turn/steer accepted; follow-up promoted") + notifyStateChanged() + } + + private func beginPendingSubmissionAsFreshTurn(_ submission: PendingSubmission) { + eventHandler = submission.handler + latestRequest = submission.request + conversationState.startRequest(submission.request.id) + pendingPrompt = wrappedPrompt(for: submission.request) + latestAgentMessageText = nil + latestFinalAnswerText = nil + streamedCommentaryBuffer = "" + hasEmittedEarlyCommentary = false + commentaryFlushTask?.cancel() + commentaryFlushTask = nil + lastEmittedLiveCommentary = nil + status = .running + submission.handler( + .phase( + OrbitActionProgress( + phase: .thinking, + detail: "starting your follow-up in the same Codex thread.", + rawSource: "previous turn completed before steer" + ) + ) ) + attemptPendingTurnStart() } private func activeBundledSkillsForCurrentRequest() -> [OrbitBundledSkill] { guard let latestRequest else { return [] } + return activeBundledSkills(for: latestRequest) + } + + private func activeBundledSkills(for request: OrbitActionRequest) -> [OrbitBundledSkill] { return OrbitBundledSkills.activeSkills( - for: latestRequest, + for: request, preparedCodexHome: preparedCodexHome ) } @@ -1337,8 +1472,12 @@ final class CodexAppServerActionProvider: ActionProvider { ? "Codex action process exited: \(stderrText!)" : "Codex action process exited unexpectedly." status = .failed(message) + conversationState.connectionLost() terminalizePendingAction(.failed(message)) teardownProcess() + Task { @MainActor [weak self] in + _ = await self?.prewarmSession() + } } private func summarizedCompletionMessage(from message: [String: Any]) -> String? { @@ -1670,6 +1809,8 @@ final class CodexAppServerActionProvider: ActionProvider { eventHandler?(.liveUpdate(update)) } + scheduleCommentaryFlush() + guard !hasEmittedEarlyCommentary, let snippet = Self.speakableCommentarySnippet(from: streamedCommentaryBuffer) else { @@ -1677,9 +1818,25 @@ final class CodexAppServerActionProvider: ActionProvider { } hasEmittedEarlyCommentary = true + commentaryFlushTask?.cancel() + commentaryFlushTask = nil eventHandler?(.commentary(snippet)) } + private func scheduleCommentaryFlush() { + guard !hasEmittedEarlyCommentary else { return } + commentaryFlushTask?.cancel() + commentaryFlushTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .milliseconds(700)) + guard let self, !Task.isCancelled, !hasEmittedEarlyCommentary, + let snippet = Self.flushableCommentarySnippet(from: streamedCommentaryBuffer) + else { return } + hasEmittedEarlyCommentary = true + commentaryFlushTask = nil + eventHandler?(.commentary(snippet)) + } + } + private func emitCompletedCommentaryIfNeeded(text: String) { if let update = Self.visibleCommentaryUpdate(from: text), update != lastEmittedLiveCommentary @@ -1695,6 +1852,8 @@ final class CodexAppServerActionProvider: ActionProvider { } hasEmittedEarlyCommentary = true + commentaryFlushTask?.cancel() + commentaryFlushTask = nil eventHandler?(.commentary(snippet)) } @@ -2052,36 +2211,11 @@ final class CodexAppServerActionProvider: ActionProvider { } static func speakableCommentarySnippet(from text: String) -> String? { - let cleaned = - text - .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) - .trimmingCharacters(in: .whitespacesAndNewlines) - - guard cleaned.count >= 28 else { return nil } - - if let firstCompletedSentence = firstCompletedSentence(in: cleaned) { - let candidate = firstCompletedSentence.trimmingCharacters(in: .whitespacesAndNewlines) - guard candidate.split(separator: " ").count >= 4 else { return nil } - return candidate - } - - guard cleaned.count >= 42, cleaned.split(separator: " ").count >= 7 else { return nil } + OrbitFirstLineExtractor.stableLine(from: text) + } - let maximumLength = 120 - let wasTruncated = cleaned.count > maximumLength - let prefix = String(cleaned.prefix(maximumLength)) - let trimmed = - (wasTruncated - ? prefix.replacingOccurrences(of: "\\s+\\S*$", with: "", options: .regularExpression) - : prefix) - .replacingOccurrences( - of: "\\b(?:and|or|to|for|with|of|in|on|at|by|from|about|into|over|after|before|without|using)$", - with: "", - options: .regularExpression - ) - .trimmingCharacters(in: .whitespacesAndNewlines) - guard trimmed.split(separator: " ").count >= 4 else { return nil } - return trimmed.hasSuffix(".") ? trimmed : "\(trimmed)." + static func flushableCommentarySnippet(from text: String) -> String? { + OrbitFirstLineExtractor.stableLine(from: text, allowUnterminated: true) } static func visibleCommentaryUpdate(from text: String) -> String? { @@ -2253,6 +2387,12 @@ final class CodexAppServerActionProvider: ActionProvider { private func teardownProcess() { let retiredTransport = transport + if let pendingSubmission { + pendingSubmission.handler(.interrupted("the Codex connection restarted before the follow-up was sent.")) + } + pendingSubmission = nil + pendingSteerRequestID = nil + cancellationPendingTurnID = false if let process, process.isRunning { gracefullyStopProcess(process) } @@ -2265,6 +2405,8 @@ final class CodexAppServerActionProvider: ActionProvider { pendingTurnStartRetryTask = nil interruptTimeoutTask?.cancel() interruptTimeoutTask = nil + commentaryFlushTask?.cancel() + commentaryFlushTask = nil stdoutHandle?.readabilityHandler = nil stderrHandle?.readabilityHandler = nil try? stdinHandle?.close() diff --git a/Orbit/DesignSystem.swift b/Orbit/DesignSystem.swift index 3c15922..2cbbfd6 100644 --- a/Orbit/DesignSystem.swift +++ b/Orbit/DesignSystem.swift @@ -2,9 +2,9 @@ // DesignSystem.swift // Orbit // -// Centralized design system for Orbit's soft-glass graphite shell. -// It defines the neutral palette, glass surfaces, and shared control -// styling used across the panel, HUD, onboarding, and overlay system. +// Centralized tokens for Orbit's quiet graphite-and-blue instrument panel. +// Tonal surfaces carry hierarchy; shadows are reserved for detached helpers +// and blue is reserved for focus, progress, and primary actions. // import AppKit @@ -306,74 +306,23 @@ struct DSPrimaryButtonStyle: ButtonStyle { @Environment(\.accessibilityReduceMotion) private var reduceMotion @State private var isHovered = false - // Separate state for the scale expansion so it animates on a slower, - // more gradual timeline (0.6s) than the background color snap (0.15s). - @State private var isHoverScaleExpanded = false - - // Whether the hover glow shadow is active. Builds up gradually (0.6s) - // on hover entry, fades out faster (0.3s) on exit. - @State private var isHoverGlowActive = false - - // Continuously toggles while hovered to drive a gentle breathing pulse - // in the glow shadow. Creates a living, organic feel — like the button - // is softly glowing, not just statically lit. - @State private var isGlowBreathingIn = false - func makeBody(configuration: Configuration) -> some View { configuration.label - .font(.system(size: 16, weight: .medium)) + .font(.body.weight(.semibold)) .foregroundColor(DS.Colors.textOnAccent) .frame(maxWidth: isFullWidth ? .infinity : nil) - .padding(.vertical, 14) - .padding(.horizontal, isFullWidth ? 0 : 20) + .frame(minHeight: 36) + .padding(.horizontal, isFullWidth ? 12 : 16) .background( - Capsule() + RoundedRectangle(cornerRadius: 8, style: .continuous) .fill(buttonBackgroundColor(isPressed: configuration.isPressed)) ) - // Hover glow — builds up gradually, then gently breathes while hovered. - // The breathing oscillates opacity and radius on a slow 2.5s loop, - // creating a candle-flame-like "alive" quality rather than a static highlight. - .shadow( - color: DS.Colors.accent.opacity( - isHoverGlowActive ? (isGlowBreathingIn ? 0.32 : 0.18) : 0 - ), - radius: isHoverGlowActive ? (isGlowBreathingIn ? 16 : 10) : 0 - ) - // Hover: gradually expand to 1.03. Press: snap down to 0.97. - .scaleEffect(reduceMotion ? 1 : (configuration.isPressed ? 0.97 : (isHoverScaleExpanded ? 1.03 : 1.0))) - .animation(reduceMotion ? nil : .easeOut(duration: 0.1), value: configuration.isPressed) + .opacity(configuration.isPressed ? 0.82 : 1) + .animation(reduceMotion ? nil : .easeOut(duration: 0.08), value: configuration.isPressed) .onHover { hovering in - // Background color — fast snap so the button feels responsive - withAnimation(.easeOut(duration: 0.15)) { + withAnimation(reduceMotion ? nil : .easeOut(duration: 0.08)) { isHovered = hovering } - - // Scale — slow, gradual expansion (like the button is swelling) - withAnimation(reduceMotion ? nil : .easeInOut(duration: hovering ? 0.6 : 0.3)) { - isHoverScaleExpanded = hovering - } - - // Glow — builds up gradually on entry, fades faster on exit - withAnimation(reduceMotion ? nil : .easeInOut(duration: hovering ? 0.6 : 0.3)) { - isHoverGlowActive = hovering - } - - // Breathing glow loop — gentle pulse while hovered. - // The 2.5s cycle keeps it feeling organic, not mechanical. - if hovering && !reduceMotion { - withAnimation( - .easeInOut(duration: 2.5) - .repeatForever(autoreverses: true) - ) { - isGlowBreathingIn = true - } - } else { - // Override the repeating animation with a finite one to stop cleanly - withAnimation(.easeOut(duration: 0.3)) { - isGlowBreathingIn = false - } - } - if hovering { NSCursor.pointingHand.push() } else { NSCursor.pop() } } } diff --git a/Orbit/Localizable.xcstrings b/Orbit/Localizable.xcstrings new file mode 100644 index 0000000..f011d0a --- /dev/null +++ b/Orbit/Localizable.xcstrings @@ -0,0 +1,81 @@ +{ + "sourceLanguage" : "en", + "strings" : { + "About" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "About" } } + } + }, + "Accessibility" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Accessibility" } } + } + }, + "Appearance" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Appearance" } } + } + }, + "Codex" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Codex" } } + } + }, + "Microphone" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Microphone" } } + } + }, + "Open System Settings" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Open System Settings" } } + } + }, + "Orbit" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Orbit" } } + } + }, + "Privacy & permissions" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Privacy & Permissions" } } + } + }, + "Quit & Reopen Orbit" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Quit & Reopen Orbit" } } + } + }, + "Reveal Orbit in Finder" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Reveal Orbit in Finder" } } + } + }, + "Screen Recording" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Screen Recording" } } + } + }, + "Settings" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Settings" } } + } + }, + "Set up Orbit" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Set up Orbit" } } + } + }, + "Voice" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Voice" } } + } + }, + "Working context" : { + "localizations" : { + "en" : { "stringUnit" : { "state" : "translated", "value" : "Working context" } } + } + } + }, + "version" : "1.0" +} diff --git a/Orbit/MenuBarPanelManager.swift b/Orbit/MenuBarPanelManager.swift index 5f14a74..09e225b 100644 --- a/Orbit/MenuBarPanelManager.swift +++ b/Orbit/MenuBarPanelManager.swift @@ -16,6 +16,7 @@ import SwiftUI extension Notification.Name { static let orbitDismissPanel = Notification.Name("orbitDismissPanel") + static let orbitShowPanel = Notification.Name("orbitShowPanel") } /// Custom NSPanel subclass that can become the key window even with @@ -30,10 +31,11 @@ final class MenuBarPanelManager: NSObject { private var panel: NSPanel? private var clickOutsideMonitor: Any? private var dismissPanelObserver: NSObjectProtocol? + private var showPanelObserver: NSObjectProtocol? private let orbitManager: OrbitManager - private let panelWidth: CGFloat = 312 - private let panelHeight: CGFloat = 380 + private let panelWidth: CGFloat = 344 + private let panelHeight: CGFloat = 480 init(orbitManager: OrbitManager) { self.orbitManager = orbitManager @@ -49,6 +51,15 @@ final class MenuBarPanelManager: NSObject { self?.hidePanel() } } + showPanelObserver = NotificationCenter.default.addObserver( + forName: .orbitShowPanel, + object: nil, + queue: .main + ) { [weak self] _ in + Task { @MainActor [weak self] in + self?.showPanel() + } + } } isolated deinit { @@ -58,6 +69,9 @@ final class MenuBarPanelManager: NSObject { if let observer = dismissPanelObserver { NotificationCenter.default.removeObserver(observer) } + if let observer = showPanelObserver { + NotificationCenter.default.removeObserver(observer) + } } // MARK: - Status Item @@ -71,6 +85,8 @@ final class MenuBarPanelManager: NSObject { button.image?.isTemplate = true button.action = #selector(statusItemClicked) button.target = self + button.setAccessibilityLabel("Orbit") + button.setAccessibilityIdentifier("menubar.orbit.open") } /// Opens the panel automatically on app launch so the user sees @@ -78,6 +94,9 @@ final class MenuBarPanelManager: NSObject { func showPanelOnLaunch() { // Small delay so the status item has time to appear in the menu bar DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + #if DEBUG + OrbitVisualQAState.renderRequestedFixtureIfNeeded() + #endif self.showPanel() } } diff --git a/Orbit/OrbitCodexConversationState.swift b/Orbit/OrbitCodexConversationState.swift new file mode 100644 index 0000000..39f402f --- /dev/null +++ b/Orbit/OrbitCodexConversationState.swift @@ -0,0 +1,102 @@ +import Foundation + +struct OrbitCodexConversationState: Equatable, Sendable { + enum Phase: Equatable, Sendable { + case disconnected + case startingThread + case ready + case startingTurn(requestID: UUID) + case running(requestID: UUID, turnID: String?) + case steering(activeRequestID: UUID, pendingRequestID: UUID) + case cancelling(requestID: UUID, waitingForTurnID: Bool) + case recovering(failedRequestID: UUID?) + } + + private(set) var contextGeneration = 0 + private(set) var threadID: String? + private(set) var activeRequestID: UUID? + private(set) var pendingRequestID: UUID? + private(set) var turnID: String? + private(set) var phase: Phase = .disconnected + + mutating func threadStarted(_ threadID: String) { + self.threadID = threadID + turnID = nil + activeRequestID = nil + pendingRequestID = nil + phase = .ready + } + + mutating func startRequest(_ requestID: UUID) { + precondition(threadID != nil) + activeRequestID = requestID + pendingRequestID = nil + turnID = nil + phase = .startingTurn(requestID: requestID) + } + + mutating func turnStarted(_ turnID: String) { + guard let activeRequestID else { return } + self.turnID = turnID + phase = .running(requestID: activeRequestID, turnID: turnID) + } + + mutating func queueSteer(_ requestID: UUID) { + guard let activeRequestID else { return } + pendingRequestID = requestID + phase = .steering(activeRequestID: activeRequestID, pendingRequestID: requestID) + } + + mutating func steerAccepted() { + guard let pendingRequestID else { return } + activeRequestID = pendingRequestID + self.pendingRequestID = nil + phase = .running(requestID: pendingRequestID, turnID: turnID) + } + + mutating func steerRejected() { + pendingRequestID = nil + guard let activeRequestID else { + phase = threadID == nil ? .disconnected : .ready + return + } + phase = .running(requestID: activeRequestID, turnID: turnID) + } + + mutating func requestCancellation() { + guard let activeRequestID else { return } + phase = .cancelling(requestID: activeRequestID, waitingForTurnID: turnID == nil) + } + + mutating func completeActiveRequest() -> UUID? { + let completed = activeRequestID + activeRequestID = nil + turnID = nil + if let pendingRequestID { + self.pendingRequestID = nil + activeRequestID = pendingRequestID + phase = .startingTurn(requestID: pendingRequestID) + } else { + phase = threadID == nil ? .disconnected : .ready + } + return completed + } + + mutating func beginNewContext() { + contextGeneration += 1 + threadID = nil + activeRequestID = nil + pendingRequestID = nil + turnID = nil + phase = .startingThread + } + + mutating func connectionLost() { + let failedRequestID = activeRequestID + threadID = nil + activeRequestID = nil + pendingRequestID = nil + turnID = nil + phase = .recovering(failedRequestID: failedRequestID) + } +} diff --git a/Orbit/OrbitManager.swift b/Orbit/OrbitManager.swift index 0c7dfea..651d604 100644 --- a/Orbit/OrbitManager.swift +++ b/Orbit/OrbitManager.swift @@ -119,11 +119,8 @@ final class OrbitManager: ObservableObject { private var onboardingTask: Task? private var onboardingLaunchTask: Task? private var codexSessionWarmupTask: Task? - private var actionAcknowledgementTask: Task? - private var completionNarrationTask: Task? private var codexWarmupGeneration: Int = 0 private var pendingCodexContextRestart = false - private var hasSpokenActionAcknowledgement = false private var escapeKeyMonitor: Any? private var lastObservedSetupStage: OrbitSetupStage? @@ -269,6 +266,7 @@ final class OrbitManager: ObservableObject { self?.refreshActionProviderPresentation() } } + configureNarrationCoordinator() self.lastObservedSetupStage = setupStage } @@ -324,9 +322,6 @@ final class OrbitManager: ObservableObject { transientHideTask?.cancel() onboardingTask?.cancel() codexSessionWarmupTask?.cancel() - actionAcknowledgementTask?.cancel() - completionNarrationTask?.cancel() - currentResponseTask?.cancel() currentResponseTask = nil currentResponseRequestID = nil @@ -643,9 +638,8 @@ final class OrbitManager: ObservableObject { private func refreshTextToSpeechProvider() { textToSpeechProvider.stopPlayback() fallbackTextToSpeechProvider.stopPlayback() - completionNarrationTask?.cancel() - completionNarrationTask = nil textToSpeechProvider = OrbitTTSProviderFactory.makePrimaryProvider(for: settings.voicePreset) + configureNarrationCoordinator() textToSpeechProviderDisplayName = textToSpeechProvider.displayName availableAppleVoices = OrbitAppleVoiceCatalog.availableVoices() selectedAppleVoiceSummary = OrbitAppleVoiceCatalog.currentSelectionSummary( @@ -823,17 +817,23 @@ final class OrbitManager: ObservableObject { } private func submitTranscriptToActionProvider(transcript: String) { + guard pendingToolPrompt == nil else { + activeActionStatus = .waitingForApproval("Answer the current Codex choice first.") + activeActionStatusSummary = OrbitActionPhase.waitingForChoice.summaryText + activeActionDetailLine = "Answer the current choice before sending another request." + appendActionUpdate("waiting for your answer") + showCodexActivityOverlayCard() + return + } + let previousRequestID = currentResponseRequestID currentResponseTask?.cancel() if let previousRequestID { releaseTemporaryCapture(for: previousRequestID) } - textToSpeechProvider.stopPlayback() - fallbackTextToSpeechProvider.stopPlayback() - completionNarrationTask?.cancel() - completionNarrationTask = nil + let requestID = UUID() + beginNarrationRequest(requestID) isRunningOnboardingTour = false - pendingToolPrompt = nil let shouldResetPresentation = !actionProvider.canInterruptCurrentAction if shouldResetPresentation { resetActionPresentationForNewRequest() @@ -849,9 +849,7 @@ final class OrbitManager: ObservableObject { voiceState = .processing clearDetectedElementLocation() showCodexActivityOverlayCard() - scheduleActionAcknowledgementFallback() - let requestID = UUID() currentResponseRequestID = requestID currentResponseTask = Task { [weak self] in guard let self else { return } @@ -917,7 +915,7 @@ final class OrbitManager: ObservableObject { applyActionProgress(progress) showCodexActivityOverlayCard() case .commentary(let commentary): - handleEarlyActionCommentary(commentary) + enqueueFirstLineNarration(commentary, requestID: requestID) case .liveUpdate(let update): activeActionDetailLine = update showCodexActivityOverlayCard() @@ -935,9 +933,6 @@ final class OrbitManager: ObservableObject { case .interrupted(let summary): releaseTemporaryCapture(for: requestID) let spokenSummary = conciseDetailLine(from: summary.isEmpty ? "stopped." : summary) - cancelActionAcknowledgementFlow() - textToSpeechProvider.stopPlayback() - fallbackTextToSpeechProvider.stopPlayback() activeActionProgress = OrbitActionProgress( phase: .interrupted, detail: spokenSummary, @@ -948,21 +943,22 @@ final class OrbitManager: ObservableObject { activeActionDetailLine = spokenSummary appendActionUpdate("interrupted") pendingToolPrompt = nil - voiceState = .idle + enqueueTerminalNarration( + spokenSummary, + fallback: "stopped.", + source: .failure, + requestID: requestID + ) showCodexActivityOverlayCard() scheduleCodexActivityOverlayDismiss() scheduleTransientHideIfNeeded() case .completed(let summary): releaseTemporaryCapture(for: requestID) - cancelActionAcknowledgementFlow() pendingToolPrompt = nil handleCompletedCodexSummary(summary, requestID: requestID) case .failed(let errorMessage): releaseTemporaryCapture(for: requestID) let shortDetail = conciseDetailLine(from: errorMessage) - cancelActionAcknowledgementFlow() - textToSpeechProvider.stopPlayback() - fallbackTextToSpeechProvider.stopPlayback() pendingToolPrompt = nil activeActionProgress = OrbitActionProgress( phase: .failed, @@ -975,15 +971,12 @@ final class OrbitManager: ObservableObject { appendActionUpdate(OrbitActionPhase.failed.summaryText) showCodexActivityOverlayCard() scheduleCodexActivityOverlayDismiss() - completionNarrationTask?.cancel() - completionNarrationTask = Task { [weak self] in - await self?.speakCompletionText( - nil, - fallback: "i could not finish that action.", - source: .failure, - turnIdentifier: requestID.uuidString - ) - } + enqueueTerminalNarration( + shortDetail, + fallback: "i could not finish that action.", + source: .failure, + requestID: requestID + ) } if event.isTerminal, currentResponseRequestID == requestID { @@ -1017,15 +1010,12 @@ final class OrbitManager: ObservableObject { } showCodexActivityOverlayCard() scheduleCodexActivityOverlayDismiss() - completionNarrationTask?.cancel() - completionNarrationTask = Task { [weak self] in - await self?.speakCompletionText( - spokenSummary, - fallback: nil, - source: .completion, - turnIdentifier: requestID.uuidString - ) - } + enqueueTerminalNarration( + spokenSummary, + fallback: nil, + source: .completion, + requestID: requestID + ) } private func buildPrimaryScreenLabel( @@ -1242,9 +1232,6 @@ final class OrbitManager: ObservableObject { activeActionDetailLine = nil pendingToolPrompt = nil codexOverlayDismissTask?.cancel() - actionAcknowledgementTask?.cancel() - actionAcknowledgementTask = nil - hasSpokenActionAcknowledgement = false } private func applyActionProgress(_ progress: OrbitActionProgress) { @@ -1263,57 +1250,49 @@ final class OrbitManager: ObservableObject { appendActionUpdate(progress.resolvedDetail ?? progress.phase.summaryText) } - private func handleEarlyActionCommentary(_ commentary: String) { - guard !hasSpokenActionAcknowledgement, - let acknowledgement = normalizedEarlyAcknowledgement(from: commentary) - else { - return - } - - hasSpokenActionAcknowledgement = true - actionAcknowledgementTask?.cancel() - actionAcknowledgementTask = Task { @MainActor [weak self] in + private func configureNarrationCoordinator() { + guard let coordinator = textToSpeechProvider as? OrbitVoiceCoordinator else { return } + coordinator.onPlaybackStateChanged = { [weak self] isPlaying in guard let self else { return } - do { - try await speakNarration( - acknowledgement, - source: .earlyCommentary, - turnIdentifier: currentResponseRequestID?.uuidString - ) - } catch { - OrbitSupportLog.append("voice", "failed early commentary speech: \(error.localizedDescription)") + self.voiceState = isPlaying ? .responding : .idle + if !isPlaying { + self.scheduleTransientHideIfNeeded() } } } - private func speakCompletionText( - _ primary: String?, - fallback: String?, - source: OrbitNarrationSource, - turnIdentifier: String? - ) async { - let trimmedPrimary = primary?.trimmingCharacters(in: .whitespacesAndNewlines) - let trimmedFallback = fallback?.trimmingCharacters(in: .whitespacesAndNewlines) - let finalText = (trimmedPrimary?.isEmpty == false ? trimmedPrimary : trimmedFallback) ?? "done." + private func beginNarrationRequest(_ requestID: UUID) { + if let coordinator = textToSpeechProvider as? OrbitVoiceCoordinator { + coordinator.beginRequest(requestID.uuidString) + } else { + textToSpeechProvider.stopPlayback() + } + fallbackTextToSpeechProvider.stopPlayback() + } - await waitForCurrentSpeechToSettle(maximumWait: 2.4) - guard !Task.isCancelled else { return } - voiceState = .responding + private func enqueueFirstLineNarration(_ text: String, requestID: UUID) { + guard let coordinator = textToSpeechProvider as? OrbitVoiceCoordinator else { return } + coordinator.enqueueFirstLine(text, requestIdentifier: requestID.uuidString) + } - do { - try await speakNarration( - finalText, + private func enqueueTerminalNarration( + _ text: String, + fallback: String?, + source: OrbitNarrationSource, + requestID: UUID + ) { + if let coordinator = textToSpeechProvider as? OrbitVoiceCoordinator { + coordinator.enqueueTerminal( + text, + fallback: fallback, source: source, - turnIdentifier: turnIdentifier + requestIdentifier: requestID.uuidString ) - } catch { - let visibleError = trimmedFallback?.isEmpty == false ? trimmedFallback! : error.localizedDescription - OrbitSupportLog.append("voice", "speech failed: \(visibleError)") + } else { + Task { [weak self] in + try? await self?.textToSpeechProvider.speakText(text.isEmpty ? (fallback ?? "done.") : text) + } } - - guard !Task.isCancelled else { return } - voiceState = .idle - scheduleTransientHideIfNeeded() } private func speakNarration( @@ -1334,58 +1313,6 @@ final class OrbitManager: ObservableObject { } } - private func waitForCurrentSpeechToSettle(maximumWait: TimeInterval) async { - let deadline = Date().addingTimeInterval(maximumWait) - while textToSpeechProvider.isPlaying || fallbackTextToSpeechProvider.isPlaying, - Date() < deadline - { - try? await Task.sleep(nanoseconds: 120_000_000) - } - - if textToSpeechProvider.isPlaying { - textToSpeechProvider.stopPlayback() - } - if fallbackTextToSpeechProvider.isPlaying { - fallbackTextToSpeechProvider.stopPlayback() - } - } - - private func scheduleActionAcknowledgementFallback() { - actionAcknowledgementTask?.cancel() - actionAcknowledgementTask = nil - } - - private func cancelActionAcknowledgementFlow() { - actionAcknowledgementTask?.cancel() - actionAcknowledgementTask = nil - } - - private func normalizedEarlyAcknowledgement(from text: String) -> String? { - let cleaned = - text - .replacingOccurrences(of: "\\s+", with: " ", options: .regularExpression) - .replacingOccurrences(of: #"\[POINT:[^\]]+\]"#, with: "", options: .regularExpression) - .trimmingCharacters(in: .whitespacesAndNewlines) - - guard !cleaned.isEmpty else { return nil } - - let firstSentence = cleaned.split(whereSeparator: { ".!?".contains($0) }).first.map(String.init) ?? cleaned - let candidate = firstSentence.trimmingCharacters(in: .whitespacesAndNewlines) - guard candidate.split(separator: " ").count >= 2 else { return nil } - - if candidate.count <= 72 { - return candidate.hasSuffix(".") ? candidate : "\(candidate)." - } - - let prefix = String(candidate.prefix(69)) - let trimmed = - prefix - .replacingOccurrences(of: "\\s+\\S*$", with: "", options: .regularExpression) - .trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return nil } - return "\(trimmed)..." - } - private func conciseDetailLine(from text: String) -> String { let cleaned = text diff --git a/Orbit/OrbitPanelView.swift b/Orbit/OrbitPanelView.swift deleted file mode 100644 index 8426e93..0000000 --- a/Orbit/OrbitPanelView.swift +++ /dev/null @@ -1,1935 +0,0 @@ -// -// OrbitPanelView.swift -// Orbit -// -// Compact menu bar panel for Orbit. Orbit is a Codex-native macOS voice -// shell, so the panel focuses on permissions, voice mode, Codex state, -// and lightweight controls. -// - -import AVFoundation -import AppKit -import SwiftUI - -private enum OrbitPanelSection { - case activity - case settings -} - -struct OrbitPanelView: View { - @ObservedObject var orbitManager: OrbitManager - @ObservedObject private var orbitSettings = OrbitSettings.shared - @State private var openAIAPIKeyDraft = "" - @State private var showAPIKeyDialog = false - @State private var showAboutPopover = false - @State private var showTeamActivity = false - @State private var panelSection: OrbitPanelSection = .activity - - private let panelShape = RoundedRectangle(cornerRadius: 24, style: .continuous) - private let cardShape = RoundedRectangle(cornerRadius: 18, style: .continuous) - - var body: some View { - ScrollView(.vertical) { - VStack(alignment: .leading, spacing: 12) { - header - - if orbitManager.setupStage == .permissions { - permissionsCard - } else if orbitManager.setupStage == .automationDisclosure { - automationDisclosureCard - } else if orbitManager.setupStage == .auth { - authCard - } else if orbitManager.setupStage == .voiceChoice { - voiceChoiceCard - } else if orbitManager.setupStage == .cloudKey { - cloudVoiceCard - } else if orbitManager.setupStage == .setupComplete { - setupCompleteCard - } else { - readyPanelContent - } - - footer - } - .padding(13) - } - .scrollIndicators(.hidden) - .frame(width: 312) - .frame(maxHeight: 720) - .background(panelBackground) - .onExitCommand { - NotificationCenter.default.post(name: .orbitDismissPanel, object: nil) - } - .onAppear { resetLegacyAppleVoiceSelection() } - } - - private var header: some View { - HStack(alignment: .center, spacing: 12) { - OrbitMarkView(size: 17) - .shadow(color: Color.white.opacity(0.14), radius: 7, x: 0, y: 0) - - VStack(alignment: .leading, spacing: 2) { - HStack(spacing: 6) { - Text("Orbit") - .font(.system(size: 16, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - - Text("|") - .font(.system(size: 11, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary.opacity(0.55)) - - Text("@4xiom_") - .font(.system(size: 10, weight: .medium, design: .rounded)) - .foregroundColor(DS.Colors.textTertiary.opacity(0.72)) - } - - Text(headerSubtitle) - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .lineLimit(1) - } - - Spacer(minLength: 8) - - if shouldShowHeaderStatus { - DSQuietStatusChip(title: statusText, tint: statusDotColor) - .accessibilityLabel("Orbit status: \(statusText)") - } - - Button { - NotificationCenter.default.post(name: .orbitDismissPanel, object: nil) - } label: { - Image(systemName: "xmark") - .font(.system(size: 10, weight: .semibold)) - .foregroundColor(DS.Colors.textSecondary) - .frame(width: 28, height: 28) - .orbitGlassCard( - shape: Circle(), - fillOpacity: 0.26, - borderOpacity: 0.14, - highlightOpacity: 0.18, - shadowOpacity: 0.14, - glowOpacity: 0.02 - ) - } - .buttonStyle(.plain) - .pointerCursor() - .accessibilityLabel("Close Orbit panel") - .accessibilityHint("Returns focus to the current application") - } - .padding(.horizontal, 2) - .padding(.top, 1) - } - - private var permissionsCard: some View { - sectionCard { - sectionHeader(title: "Permissions", subtitle: "Grant desktop access once, then Orbit is ready to talk, point, and act.") - - VStack(spacing: 0) { - microphonePermissionRow - .padding(.vertical, 4) - rowDivider - accessibilityPermissionRow - .padding(.vertical, 4) - rowDivider - screenAccessPermissionRow - .padding(.vertical, 4) - } - - } - } - - private var readyPanelContent: some View { - VStack(alignment: .leading, spacing: 10) { - segmentedControl(spacing: 3) { - settingOptionButton(label: "Activity", isSelected: panelSection == .activity) { - panelSection = .activity - } - settingOptionButton(label: "Settings", isSelected: panelSection == .settings) { - panelSection = .settings - } - } - .accessibilityLabel("Panel section") - - if panelSection == .activity { - taskActivityCard - } else { - settingsCard - } - } - } - - private var taskActivityCard: some View { - sectionCard { - sectionHeader(title: "Current task", subtitle: "Live Codex activity and explicit team-up work.") - codexCard - if !orbitManager.codexSubagentActivities.isEmpty { - rowDivider - teamActivityRow - } - } - } - - private var settingsCard: some View { - sectionCard { - sectionHeader(title: "Settings", subtitle: "Voice, model, working context, and Orbit visibility.") - - authStatusRow - rowDivider - unrestrictedAutomationRow - rowDivider - - if orbitManager.setupStage == .onboarding { - compactSetupRow - rowDivider - } - - voicePresetRow - rowDivider - codexConfigurationGroup - if let notice = orbitManager.codexModelMigrationNotice { - Text(notice) - .font(.system(size: 10.5, weight: .medium)) - .foregroundStyle(DS.Colors.warning) - .fixedSize(horizontal: false, vertical: true) - .accessibilityLabel(notice) - } - rowDivider - agentFolderRow - rowDivider - providerRow(icon: "waveform.badge.mic", title: "Speech to Text", value: panelSpeechToTextLabel) - rowDivider - microphoneInputRow - rowDivider - speechOutputRow - if orbitSettings.voicePreset == .cloudVoice { - rowDivider - openAIKeyRow - } - rowDivider - showOrbitToggleRow - } - } - - private var automationDisclosureCard: some View { - sectionCard { - sectionHeader( - title: "Unrestricted automation", - subtitle: "Orbit works like an operator you explicitly invoke, not a continuous screen recorder." - ) - - VStack(alignment: .leading, spacing: 10) { - Label("Can run commands and edit files without approval prompts", systemImage: "terminal") - Label("Takes one fresh screen capture for each request", systemImage: "camera.viewfinder") - Label("Deletes the temporary capture when the turn ends", systemImage: "trash") - } - .font(.system(size: 11, weight: .medium)) - .foregroundStyle(DS.Colors.textSecondary) - .accessibilityElement(children: .combine) - - Button("I understand — continue") { - orbitManager.acknowledgeUnrestrictedAutomation() - } - .buttonStyle(.borderedProminent) - .controlSize(.regular) - .frame(maxWidth: .infinity, alignment: .trailing) - } - } - - private var unrestrictedAutomationRow: some View { - rowLabel( - icon: "bolt.shield", - title: "Automation Access", - subtitle: "Unrestricted · approvals automatic · one temporary capture per request" - ) - .accessibilityLabel("Automation access: unrestricted, approvals automatic, one temporary screen capture per request") - } - - private var authCard: some View { - sectionCard { - sectionHeader(title: "Connect", subtitle: "Sign in with ChatGPT so Orbit can start its live Codex session.") - - VStack(alignment: .leading, spacing: 10) { - Text(authCardMessage) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .fixedSize(horizontal: false, vertical: true) - - if orbitManager.codexAuthState == .loginInProgress { - HStack(spacing: 8) { - primaryButton("Open ChatGPT Login") { - orbitManager.reopenCodexLogin() - } - - secondaryCapsuleButton("Reconnect") { - orbitManager.reconnectCodexSession() - } - } - } else { - primaryButton(primaryAuthButtonTitle) { - orbitManager.connectCodexAccount() - } - } - - if shouldShowAuthRetry { - secondaryCapsuleButton("Retry Codex") { - orbitManager.reconnectCodexSession() - } - } - } - } - } - - private var voiceChoiceCard: some View { - sectionCard { - sectionHeader(title: "Voice", subtitle: "Choose whether Orbit should use your Mac or OpenAI voice for speech.") - - VStack(spacing: 10) { - voiceChoiceOption( - title: "Use Local Voice", - subtitle: "Apple on-device recognition and speech. No API key.", - isSelected: orbitSettings.voicePreset == .localVoice - ) { - orbitManager.selectVoicePreset(.localVoice) - } - - voiceChoiceOption( - title: "Use Cloud Voice", - subtitle: "OpenAI transcription and AI-generated voice with your API key.", - isSelected: orbitSettings.voicePreset == .cloudVoice - ) { - orbitManager.selectVoicePreset(.cloudVoice) - } - } - } - } - - private var cloudVoiceCard: some View { - sectionCard { - sectionHeader(title: "Cloud Voice", subtitle: "Add an OpenAI API key to enable cloud speech.") - - VStack(alignment: .leading, spacing: 10) { - Label( - "Cloud responses use an AI-generated voice and send speech audio to OpenAI.", - systemImage: "cloud" - ) - .font(.system(size: 10.5, weight: .medium)) - .foregroundStyle(DS.Colors.textSecondary) - - SecureField("OpenAI API Key", text: $openAIAPIKeyDraft) - .textFieldStyle(.plain) - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textPrimary) - .padding(.horizontal, 12) - .padding(.vertical, 10) - .orbitGlassCard( - shape: RoundedRectangle(cornerRadius: 14, style: .continuous), - fillOpacity: 0.20, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - - HStack(spacing: 8) { - DSQuietStatusChip( - title: orbitManager.openAICloudCredentialState.summaryText, - tint: cloudVoiceStatusTint - ) - - Text(orbitManager.openAICloudCredentialState.detailText) - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .fixedSize(horizontal: false, vertical: true) - } - - HStack(spacing: 8) { - primaryButton("Save OpenAI Key") { - let draft = openAIAPIKeyDraft - Task { - let didConnect = await orbitManager.saveOpenAIAPIKey(draft) - if didConnect { - openAIAPIKeyDraft = "" - } - } - } - .opacity(canSubmitOpenAIKey ? 1.0 : 0.55) - .disabled(!canSubmitOpenAIKey) - - secondaryCapsuleButton("Use Local") { - orbitManager.selectVoicePreset(.localVoice) - } - } - } - } - } - - private func sectionCard(@ViewBuilder content: () -> Content) -> some View { - VStack(alignment: .leading, spacing: 12) { - content() - } - .padding(14) - .frame(maxWidth: .infinity, alignment: .leading) - .orbitGlassCard( - shape: cardShape, - fillOpacity: 0.34, - borderOpacity: 0.15, - highlightOpacity: 0.24, - shadowOpacity: 0.18, - glowOpacity: 0.03 - ) - } - - private func sectionHeader(title: String, subtitle: String) -> some View { - VStack(alignment: .leading, spacing: 5) { - Text(title) - .font(.system(size: 13, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - - Text(subtitle) - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .fixedSize(horizontal: false, vertical: true) - } - .frame(maxWidth: .infinity, alignment: .leading) - } - - private func voiceChoiceOption( - title: String, - subtitle: String, - isSelected: Bool, - action: @escaping () -> Void - ) -> some View { - Button(action: action) { - HStack(alignment: .center, spacing: 10) { - VStack(alignment: .leading, spacing: 3) { - Text(title) - .font(.system(size: 12.5, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - - Text(subtitle) - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .fixedSize(horizontal: false, vertical: true) - } - - Spacer(minLength: 8) - - Image(systemName: isSelected ? "checkmark.circle.fill" : "circle") - .font(.system(size: 14, weight: .semibold)) - .foregroundColor(isSelected ? Color.white.opacity(0.90) : DS.Colors.textTertiary.opacity(0.7)) - } - .padding(.horizontal, 12) - .padding(.vertical, 11) - .frame(maxWidth: .infinity, alignment: .leading) - .orbitGlassCard( - shape: RoundedRectangle(cornerRadius: 16, style: .continuous), - fillOpacity: isSelected ? 0.24 : 0.16, - borderOpacity: isSelected ? 0.18 : 0.10, - highlightOpacity: 0.18, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .buttonStyle(.plain) - .pointerCursor() - } - - private var compactSetupRow: some View { - VStack(alignment: .leading, spacing: 10) { - VStack(alignment: .leading, spacing: 3) { - Text("Hold Control+Option to talk") - .font(.system(size: 12, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - - Text("Run the intro once to see Orbit speak and point.") - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .fixedSize(horizontal: false, vertical: true) - } - - Button("Start Tour") { - orbitManager.triggerOnboarding() - } - .font(.system(size: 11, weight: .semibold)) - .foregroundColor(DS.Colors.textOnAccent) - .frame(maxWidth: .infinity) - .padding(.horizontal, 11) - .padding(.vertical, 8) - .background( - Capsule(style: .continuous) - .fill(DS.Colors.accent) - ) - .buttonStyle(.plain) - .pointerCursor() - } - } - - private var setupCompleteCard: some View { - sectionCard { - VStack(alignment: .center, spacing: 6) { - Image(systemName: "checkmark.circle.fill") - .font(.system(size: 28)) - .foregroundColor(DS.Colors.success) - - Text("You're all set") - .font(.system(size: 15, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - } - .frame(maxWidth: .infinity) - .padding(.vertical, 4) - - VStack(alignment: .leading, spacing: 6) { - Text("WHAT ORBIT CAN DO") - .font(.system(size: 10, weight: .semibold, design: .rounded)) - .foregroundColor(DS.Colors.textTertiary) - - bulletItem("Learn any software on your screen") - bulletItem("Control your browser hands-free") - bulletItem("Create documents and presentations") - bulletItem("Get pointed to the right button") - } - - VStack(alignment: .leading, spacing: 4) { - Text("TRY THIS FIRST") - .font(.system(size: 10, weight: .semibold, design: .rounded)) - .foregroundColor(DS.Colors.textTertiary) - - Text("Hold Control+Option and ask\n\"what's on my screen?\"") - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .fixedSize(horizontal: false, vertical: true) - } - - primaryButton("Get Started") { - orbitManager.dismissSetupComplete() - } - } - } - - private func bulletItem(_ text: String) -> some View { - HStack(alignment: .top, spacing: 6) { - Text("·") - .font(.system(size: 12, weight: .bold)) - .foregroundColor(DS.Colors.textTertiary) - Text(text) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - } - } - - private var voicePresetRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel( - icon: "waveform", - title: "Voice Mode", - subtitle: orbitSettings.voicePreset == .cloudVoice - ? "OpenAI cloud voice" - : "Apple local voice" - ) - - Spacer(minLength: 12) - - segmentedControl { - ForEach(OrbitVoicePreset.allCases) { preset in - settingOptionButton( - label: preset.displayName, - isSelected: orbitSettings.voicePreset == preset - ) { - orbitManager.selectVoicePreset(preset) - } - } - } - } - } - - private var teamActivityRow: some View { - DisclosureGroup(isExpanded: $showTeamActivity) { - VStack(alignment: .leading, spacing: 7) { - ForEach(orbitManager.codexSubagentActivities) { activity in - HStack(spacing: 7) { - Circle() - .fill(activity.status == "running" ? DS.Colors.accent : DS.Colors.textTertiary) - .frame(width: 6, height: 6) - VStack(alignment: .leading, spacing: 1) { - Text(activity.agentPath) - .font(.system(size: 10.5, weight: .semibold)) - Text(activity.message ?? activity.status) - .font(.system(size: 10)) - .foregroundStyle(DS.Colors.textTertiary) - .lineLimit(2) - } - } - } - } - .padding(.top, 7) - } label: { - Label("Team-up activity", systemImage: "person.2") - .font(.system(size: 11, weight: .semibold)) - .foregroundStyle(DS.Colors.textSecondary) - } - .accessibilityHint("Expands child-agent activity for this task") - } - - private var agentFolderRow: some View { - HStack(spacing: 10) { - rowLabel( - icon: "folder", - title: "Agent Folder", - subtitle: "Starting context only; filesystem access remains unrestricted" - ) - Spacer(minLength: 6) - Button(agentFolderLabel) { chooseAgentFolder() } - .buttonStyle(.borderless) - .lineLimit(1) - .help(orbitSettings.codexAgentFolder.isEmpty ? "Uses your home folder" : orbitSettings.codexAgentFolder) - } - } - - private var agentFolderLabel: String { - let path = orbitSettings.codexAgentFolder.trimmingCharacters(in: .whitespacesAndNewlines) - return path.isEmpty ? "Home" : URL(fileURLWithPath: path).lastPathComponent - } - - private func chooseAgentFolder() { - let panel = NSOpenPanel() - panel.title = "Choose Orbit Agent Folder" - panel.message = "This sets the starting context. Orbit still has unrestricted filesystem access." - panel.canChooseDirectories = true - panel.canChooseFiles = false - panel.allowsMultipleSelection = false - if panel.runModal() == .OK, let url = panel.url { - orbitSettings.codexAgentFolder = url.path - } - } - - private var codexCard: some View { - VStack(alignment: .leading, spacing: 10) { - HStack(alignment: .center, spacing: 10) { - OrbitMarkView(size: 14) - .shadow(color: Color.white.opacity(0.10), radius: 4, x: 0, y: 0) - - VStack(alignment: .leading, spacing: 2) { - Text("Codex Session") - .font(.system(size: 13, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - - Text("Persistent Codex session") - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .lineLimit(1) - } - - Spacer(minLength: 8) - - HStack(spacing: 6) { - if orbitManager.canInterruptCodexAction { - Button { - orbitManager.interruptCurrentAction() - } label: { - Image(systemName: "stop.fill") - .font(.system(size: 9.5, weight: .bold)) - .foregroundColor(DS.Colors.textSecondary) - .frame(width: 28, height: 28) - .orbitGlassCard( - shape: Circle(), - fillOpacity: 0.24, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .buttonStyle(.plain) - .pointerCursor() - .help("Interrupt Codex") - .accessibilityLabel("Interrupt current Codex task") - } - - Button { - orbitManager.reconnectCodexSession() - } label: { - Image(systemName: "arrow.clockwise") - .font(.system(size: 10.5, weight: .semibold)) - .foregroundColor(DS.Colors.textSecondary) - .frame(width: 28, height: 28) - .orbitGlassCard( - shape: Circle(), - fillOpacity: 0.24, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .buttonStyle(.plain) - .pointerCursor() - .help("Reconnect Codex") - .accessibilityLabel("Reconnect Codex session") - } - - if shouldShowCodexStatusChip { - DSQuietStatusChip(title: actionStatusLabel, tint: codexStatusColor) - .accessibilityLabel("Codex status: \(actionStatusLabel)") - } - } - - Text(codexSummaryLine) - .font(.system(size: 13, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .lineLimit(1) - - if let detailLine = codexDetailLine { - Text(detailLine) - .font(.system(size: 11, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .lineLimit(2) - } - - if let activeTurnSummary = orbitManager.codexActiveTurnSummary, !activeTurnSummary.isEmpty { - Text(activeTurnSummary) - .font(.system(size: 10, weight: .medium, design: .monospaced)) - .foregroundColor(DS.Colors.textTertiary.opacity(0.92)) - .lineLimit(1) - } - - if let pendingToolPrompt = orbitManager.pendingToolPrompt { - VStack(alignment: .leading, spacing: 7) { - Text(pendingToolPrompt.title) - .font(.system(size: 11.5, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - .fixedSize(horizontal: false, vertical: true) - - if let detail = pendingToolPrompt.detail, !detail.isEmpty { - Text(detail) - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .fixedSize(horizontal: false, vertical: true) - } - - VStack(spacing: 6) { - ForEach(pendingToolPrompt.options, id: \.self) { option in - primaryChoiceButton(option) { - orbitManager.answerToolPrompt(with: option) - } - } - } - } - .padding(.top, 2) - } else if !visibleRecentActionUpdates.isEmpty { - VStack(alignment: .leading, spacing: 5) { - ForEach(Array(visibleRecentActionUpdates.enumerated()), id: \.offset) { _, update in - HStack(alignment: .top, spacing: 6) { - Circle() - .fill(Color.white.opacity(0.55)) - .frame(width: 4, height: 4) - .padding(.top, 5) - Text(update) - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .fixedSize(horizontal: false, vertical: true) - } - } - } - } - } - .padding(.vertical, 2) - } - - private var codexConfigurationGroup: some View { - VStack(alignment: .leading, spacing: 9) { - HStack { - Label("Codex configuration", systemImage: "cpu") - .font(.system(size: 11, weight: .semibold)) - .foregroundStyle(DS.Colors.textSecondary) - Spacer(minLength: 8) - Text("Account catalog") - .font(.system(size: 9.5, weight: .medium)) - .foregroundStyle(DS.Colors.textTertiary) - } - - codexModelRow - rowDivider.padding(.leading, -26) - - if availableServiceTiers.count > 1 { - codexServiceTierRow - rowDivider.padding(.leading, -26) - } - - codexReasoningEffortRow - } - .padding(10) - .background( - RoundedRectangle(cornerRadius: 10, style: .continuous) - .fill(DS.Colors.surface2) - ) - .accessibilityElement(children: .contain) - .accessibilityLabel("Codex configuration") - } - - private var codexModelRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel( - icon: "cpu", - title: "Model", - subtitle: nil - ) - - Spacer(minLength: 12) - - modelSelectorMenu - } - .accessibilityElement(children: .contain) - .accessibilityLabel("Codex model") - .accessibilityValue(selectedModelShortLabel) - } - - private var codexReasoningEffortRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel(icon: "dial.medium", title: "Effort", subtitle: "Options supported by the selected model") - - Spacer(minLength: 8) - - codexEffortSelector - } - } - - @ViewBuilder - private var codexEffortSelector: some View { - if orbitManager.availableCodexEfforts.count <= 4 { - segmentedControl(spacing: 2) { - ForEach(orbitManager.availableCodexEfforts) { effort in - settingOptionButton( - label: effort.displayName, - isSelected: orbitSettings.codexReasoningEffort == effort - ) { - orbitSettings.codexReasoningEffort = effort - } - } - } - } else { - Menu { - ForEach(orbitManager.availableCodexEfforts) { effort in - Button { - orbitSettings.codexReasoningEffort = effort - } label: { - if orbitSettings.codexReasoningEffort == effort { - Label(effort.displayName, systemImage: "checkmark") - } else { - Text(effort.displayName) - } - } - } - } label: { - selectorLabel(orbitSettings.codexReasoningEffort.displayName) - } - .menuStyle(.borderlessButton) - .accessibilityLabel("Codex effort") - .accessibilityValue(orbitSettings.codexReasoningEffort.displayName) - } - } - - private var codexServiceTierRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel(icon: "bolt", title: "Service Tier", subtitle: nil) - - Spacer(minLength: 8) - - if availableServiceTiers.count <= 3 { - segmentedControl(spacing: 2) { - ForEach(availableServiceTiers) { tier in - settingOptionButton( - label: tier.displayName, - isSelected: orbitSettings.codexServiceTier == tier - ) { - orbitSettings.codexServiceTier = tier - } - } - } - } else { - Menu { - ForEach(availableServiceTiers) { tier in - Button(tier.displayName) { orbitSettings.codexServiceTier = tier } - } - } label: { - selectorLabel(orbitSettings.codexServiceTier.displayName) - } - .menuStyle(.borderlessButton) - .accessibilityLabel("Codex service tier") - .accessibilityValue(orbitSettings.codexServiceTier.displayName) - } - } - } - - private func selectorLabel(_ title: String) -> some View { - HStack(spacing: 5) { - Text(title) - .lineLimit(1) - Image(systemName: "chevron.up.chevron.down") - .font(.system(size: 8.5, weight: .semibold)) - } - .font(.system(size: 10.5, weight: .semibold)) - .foregroundStyle(DS.Colors.textSecondary) - .padding(.horizontal, 9) - .padding(.vertical, 6) - .background(Capsule().fill(DS.Colors.surface3)) - } - - private var availableServiceTiers: [OrbitCodexServiceTier] { - let serverTiers = - orbitManager.availableCodexModels - .first(where: { $0.model == orbitSettings.codexActionModel })? - .supportedServiceTiers ?? [] - return [.serverDefault] + serverTiers.filter { !$0.rawValue.isEmpty } - } - - private func providerRow(icon: String, title: String, value: String) -> some View { - HStack(alignment: .center, spacing: 12) { - rowLabel(icon: icon, title: title, subtitle: nil) - - Spacer(minLength: 8) - - Text(value) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.22, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - } - - private var authStatusRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel( - icon: "person.crop.circle", - title: "ChatGPT", - subtitle: orbitManager.codexAccountSummary ?? "Connected" - ) - - Spacer(minLength: 8) - - Button("Sign Out") { - orbitManager.signOutCodexAccount() - } - .font(.system(size: 10.5, weight: .semibold)) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.18, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - .buttonStyle(.plain) - .pointerCursor() - } - } - - private var modelSelectorMenu: some View { - Menu { - ForEach(orbitManager.availableCodexModels) { model in - Button { - orbitSettings.codexActionModel = model.model - } label: { - if orbitSettings.codexActionModel == model.model { - Label(model.displayName, systemImage: "checkmark") - } else { - Text(model.displayName) - } - } - } - } label: { - HStack(spacing: 6) { - Text(selectedModelShortLabel) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .lineLimit(1) - .minimumScaleFactor(0.8) - - Image(systemName: "chevron.up.chevron.down") - .font(.system(size: 9.5, weight: .semibold)) - .foregroundColor(DS.Colors.textTertiary) - } - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.22, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .menuStyle(.borderlessButton) - .pointerCursor() - .accessibilityLabel("Codex model") - .accessibilityValue(selectedModelShortLabel) - } - - private var speechOutputRow: some View { - VStack(alignment: .leading, spacing: 6) { - HStack(alignment: .center, spacing: 12) { - rowLabel( - icon: "speaker.wave.2.fill", - title: "Voice", - subtitle: orbitSettings.voicePreset == .localVoice ? "Siri Natural Nora · on device" : "OpenAI speech" - ) - - Spacer(minLength: 8) - - if orbitSettings.voicePreset == .localVoice { - VStack(alignment: .trailing, spacing: 5) { - Text(noraVoiceStatusLabel) - .font(.system(size: 11, weight: .medium)) - .foregroundColor( - orbitManager.isNoraVoiceAvailable - ? DS.Colors.textSecondary - : DS.Colors.warningText - ) - .lineLimit(1) - .accessibilityLabel("Local narrator") - .accessibilityValue(noraVoiceStatusLabel) - - Button(orbitManager.isPreviewingAppleVoice ? "Stop" : "Preview") { - orbitManager.toggleAppleVoicePreview() - } - .buttonStyle(.borderless) - .font(.system(size: 10, weight: .semibold)) - .accessibilityLabel(orbitManager.isPreviewingAppleVoice ? "Stop voice preview" : "Preview selected voice") - } - } else { - Text(panelTextToSpeechLabel) - .font(.system(size: 11.5, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.22, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - } - - if orbitSettings.voicePreset == .localVoice, !orbitManager.isNoraVoiceAvailable, - !orbitManager.isCheckingNoraVoice - { - Text("Siri Natural Nora (Voice 4) is unavailable to Orbit. Local narration will use the best compatible Apple fallback voice.") - .font(.system(size: 9.5, weight: .medium)) - .foregroundColor(DS.Colors.warningText.opacity(0.88)) - .fixedSize(horizontal: false, vertical: true) - .accessibilityLabel( - "Siri Natural Nora Voice 4 is unavailable. Orbit will use the best compatible Apple fallback voice." - ) - } - } - } - - private var noraVoiceStatusLabel: String { - if orbitManager.isCheckingNoraVoice { return "Checking Nora…" } - return orbitManager.isNoraVoiceAvailable ? "Voice 4 · Natural Nora" : "Fallback voice" - } - - private func resetLegacyAppleVoiceSelection() { - let selected = orbitSettings.appleTTSVoiceIdentifier.trimmingCharacters(in: .whitespacesAndNewlines) - guard !selected.isEmpty else { return } - orbitManager.selectAppleVoice("") - } - - private var microphoneInputRow: some View { - VStack(alignment: .leading, spacing: 6) { - HStack(spacing: 10) { - rowLabel( - icon: "mic", - title: "Microphone", - subtitle: orbitManager.microphoneDeviceNotice ?? microphonePrivacyLabel - ) - Spacer(minLength: 6) - Menu { - Button("System Default") { orbitManager.selectMicrophone("") } - Divider() - ForEach(orbitManager.availableMicrophones) { microphone in - Button(microphone.name) { orbitManager.selectMicrophone(microphone.uid) } - } - } label: { - Text(selectedMicrophoneLabel) - .font(.system(size: 10.5, weight: .medium)) - .lineLimit(1) - .frame(maxWidth: 100, alignment: .trailing) - } - .menuStyle(.borderlessButton) - } - HStack(spacing: 8) { - ProgressView(value: orbitManager.microphoneTestLevel) - .progressViewStyle(.linear) - .accessibilityLabel("Microphone input level") - .accessibilityValue("\(Int(orbitManager.microphoneTestLevel * 100)) percent") - Button(orbitManager.isTestingMicrophone ? "Stop" : "Test") { - orbitManager.toggleMicrophoneTest() - } - .buttonStyle(.borderless) - .font(.system(size: 10, weight: .semibold)) - .accessibilityLabel(orbitManager.isTestingMicrophone ? "Stop microphone test" : "Test microphone") - } - } - .onAppear { orbitManager.refreshMicrophones() } - } - - private var microphonePrivacyLabel: String { - switch orbitSettings.voicePreset { - case .localVoice: - return "On-device recognition · audio stays on this Mac" - case .cloudVoice: - return "Cloud recognition · speech audio is sent to OpenAI" - } - } - - private var selectedMicrophoneLabel: String { - let selectedUID = orbitSettings.microphoneDeviceUID.trimmingCharacters(in: .whitespacesAndNewlines) - guard !selectedUID.isEmpty else { return "System" } - return orbitManager.availableMicrophones.first(where: { $0.uid == selectedUID })?.name ?? "Disconnected" - } - - private var openAIKeyRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel( - icon: "key", - title: "OpenAI Key", - subtitle: orbitManager.openAICloudCredentialState.detailText - ) - - Spacer(minLength: 8) - - HStack(spacing: 8) { - DSQuietStatusChip( - title: orbitManager.openAICloudCredentialState.summaryText, - tint: cloudVoiceStatusTint - ) - - Button(openAIKeyActionTitle) { - openAIAPIKeyDraft = "" - showAPIKeyDialog = true - } - .font(.system(size: 10.5, weight: .semibold)) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.18, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - .buttonStyle(.plain) - .pointerCursor() - .popover(isPresented: $showAPIKeyDialog, arrowEdge: .bottom) { - apiKeyDialogContent - } - } - } - } - - private var apiKeyDialogContent: some View { - VStack(alignment: .leading, spacing: 12) { - Text(openAIKeyDialogTitle) - .font(.system(size: 13, weight: .semibold)) - .foregroundColor(.primary) - - SecureField("sk-...", text: $openAIAPIKeyDraft) - .textFieldStyle(.roundedBorder) - .font(.system(size: 12, weight: .medium)) - .frame(width: 240) - - HStack(spacing: 8) { - Spacer() - Button("Cancel") { - showAPIKeyDialog = false - openAIAPIKeyDraft = "" - } - .buttonStyle(.plain) - .font(.system(size: 12, weight: .medium)) - .foregroundColor(.secondary) - - Button("Save") { - let draft = openAIAPIKeyDraft - Task { - let didConnect = await orbitManager.saveOpenAIAPIKey(draft) - if didConnect { - openAIAPIKeyDraft = "" - showAPIKeyDialog = false - } - } - } - .buttonStyle(.borderedProminent) - .font(.system(size: 12, weight: .semibold)) - .disabled(openAIAPIKeyDraft.trimmingCharacters(in: .whitespaces).isEmpty) - } - } - .padding(16) - } - - private var showOrbitToggleRow: some View { - HStack(alignment: .center, spacing: 12) { - rowLabel(icon: "cursorarrow.motionlines", title: "Show Orbit", subtitle: "Keep the cursor visible") - - Spacer(minLength: 8) - - Toggle( - "Show Orbit cursor", - isOn: Binding( - get: { orbitManager.isOrbitCursorEnabled }, - set: { orbitManager.setOrbitCursorEnabled($0) } - ) - ) - .toggleStyle(.switch) - .labelsHidden() - .tint(Color.white.opacity(0.8)) - .scaleEffect(0.8) - .accessibilityLabel("Show Orbit cursor") - .accessibilityValue(orbitManager.isOrbitCursorEnabled ? "On" : "Off") - } - } - - private var openAIKeyActionTitle: String { - switch orbitManager.openAICloudCredentialState { - case .connected: - return "Replace" - case .missing, .validating, .invalid, .networkError: - return "Add" - } - } - - private var openAIKeyDialogTitle: String { - switch orbitManager.openAICloudCredentialState { - case .connected: - return "Replace OpenAI Key" - case .missing, .validating, .invalid, .networkError: - return "Add OpenAI Key" - } - } - - private func rowLabel(icon: String, title: String, subtitle: String?) -> some View { - HStack(alignment: .center, spacing: 10) { - Image(systemName: icon) - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .frame(width: 16) - - VStack(alignment: .leading, spacing: subtitle == nil ? 0 : 2) { - Text(title) - .font(.system(size: 13, weight: .medium)) - .foregroundColor(DS.Colors.textSecondary) - .lineLimit(1) - .minimumScaleFactor(0.86) - - if let subtitle, !subtitle.isEmpty { - Text(subtitle) - .font(.system(size: 10.5, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - .lineLimit(2) - .fixedSize(horizontal: false, vertical: true) - } - } - .layoutPriority(1) - } - } - - private func segmentedControl( - spacing: CGFloat = 4, - @ViewBuilder content: () -> Content - ) -> some View { - HStack(spacing: spacing) { - content() - } - .padding(2) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.18, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.10, - glowOpacity: 0.01 - ) - } - - private var footer: some View { - HStack(alignment: .center) { - Button { - NSApp.terminate(nil) - } label: { - Label("Quit Orbit", systemImage: "power") - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - } - .buttonStyle(.plain) - .pointerCursor() - - Spacer() - - if orbitManager.hasCompletedOnboarding { - Button { - orbitManager.replayOnboarding() - } label: { - Label("Replay Tour", systemImage: "sparkles") - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - } - .buttonStyle(.plain) - .pointerCursor() - } - - Spacer() - - Button { - showAboutPopover.toggle() - } label: { - Label("About", systemImage: "info.circle") - .font(.system(size: 12, weight: .medium)) - .foregroundColor(DS.Colors.textTertiary) - } - .buttonStyle(.plain) - .pointerCursor() - .popover(isPresented: $showAboutPopover, arrowEdge: .bottom) { - aboutPopoverContent - } - } - .padding(.horizontal, 4) - .padding(.top, 2) - } - - private var aboutPopoverContent: some View { - VStack(alignment: .center, spacing: 12) { - OrbitMarkView() - .frame(width: 32, height: 32) - .foregroundColor(.primary) - - VStack(spacing: 2) { - Text("Orbit") - .font(.system(size: 15, weight: .semibold)) - .foregroundColor(.primary) - Text("v\(Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "1.0.0")") - .font(.system(size: 11, weight: .medium)) - .foregroundColor(.secondary) - Text("by @4xiom_") - .font(.system(size: 11, weight: .medium)) - .foregroundColor(.secondary) - } - - Divider() - - VStack(alignment: .leading, spacing: 8) { - aboutLink(icon: "globe", title: "Website", url: "https://www.orbitcodex.com") - aboutLink(icon: "chevron.left.forwardslash.chevron.right", title: "GitHub", url: "https://github.com/4xiomdev") - aboutLink(icon: "at", title: "X / Twitter", url: "https://x.com/4xiom_") - aboutLink(icon: "envelope", title: "Email", url: "mailto:4xiomdev@gmail.com") - aboutLink(icon: "heart", title: "Sponsor", url: "https://github.com/sponsors/4xiomdev") - aboutLink(icon: "cup.and.saucer", title: "Buy Me a Coffee", url: "https://buymeacoffee.com/4xiom") - } - - if AppBundleConfiguration.showsCodexDebugInfo { - Divider() - - VStack(alignment: .leading, spacing: 8) { - Text("Codex Debug") - .font(.system(size: 10, weight: .semibold, design: .rounded)) - .foregroundColor(.secondary) - - if let activeTurnSummary = orbitManager.codexActiveTurnSummary, !activeTurnSummary.isEmpty { - Text(activeTurnSummary) - .font(.system(size: 10, weight: .medium, design: .monospaced)) - .foregroundColor(.secondary) - } - - if !orbitManager.codexDebugEvents.isEmpty { - Text(orbitManager.codexDebugEvents.suffix(4).joined(separator: "\n")) - .font(.system(size: 9, weight: .medium, design: .monospaced)) - .foregroundColor(.secondary) - .fixedSize(horizontal: false, vertical: true) - } - } - - Divider() - } - - Text("Open source under MIT license") - .font(.system(size: 10, weight: .medium)) - .foregroundColor(.secondary) - } - .padding(16) - .frame(width: 200) - } - - private func aboutLink(icon: String, title: String, url: String) -> some View { - Button { - if let linkURL = URL(string: url) { - NSWorkspace.shared.open(linkURL) - } - } label: { - HStack(spacing: 8) { - Image(systemName: icon) - .font(.system(size: 11)) - .foregroundColor(.secondary) - .frame(width: 16) - Text(title) - .font(.system(size: 12, weight: .medium)) - .foregroundColor(.primary) - Spacer() - Image(systemName: "arrow.up.forward") - .font(.system(size: 9, weight: .semibold)) - .foregroundColor(Color.secondary.opacity(0.6)) - } - } - .buttonStyle(.plain) - .pointerCursor() - } - - private var selectedModelShortLabel: String { - orbitManager.availableCodexModels.first(where: { $0.model == orbitSettings.codexActionModel })?.shortDisplayName - ?? OrbitCodexModelOption.fallbackOption(for: orbitSettings.codexActionModel)?.shortDisplayName - ?? orbitSettings.codexActionModel - } - - private func settingOptionButton( - label: String, - isSelected: Bool, - action: @escaping () -> Void - ) -> some View { - Button(action: action) { - Text(label) - .font(.system(size: 11, weight: .semibold)) - .foregroundColor(isSelected ? DS.Colors.textPrimary : DS.Colors.textTertiary) - .lineLimit(1) - .minimumScaleFactor(0.9) - .fixedSize(horizontal: true, vertical: false) - .padding(.horizontal, 10) - .padding(.vertical, 7) - .background( - Capsule(style: .continuous) - .fill(isSelected ? Color.white.opacity(0.10) : Color.clear) - ) - } - .buttonStyle(.plain) - .pointerCursor() - .accessibilityLabel(label) - .accessibilityValue(isSelected ? "Selected" : "Not selected") - .accessibilityAddTraits(isSelected ? .isSelected : []) - } - - private func primaryButton(_ title: String, action: @escaping () -> Void) -> some View { - Button(action: action) { - Text(title) - .font(.system(size: 13.5, weight: .semibold)) - .foregroundColor(DS.Colors.textOnAccent) - .frame(maxWidth: .infinity) - .padding(.vertical, 12) - .background( - Capsule(style: .continuous) - .fill(DS.Colors.accent) - ) - } - .buttonStyle(.plain) - .pointerCursor() - } - - private func secondaryCapsuleButton(_ title: String, action: @escaping () -> Void) -> some View { - Button(action: action) { - Text(title) - .font(.system(size: 11, weight: .semibold)) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 12) - .padding(.vertical, 8) - .frame(maxWidth: .infinity) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.18, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .buttonStyle(.plain) - .pointerCursor() - } - - private func primaryChoiceButton(_ title: String, action: @escaping () -> Void) -> some View { - Button(action: action) { - Text(title) - .font(.system(size: 11, weight: .semibold)) - .foregroundColor(DS.Colors.textPrimary) - .frame(maxWidth: .infinity, alignment: .leading) - .padding(.horizontal, 11) - .padding(.vertical, 9) - .orbitGlassCard( - shape: RoundedRectangle(cornerRadius: 12, style: .continuous), - fillOpacity: 0.18, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .buttonStyle(.plain) - .pointerCursor() - } - - private var rowDivider: some View { - Rectangle() - .fill(Color.white.opacity(0.08)) - .frame(height: 1) - .padding(.leading, 26) - } - - private var panelBackground: some View { - panelShape - .fill(Color.clear) - .orbitGlassCard( - shape: panelShape, - fillOpacity: 0.55, - borderOpacity: 0.18, - highlightOpacity: 0.27, - shadowOpacity: 0.24, - glowOpacity: 0.03 - ) - } - - private var headerSubtitle: String { - if orbitManager.isRunningOnboardingTour { - return "Orbit intro in progress" - } - switch orbitManager.setupStage { - case .permissions: - return "Codex voice shell" - case .automationDisclosure: - return "Review automation access" - case .auth: - return "Connect ChatGPT" - case .voiceChoice: - return "Choose your voice mode" - case .cloudKey: - return "Connect OpenAI voice" - case .onboarding: - return "Orbit setup" - case .setupComplete: - return "Setup complete" - case .ready: - break - } - switch orbitManager.activeActionStatus { - case .idle: - return "Codex voice shell" - case .failed: - return "Codex needs attention" - case .interrupted: - return "Codex interrupted" - case .running, .waitingForApproval, .completed: - return "Live Codex session" - } - } - - private var shouldShowHeaderStatus: Bool { - orbitManager.setupStage != .ready || panelSection != .activity - } - - private var shouldShowCodexStatusChip: Bool { - switch orbitManager.activeActionStatus { - case .idle: - return false - case .running, .waitingForApproval, .completed, .interrupted, .failed: - return true - } - } - - private var statusDotColor: Color { - if orbitManager.isRunningOnboardingTour { - return Color.white.opacity(0.88) - } - - switch orbitManager.activeActionStatus { - case .running: - return Color.white.opacity(0.88) - case .waitingForApproval: - return DS.Colors.warning - case .completed: - return DS.Colors.success - case .interrupted: - return DS.Colors.warning - case .failed: - return DS.Colors.destructive - case .idle: - break - } - - if !orbitManager.isOverlayVisible { - return DS.Colors.textTertiary - } - - switch orbitManager.voiceState { - case .idle: - return Color.white.opacity(0.82) - case .listening, .processing, .responding: - return Color.white.opacity(0.96) - } - } - - private var statusText: String { - if orbitManager.isRunningOnboardingTour { - return "Tour" - } - - switch orbitManager.setupStage { - case .permissions, .automationDisclosure, .auth, .voiceChoice, .cloudKey, .onboarding, .setupComplete: - return "Setup" - case .ready: - break - } - - switch orbitManager.activeActionStatus { - case .running: - return "Working" - case .waitingForApproval: - return "Waiting" - case .completed: - return "Done" - case .interrupted: - return "Stopped" - case .failed: - return "Issue" - case .idle: - break - } - - if !orbitManager.isOverlayVisible { - return "Ready" - } - - switch orbitManager.voiceState { - case .idle: - return "Ready" - case .listening: - return "Listening" - case .processing: - return "Thinking" - case .responding: - return "Speaking" - } - } - - private var actionStatusLabel: String { - switch orbitManager.activeActionStatus { - case .idle: - return "Live" - case .running: - return "Working" - case .waitingForApproval: - return "Waiting" - case .completed: - return "Done" - case .interrupted: - return "Stopped" - case .failed: - return "Issue" - } - } - - private var codexStatusColor: Color { - switch orbitManager.activeActionStatus { - case .idle: - return Color.white.opacity(0.70) - case .running: - return Color.white.opacity(0.92) - case .waitingForApproval: - return DS.Colors.warning - case .completed: - return DS.Colors.success - case .interrupted: - return DS.Colors.warning - case .failed: - return DS.Colors.destructive - } - } - - private var visibleRecentActionUpdates: [String] { - let activeDetail = orbitManager.activeActionDetailLine? - .trimmingCharacters(in: .whitespacesAndNewlines) - - let filtered = orbitManager.recentActionUpdates.filter { update in - guard let activeDetail, !activeDetail.isEmpty else { return true } - return update.trimmingCharacters(in: .whitespacesAndNewlines) != activeDetail - } - - return Array(filtered.suffix(4)) - } - - private var codexDetailLine: String? { - if let detail = orbitManager.activeActionDetailLine { - return detail - } - - if orbitManager.isRunningOnboardingTour { - return "showing how Orbit points and speaks." - } - - switch orbitManager.activeActionStatus { - case .idle: - if let accountSummary = orbitManager.codexAccountSummary, - !accountSummary.isEmpty - { - return accountSummary.lowercased() - } - if orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("ready in session") - || orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("connected to codex") - { - return "live codex session connected." - } - if orbitManager.codexSessionSummary.localizedCaseInsensitiveContains("starting codex") { - return "starting the live codex session." - } - return nil - case .running, .waitingForApproval, .completed, .failed, .interrupted: - return nil - } - } - - private var codexSummaryLine: String { - if orbitManager.isRunningOnboardingTour { - return "orbit intro in progress" - } - - if let summary = orbitManager.activeActionStatusSummary, !summary.isEmpty { - return summary - } - - switch orbitManager.activeActionStatus { - case .idle: - return "ready" - case .running: - return "thinking" - case .waitingForApproval: - return "waiting for approval" - case .completed: - return "done" - case .interrupted: - return "interrupted" - case .failed: - return "failed" - } - } - - private var authCardMessage: String { - switch orbitManager.codexAuthState { - case .unknown, .checking: - return "Orbit is checking whether you're already signed in to Orbit." - case .authRequired: - return "Orbit uses your ChatGPT account with its own Orbit-managed session. Connect once and future launches can reuse Orbit's saved session." - case .loginInProgress: - return "Finish the ChatGPT browser sign-in, then come back to Orbit." - case .authFailed(let message): - return message - case .runtimeUnavailable(let message): - return message - case .authenticated(let email, let plan): - let emailPart = email ?? "Connected" - if let plan, !plan.isEmpty { - return "\(emailPart) · ChatGPT \(plan.capitalized)" - } - return emailPart - } - } - - private var primaryAuthButtonTitle: String { - switch orbitManager.codexAuthState { - case .authFailed, .runtimeUnavailable: - return "Connect ChatGPT" - case .loginInProgress: - return "Open ChatGPT Login" - default: - return "Connect ChatGPT" - } - } - - private var shouldShowAuthRetry: Bool { - switch orbitManager.codexAuthState { - case .authFailed, .runtimeUnavailable: - return true - default: - return false - } - } - - private var panelSpeechToTextLabel: String { - switch orbitManager.orbitDictationManager.transcriptionProviderDisplayName { - case "OpenAI Transcribe": - return "OpenAI" - default: - return orbitManager.orbitDictationManager.transcriptionProviderDisplayName - } - } - - private var panelTextToSpeechLabel: String { - switch orbitManager.textToSpeechProviderDisplayName { - case "OpenAI Voice": - return "OpenAI" - case "Apple Speech": - return "Siri" - default: - return orbitManager.textToSpeechProviderDisplayName - } - } - - private var cloudVoiceStatusTint: Color { - switch orbitManager.openAICloudCredentialState { - case .connected: - return DS.Colors.success - case .validating: - return Color.white.opacity(0.88) - case .missing: - return DS.Colors.textTertiary - case .invalid, .networkError: - return DS.Colors.warning - } - } - - private var canSubmitOpenAIKey: Bool { - let trimmed = openAIAPIKeyDraft.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return false } - if orbitManager.openAICloudCredentialState == .validating { - return false - } - return true - } - - private var accessibilityPermissionRow: some View { - permissionRow( - label: "Accessibility", - iconName: "hand.raised", - isGranted: orbitManager.hasAccessibilityPermission, - subtitle: orbitManager.hasAccessibilityPermission - ? nil - : "If Orbit is missing, use Find App.", - action: { - orbitManager.permissionCoordinator.begin(.accessibility) - }, - alternateAction: { - orbitManager.permissionCoordinator.revealOrbitInFinder() - orbitManager.permissionCoordinator.openSettings(for: .accessibility) - }, - alternateTitle: "Reveal Orbit" - ) - } - - private var screenAccessPermissionRow: some View { - permissionRow( - label: "Screen Access", - iconName: "rectangle.dashed.badge.record", - isGranted: orbitManager.hasUsableScreenAccessPermission, - subtitle: orbitManager.hasUsableScreenAccessPermission - ? nil - : "Grant once so Orbit can see your current screen.", - action: { - orbitManager.permissionCoordinator.begin(.screenRecording) - }, - alternateAction: { - orbitManager.permissionCoordinator.openSettings(for: .screenRecording) - }, - alternateTitle: "Open Settings" - ) - } - - private func relaunchOrbit() { - let url = URL(fileURLWithPath: Bundle.main.resourcePath!) - .deletingLastPathComponent().deletingLastPathComponent() - let task = Process() - task.launchPath = "/usr/bin/open" - task.arguments = [url.path] - try? task.run() - NSApp.terminate(nil) - } - - private var microphonePermissionRow: some View { - permissionRow( - label: "Microphone", - iconName: "mic", - isGranted: orbitManager.hasMicrophonePermission, - subtitle: nil - ) { - orbitManager.permissionCoordinator.begin(.microphone) - } - } - - private func permissionRow( - label: String, - iconName: String, - isGranted: Bool, - subtitle: String?, - action: @escaping () -> Void, - alternateAction: (() -> Void)? = nil, - alternateTitle: String? = nil - ) -> some View { - HStack(alignment: .center, spacing: 12) { - rowLabel(icon: iconName, title: label, subtitle: subtitle) - - Spacer(minLength: 8) - - if isGranted { - DSQuietStatusChip(title: "Granted", tint: DS.Colors.success) - } else { - VStack(alignment: .trailing, spacing: 8) { - Button(action: action) { - Text("Grant") - .font(.system(size: 11, weight: .semibold)) - .lineLimit(1) - .fixedSize(horizontal: true, vertical: false) - .foregroundColor(DS.Colors.textOnAccent) - .padding(.horizontal, 12) - .padding(.vertical, 7) - .background( - Capsule(style: .continuous) - .fill(DS.Colors.accent) - ) - } - .buttonStyle(.plain) - .pointerCursor() - - if let alternateAction, let alternateTitle { - Button(action: alternateAction) { - Text(alternateTitle) - .font(.system(size: 11, weight: .semibold)) - .lineLimit(1) - .fixedSize(horizontal: true, vertical: false) - .foregroundColor(DS.Colors.textSecondary) - .padding(.horizontal, 12) - .padding(.vertical, 7) - .orbitGlassCard( - shape: Capsule(style: .continuous), - fillOpacity: 0.18, - borderOpacity: 0.12, - highlightOpacity: 0.16, - shadowOpacity: 0.08, - glowOpacity: 0.01 - ) - } - .buttonStyle(.plain) - .pointerCursor() - } - } - } - } - } -} diff --git a/Orbit/OrbitPanelViewV2.swift b/Orbit/OrbitPanelViewV2.swift new file mode 100644 index 0000000..b531d9c --- /dev/null +++ b/Orbit/OrbitPanelViewV2.swift @@ -0,0 +1,1101 @@ +import AppKit +import SwiftUI + +enum OrbitPanelRoute: Equatable, Sendable { + case task + case settings + case voice + case codex + case context + case privacy + case appearance + case about +} + +enum OrbitPanelIntent: Sendable { + case openSettings + case goBack + case interruptTask + case reconnectCodex + case replayTour + case quit +} + +struct OrbitPanelPresentationState: Equatable, Sendable { + let title: String + let summary: String + let detail: String? + let statusLabel: String + let statusColor: OrbitPanelStatusColor + let canInterrupt: Bool + + @MainActor + init(manager: OrbitManager) { + canInterrupt = manager.canInterruptCodexAction + detail = manager.activeActionDetailLine + switch manager.activeActionStatus { + case .idle: + title = "Ready" + summary = "Hold Control + Option and ask Orbit anything about your screen." + statusLabel = "Ready" + statusColor = .neutral + case .running: + title = "Working" + summary = manager.activeActionStatusSummary ?? "Codex is working on your request." + statusLabel = "Working" + statusColor = .active + case .waitingForApproval(let message): + title = "Needs your answer" + summary = message + statusLabel = "Waiting" + statusColor = .warning + case .interrupted(let message): + title = "Stopped" + summary = message + statusLabel = "Stopped" + statusColor = .warning + case .completed(let message): + title = "Finished" + summary = message + statusLabel = "Done" + statusColor = .success + case .failed(let message): + title = "Needs attention" + summary = message + statusLabel = "Issue" + statusColor = .failure + } + } +} + +enum OrbitPanelStatusColor: Equatable, Sendable { + case neutral + case active + case success + case warning + case failure + + var color: Color { + switch self { + case .neutral: DS.Colors.textTertiary + case .active: DS.Colors.accent + case .success: DS.Colors.success + case .warning: DS.Colors.warning + case .failure: DS.Colors.destructive + } + } +} + +struct OrbitPanelView: View { + @ObservedObject var orbitManager: OrbitManager + @ObservedObject private var settings = OrbitSettings.shared + @State private var route: OrbitPanelRoute = .task + @State private var openAIAPIKeyDraft = "" + @State private var permissionActionScreenFrame: CGRect? + @State private var showRecentActivity = false + @State private var showTeamActivity = false + + private let panelShape = RoundedRectangle(cornerRadius: 16, style: .continuous) + + @ViewBuilder + var body: some View { + #if DEBUG + if let fixture = OrbitVisualQAState.requested { + OrbitVisualQAPanel(state: fixture) + } else { + livePanel + } + #else + livePanel + #endif + } + + private var livePanel: some View { + ScrollView(.vertical) { + VStack(alignment: .leading, spacing: 16) { + header + content + } + .padding(16) + } + .scrollIndicators(.hidden) + .frame(width: 344) + .frame(maxHeight: 720) + .background(panelShape.fill(DS.Colors.background)) + .clipShape(panelShape) + .overlay(panelShape.stroke(DS.Colors.borderSubtle, lineWidth: 1)) + .coordinateSpace(name: "orbit.panel") + .onExitCommand { handle(.goBack) } + .onChange(of: orbitManager.setupStage) { _, stage in + if stage != .ready { route = .task } + } + .accessibilityIdentifier("panel.orbit.root") + } + + private var header: some View { + HStack(spacing: 10) { + if route != .task, orbitManager.setupStage == .ready { + iconButton("Back", systemImage: "chevron.left", identifier: "panel.navigation.back") { + handle(.goBack) + } + } else { + OrbitMarkView(size: 18) + .frame(width: 28, height: 28) + .accessibilityHidden(true) + } + + VStack(alignment: .leading, spacing: 1) { + Text(headerTitle) + .font(.headline) + .foregroundStyle(DS.Colors.textPrimary) + Text(headerSubtitle) + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + .lineLimit(1) + } + + Spacer(minLength: 8) + + if orbitManager.setupStage == .ready, route == .task { + iconButton("Settings", systemImage: "gearshape", identifier: "panel.navigation.settings") { + handle(.openSettings) + } + } + + Menu { + Button("Replay tour") { handle(.replayTour) } + Button("About Orbit") { route = .about } + Divider() + Button("Quit Orbit") { handle(.quit) } + } label: { + Image(systemName: "ellipsis") + .frame(width: 28, height: 28) + .contentShape(Rectangle()) + } + .menuStyle(.borderlessButton) + .accessibilityLabel("More Orbit options") + .accessibilityIdentifier("panel.navigation.more") + + iconButton("Close", systemImage: "xmark", identifier: "panel.navigation.close") { + NotificationCenter.default.post(name: .orbitDismissPanel, object: nil) + } + } + .accessibilityElement(children: .contain) + .accessibilityIdentifier("panel.header") + } + + @ViewBuilder + private var content: some View { + switch orbitManager.setupStage { + case .permissions: + permissionOnboarding + case .automationDisclosure: + automationDisclosure + case .auth: + authSetup + case .voiceChoice: + voiceChoiceSetup + case .cloudKey: + cloudKeySetup + case .onboarding: + tourSetup + case .setupComplete: + setupComplete + case .ready: + readyContent + } + } + + @ViewBuilder + private var readyContent: some View { + switch route { + case .task: taskView + case .settings: settingsRoot + case .voice: voiceSettings + case .codex: codexSettings + case .context: contextSettings + case .privacy: privacySettings + case .appearance: appearanceSettings + case .about: aboutView + } + } + + private var taskView: some View { + let presentation = OrbitPanelPresentationState(manager: orbitManager) + return VStack(alignment: .leading, spacing: 14) { + HStack(alignment: .firstTextBaseline) { + Text(presentation.title) + .font(.title3.weight(.semibold)) + .foregroundStyle(DS.Colors.textPrimary) + Spacer() + statusBadge(presentation.statusLabel, color: presentation.statusColor.color) + } + + Text(presentation.summary) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("activity.summary") + + if let detail = presentation.detail, detail != presentation.summary { + Text(detail) + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + .fixedSize(horizontal: false, vertical: true) + .accessibilityIdentifier("activity.detail") + } + + if let prompt = orbitManager.pendingToolPrompt { + VStack(alignment: .leading, spacing: 10) { + Text(prompt.title) + .font(.body.weight(.semibold)) + .foregroundStyle(DS.Colors.textPrimary) + if let detail = prompt.detail { + Text(detail) + .font(.caption) + .foregroundStyle(DS.Colors.textSecondary) + } + ForEach(prompt.options, id: \.self) { option in + primaryButton(option, identifier: "activity.choice.\(stableIdentifier(option))") { + orbitManager.answerToolPrompt(with: option) + } + } + } + .tonalGroup() + .accessibilityIdentifier("activity.choice.prompt") + } + + if presentation.canInterrupt { + secondaryButton("Stop current task", systemImage: "stop.fill", identifier: "activity.task.stop") { + handle(.interruptTask) + } + } else if case .failed = orbitManager.activeActionStatus { + primaryButton("Reconnect Codex", identifier: "activity.task.retry") { + handle(.reconnectCodex) + } + } + + if !orbitManager.recentActionUpdates.isEmpty { + disclosure( + "Recent activity", + isExpanded: $showRecentActivity, + identifier: "activity.recent.toggle" + ) { + ForEach(Array(orbitManager.recentActionUpdates.suffix(4).enumerated()), id: \.offset) { _, update in + Label(update, systemImage: "circle.fill") + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + .labelStyle(OrbitActivityLabelStyle()) + } + } + } + + if !orbitManager.codexSubagentActivities.isEmpty { + disclosure( + "Team-up activity", + isExpanded: $showTeamActivity, + identifier: "activity.team.toggle" + ) { + ForEach(orbitManager.codexSubagentActivities) { activity in + VStack(alignment: .leading, spacing: 2) { + Text(activity.agentPath) + .font(.caption.weight(.semibold)) + Text(activity.message ?? activity.status) + .font(.caption2) + .foregroundStyle(DS.Colors.textTertiary) + } + } + } + } + + HStack(spacing: 8) { + Circle() + .fill(DS.Colors.success) + .frame(width: 6, height: 6) + Text(orbitManager.codexSessionSummary) + .font(.caption2) + .foregroundStyle(DS.Colors.textTertiary) + .lineLimit(1) + } + .accessibilityLabel("Codex: \(orbitManager.codexSessionSummary)") + } + .tonalGroup() + .accessibilityIdentifier("activity.currentTask") + } + + private var settingsRoot: some View { + VStack(spacing: 1) { + settingsRow("Voice", detail: settings.voicePreset == .localVoice ? "Local · Nora Voice 4" : "OpenAI cloud", icon: "waveform", route: .voice) + divider + settingsRow("Codex", detail: orbitManager.codexConfigurationSummary, icon: "cpu", route: .codex) + divider + settingsRow("Working context", detail: agentFolderLabel, icon: "folder", route: .context) + divider + settingsRow("Privacy & permissions", detail: "Unrestricted · one capture per request", icon: "hand.raised", route: .privacy) + divider + settingsRow("Appearance", detail: settings.showCursor ? "Orbit cursor shown" : "Orbit cursor hidden", icon: "cursorarrow", route: .appearance) + divider + settingsRow("About Orbit", detail: appVersion, icon: "info.circle", route: .about) + } + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + .accessibilityIdentifier("settings.root") + } + + private var permissionOnboarding: some View { + let active = activePermissionStep + let state = orbitManager.permissionCoordinator.viewState + return VStack(alignment: .leading, spacing: 16) { + HStack(spacing: 8) { + ForEach(visiblePermissionSteps) { permission in + permissionProgress(permission, isActive: permission == active) + } + } + .accessibilityLabel("Permission setup progress") + + VStack(alignment: .leading, spacing: 12) { + Label(active.title, systemImage: permissionIcon(active)) + .font(.title3.weight(.semibold)) + .foregroundStyle(DS.Colors.textPrimary) + + Text(permissionInstruction(active)) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + + if let message = state.message { + Text(message) + .font(.caption) + .foregroundStyle(permissionMessageColor(active)) + .fixedSize(horizontal: false, vertical: true) + } + + permissionPrimaryAction(active) + + if active != .microphone || state.showsManualFallback { + HStack(spacing: 8) { + if active != .microphone { + secondaryButton("Reveal Orbit", systemImage: "finder", identifier: "permissions.revealFinder") { + orbitManager.permissionCoordinator.revealOrbitInFinder() + } + } + secondaryButton("Open Settings", systemImage: "gearshape", identifier: "permissions.openSettings") { + orbitManager.permissionCoordinator.openSettings(for: active) + } + } + } + } + .tonalGroup() + .accessibilityIdentifier("permissions.activeStep") + } + } + + @ViewBuilder + private func permissionPrimaryAction(_ permission: OrbitPermissionKind) -> some View { + let status = orbitManager.permissionCoordinator.statusByPermission[permission] ?? .notDetermined + switch status { + case .granted: + statusBadge("Granted", color: DS.Colors.success) + case .restartRequired where permission == .screenRecording: + primaryButton("Quit & Reopen Orbit", identifier: "permissions.screenRecording.relaunch") { + orbitManager.permissionCoordinator.quitAndReopenOrbit() + } + case .prompting: + ProgressView("Waiting for macOS…") + .controlSize(.small) + .accessibilityIdentifier("permissions.prompting") + case .waitingInSettings: + statusBadge("Waiting in Settings", color: DS.Colors.warning) + case .waitingForPicker: + ProgressView("Waiting for screen confirmation…") + .controlSize(.small) + case .notDetermined, .denied, .restartRequired: + primaryButton("Grant \(permission.title)", identifier: "permissions.\(permission.rawValue).grant") { + orbitManager.permissionCoordinator.begin( + permission, + sourceFrame: permissionActionScreenFrame, + entryContext: status == .denied ? .recovery : .firstRun + ) + } + .background(OrbitScreenFrameReader(frame: $permissionActionScreenFrame)) + } + } + + private var automationDisclosure: some View { + setupCard( + title: "Unrestricted automation", + detail: + "Orbit can run commands and edit files without approval prompts. It captures the current screen once for each request and deletes that temporary image when the turn ends." + ) { + Label("Automatic command and file access", systemImage: "terminal") + Label("One fresh capture per request", systemImage: "camera.viewfinder") + Label("No continuous recording indicator", systemImage: "eye.slash") + primaryButton("I understand — continue", identifier: "setup.automation.continue") { + orbitManager.acknowledgeUnrestrictedAutomation() + } + } + } + + private var authSetup: some View { + setupCard(title: "Connect ChatGPT", detail: authMessage) { + if orbitManager.codexAuthState == .loginInProgress { + primaryButton("Open ChatGPT login", identifier: "setup.auth.openLogin") { + orbitManager.reopenCodexLogin() + } + } else { + primaryButton("Connect ChatGPT", identifier: "setup.auth.connect") { + orbitManager.connectCodexAccount() + } + } + secondaryButton("Retry connection", systemImage: "arrow.clockwise", identifier: "setup.auth.retry") { + orbitManager.reconnectCodexSession() + } + } + } + + private var voiceChoiceSetup: some View { + setupCard(title: "Choose voice mode", detail: "Local keeps recognition and narration on this Mac. Cloud sends speech audio to OpenAI.") { + choiceButton("Local", detail: "Apple on-device recognition · Siri Natural Nora Voice 4", selected: settings.voicePreset == .localVoice) { + orbitManager.selectVoicePreset(.localVoice) + } + choiceButton("Cloud", detail: "OpenAI transcription and AI-generated voice", selected: settings.voicePreset == .cloudVoice) { + orbitManager.selectVoicePreset(.cloudVoice) + } + } + } + + private var cloudKeySetup: some View { + setupCard(title: "Connect cloud voice", detail: "Cloud mode sends speech audio to OpenAI and uses an AI-generated voice.") { + apiKeyField + primaryButton("Save API key", identifier: "setup.cloudKey.save") { saveAPIKey() } + .disabled(openAIAPIKeyDraft.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + secondaryButton("Use local instead", systemImage: "desktopcomputer", identifier: "setup.cloudKey.local") { + orbitManager.selectVoicePreset(.localVoice) + } + } + } + + private var tourSetup: some View { + setupCard(title: "Meet Orbit", detail: "A short local demo shows how to speak, follow task activity, and use visual pointing.") { + primaryButton("Start tour", identifier: "setup.tour.start") { orbitManager.triggerOnboarding() } + } + } + + private var setupComplete: some View { + setupCard(title: "Orbit is ready", detail: "Hold Control + Option and ask “what’s on my screen?”") { + Label("One warm Codex thread", systemImage: "checkmark.circle") + Label("Local Nora Voice 4 narration", systemImage: "checkmark.circle") + Label("Fresh visual context per request", systemImage: "checkmark.circle") + primaryButton("Get started", identifier: "setup.complete.finish") { orbitManager.dismissSetupComplete() } + } + } + + private var voiceSettings: some View { + VStack(alignment: .leading, spacing: 12) { + settingSection("Mode") { + choiceButton("Local", detail: "On-device recognition and Nora Voice 4", selected: settings.voicePreset == .localVoice) { + orbitManager.selectVoicePreset(.localVoice) + } + choiceButton("Cloud", detail: "Speech audio is sent to OpenAI", selected: settings.voicePreset == .cloudVoice) { + orbitManager.selectVoicePreset(.cloudVoice) + } + } + + settingSection("Narrator") { + HStack { + Text(settings.voicePreset == .localVoice ? "Siri Natural Nora · Voice 4" : "OpenAI AI-generated voice") + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + Spacer() + Button(orbitManager.isPreviewingAppleVoice ? "Stop" : "Preview") { + orbitManager.toggleAppleVoicePreview() + } + .disabled(settings.voicePreset != .localVoice) + .accessibilityIdentifier("settings.voice.preview") + } + } + + settingSection("Microphone") { + Menu { + Button("System default") { orbitManager.selectMicrophone("") } + Divider() + ForEach(orbitManager.availableMicrophones) { microphone in + Button(microphone.name) { orbitManager.selectMicrophone(microphone.uid) } + } + } label: { + HStack { + Text(selectedMicrophoneLabel) + Spacer() + Image(systemName: "chevron.up.chevron.down") + } + } + .accessibilityIdentifier("settings.voice.microphone") + + HStack { + ProgressView(value: orbitManager.microphoneTestLevel) + .accessibilityLabel("Microphone input level") + Button(orbitManager.isTestingMicrophone ? "Stop test" : "Test microphone") { + orbitManager.toggleMicrophoneTest() + } + .accessibilityIdentifier("settings.voice.microphoneTest") + } + if let notice = orbitManager.microphoneDeviceNotice { + Text(notice).font(.caption).foregroundStyle(DS.Colors.warning) + } + } + + if settings.voicePreset == .cloudVoice { + settingSection("OpenAI API key") { + apiKeyField + primaryButton("Save API key", identifier: "settings.voice.apiKey.save") { saveAPIKey() } + } + } + } + .onAppear { orbitManager.refreshMicrophones() } + .accessibilityIdentifier("settings.voice") + } + + private var codexSettings: some View { + VStack(alignment: .leading, spacing: 12) { + settingSection("Account") { + Text(orbitManager.codexAccountSummary ?? orbitManager.codexSessionSummary) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + HStack { + secondaryButton("Reconnect", systemImage: "arrow.clockwise", identifier: "settings.codex.reconnect") { + orbitManager.reconnectCodexSession() + } + secondaryButton("Sign out", systemImage: "rectangle.portrait.and.arrow.right", identifier: "settings.codex.signout") { + orbitManager.signOutCodexAccount() + } + } + } + + settingSection("Configuration") { + labeledMenu("Model", identifier: "settings.codex.model", value: selectedModelLabel) { + ForEach(orbitManager.availableCodexModels) { model in + Button(model.displayName) { settings.codexActionModel = model.model } + } + } + labeledMenu("Effort", identifier: "settings.codex.effort", value: settings.codexReasoningEffort.displayName) { + ForEach(orbitManager.availableCodexEfforts) { effort in + Button(effort.displayName) { settings.codexReasoningEffort = effort } + } + } + labeledMenu("Service tier", identifier: "settings.codex.tier", value: settings.codexServiceTier.displayName) { + ForEach(availableServiceTiers) { tier in + Button(tier.displayName) { settings.codexServiceTier = tier } + } + } + } + if let notice = orbitManager.codexModelMigrationNotice { + Text(notice).font(.caption).foregroundStyle(DS.Colors.warning) + } + } + .accessibilityIdentifier("settings.codex") + } + + private var contextSettings: some View { + settingSection("Agent Folder") { + Text("Changing this folder begins fresh Codex context. Orbit still retains unrestricted filesystem access.") + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + primaryButton("Choose folder…", identifier: "settings.context.chooseFolder") { chooseAgentFolder() } + Text(settings.codexAgentFolder.isEmpty ? NSHomeDirectory() : settings.codexAgentFolder) + .font(.caption.monospaced()) + .foregroundStyle(DS.Colors.textTertiary) + .textSelection(.enabled) + } + .accessibilityIdentifier("settings.context") + } + + private var privacySettings: some View { + VStack(alignment: .leading, spacing: 12) { + settingSection("Automation") { + Label("Unrestricted filesystem and command access", systemImage: "bolt.shield") + Label("App-server approvals are accepted automatically", systemImage: "checkmark.shield") + Label("One temporary screen capture per request", systemImage: "camera.viewfinder") + Label("Capture is deleted at every terminal state", systemImage: "trash") + } + settingSection("Permissions") { + ForEach(visiblePermissionSteps) { permission in + HStack { + Label(permission.title, systemImage: permissionIcon(permission)) + Spacer() + Text(permissionStatusLabel(permission)) + .foregroundStyle(permissionStatusColor(permission)) + } + } + } + } + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .accessibilityIdentifier("settings.privacy") + } + + private var appearanceSettings: some View { + settingSection("Desktop") { + Toggle( + "Show Orbit cursor", + isOn: Binding( + get: { orbitManager.isOrbitCursorEnabled }, + set: { orbitManager.setOrbitCursorEnabled($0) } + ) + ) + .toggleStyle(.switch) + .accessibilityIdentifier("settings.appearance.cursor") + Text("Motion follows the macOS Reduce Motion setting.") + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + } + .accessibilityIdentifier("settings.appearance") + } + + private var aboutView: some View { + VStack(alignment: .leading, spacing: 12) { + HStack(spacing: 12) { + OrbitMarkView(size: 28) + VStack(alignment: .leading) { + Text("Orbit").font(.title3.weight(.semibold)) + Text(appVersion).font(.caption).foregroundStyle(DS.Colors.textTertiary) + } + } + Text("A quiet, screen-aware Codex instrument for macOS.") + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + Text("Local narration uses Siri Natural Nora Voice 4 when installed. Orbit never continuously records your screen.") + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + } + .tonalGroup() + .accessibilityIdentifier("settings.about") + } + + private func setupCard( + title: String, + detail: String, + @ViewBuilder content: () -> Content + ) -> some View { + VStack(alignment: .leading, spacing: 14) { + Text(title).font(.title3.weight(.semibold)).foregroundStyle(DS.Colors.textPrimary) + Text(detail).font(.body).foregroundStyle(DS.Colors.textSecondary).fixedSize(horizontal: false, vertical: true) + content() + } + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .tonalGroup() + .accessibilityIdentifier("setup.currentStep") + } + + private func settingSection( + _ title: String, + @ViewBuilder content: () -> Content + ) -> some View { + VStack(alignment: .leading, spacing: 10) { + Text(title) + .font(.caption.weight(.semibold)) + .foregroundStyle(DS.Colors.textTertiary) + content() + } + .tonalGroup() + } + + private func settingsRow( + _ title: String, + detail: String, + icon: String, + route: OrbitPanelRoute + ) -> some View { + Button { + self.route = route + } label: { + HStack(spacing: 12) { + Image(systemName: icon).frame(width: 20).foregroundStyle(DS.Colors.textSecondary) + VStack(alignment: .leading, spacing: 2) { + Text(title).font(.body.weight(.medium)).foregroundStyle(DS.Colors.textPrimary) + Text(detail).font(.caption).foregroundStyle(DS.Colors.textTertiary).lineLimit(2) + } + Spacer() + Image(systemName: "chevron.right").font(.caption.weight(.semibold)).foregroundStyle(DS.Colors.textTertiary) + } + .padding(12) + .contentShape(Rectangle()) + } + .buttonStyle(OrbitTonalButtonStyle()) + .accessibilityIdentifier("settings.route.\(stableIdentifier(title))") + } + + private func choiceButton( + _ title: String, + detail: String, + selected: Bool, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { + HStack(alignment: .top, spacing: 10) { + VStack(alignment: .leading, spacing: 3) { + Text(title).font(.body.weight(.semibold)).foregroundStyle(DS.Colors.textPrimary) + Text(detail).font(.caption).foregroundStyle(DS.Colors.textTertiary).fixedSize(horizontal: false, vertical: true) + } + Spacer() + Image(systemName: selected ? "checkmark.circle.fill" : "circle") + .foregroundStyle(selected ? DS.Colors.accent : DS.Colors.textTertiary) + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 8) + .fill(selected ? DS.Colors.accent.opacity(0.12) : DS.Colors.surface2) + ) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(selected ? DS.Colors.accent.opacity(0.7) : DS.Colors.borderSubtle) + ) + } + .buttonStyle(OrbitTonalButtonStyle()) + .accessibilityIdentifier("choice.\(stableIdentifier(title))") + } + + private func primaryButton( + _ title: String, + identifier: String, + action: @escaping () -> Void + ) -> some View { + Button(title, action: action) + .font(.body.weight(.semibold)) + .foregroundStyle(DS.Colors.textOnAccent) + .padding(.horizontal, 12) + .frame(minHeight: 32) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.accent)) + .buttonStyle(OrbitTonalButtonStyle()) + .accessibilityIdentifier(identifier) + } + + private func secondaryButton( + _ title: String, + systemImage: String, + identifier: String, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { Label(title, systemImage: systemImage) } + .font(.body.weight(.medium)) + .foregroundStyle(DS.Colors.textSecondary) + .padding(.horizontal, 10) + .frame(minHeight: 32) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.surface2)) + .overlay(RoundedRectangle(cornerRadius: 8).stroke(DS.Colors.borderSubtle)) + .buttonStyle(OrbitTonalButtonStyle()) + .accessibilityIdentifier(identifier) + } + + private func iconButton( + _ label: String, + systemImage: String, + identifier: String, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { + Image(systemName: systemImage) + .frame(width: 28, height: 28) + .contentShape(Rectangle()) + } + .buttonStyle(OrbitTonalButtonStyle()) + .foregroundStyle(DS.Colors.textSecondary) + .accessibilityLabel(label) + .accessibilityIdentifier(identifier) + } + + private func disclosure( + _ title: String, + isExpanded: Binding, + identifier: String, + @ViewBuilder content: @escaping () -> Content + ) -> some View { + DisclosureGroup(isExpanded: isExpanded) { + VStack(alignment: .leading, spacing: 7) { content() } + .padding(.top, 8) + } label: { + Text(title).font(.caption.weight(.semibold)).foregroundStyle(DS.Colors.textSecondary) + } + .accessibilityIdentifier(identifier) + } + + private func labeledMenu( + _ title: String, + identifier: String, + value: String, + @ViewBuilder content: () -> Content + ) -> some View { + HStack { + Text(title).font(.body).foregroundStyle(DS.Colors.textSecondary) + Spacer() + Menu(content: content) { + HStack(spacing: 5) { + Text(value).lineLimit(1) + Image(systemName: "chevron.up.chevron.down").font(.caption2) + } + } + .menuStyle(.borderlessButton) + .accessibilityIdentifier(identifier) + } + } + + private func statusBadge(_ title: String, color: Color) -> some View { + HStack(spacing: 6) { + Circle().fill(color).frame(width: 6, height: 6) + Text(title).font(.caption.weight(.semibold)).foregroundStyle(DS.Colors.textSecondary) + } + .padding(.horizontal, 8) + .frame(minHeight: 26) + .background(Capsule().fill(DS.Colors.surface2)) + } + + private func permissionProgress(_ permission: OrbitPermissionKind, isActive: Bool) -> some View { + let granted = orbitManager.permissionCoordinator.statusByPermission[permission] == .granted + return HStack(spacing: 5) { + Image(systemName: granted ? "checkmark.circle.fill" : permissionIcon(permission)) + Text(permission == .screenRecording ? "Screen" : permission.title) + .lineLimit(1) + } + .font(.caption.weight(.semibold)) + .foregroundStyle(granted ? DS.Colors.success : (isActive ? DS.Colors.textPrimary : DS.Colors.textTertiary)) + .padding(.horizontal, 8) + .frame(maxWidth: .infinity, minHeight: 30) + .background(RoundedRectangle(cornerRadius: 8).fill(isActive ? DS.Colors.surface2 : DS.Colors.surface1)) + } + + private var apiKeyField: some View { + SecureField("OpenAI API key", text: $openAIAPIKeyDraft) + .textFieldStyle(.plain) + .padding(10) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.surface2)) + .overlay(RoundedRectangle(cornerRadius: 8).stroke(DS.Colors.borderSubtle)) + .accessibilityIdentifier("voice.apiKey.field") + } + + private var divider: some View { + Rectangle().fill(DS.Colors.borderSubtle).frame(height: 1).padding(.leading, 44) + } + + private var visiblePermissionSteps: [OrbitPermissionKind] { + OrbitPermissionKind.allCases.filter(\.isVisibleOnboardingStep) + } + + private var activePermissionStep: OrbitPermissionKind { + if let active = orbitManager.permissionCoordinator.activePermission, active.isVisibleOnboardingStep { + return active + } + return visiblePermissionSteps.first(where: { + orbitManager.permissionCoordinator.statusByPermission[$0] != .granted + }) ?? .microphone + } + + private func permissionIcon(_ permission: OrbitPermissionKind) -> String { + switch permission { + case .microphone: "mic" + case .accessibility: "hand.raised" + case .screenRecording, .screenContent: "rectangle.dashed.badge.record" + } + } + + private func permissionInstruction(_ permission: OrbitPermissionKind) -> String { + switch permission { + case .microphone: + "Allow Orbit to hear push-to-talk requests. Local mode keeps recognition on this Mac." + case .accessibility: + "Allow Orbit to follow the pointer and interact with desktop controls. The guide will attach to System Settings." + case .screenRecording, .screenContent: + "Allow one fresh screen capture for each submitted request. Orbit does not continuously record." + } + } + + private func permissionStatusLabel(_ permission: OrbitPermissionKind) -> String { + switch orbitManager.permissionCoordinator.statusByPermission[permission] ?? .notDetermined { + case .notDetermined: "Not granted" + case .prompting: "Prompting" + case .waitingInSettings: "Waiting" + case .waitingForPicker: "Confirming" + case .restartRequired: "Reopen required" + case .granted: "Granted" + case .denied: "Denied" + } + } + + private func permissionStatusColor(_ permission: OrbitPermissionKind) -> Color { + switch orbitManager.permissionCoordinator.statusByPermission[permission] ?? .notDetermined { + case .granted: DS.Colors.success + case .denied: DS.Colors.destructive + case .restartRequired, .waitingInSettings, .waitingForPicker: DS.Colors.warning + case .notDetermined, .prompting: DS.Colors.textTertiary + } + } + + private func permissionMessageColor(_ permission: OrbitPermissionKind) -> Color { + switch orbitManager.permissionCoordinator.statusByPermission[permission] { + case .denied: DS.Colors.destructiveText + case .restartRequired: DS.Colors.warningText + default: DS.Colors.textTertiary + } + } + + private var headerTitle: String { + if orbitManager.setupStage != .ready { return "Set up Orbit" } + return switch route { + case .task: "Orbit" + case .settings: "Settings" + case .voice: "Voice" + case .codex: "Codex" + case .context: "Working context" + case .privacy: "Privacy & permissions" + case .appearance: "Appearance" + case .about: "About" + } + } + + private var headerSubtitle: String { + if orbitManager.setupStage != .ready { return "One clear step at a time" } + return switch route { + case .task: OrbitPanelPresentationState(manager: orbitManager).statusLabel + case .settings: "Voice, Codex, context, and privacy" + case .voice: "Input and narration" + case .codex: "Account-backed model catalog" + case .context: "Where new work begins" + case .privacy: "Automation and screen capture" + case .appearance: "Desktop presentation" + case .about: appVersion + } + } + + private var authMessage: String { + switch orbitManager.codexAuthState { + case .unknown, .checking: "Orbit is checking your ChatGPT account." + case .authRequired: "Sign in once so Orbit can keep one warm Codex thread ready." + case .loginInProgress: "Finish signing in through the browser window." + case .authFailed(let message), .runtimeUnavailable(let message): message + case .authenticated: "Your ChatGPT account is connected." + } + } + + private var selectedModelLabel: String { + orbitManager.availableCodexModels.first(where: { $0.model == settings.codexActionModel })?.shortDisplayName + ?? settings.codexActionModel.nilIfBlank + ?? "Server default" + } + + private var availableServiceTiers: [OrbitCodexServiceTier] { + let advertised = + orbitManager.availableCodexModels + .first(where: { $0.model == settings.codexActionModel })? + .supportedServiceTiers ?? [] + return [.serverDefault] + advertised.filter { !$0.rawValue.isEmpty } + } + + private var selectedMicrophoneLabel: String { + let uid = settings.microphoneDeviceUID.trimmingCharacters(in: .whitespacesAndNewlines) + guard !uid.isEmpty else { return "System default" } + return orbitManager.availableMicrophones.first(where: { $0.uid == uid })?.name ?? "Disconnected" + } + + private var agentFolderLabel: String { + let path = settings.codexAgentFolder.trimmingCharacters(in: .whitespacesAndNewlines) + return path.isEmpty ? "Home folder" : URL(fileURLWithPath: path).lastPathComponent + } + + private var appVersion: String { + "Version \(Bundle.main.infoDictionary?["CFBundleShortVersionString"] as? String ?? "1.1.0")" + } + + private func chooseAgentFolder() { + let panel = NSOpenPanel() + panel.title = "Choose Orbit Agent Folder" + panel.message = "This changes starting context. Filesystem access remains unrestricted." + panel.canChooseDirectories = true + panel.canChooseFiles = false + panel.allowsMultipleSelection = false + if panel.runModal() == .OK, let url = panel.url { settings.codexAgentFolder = url.path } + } + + private func saveAPIKey() { + let draft = openAIAPIKeyDraft + Task { + if await orbitManager.saveOpenAIAPIKey(draft) { openAIAPIKeyDraft = "" } + } + } + + private func handle(_ intent: OrbitPanelIntent) { + switch intent { + case .openSettings: route = .settings + case .goBack: + if route == .task { + NotificationCenter.default.post(name: .orbitDismissPanel, object: nil) + } else if route == .settings { + route = .task + } else { + route = .settings + } + case .interruptTask: orbitManager.interruptCurrentAction() + case .reconnectCodex: orbitManager.reconnectCodexSession() + case .replayTour: orbitManager.replayOnboarding() + case .quit: NSApp.terminate(nil) + } + } + + private func stableIdentifier(_ value: String) -> String { + value.lowercased() + .replacingOccurrences(of: "[^a-z0-9]+", with: "-", options: .regularExpression) + .trimmingCharacters(in: CharacterSet(charactersIn: "-")) + } +} + +private struct OrbitTonalButtonStyle: ButtonStyle { + @Environment(\.accessibilityReduceMotion) private var reduceMotion + + func makeBody(configuration: Configuration) -> some View { + configuration.label + .contentShape(Rectangle()) + .opacity(configuration.isPressed ? 0.78 : 1) + .animation(reduceMotion ? nil : .easeOut(duration: 0.08), value: configuration.isPressed) + } +} + +private struct OrbitActivityLabelStyle: LabelStyle { + func makeBody(configuration: Configuration) -> some View { + HStack(alignment: .firstTextBaseline, spacing: 7) { + configuration.icon.font(.system(size: 4)) + configuration.title + } + } +} + +extension View { + fileprivate func tonalGroup() -> some View { + padding(14) + .frame(maxWidth: .infinity, alignment: .leading) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + } +} + +private struct OrbitScreenFrameReader: NSViewRepresentable { + @Binding var frame: CGRect? + + func makeNSView(context: Context) -> NSView { + let view = NSView() + DispatchQueue.main.async { updateFrame(view) } + return view + } + + func updateNSView(_ nsView: NSView, context: Context) { + DispatchQueue.main.async { updateFrame(nsView) } + } + + private func updateFrame(_ view: NSView) { + guard let window = view.window else { return } + let windowRect = view.convert(view.bounds, to: nil) + frame = window.convertToScreen(windowRect) + } +} + +extension String { + fileprivate var nilIfBlank: String? { + let value = trimmingCharacters(in: .whitespacesAndNewlines) + return value.isEmpty ? nil : value + } +} diff --git a/Orbit/OrbitPermissionCoordinator.swift b/Orbit/OrbitPermissionCoordinator.swift index 6b8d514..f156944 100644 --- a/Orbit/OrbitPermissionCoordinator.swift +++ b/Orbit/OrbitPermissionCoordinator.swift @@ -1,96 +1,130 @@ import AVFoundation import AppKit import Combine +import QuartzCore import SwiftUI -enum OrbitPermissionKind: String, CaseIterable, Identifiable, Sendable { - case microphone - case accessibility - case screenRecording - - var id: String { rawValue } - - var title: String { - switch self { - case .microphone: "Microphone" - case .accessibility: "Accessibility" - case .screenRecording: "Screen Recording" - } - } -} - -enum OrbitPermissionStatus: Equatable, Sendable { - case notDetermined - case requesting - case denied - case restartRequired - case granted -} - @MainActor -final class OrbitPermissionCoordinator: ObservableObject { +final class OrbitPermissionCoordinator: NSObject, ObservableObject { @Published private(set) var activePermission: OrbitPermissionKind? @Published private(set) var statusByPermission: [OrbitPermissionKind: OrbitPermissionStatus] = [:] + @Published private(set) var viewState: OrbitPermissionFlowViewState = .initial + + private struct PendingSettingsHandoff { + let permission: OrbitPermissionKind + let sourceFrame: CGRect? + let minimumActivationSequence: Int + let expiresAt: Date + } - private weak var orbitManager: OrbitManager? - private lazy var guide = OrbitPermissionGuideWindowController(coordinator: self) + private let driver: any OrbitPermissionDriving + private let clock: any OrbitPermissionClock + private let settingsLocator: any OrbitSettingsWindowLocating + private lazy var guide = OrbitPermissionGuideWindowController( + coordinator: self, + settingsLocator: settingsLocator + ) + private var activationSequence = 0 + private var pendingHandoff: PendingSettingsHandoff? + private var manualFallbackTask: Task? + private var handoffExpiryTask: Task? + private var paneSettleTask: Task? + + init( + orbitManager: OrbitManager, + driver: (any OrbitPermissionDriving)? = nil, + clock: any OrbitPermissionClock = OrbitSystemPermissionClock(), + settingsLocator: any OrbitSettingsWindowLocating = OrbitSystemSettingsLocator() + ) { + self.driver = driver ?? OrbitSystemPermissionDriver(orbitManager: orbitManager) + self.clock = clock + self.settingsLocator = settingsLocator + super.init() + bindWorkspaceEvents() + refresh() + } - init(orbitManager: OrbitManager) { - self.orbitManager = orbitManager + init( + driver: any OrbitPermissionDriving, + clock: any OrbitPermissionClock = OrbitSystemPermissionClock(), + settingsLocator: any OrbitSettingsWindowLocating = OrbitSystemSettingsLocator() + ) { + self.driver = driver + self.clock = clock + self.settingsLocator = settingsLocator + super.init() + bindWorkspaceEvents() refresh() } - func begin(_ permission: OrbitPermissionKind) { - activePermission = permission - statusByPermission[permission] = .requesting + deinit { + NSWorkspace.shared.notificationCenter.removeObserver(self) + } + + func begin( + _ permission: OrbitPermissionKind, + sourceFrame: CGRect? = nil, + entryContext: OrbitPermissionEntryContext = .firstRun + ) { + cancelHandoff() + activePermission = permission == .screenContent ? .screenRecording : permission + updateViewState(entryContext: entryContext, showsManualFallback: false, message: nil) switch permission { case .microphone: requestMicrophone() case .accessibility: - _ = WindowPositionManager.requestAccessibilityPermission() - orbitManager?.refreshAllPermissions() - if orbitManager?.hasAccessibilityPermission == true { - markGranted(permission) - } else { - guide.show(for: permission) - } + beginAccessibility(sourceFrame: sourceFrame) case .screenRecording: - WindowPositionManager.openScreenRecordingSettings() - guide.show(for: permission) - orbitManager?.requestScreenContentPermission() + beginScreenRecording(sourceFrame: sourceFrame) + case .screenContent: + beginScreenContentProbe() } } func refresh() { - orbitManager?.refreshAllPermissions() - guard let orbitManager else { return } - statusByPermission[.microphone] = - orbitManager.hasMicrophonePermission + driver.refresh() + + statusByPermission[.microphone] = driver.microphoneStatus + statusByPermission[.accessibility] = + driver.hasAccessibilityPermission ? .granted - : microphoneStatus - statusByPermission[.accessibility] = orbitManager.hasAccessibilityPermission ? .granted : .denied - if orbitManager.hasUsableScreenAccessPermission { + : (statusByPermission[.accessibility] ?? .notDetermined) + + if driver.hasScreenContentPermission { + statusByPermission[.screenContent] = .granted statusByPermission[.screenRecording] = .granted - } else if CGPreflightScreenCaptureAccess() { + } else if driver.hasScreenRecordingPermission || CGPreflightScreenCaptureAccess() { + statusByPermission[.screenContent] = + statusByPermission[.screenContent] == .waitingForPicker + ? .waitingForPicker : .notDetermined statusByPermission[.screenRecording] = .restartRequired } else { - statusByPermission[.screenRecording] = .denied + statusByPermission[.screenContent] = .notDetermined + statusByPermission[.screenRecording] = statusByPermission[.screenRecording] ?? .notDetermined } - if let activePermission, statusByPermission[activePermission] == .granted { - guide.scheduleSuccessDismiss() + if activePermission == .accessibility, + statusByPermission[.accessibility] == .granted + { + completeActivePermission(.accessibility) + } else if activePermission == .screenRecording, + statusByPermission[.screenRecording] == .granted + { + completeActivePermission(.screenRecording) } + + updateViewState() } func openSettings(for permission: OrbitPermissionKind) { switch permission { case .microphone: - openPrivacyPane("Privacy_Microphone") + openPrivacyPane(modernPath: "Privacy_Microphone", legacyAnchor: "Privacy_Microphone") case .accessibility: - WindowPositionManager.openAccessibilitySettings() - case .screenRecording: - WindowPositionManager.openScreenRecordingSettings() + openPrivacyPane(modernPath: "Privacy_Accessibility", legacyAnchor: "Privacy_Accessibility") + case .screenRecording, .screenContent: + openPrivacyPane(modernPath: "Privacy_ScreenCapture", legacyAnchor: "Privacy_ScreenCapture") } } @@ -100,144 +134,434 @@ final class OrbitPermissionCoordinator: ObservableObject { func dismissGuide() { guide.hide() + cancelHandoff() activePermission = nil + updateViewState() } - private var microphoneStatus: OrbitPermissionStatus { - switch AVCaptureDevice.authorizationStatus(for: .audio) { - case .notDetermined: .notDetermined - case .authorized: .granted - case .denied, .restricted: .denied - @unknown default: .denied + func quitAndReopenOrbit() { + guard statusByPermission[.screenRecording] == .restartRequired else { return } + updateViewState(message: "Reopening Orbit to finish Screen Recording…") + let configuration = NSWorkspace.OpenConfiguration() + configuration.activates = true + configuration.createsNewApplicationInstance = true + NSWorkspace.shared.openApplication( + at: Bundle.main.bundleURL, + configuration: configuration + ) { [weak self] application, error in + Task { @MainActor [weak self] in + guard let self else { return } + if application != nil, error == nil { + NSApp.terminate(nil) + } else { + self.updateViewState( + entryContext: .recovery, + showsManualFallback: true, + message: "Orbit could not reopen automatically. Try again or reopen it from Finder." + ) + } + } } } - private func requestMicrophone() { - guard AVCaptureDevice.authorizationStatus(for: .audio) == .notDetermined else { - openSettings(for: .microphone) - refresh() + fileprivate func guideDragCompleted(permission: OrbitPermissionKind, succeeded: Bool) { + guard succeeded else { + statusByPermission[permission] = .waitingInSettings + updateViewState(message: "The drag was cancelled. Drag Orbit again or use the alternatives below.") return } - AVCaptureDevice.requestAccess(for: .audio) { [weak self] granted in - Task { @MainActor [weak self] in - guard let self else { return } - self.orbitManager?.refreshAllPermissions() - self.statusByPermission[.microphone] = granted ? .granted : .denied - self.activePermission = nil + + switch permission { + case .accessibility: + statusByPermission[.accessibility] = .waitingInSettings + updateViewState(message: "Orbit was added. Waiting for macOS to confirm Accessibility…") + case .screenRecording, .screenContent: + statusByPermission[.screenRecording] = .restartRequired + guide.showSuccess(message: "Orbit was added. Reopen once to finish.") + updateViewState(message: "Screen Recording needs one quick relaunch.") + NotificationCenter.default.post(name: .orbitShowPanel, object: nil) + case .microphone: + break + } + } + + private func beginAccessibility(sourceFrame: CGRect?) { + let destination = driver.requestAccessibilityPermission() + switch destination { + case .alreadyGranted: + completeActivePermission(.accessibility) + case .systemPrompt: + statusByPermission[.accessibility] = .prompting + armSettingsHandoff(for: .accessibility, sourceFrame: sourceFrame) + case .systemSettings: + statusByPermission[.accessibility] = .waitingInSettings + armSettingsHandoff(for: .accessibility, sourceFrame: sourceFrame) + handleSettingsActivationIfReady() + } + updateViewState() + } + + private func beginScreenRecording(sourceFrame: CGRect?) { + let destination = driver.requestScreenRecordingPermission() + switch destination { + case .alreadyGranted: + beginScreenContentProbe() + case .systemPrompt: + statusByPermission[.screenRecording] = .prompting + armSettingsHandoff(for: .screenRecording, sourceFrame: sourceFrame) + case .systemSettings: + statusByPermission[.screenRecording] = .waitingInSettings + armSettingsHandoff(for: .screenRecording, sourceFrame: sourceFrame) + handleSettingsActivationIfReady() + } + updateViewState() + } + + private func beginScreenContentProbe() { + activePermission = .screenRecording + statusByPermission[.screenContent] = .waitingForPicker + statusByPermission[.screenRecording] = .waitingForPicker + updateViewState(message: "Confirm the screen picker once so Orbit can verify live access.") + driver.requestScreenContentPermission() + } + + private func requestMicrophone() { + guard driver.microphoneStatus == .notDetermined else { + statusByPermission[.microphone] = driver.microphoneStatus + if driver.microphoneStatus != .granted { + openSettings(for: .microphone) } + updateViewState(entryContext: .recovery, showsManualFallback: driver.microphoneStatus != .granted) + return + } + + statusByPermission[.microphone] = .prompting + Task { @MainActor [weak self] in + guard let self else { return } + let granted = await driver.requestMicrophoneAccess() + guard !Task.isCancelled else { return } + driver.refresh() + statusByPermission[.microphone] = granted ? .granted : .denied + activePermission = granted ? nextIncompletePermission(after: .microphone) : .microphone + updateViewState( + entryContext: granted ? .firstRun : .recovery, + showsManualFallback: !granted, + message: granted ? nil : "Microphone access was denied. Open Settings to try again." + ) + } + } + + private func armSettingsHandoff(for permission: OrbitPermissionKind, sourceFrame: CGRect?) { + let permissionClock = clock + pendingHandoff = PendingSettingsHandoff( + permission: permission, + sourceFrame: sourceFrame, + minimumActivationSequence: activationSequence + 1, + expiresAt: clock.now.addingTimeInterval(20) + ) + + manualFallbackTask?.cancel() + manualFallbackTask = Task { @MainActor [weak self] in + try? await permissionClock.sleep(for: .seconds(1)) + guard let self, !Task.isCancelled, pendingHandoff != nil else { return } + updateViewState(showsManualFallback: true) + } + + handoffExpiryTask?.cancel() + handoffExpiryTask = Task { @MainActor [weak self] in + try? await permissionClock.sleep(for: .seconds(20)) + guard let self, !Task.isCancelled, let handoff = pendingHandoff, + handoff.expiresAt <= permissionClock.now + else { return } + pendingHandoff = nil + statusByPermission[handoff.permission] = .denied + guide.hide() + updateViewState( + entryContext: .recovery, + showsManualFallback: true, + message: "System Settings did not open. Use Open Settings to continue." + ) } } - private func markGranted(_ permission: OrbitPermissionKind) { + private func bindWorkspaceEvents() { + NSWorkspace.shared.notificationCenter.addObserver( + self, + selector: #selector(workspaceDidActivate(_:)), + name: NSWorkspace.didActivateApplicationNotification, + object: nil + ) + NSWorkspace.shared.notificationCenter.addObserver( + self, + selector: #selector(workspaceDidTerminate(_:)), + name: NSWorkspace.didTerminateApplicationNotification, + object: nil + ) + } + + @objc private func workspaceDidActivate(_ notification: Notification) { + activationSequence += 1 + guard + let application = notification.userInfo?[NSWorkspace.applicationUserInfoKey] + as? NSRunningApplication + else { return } + if application.bundleIdentifier == OrbitSystemSettingsWindowLocator.bundleIdentifier { + handleSettingsActivationIfReady() + } else if application.bundleIdentifier == Bundle.main.bundleIdentifier { + guide.hide() + refresh() + } + } + + @objc private func workspaceDidTerminate(_ notification: Notification) { + guard + let application = notification.userInfo?[NSWorkspace.applicationUserInfoKey] + as? NSRunningApplication, + application.bundleIdentifier == OrbitSystemSettingsWindowLocator.bundleIdentifier + else { return } + dismissGuide() + } + + private func handleSettingsActivationIfReady() { + guard let handoff = pendingHandoff, + activationSequence >= handoff.minimumActivationSequence + || NSWorkspace.shared.frontmostApplication?.bundleIdentifier + == OrbitSystemSettingsWindowLocator.bundleIdentifier + else { return } + + openSettings(for: handoff.permission) + statusByPermission[handoff.permission] = .waitingInSettings + paneSettleTask?.cancel() + let permissionClock = clock + paneSettleTask = Task { @MainActor [weak self] in + try? await permissionClock.sleep(for: .milliseconds(400)) + guard let self, !Task.isCancelled, pendingHandoff?.permission == handoff.permission else { return } + guide.show(for: handoff.permission, sourceFrame: handoff.sourceFrame) + updateViewState(showsManualFallback: true) + } + } + + private func completeActivePermission(_ permission: OrbitPermissionKind) { statusByPermission[permission] = .granted - activePermission = nil - guide.scheduleSuccessDismiss() + cancelHandoff() + guide.showSuccess(message: "\(permission.title) is ready.") + activePermission = nextIncompletePermission(after: permission) + updateViewState(message: nil) + } + + private func nextIncompletePermission(after permission: OrbitPermissionKind) -> OrbitPermissionKind? { + let visible = OrbitPermissionKind.allCases.filter(\.isVisibleOnboardingStep) + guard let index = visible.firstIndex(of: permission) else { return visible.first } + return visible.dropFirst(index + 1).first(where: { statusByPermission[$0] != .granted }) + ?? visible.first(where: { statusByPermission[$0] != .granted }) } - private func openPrivacyPane(_ anchor: String) { - guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?\(anchor)") else { return } - NSWorkspace.shared.open(url) + private func cancelHandoff() { + pendingHandoff = nil + manualFallbackTask?.cancel() + manualFallbackTask = nil + handoffExpiryTask?.cancel() + handoffExpiryTask = nil + paneSettleTask?.cancel() + paneSettleTask = nil + } + + private func updateViewState( + entryContext: OrbitPermissionEntryContext? = nil, + showsManualFallback: Bool? = nil, + message: String? = nil + ) { + viewState = OrbitPermissionFlowViewState( + activePermission: activePermission, + statuses: statusByPermission, + entryContext: entryContext ?? viewState.entryContext, + showsManualFallback: showsManualFallback ?? viewState.showsManualFallback, + message: message + ) + } + + private func openPrivacyPane(modernPath: String, legacyAnchor: String) { + let modern = "x-apple.systempreferences:com.apple.settings.PrivacySecurity.extension?\(modernPath)" + if let url = URL(string: modern), NSWorkspace.shared.open(url) { return } + guard + let legacy = URL( + string: "x-apple.systempreferences:com.apple.preference.security?\(legacyAnchor)" + ) + else { return } + NSWorkspace.shared.open(legacy) } } @MainActor private final class OrbitPermissionGuideModel: ObservableObject { @Published var permission: OrbitPermissionKind = .accessibility - @Published var isAttachedToSettings = false @Published var isDragging = false + @Published var isSettled = false + @Published var successMessage: String? } @MainActor private final class OrbitPermissionGuideWindowController { private weak var coordinator: OrbitPermissionCoordinator? + private let settingsLocator: any OrbitSettingsWindowLocating private let model = OrbitPermissionGuideModel() private var panel: NSPanel? private var trackingTimer: Timer? - private var successDismissTask: Task? + private var successTask: Task? + private var revealTask: Task? private var escapeMonitor: Any? private var localEscapeMonitor: Any? - private var hasSeenSettingsWindow = false + private var sourceFrame: CGRect? + private var hasAttached = false - init(coordinator: OrbitPermissionCoordinator) { + init( + coordinator: OrbitPermissionCoordinator, + settingsLocator: any OrbitSettingsWindowLocating + ) { self.coordinator = coordinator + self.settingsLocator = settingsLocator } - func show(for permission: OrbitPermissionKind) { + func show(for permission: OrbitPermissionKind, sourceFrame: CGRect?) { model.permission = permission - hasSeenSettingsWindow = false - if panel == nil, let coordinator { - let guidePanel = NSPanel( - contentRect: NSRect(x: 0, y: 0, width: 520, height: 78), - styleMask: [.borderless, .nonactivatingPanel], - backing: .buffered, - defer: false - ) - guidePanel.contentView = NSHostingView( - rootView: OrbitPermissionGuideView( - coordinator: coordinator, - model: model, - appURL: Bundle.main.bundleURL - ) - ) - guidePanel.isFloatingPanel = true - guidePanel.level = .statusBar - guidePanel.isOpaque = false - guidePanel.backgroundColor = .clear - guidePanel.hasShadow = true - guidePanel.hidesOnDeactivate = false - guidePanel.isExcludedFromWindowsMenu = true - guidePanel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] - guidePanel.becomesKeyOnlyIfNeeded = true - panel = guidePanel - } - - positionPanel() - panel?.orderFrontRegardless() + model.successMessage = nil + model.isSettled = false + self.sourceFrame = sourceFrame + hasAttached = false + makePanelIfNeeded() startTracking() installEscapeMonitor() + updatePosition() } func hide() { trackingTimer?.invalidate() trackingTimer = nil - successDismissTask?.cancel() - successDismissTask = nil + successTask?.cancel() + successTask = nil + revealTask?.cancel() + revealTask = nil model.isDragging = false + model.isSettled = false panel?.orderOut(nil) - if let escapeMonitor { - NSEvent.removeMonitor(escapeMonitor) - self.escapeMonitor = nil - } - if let localEscapeMonitor { - NSEvent.removeMonitor(localEscapeMonitor) - self.localEscapeMonitor = nil - } + removeEscapeMonitors() } - func scheduleSuccessDismiss() { - guard panel?.isVisible == true, successDismissTask == nil else { return } - successDismissTask = Task { [weak self] in - try? await Task.sleep(for: .milliseconds(750)) - self?.hide() + func showSuccess(message: String) { + guard panel?.isVisible == true else { return } + model.successMessage = message + successTask?.cancel() + successTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(1.2)) + guard let self, !Task.isCancelled else { return } + hide() } } + private func makePanelIfNeeded() { + guard panel == nil, let coordinator else { return } + let guideModel = model + let guidePanel = NSPanel( + contentRect: CGRect(origin: .zero, size: OrbitPermissionGuideAnchorResolver.guideSize), + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + guidePanel.contentView = NSHostingView( + rootView: OrbitPermissionGuideView( + model: model, + appURL: Bundle.main.bundleURL, + onDragCompleted: { [weak coordinator] succeeded in + coordinator?.guideDragCompleted(permission: guideModel.permission, succeeded: succeeded) + } + ) + ) + guidePanel.isFloatingPanel = true + guidePanel.level = .floating + guidePanel.isOpaque = false + guidePanel.backgroundColor = .clear + guidePanel.hasShadow = true + guidePanel.hidesOnDeactivate = false + guidePanel.isExcludedFromWindowsMenu = true + guidePanel.collectionBehavior = [.moveToActiveSpace, .fullScreenAuxiliary] + guidePanel.becomesKeyOnlyIfNeeded = true + panel = guidePanel + } + private func startTracking() { trackingTimer?.invalidate() - trackingTimer = Timer.scheduledTimer(withTimeInterval: 0.45, repeats: true) { [weak self] _ in + trackingTimer = Timer.scheduledTimer(withTimeInterval: 0.15, repeats: true) { [weak self] _ in Task { @MainActor [weak self] in - guard let self else { return } - self.coordinator?.refresh() - if Self.systemSettingsWindowFrame() != nil { - self.hasSeenSettingsWindow = true - self.positionPanel() - } else if self.hasSeenSettingsWindow { - self.coordinator?.dismissGuide() + self?.coordinator?.refresh() + self?.updatePosition() + } + } + } + + private func updatePosition() { + guard let panel else { return } + guard let snapshot = settingsLocator.frontmostSnapshot(), + snapshot.blockingModalFrame == nil + else { + panel.orderOut(nil) + model.isSettled = false + return + } + + let target = OrbitPermissionGuideAnchorResolver.anchoredFrame(snapshot: snapshot) + let reduceMotion = NSWorkspace.shared.accessibilityDisplayShouldReduceMotion + if !hasAttached { + hasAttached = true + let launchFrame = + sourceFrame.map { + CGRect( + x: $0.midX - target.width / 2, + y: $0.midY - target.height / 2, + width: target.width, + height: target.height + ) + } ?? target + panel.setFrame(launchFrame, display: false) + panel.alphaValue = reduceMotion ? 0 : 1 + panel.orderFrontRegardless() + revealTask?.cancel() + revealTask = Task { @MainActor [weak self] in + try? await Task.sleep(for: .milliseconds(180)) + guard let self, !Task.isCancelled else { return } + if reduceMotion { + panel.alphaValue = 1 + panel.setFrame(target, display: true) } else { - self.positionPanel() + NSAnimationContext.runAnimationGroup( + { context in + context.duration = 0.32 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + panel.animator().setFrame(target, display: true) + }, + completionHandler: { [weak self] in + Task { @MainActor [weak self] in self?.model.isSettled = true } + }) } + model.isSettled = true } + return } + + panel.orderFrontRegardless() + guard !panel.frame.equalTo(target) else { + model.isSettled = true + return + } + if reduceMotion { + panel.setFrame(target, display: true) + } else { + NSAnimationContext.runAnimationGroup { context in + context.duration = 0.16 + context.timingFunction = CAMediaTimingFunction(name: .easeOut) + panel.animator().setFrame(target, display: true) + } + } + model.isSettled = true } private func installEscapeMonitor() { @@ -253,133 +577,91 @@ private final class OrbitPermissionGuideWindowController { } } - private func positionPanel() { - guard let panel else { return } - let settingsFrame = Self.systemSettingsWindowFrame() - let screen = - NSScreen.screens.first(where: { settingsFrame.map($0.frame.intersects) ?? false }) - ?? NSScreen.main - guard let visibleFrame = screen?.visibleFrame else { return } - - model.isAttachedToSettings = settingsFrame != nil - let width = settingsFrame.map { min(560, max(420, $0.width - 170)) } ?? 520 - let proposed = - settingsFrame.map { - NSRect(x: $0.maxX - width, y: $0.minY + 8, width: width, height: 78) - } - ?? NSRect( - x: visibleFrame.midX - width / 2, - y: visibleFrame.minY + 18, - width: width, - height: 78 - ) - let x = min(max(proposed.minX, visibleFrame.minX + 10), visibleFrame.maxX - proposed.width - 10) - let y = min(max(proposed.minY, visibleFrame.minY + 10), visibleFrame.maxY - proposed.height - 10) - panel.setFrame(NSRect(x: x, y: y, width: proposed.width, height: proposed.height), display: true) - } - - private static func systemSettingsWindowFrame() -> NSRect? { - guard - let windows = CGWindowListCopyWindowInfo( - [.optionOnScreenOnly, .excludeDesktopElements], - kCGNullWindowID - ) as? [[String: Any]], - let window = windows.first(where: { - let owner = $0[kCGWindowOwnerName as String] as? String - return ($0[kCGWindowLayer as String] as? Int) == 0 - && (owner == "System Settings" || owner == "System Preferences") - }), - let bounds = window[kCGWindowBounds as String] as? [String: CGFloat], - let x = bounds["X"], let y = bounds["Y"], - let width = bounds["Width"], let height = bounds["Height"] - else { return nil } - - let cgWindow = CGRect(x: x, y: y, width: width, height: height) - for screen in NSScreen.screens { - guard let number = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID else { continue } - let cgDisplay = CGDisplayBounds(number) - if cgDisplay.intersects(cgWindow) { - let convertedY = screen.frame.maxY - (cgWindow.minY - cgDisplay.minY) - cgWindow.height - return NSRect(x: cgWindow.minX, y: convertedY, width: cgWindow.width, height: cgWindow.height) - } - } - let primaryTop = NSScreen.screens.first?.frame.maxY ?? 0 - return NSRect(x: x, y: primaryTop - y - height, width: width, height: height) + private func removeEscapeMonitors() { + if let escapeMonitor { NSEvent.removeMonitor(escapeMonitor) } + if let localEscapeMonitor { NSEvent.removeMonitor(localEscapeMonitor) } + escapeMonitor = nil + localEscapeMonitor = nil } } private struct OrbitPermissionGuideView: View { - @ObservedObject var coordinator: OrbitPermissionCoordinator @ObservedObject var model: OrbitPermissionGuideModel let appURL: URL + let onDragCompleted: (Bool) -> Void @Environment(\.accessibilityReduceMotion) private var reduceMotion var body: some View { - HStack(spacing: 12) { - OrbitDraggablePermissionTile(appURL: appURL, isDragging: $model.isDragging) - .frame(width: 44, height: 44) - .accessibilityLabel("Orbit app tile") - .accessibilityHint("Drag Orbit into the \(model.permission.title) list in System Settings.") - - VStack(alignment: .leading, spacing: 3) { - Text("Add Orbit to \(model.permission.title)") - .font(.system(size: 13, weight: .semibold)) + HStack(spacing: 14) { + Image(systemName: model.successMessage == nil ? "arrow.up" : "checkmark") + .font(.system(size: 13, weight: .bold)) + .foregroundStyle(model.successMessage == nil ? DS.Colors.accent : DS.Colors.success) + .frame(width: 28, height: 28) + .background(Circle().fill(DS.Colors.surface3)) + .scaleEffect(reduceMotion || !model.isSettled ? 1 : 1.04) + + OrbitDraggablePermissionPill( + appURL: appURL, + isDragging: $model.isDragging, + onCompleted: onDragCompleted + ) + .frame(width: 214, height: 54) + .accessibilityLabel("Orbit app") + .accessibilityHint("Drag Orbit into the \(model.permission.title) list in System Settings.") + + VStack(alignment: .leading, spacing: 4) { + Text(model.successMessage ?? "Drag Orbit into the list") + .font(.headline) .foregroundStyle(DS.Colors.textPrimary) - Text(guideMessage) - .font(.system(size: 11)) - .foregroundStyle(DS.Colors.textSecondary) - .lineLimit(2) + Text( + model.successMessage == nil + ? "Only the Orbit app moves. This guide stays attached." + : "You can return to Orbit." + ) + .font(.caption) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) } - Spacer(minLength: 8) - Button("Reveal Orbit") { coordinator.revealOrbitInFinder() } - .accessibilityLabel("Reveal Orbit in Finder") - Button("Open Settings") { coordinator.openSettings(for: model.permission) } } - .buttonStyle(.borderless) - .padding(.horizontal, 14) + .padding(16) .frame(maxWidth: .infinity, maxHeight: .infinity) .background( - RoundedRectangle(cornerRadius: model.isAttachedToSettings ? 12 : 18, style: .continuous) - .fill(Color(nsColor: .windowBackgroundColor).opacity(0.97)) + RoundedRectangle(cornerRadius: 12, style: .continuous) + .fill(DS.Colors.surface1.opacity(0.98)) .overlay( - RoundedRectangle(cornerRadius: model.isAttachedToSettings ? 12 : 18, style: .continuous) - .stroke(DS.Colors.accent.opacity(0.35), lineWidth: 1) + RoundedRectangle(cornerRadius: 12, style: .continuous) + .stroke(DS.Colors.accent.opacity(0.28), lineWidth: 1) ) ) - .opacity(model.isDragging && !reduceMotion ? 0.82 : 1) + .opacity(model.isDragging && !reduceMotion ? 0.96 : 1) .animation(reduceMotion ? nil : .easeOut(duration: 0.16), value: model.isDragging) } - - private var guideMessage: String { - if model.permission == .screenRecording, - coordinator.statusByPermission[.screenRecording] == .restartRequired - { - return "Screen access was enabled. Quit and reopen Orbit, then retry the live capture check." - } - return "Drag only the Orbit tile. This guide stays attached to System Settings." - } } -private struct OrbitDraggablePermissionTile: NSViewRepresentable { +private struct OrbitDraggablePermissionPill: NSViewRepresentable { let appURL: URL @Binding var isDragging: Bool + let onCompleted: (Bool) -> Void - func makeNSView(context: Context) -> OrbitDraggablePermissionTileView { - let view = OrbitDraggablePermissionTileView(appURL: appURL) + func makeNSView(context: Context) -> OrbitDraggablePermissionPillView { + let view = OrbitDraggablePermissionPillView(appURL: appURL) view.onDraggingChanged = { isDragging = $0 } + view.onCompleted = onCompleted return view } - func updateNSView(_ nsView: OrbitDraggablePermissionTileView, context: Context) { + func updateNSView(_ nsView: OrbitDraggablePermissionPillView, context: Context) { nsView.appURL = appURL nsView.onDraggingChanged = { isDragging = $0 } + nsView.onCompleted = onCompleted } } -private final class OrbitDraggablePermissionTileView: NSView, NSDraggingSource { +private final class OrbitDraggablePermissionPillView: NSView, NSDraggingSource { var appURL: URL { didSet { needsDisplay = true } } var onDraggingChanged: ((Bool) -> Void)? - private var mouseDownEvent: NSEvent? + var onCompleted: ((Bool) -> Void)? + private var isHovering = false init(appURL: URL) { self.appURL = appURL @@ -387,32 +669,74 @@ private final class OrbitDraggablePermissionTileView: NSView, NSDraggingSource { toolTip = "Drag Orbit into System Settings" setAccessibilityElement(true) setAccessibilityRole(.button) - setAccessibilityLabel("Orbit app tile") + setAccessibilityLabel("Orbit app drag source") + addTrackingArea( + NSTrackingArea( + rect: bounds, + options: [.mouseEnteredAndExited, .activeAlways, .inVisibleRect], + owner: self, + userInfo: nil + ) + ) } required init?(coder: NSCoder) { nil } - override func mouseDown(with event: NSEvent) { mouseDownEvent = event } + override func mouseEntered(with event: NSEvent) { + isHovering = true + needsDisplay = true + } + + override func mouseExited(with event: NSEvent) { + isHovering = false + needsDisplay = true + } - override func mouseDragged(with event: NSEvent) { - guard let mouseDownEvent else { return } + override func mouseDown(with event: NSEvent) { onDraggingChanged?(true) - let item = NSDraggingItem(pasteboardWriter: appURL as NSURL) + let pasteboardItem = NSPasteboardItem() + pasteboardItem.setString(appURL.absoluteString, forType: .fileURL) + let item = NSDraggingItem(pasteboardWriter: pasteboardItem) let image = NSWorkspace.shared.icon(forFile: appURL.path) - image.size = NSSize(width: 48, height: 48) + image.size = NSSize(width: 40, height: 40) item.setDraggingFrame(bounds, contents: image) - beginDraggingSession(with: [item], event: mouseDownEvent, source: self) - self.mouseDownEvent = nil + beginDraggingSession(with: [item], event: event, source: self) } override func draw(_ dirtyRect: NSRect) { - NSColor.controlAccentColor.withAlphaComponent(0.16).setFill() - NSBezierPath(roundedRect: bounds.insetBy(dx: 1, dy: 1), xRadius: 11, yRadius: 11).fill() - let image = NSWorkspace.shared.icon(forFile: appURL.path) - image.draw(in: bounds.insetBy(dx: 6, dy: 6)) + let background = + isHovering + ? NSColor.controlAccentColor.withAlphaComponent(0.22) + : NSColor.controlAccentColor.withAlphaComponent(0.13) + background.setFill() + NSBezierPath(roundedRect: bounds.insetBy(dx: 1, dy: 1), xRadius: 8, yRadius: 8).fill() + + let iconRect = NSRect(x: 9, y: (bounds.height - 36) / 2, width: 36, height: 36) + NSWorkspace.shared.icon(forFile: appURL.path).draw(in: iconRect) + + let attributes: [NSAttributedString.Key: Any] = [ + .font: NSFont.systemFont(ofSize: 13, weight: .semibold), + .foregroundColor: NSColor.labelColor, + ] + NSString(string: "Orbit.app").draw( + at: NSPoint(x: 55, y: bounds.midY - 8), + withAttributes: attributes + ) + } + + func draggingSession( + _ session: NSDraggingSession, + sourceOperationMaskFor context: NSDraggingContext + ) -> NSDragOperation { .copy } + + func draggingSession( + _ session: NSDraggingSession, + endedAt screenPoint: NSPoint, + operation: NSDragOperation + ) { + onDraggingChanged?(false) + onCompleted?(!operation.isEmpty) } - func draggingSession(_ session: NSDraggingSession, sourceOperationMaskFor context: NSDraggingContext) -> NSDragOperation { .copy } - func draggingSession(_ session: NSDraggingSession, endedAt screenPoint: NSPoint, operation: NSDragOperation) { onDraggingChanged?(false) } func ignoreModifierKeys(for session: NSDraggingSession) -> Bool { true } } diff --git a/Orbit/OrbitPermissionModels.swift b/Orbit/OrbitPermissionModels.swift new file mode 100644 index 0000000..69b16b7 --- /dev/null +++ b/Orbit/OrbitPermissionModels.swift @@ -0,0 +1,274 @@ +import AVFoundation +import AppKit +import CoreGraphics +import Foundation + +protocol OrbitPermissionClock: Sendable { + var now: Date { get } + func sleep(for duration: Duration) async throws +} + +struct OrbitSystemPermissionClock: OrbitPermissionClock { + var now: Date { Date() } + + func sleep(for duration: Duration) async throws { + try await Task.sleep(for: duration) + } +} + +@MainActor +protocol OrbitPermissionDriving: AnyObject { + var microphoneStatus: OrbitPermissionStatus { get } + var hasAccessibilityPermission: Bool { get } + var hasScreenRecordingPermission: Bool { get } + var hasScreenContentPermission: Bool { get } + + func refresh() + func requestMicrophoneAccess() async -> Bool + func requestAccessibilityPermission() -> PermissionRequestPresentationDestination + func requestScreenRecordingPermission() -> PermissionRequestPresentationDestination + func requestScreenContentPermission() +} + +@MainActor +final class OrbitSystemPermissionDriver: OrbitPermissionDriving { + private weak var orbitManager: OrbitManager? + + init(orbitManager: OrbitManager) { + self.orbitManager = orbitManager + } + + var microphoneStatus: OrbitPermissionStatus { + switch AVCaptureDevice.authorizationStatus(for: .audio) { + case .notDetermined: .notDetermined + case .authorized: .granted + case .denied, .restricted: .denied + @unknown default: .denied + } + } + + var hasAccessibilityPermission: Bool { orbitManager?.hasAccessibilityPermission == true } + var hasScreenRecordingPermission: Bool { orbitManager?.hasScreenRecordingPermission == true } + var hasScreenContentPermission: Bool { orbitManager?.hasScreenContentPermission == true } + + func refresh() { + orbitManager?.refreshAllPermissions() + } + + func requestMicrophoneAccess() async -> Bool { + await AVCaptureDevice.requestAccess(for: .audio) + } + + func requestAccessibilityPermission() -> PermissionRequestPresentationDestination { + WindowPositionManager.requestAccessibilityPermission() + } + + func requestScreenRecordingPermission() -> PermissionRequestPresentationDestination { + WindowPositionManager.requestScreenRecordingPermission() + } + + func requestScreenContentPermission() { + orbitManager?.requestScreenContentPermission() + } +} + +enum OrbitPermissionKind: String, CaseIterable, Identifiable, Sendable { + case microphone + case accessibility + case screenRecording + case screenContent + + var id: String { rawValue } + + var title: String { + switch self { + case .microphone: "Microphone" + case .accessibility: "Accessibility" + case .screenRecording, .screenContent: "Screen Recording" + } + } + + var isVisibleOnboardingStep: Bool { self != .screenContent } +} + +enum OrbitPermissionStatus: Equatable, Sendable { + case notDetermined + case prompting + case waitingInSettings + case waitingForPicker + case restartRequired + case granted + case denied +} + +enum OrbitPermissionEntryContext: Equatable, Sendable { + case firstRun + case recovery +} + +struct OrbitPermissionSnapshot: Equatable, Sendable { + let microphone: OrbitPermissionStatus + let accessibility: OrbitPermissionStatus + let screenRecording: OrbitPermissionStatus + let screenContent: OrbitPermissionStatus + + subscript(_ kind: OrbitPermissionKind) -> OrbitPermissionStatus { + switch kind { + case .microphone: microphone + case .accessibility: accessibility + case .screenRecording: screenRecording + case .screenContent: screenContent + } + } +} + +struct OrbitPermissionFlowViewState: Equatable, Sendable { + let activePermission: OrbitPermissionKind? + let statuses: [OrbitPermissionKind: OrbitPermissionStatus] + let entryContext: OrbitPermissionEntryContext + let showsManualFallback: Bool + let message: String? + + static let initial = OrbitPermissionFlowViewState( + activePermission: nil, + statuses: [:], + entryContext: .firstRun, + showsManualFallback: false, + message: nil + ) +} + +struct OrbitSystemSettingsWindowSnapshot: Equatable, Sendable { + let processIdentifier: pid_t + let windowFrame: CGRect + let blockingModalFrame: CGRect? + let visibleFrame: CGRect +} + +enum OrbitSystemSettingsWindowLocator { + static let bundleIdentifier = "com.apple.systempreferences" + + @MainActor + static func frontmostSnapshot() -> OrbitSystemSettingsWindowSnapshot? { + guard let frontmost = NSWorkspace.shared.frontmostApplication, + frontmost.bundleIdentifier == bundleIdentifier + else { return nil } + return snapshot(processIdentifier: frontmost.processIdentifier) + } + + @MainActor + static func snapshot(processIdentifier: pid_t) -> OrbitSystemSettingsWindowSnapshot? { + guard + let rawWindows = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] + else { return nil } + + let candidates = rawWindows.compactMap { info -> CGRect? in + guard (info[kCGWindowOwnerPID as String] as? pid_t) == processIdentifier, + (info[kCGWindowLayer as String] as? Int) == 0, + (info[kCGWindowAlpha as String] as? Double ?? 1) > 0.01, + let bounds = info[kCGWindowBounds as String] as? [String: CGFloat], + let x = bounds["X"], let y = bounds["Y"], + let width = bounds["Width"], let height = bounds["Height"], + width >= 220, height >= 120 + else { return nil } + return appKitFrame(fromQuartzFrame: CGRect(x: x, y: y, width: width, height: height)) + } + + guard + let primary = + candidates + .filter({ $0.width >= 500 && $0.height >= 350 }) + .max(by: { $0.width * $0.height < $1.width * $1.height }) + ?? candidates.max(by: { $0.width * $0.height < $1.width * $1.height }) + else { return nil } + + let blockingModal = + candidates + .filter { candidate in + candidate != primary + && primary.intersects(candidate) + && candidate.width < primary.width * 0.9 + && candidate.height < primary.height * 0.9 + && abs(candidate.midX - primary.midX) < 90 + && abs(candidate.midY - primary.midY) < 90 + } + .max(by: { $0.width * $0.height < $1.width * $1.height }) + + let screen = + NSScreen.screens.max { lhs, rhs in + lhs.frame.intersection(primary).area < rhs.frame.intersection(primary).area + } ?? NSScreen.main + guard let visibleFrame = screen?.visibleFrame else { return nil } + + return OrbitSystemSettingsWindowSnapshot( + processIdentifier: processIdentifier, + windowFrame: primary, + blockingModalFrame: blockingModal, + visibleFrame: visibleFrame + ) + } + + @MainActor + private static func appKitFrame(fromQuartzFrame frame: CGRect) -> CGRect { + let primaryTop = + NSScreen.screens + .first(where: { $0.displayID == CGMainDisplayID() })? + .frame.maxY ?? 0 + return CGRect(x: frame.minX, y: primaryTop - frame.maxY, width: frame.width, height: frame.height) + } +} + +@MainActor +protocol OrbitSettingsWindowLocating: Sendable { + func frontmostSnapshot() -> OrbitSystemSettingsWindowSnapshot? +} + +struct OrbitSystemSettingsLocator: OrbitSettingsWindowLocating { + @MainActor + func frontmostSnapshot() -> OrbitSystemSettingsWindowSnapshot? { + OrbitSystemSettingsWindowLocator.frontmostSnapshot() + } +} + +enum OrbitPermissionGuideAnchorResolver { + static let guideSize = CGSize(width: 518, height: 112) + static let sidebarExclusion: CGFloat = 170 + static let bottomInset: CGFloat = 14 + + static func anchoredFrame( + snapshot: OrbitSystemSettingsWindowSnapshot, + size: CGSize = guideSize + ) -> CGRect { + let contentMinX = min(snapshot.windowFrame.maxX, snapshot.windowFrame.minX + sidebarExclusion) + let contentWidth = max(0, snapshot.windowFrame.maxX - contentMinX) + let proposedX = contentMinX + max(0, (contentWidth - size.width) / 2) + let proposedY = snapshot.windowFrame.minY + bottomInset + return clamp( + CGRect(origin: CGPoint(x: proposedX, y: proposedY), size: size), + to: snapshot.visibleFrame, + margin: 10 + ) + } + + static func clamp(_ frame: CGRect, to visibleFrame: CGRect, margin: CGFloat) -> CGRect { + let x = min( + max(frame.minX, visibleFrame.minX + margin), + max(visibleFrame.minX + margin, visibleFrame.maxX - frame.width - margin) + ) + let y = min( + max(frame.minY, visibleFrame.minY + margin), + max(visibleFrame.minY + margin, visibleFrame.maxY - frame.height - margin) + ) + return CGRect(x: x, y: y, width: frame.width, height: frame.height) + } +} + +extension CGRect { + fileprivate var area: CGFloat { + guard !isNull, !isInfinite else { return 0 } + return max(0, width) * max(0, height) + } +} diff --git a/Orbit/OrbitVisualQA.swift b/Orbit/OrbitVisualQA.swift new file mode 100644 index 0000000..f6209cd --- /dev/null +++ b/Orbit/OrbitVisualQA.swift @@ -0,0 +1,598 @@ +import AppKit +import SwiftUI + +#if DEBUG + enum OrbitVisualQAState: String, CaseIterable, Sendable { + case permissionMicrophone = "permission-microphone" + case permissionAccessibility = "permission-accessibility" + case permissionScreen = "permission-screen" + case permissionRestart = "permission-restart" + case coachAnchored = "coach-anchored" + case coachFallback = "coach-fallback" + case coachDragging = "coach-dragging" + case coachSuccess = "coach-success" + case onboardingAutomation = "onboarding-automation" + case onboardingAuth = "onboarding-auth" + case onboardingLocalVoice = "onboarding-local-voice" + case onboardingCloudVoice = "onboarding-cloud-voice" + case onboardingComplete = "onboarding-complete" + case taskReady = "task-ready" + case taskListening = "task-listening" + case taskWorking = "task-working" + case taskToolPrompt = "task-tool-prompt" + case taskTeamUp = "task-team-up" + case taskCompleted = "task-completed" + case taskInterrupted = "task-interrupted" + case taskFailed = "task-failed" + case settingsRoot = "settings-root" + case settingsVoice = "settings-voice" + case settingsCloudVoiceError = "settings-cloud-voice-error" + case settingsModel = "settings-model" + case settingsUnknownModel = "settings-unknown-model" + case settingsDeviceError = "settings-device-error" + case recovery = "recovery" + case stressLongCopy = "stress-long-copy" + + static var requested: OrbitVisualQAState? { + let arguments = ProcessInfo.processInfo.arguments + guard let index = arguments.firstIndex(of: "-OrbitVisualQAState"), + arguments.indices.contains(index + 1) + else { return nil } + return OrbitVisualQAState(rawValue: arguments[index + 1]) + } + + static var requestedOutputURL: URL? { + let arguments = ProcessInfo.processInfo.arguments + guard let index = arguments.firstIndex(of: "-OrbitVisualQAOutput"), + arguments.indices.contains(index + 1) + else { return nil } + return URL(fileURLWithPath: arguments[index + 1]) + } + + @MainActor + static func renderRequestedFixtureIfNeeded() { + guard let state = requested, let outputURL = requestedOutputURL else { return } + + let hostingView = NSHostingView( + rootView: OrbitVisualQAPanel(state: state) + .environment(\.colorScheme, .dark) + ) + let fittingSize = hostingView.fittingSize + hostingView.frame = CGRect(origin: .zero, size: fittingSize) + hostingView.layoutSubtreeIfNeeded() + + let scale: CGFloat = 2 + guard + let representation = NSBitmapImageRep( + bitmapDataPlanes: nil, + pixelsWide: Int(fittingSize.width * scale), + pixelsHigh: Int(fittingSize.height * scale), + bitsPerSample: 8, + samplesPerPixel: 4, + hasAlpha: true, + isPlanar: false, + colorSpaceName: .deviceRGB, + bytesPerRow: 0, + bitsPerPixel: 0 + ) + else { return } + + representation.size = fittingSize + hostingView.cacheDisplay(in: hostingView.bounds, to: representation) + guard let data = representation.representation(using: .png, properties: [:]) else { return } + do { + try data.write(to: outputURL, options: .atomic) + print("Orbit visual fixture rendered: \(outputURL.path)") + } catch { + print("Orbit visual fixture failed: \(error.localizedDescription)") + } + } + } + + struct OrbitVisualQAPanel: View { + let state: OrbitVisualQAState + + private let shape = RoundedRectangle(cornerRadius: 16, style: .continuous) + + @ViewBuilder + var body: some View { + if state.isCoach { + coachContent + } else { + VStack(alignment: .leading, spacing: 16) { + header + content + } + .padding(16) + .frame(width: 344, alignment: .topLeading) + .background(shape.fill(DS.Colors.background)) + .clipShape(shape) + .overlay(shape.stroke(DS.Colors.borderSubtle, lineWidth: 1)) + .accessibilityIdentifier("visual.fixture.\(state.rawValue)") + } + } + + private var header: some View { + HStack(spacing: 10) { + OrbitMarkView(size: 18) + .frame(width: 28, height: 28) + .accessibilityHidden(true) + VStack(alignment: .leading, spacing: 1) { + Text(headerTitle) + .font(.headline) + .foregroundStyle(DS.Colors.textPrimary) + Text(headerSubtitle) + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + .lineLimit(2) + } + Spacer(minLength: 8) + Image(systemName: "gearshape") + .frame(width: 28, height: 28) + .foregroundStyle(DS.Colors.textSecondary) + Image(systemName: "ellipsis") + .frame(width: 28, height: 28) + .foregroundStyle(DS.Colors.textSecondary) + Image(systemName: "xmark") + .frame(width: 28, height: 28) + .foregroundStyle(DS.Colors.textSecondary) + } + } + + @ViewBuilder + private var content: some View { + if state.isPermission { + permissionContent + } else if state.isOnboarding { + onboardingContent + } else if state.isSettings { + settingsContent + } else { + taskContent + } + } + + private var coachContent: some View { + HStack(spacing: 14) { + Image(systemName: state == .coachSuccess ? "checkmark" : "arrow.up") + .font(.system(size: 13, weight: .bold)) + .foregroundStyle(state == .coachSuccess ? DS.Colors.success : DS.Colors.accent) + .frame(width: 28, height: 28) + .background(Circle().fill(DS.Colors.surface3)) + HStack(spacing: 10) { + OrbitMarkView(size: 24) + Text(state == .coachDragging ? "Dragging Orbit.app" : "Orbit.app") + .font(.body.weight(.semibold)) + } + .foregroundStyle(DS.Colors.textPrimary) + .padding(.horizontal, 12) + .frame(width: 214, height: 54, alignment: .leading) + .background( + RoundedRectangle(cornerRadius: 8) + .fill(DS.Colors.accent.opacity(state == .coachDragging ? 0.24 : 0.14)) + ) + VStack(alignment: .leading, spacing: 4) { + Text(coachTitle).font(.headline).foregroundStyle(DS.Colors.textPrimary) + Text(coachDetail) + .font(.caption) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + } + } + .padding(16) + .frame(width: 518, height: 112) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + .overlay(RoundedRectangle(cornerRadius: 12).stroke(DS.Colors.accent.opacity(0.28))) + .accessibilityIdentifier("visual.fixture.\(state.rawValue)") + } + + private var onboardingContent: some View { + VStack(alignment: .leading, spacing: 12) { + Label(onboardingTitle, systemImage: onboardingIcon) + .font(.title3.weight(.semibold)) + .foregroundStyle(DS.Colors.textPrimary) + Text(onboardingDetail) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + if state == .onboardingAutomation { + Label("Automatic command and file access", systemImage: "terminal") + Label("One fresh capture per request", systemImage: "camera.viewfinder") + Label("No continuous recording indicator", systemImage: "eye.slash") + } else if state == .onboardingLocalVoice || state == .onboardingCloudVoice { + settingsRow( + "Local", + detail: "On-device · Nora Voice 4", + icon: state == .onboardingLocalVoice ? "checkmark.circle.fill" : "circle" + ) + settingsRow( + "Cloud", + detail: "OpenAI · AI-generated voice", + icon: state == .onboardingCloudVoice ? "checkmark.circle.fill" : "circle" + ) + } else if state == .onboardingComplete { + Label("One warm Codex thread", systemImage: "checkmark.circle") + Label("Local Nora Voice 4 narration", systemImage: "checkmark.circle") + Label("Fresh visual context per request", systemImage: "checkmark.circle") + } + Text(onboardingAction) + .font(.body.weight(.semibold)) + .foregroundStyle(DS.Colors.textOnAccent) + .frame(maxWidth: .infinity, minHeight: 38) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.accent)) + } + .padding(14) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + } + + private var permissionContent: some View { + VStack(alignment: .leading, spacing: 14) { + HStack(spacing: 6) { + permissionStep("Microphone", index: 0) + permissionStep("Accessibility", index: 1) + permissionStep("Screen", index: 2) + } + VStack(alignment: .leading, spacing: 10) { + Label(permissionTitle, systemImage: permissionIcon) + .font(.title3.weight(.semibold)) + .foregroundStyle(DS.Colors.textPrimary) + Text(permissionMessage) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + if state == .permissionAccessibility || state == .permissionScreen { + Label("Drag the Orbit app pill into the open privacy pane.", systemImage: "arrow.up.right") + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + } + Text(permissionAction) + .font(.body.weight(.semibold)) + .foregroundStyle(DS.Colors.textOnAccent) + .frame(maxWidth: .infinity, minHeight: 38) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.accent)) + HStack { + Text("Reveal Orbit in Finder") + Spacer() + Text("Open System Settings") + } + .font(.caption.weight(.medium)) + .foregroundStyle(DS.Colors.textSecondary) + } + .padding(14) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + } + } + + private var taskContent: some View { + VStack(alignment: .leading, spacing: 12) { + HStack(spacing: 7) { + Circle().fill(taskColor).frame(width: 7, height: 7) + Text(taskStatus).font(.caption.weight(.semibold)) + } + .foregroundStyle(DS.Colors.textSecondary) + Text(taskTitle) + .font(.title3.weight(.semibold)) + .foregroundStyle(DS.Colors.textPrimary) + Text(taskMessage) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + if state == .taskToolPrompt { + VStack(alignment: .leading, spacing: 8) { + Text("Codex needs a tool choice before another request can begin.") + Text("Continue").fontWeight(.semibold) + .frame(maxWidth: .infinity, minHeight: 36) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.accent)) + } + .foregroundStyle(DS.Colors.textOnAccent) + .padding(12) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface2)) + } + Divider().overlay(DS.Colors.borderSubtle) + Label("Recent activity", systemImage: "chevron.right") + .font(.caption.weight(.semibold)) + .foregroundStyle(DS.Colors.textTertiary) + } + .padding(14) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + } + + private var settingsContent: some View { + VStack(alignment: .leading, spacing: 10) { + if state == .settingsVoice || state == .settingsDeviceError || state == .settingsCloudVoiceError { + settingsRow("Voice", detail: "Local · Nora (Voice 4)", icon: "waveform") + settingsRow( + "Microphone", + detail: state == .settingsDeviceError ? "Device disconnected" : "MacBook Microphone", + icon: "mic" + ) + if state == .settingsDeviceError { + Text("Reconnect the selected microphone or choose another input device.") + .font(.caption) + .foregroundStyle(DS.Colors.destructiveText) + .fixedSize(horizontal: false, vertical: true) + } + if state == .settingsCloudVoiceError { + Text("Cloud voice is selected, but the saved OpenAI key could not be verified.") + .font(.caption) + .foregroundStyle(DS.Colors.destructiveText) + .fixedSize(horizontal: false, vertical: true) + } + settingsRow("Preview", detail: "Play Nora", icon: "play.fill") + } else if state == .settingsModel || state == .settingsUnknownModel { + settingsRow( + "Model", + detail: state == .settingsUnknownModel ? "future-model-ultra-long-account-value" : "Server default", + icon: "cpu" + ) + settingsRow("Reasoning", detail: "High", icon: "dial.medium") + settingsRow("Service tier", detail: "Automatic", icon: "speedometer") + Text("Options come directly from your signed-in Codex account.") + .font(.caption) + .foregroundStyle(DS.Colors.textTertiary) + } else if state == .recovery || state == .stressLongCopy { + settingsRow("Codex", detail: "Reconnecting", icon: "arrow.triangle.2.circlepath") + Text( + state == .stressLongCopy + ? "Orbit is warming exactly one replacement session while preserving your current working context, model selection, reasoning effort, and service tier." + : "Orbit is warming one replacement session. Your working context is unchanged." + ) + .font(.body) + .foregroundStyle(DS.Colors.textSecondary) + .fixedSize(horizontal: false, vertical: true) + Text("Retry now") + .font(.body.weight(.semibold)) + .foregroundStyle(DS.Colors.textOnAccent) + .frame(maxWidth: .infinity, minHeight: 38) + .background(RoundedRectangle(cornerRadius: 8).fill(DS.Colors.accent)) + } else { + settingsRow("Voice", detail: "Local · Nora", icon: "waveform") + settingsRow("Codex", detail: "Connected", icon: "sparkles") + settingsRow("Working context", detail: "Selected folder", icon: "folder") + settingsRow("Privacy & permissions", detail: "Review", icon: "hand.raised") + settingsRow("Appearance & About", detail: "Orbit 1.1.0", icon: "circle.lefthalf.filled") + } + } + .padding(14) + .background(RoundedRectangle(cornerRadius: 12).fill(DS.Colors.surface1)) + } + + private func settingsRow(_ title: String, detail: String, icon: String) -> some View { + HStack(spacing: 10) { + Image(systemName: icon).frame(width: 18).foregroundStyle(DS.Colors.textSecondary) + Text(title).foregroundStyle(DS.Colors.textPrimary) + Spacer(minLength: 8) + Text(detail).font(.caption).foregroundStyle(DS.Colors.textTertiary).lineLimit(1) + Image(systemName: "chevron.right").font(.caption2).foregroundStyle(DS.Colors.textTertiary) + } + .frame(minHeight: 36) + } + + private func permissionStep(_ label: String, index: Int) -> some View { + let active = permissionIndex == index + let complete = permissionIndex > index + return HStack(spacing: 4) { + Image(systemName: complete ? "checkmark.circle.fill" : "circle") + Text(label).lineLimit(1) + } + .font(.caption2.weight(.semibold)) + .foregroundStyle(complete ? DS.Colors.success : (active ? DS.Colors.textPrimary : DS.Colors.textTertiary)) + .frame(maxWidth: .infinity, minHeight: 28) + .background(RoundedRectangle(cornerRadius: 8).fill(active ? DS.Colors.surface2 : DS.Colors.surface1)) + } + + private var permissionIndex: Int { + switch state { + case .permissionMicrophone: 0 + case .permissionAccessibility: 1 + case .permissionScreen, .permissionRestart: 2 + default: 0 + } + } + + private var permissionTitle: String { + switch state { + case .permissionMicrophone: "Let Orbit hear your request" + case .permissionAccessibility: "Allow Orbit to work with your Mac" + case .permissionScreen: "Give each request visual context" + case .permissionRestart: "Screen Recording is ready" + default: "Set up Orbit" + } + } + + private var permissionMessage: String { + switch state { + case .permissionMicrophone: "Speech stays on this Mac when Local voice is selected." + case .permissionAccessibility: "Orbit uses Accessibility to carry out the commands you ask Codex to perform." + case .permissionScreen: "Orbit captures the current screen once per request, then deletes the temporary image." + case .permissionRestart: "macOS requires a safe relaunch before Orbit can confirm the permission." + default: "" + } + } + + private var permissionAction: String { + switch state { + case .permissionMicrophone: "Allow Microphone" + case .permissionAccessibility: "Open Accessibility Settings" + case .permissionScreen: "Open Screen Recording Settings" + case .permissionRestart: "Quit & Reopen Orbit" + default: "Continue" + } + } + + private var permissionIcon: String { + switch state { + case .permissionMicrophone: "mic" + case .permissionAccessibility: "cursorarrow.motionlines" + case .permissionScreen, .permissionRestart: "rectangle.on.rectangle" + default: "checkmark.shield" + } + } + + private var taskTitle: String { + switch state { + case .taskReady: "Ready for your next request" + case .taskListening: "Listening on this Mac" + case .taskWorking: "Reviewing the current screen" + case .taskToolPrompt: "Choose how Codex should continue" + case .taskTeamUp: "Codex teamed up for this task" + case .taskCompleted: "Finished" + case .taskInterrupted: "Stopped safely" + case .taskFailed: "The request could not start" + default: "Orbit" + } + } + + private var taskStatus: String { + switch state { + case .taskReady: "Ready" + case .taskListening: "Listening" + case .taskWorking: "Working" + case .taskToolPrompt: "Waiting" + case .taskTeamUp: "Team-up" + case .taskCompleted: "Done" + case .taskInterrupted: "Stopped" + case .taskFailed: "Needs attention" + default: "Ready" + } + } + + private var taskMessage: String { + switch state { + case .taskReady: "Hold Control + Option and ask Orbit anything about your screen." + case .taskListening: "Local speech recognition is active. Release the shortcut when you finish speaking." + case .taskWorking: "Codex is keeping this working thread warm while it checks the relevant files." + case .taskToolPrompt: "Your active request remains attached to the same Codex thread." + case .taskTeamUp: "Two child agents are checking permission reliability and narration behavior in parallel." + case .taskCompleted: "The requested changes are built, tested, and ready for review." + case .taskInterrupted: "Orbit kept the Codex thread warm and cleaned the temporary screen capture." + case .taskFailed: "Screen capture failed, so Orbit did not send a request without visual context. Retry when ready." + default: "" + } + } + + private var taskColor: Color { + switch state { + case .taskReady: DS.Colors.textTertiary + case .taskListening: DS.Colors.accent + case .taskWorking: DS.Colors.accent + case .taskToolPrompt: DS.Colors.warning + case .taskTeamUp: DS.Colors.accent + case .taskCompleted: DS.Colors.success + case .taskInterrupted: DS.Colors.warning + case .taskFailed: DS.Colors.destructive + default: DS.Colors.textTertiary + } + } + + private var headerTitle: String { + if state.isPermission { return "Set up Orbit" } + if state.isOnboarding { return "Set up Orbit" } + if state.isSettings { return state == .settingsRoot ? "Settings" : "Settings" } + return "Orbit" + } + + private var headerSubtitle: String { + if state.isPermission { return "One clear step at a time" } + if state.isOnboarding { return "A private, reliable default" } + if state.isSettings { return "Voice, Codex, context, and privacy" } + return taskStatus + } + + private var coachTitle: String { + switch state { + case .coachSuccess: "Accessibility is ready" + case .coachDragging: "Drop Orbit in System Settings" + default: "Drag Orbit into the list" + } + } + + private var coachDetail: String { + switch state { + case .coachSuccess: "You can return to Orbit." + case .coachFallback: "If dragging is unavailable, use the keyboard fallback in Orbit." + default: "Only the Orbit app moves. This guide stays attached." + } + } + + private var onboardingTitle: String { + switch state { + case .onboardingAutomation: "Unrestricted automation" + case .onboardingAuth: "Connect ChatGPT" + case .onboardingLocalVoice, .onboardingCloudVoice: "Choose voice mode" + case .onboardingComplete: "Orbit is ready" + default: "Set up Orbit" + } + } + + private var onboardingDetail: String { + switch state { + case .onboardingAutomation: + "Orbit can run commands and edit files without approval prompts. Each request gets one fresh capture that is deleted at the end of the turn." + case .onboardingAuth: + "Sign in once so Orbit can keep one Codex thread warm for this working context." + case .onboardingLocalVoice: + "Local keeps recognition and Siri Natural Nora Voice 4 on this Mac." + case .onboardingCloudVoice: + "Cloud sends speech audio to OpenAI and clearly labels the AI-generated voice." + case .onboardingComplete: + "Hold Control + Option and ask what is on your screen." + default: "" + } + } + + private var onboardingAction: String { + switch state { + case .onboardingAutomation: "I understand — continue" + case .onboardingAuth: "Connect ChatGPT" + case .onboardingLocalVoice, .onboardingCloudVoice: "Continue with this voice mode" + case .onboardingComplete: "Get started" + default: "Continue" + } + } + + private var onboardingIcon: String { + switch state { + case .onboardingAutomation: "terminal" + case .onboardingAuth: "person.crop.circle.badge.checkmark" + case .onboardingLocalVoice, .onboardingCloudVoice: "waveform" + case .onboardingComplete: "checkmark.circle" + default: "sparkles" + } + } + } + + extension OrbitVisualQAState { + fileprivate var isPermission: Bool { + switch self { + case .permissionMicrophone, .permissionAccessibility, .permissionScreen, .permissionRestart: true + default: false + } + } + + fileprivate var isCoach: Bool { + switch self { + case .coachAnchored, .coachFallback, .coachDragging, .coachSuccess: true + default: false + } + } + + fileprivate var isOnboarding: Bool { + switch self { + case .onboardingAutomation, .onboardingAuth, .onboardingLocalVoice, .onboardingCloudVoice, + .onboardingComplete: + true + default: false + } + } + + fileprivate var isSettings: Bool { + switch self { + case .settingsRoot, .settingsVoice, .settingsCloudVoiceError, .settingsModel, + .settingsUnknownModel, .settingsDeviceError, .recovery, .stressLongCopy: + true + default: false + } + } + } +#endif diff --git a/Orbit/OrbitVoiceCoordinator.swift b/Orbit/OrbitVoiceCoordinator.swift index b05e785..5d842a4 100644 --- a/Orbit/OrbitVoiceCoordinator.swift +++ b/Orbit/OrbitVoiceCoordinator.swift @@ -1,9 +1,9 @@ import Foundation -enum OrbitNarrationSource: Sendable { +enum OrbitNarrationSource: Sendable, Equatable { case preview case onboarding - case earlyCommentary + case firstLine case completion case failure } @@ -24,6 +24,54 @@ struct OrbitNarrationRequest: Sendable { } } +enum OrbitFirstLineExtractor { + static func stableLine(from rawText: String, allowUnterminated: Bool = false) -> String? { + let normalizedNewlines = rawText.replacingOccurrences(of: "\r\n", with: "\n") + let hasCompletedLine = normalizedNewlines.contains("\n") + let firstRawLine = + hasCompletedLine + ? normalizedNewlines + .split(separator: "\n", omittingEmptySubsequences: true) + .map(String.init) + .first(where: { !OrbitNarrationFormatter.spokenText(from: $0).isEmpty }) : nil + + let source = firstRawLine ?? normalizedNewlines + let cleaned = OrbitNarrationFormatter.spokenText(from: source, maximumLength: 180) + + if let boundary = cleaned.firstIndex(where: { ".!?".contains($0) }) { + return String(cleaned[...boundary]).trimmingCharacters(in: .whitespacesAndNewlines) + } + + guard hasCompletedLine || allowUnterminated, + cleaned.split(separator: " ").count >= 4 + else { return nil } + return cleaned.hasSuffix("…") ? cleaned : "\(cleaned)." + } + + static func removingSpokenPrefix(_ spokenPrefix: String?, from completion: String) -> String { + let formattedCompletion = OrbitNarrationFormatter.spokenText(from: completion) + guard let spokenPrefix else { return formattedCompletion } + + let formattedPrefix = OrbitNarrationFormatter.spokenText(from: spokenPrefix, maximumLength: 180) + guard !formattedPrefix.isEmpty else { return formattedCompletion } + + let prefixWithoutTerminal = formattedPrefix.trimmingCharacters(in: CharacterSet(charactersIn: ".!?… ")) + guard !prefixWithoutTerminal.isEmpty else { return formattedCompletion } + + if formattedCompletion.lowercased().hasPrefix(prefixWithoutTerminal.lowercased()) { + let index = formattedCompletion.index(formattedCompletion.startIndex, offsetBy: prefixWithoutTerminal.count) + var remainder = String(formattedCompletion[index...]) + .trimmingCharacters(in: .whitespacesAndNewlines) + while let first = remainder.first, ".!?…:;-".contains(first) { + remainder.removeFirst() + remainder = remainder.trimmingCharacters(in: .whitespacesAndNewlines) + } + return remainder + } + return formattedCompletion + } +} + enum OrbitNarrationFormatter { static func spokenText(from rawText: String, maximumLength: Int = 560) -> String { guard maximumLength > 0 else { return "" } @@ -73,6 +121,7 @@ struct OrbitNarrationDeduplicator { private struct Entry { let key: String let turnIdentifier: String? + let source: OrbitNarrationSource let timestamp: Date } @@ -83,23 +132,35 @@ struct OrbitNarrationDeduplicator { self.retentionInterval = retentionInterval } - mutating func shouldSpeak( + mutating func contains( _ text: String, turnIdentifier: String?, + source: OrbitNarrationSource, now: Date = Date() ) -> Bool { let key = OrbitNarrationFormatter.comparisonKey(for: text) - guard !key.isEmpty else { return false } + guard !key.isEmpty else { return true } recentEntries.removeAll { now.timeIntervalSince($0.timestamp) > retentionInterval } - let isDuplicate = recentEntries.contains { entry in - guard entry.turnIdentifier == turnIdentifier else { return false } - return entry.key == key || entry.key.hasPrefix(key) || key.hasPrefix(entry.key) + return recentEntries.contains { entry in + entry.turnIdentifier == turnIdentifier + && entry.source == source + && entry.key == key } - guard !isDuplicate else { return false } + } - recentEntries.append(Entry(key: key, turnIdentifier: turnIdentifier, timestamp: now)) - return true + mutating func record( + _ text: String, + turnIdentifier: String?, + source: OrbitNarrationSource, + now: Date = Date() + ) { + let key = OrbitNarrationFormatter.comparisonKey(for: text) + guard !key.isEmpty else { return } + recentEntries.removeAll { now.timeIntervalSince($0.timestamp) > retentionInterval } + recentEntries.append( + Entry(key: key, turnIdentifier: turnIdentifier, source: source, timestamp: now) + ) } mutating func reset() { @@ -109,10 +170,20 @@ struct OrbitNarrationDeduplicator { @MainActor final class OrbitVoiceCoordinator: TextToSpeechProvider { + private struct QueuedNarration { + let request: OrbitNarrationRequest + let generation: Int + } + private let primaryProvider: any TextToSpeechProvider private let fallbackProvider: (any TextToSpeechProvider)? private var generation = 0 private var deduplicator = OrbitNarrationDeduplicator() + private var queue: [QueuedNarration] = [] + private var queueTask: Task? + private var firstLineByRequest: [String: String] = [:] + + var onPlaybackStateChanged: ((Bool) -> Void)? init( primary: any TextToSpeechProvider, @@ -148,34 +219,142 @@ final class OrbitVoiceCoordinator: TextToSpeechProvider { func speak(_ request: OrbitNarrationRequest) async throws { let formattedText = OrbitNarrationFormatter.spokenText(from: request.text) + guard !formattedText.isEmpty else { return } guard - deduplicator.shouldSpeak( + !deduplicator.contains( formattedText, - turnIdentifier: request.turnIdentifier + turnIdentifier: request.turnIdentifier, + source: request.source ) else { return } stopPlayback() let requestGeneration = generation + try await play(formattedText, request: request, generation: requestGeneration) + guard requestGeneration == generation else { throw CancellationError() } + } + + func beginRequest(_ requestIdentifier: String) { + stopPlayback() + firstLineByRequest = firstLineByRequest.filter { $0.key == requestIdentifier } + } + + func enqueueFirstLine(_ text: String, requestIdentifier: String) { + guard firstLineByRequest[requestIdentifier] == nil, + let firstLine = OrbitFirstLineExtractor.stableLine(from: text, allowUnterminated: true) + else { return } + + firstLineByRequest[requestIdentifier] = firstLine + enqueue( + OrbitNarrationRequest( + text: firstLine, + source: .firstLine, + turnIdentifier: requestIdentifier + ) + ) + } + func enqueueTerminal( + _ text: String, + fallback: String? = nil, + source: OrbitNarrationSource, + requestIdentifier: String + ) { + let resolved = OrbitNarrationFormatter.spokenText(from: text).isEmpty ? (fallback ?? "done.") : text + if firstLineByRequest[requestIdentifier] == nil, + let firstLine = OrbitFirstLineExtractor.stableLine(from: resolved, allowUnterminated: true) + { + enqueueFirstLine(firstLine, requestIdentifier: requestIdentifier) + } + + let remainder = OrbitFirstLineExtractor.removingSpokenPrefix( + firstLineByRequest[requestIdentifier], + from: resolved + ) + guard !remainder.isEmpty else { return } + enqueue( + OrbitNarrationRequest( + text: remainder, + source: source, + turnIdentifier: requestIdentifier + ) + ) + } + + private func enqueue(_ request: OrbitNarrationRequest) { + let formatted = OrbitNarrationFormatter.spokenText(from: request.text) + guard !formatted.isEmpty, + !deduplicator.contains( + formatted, + turnIdentifier: request.turnIdentifier, + source: request.source + ), + !queue.contains(where: { + $0.request.turnIdentifier == request.turnIdentifier + && $0.request.source == request.source + && OrbitNarrationFormatter.comparisonKey(for: $0.request.text) + == OrbitNarrationFormatter.comparisonKey(for: formatted) + }) + else { return } + + queue.append(QueuedNarration(request: request, generation: generation)) + startQueueIfNeeded() + } + + private func startQueueIfNeeded() { + guard queueTask == nil else { return } + queueTask = Task { @MainActor [weak self] in + guard let self else { return } + while !queue.isEmpty { + let queued = queue.removeFirst() + guard queued.generation == generation else { continue } + let text = OrbitNarrationFormatter.spokenText(from: queued.request.text) + guard !text.isEmpty else { continue } + do { + try await play(text, request: queued.request, generation: queued.generation) + } catch is CancellationError { + break + } catch { + OrbitSupportLog.append("voice", "queued narration failed: \(error.localizedDescription)") + } + } + queueTask = nil + onPlaybackStateChanged?(false) + } + } + + private func play( + _ text: String, + request: OrbitNarrationRequest, + generation requestGeneration: Int + ) async throws { + onPlaybackStateChanged?(true) do { - try await primaryProvider.speakText(formattedText) + try await primaryProvider.speakText(text) } catch is CancellationError { throw CancellationError() } catch { guard requestGeneration == generation, let fallbackProvider, fallbackProvider.isConfigured else { throw error } - try await fallbackProvider.speakText(formattedText) + try await fallbackProvider.speakText(text) } - guard requestGeneration == generation else { throw CancellationError() } + deduplicator.record( + text, + turnIdentifier: request.turnIdentifier, + source: request.source + ) } func stopPlayback() { generation &+= 1 + queueTask?.cancel() + queueTask = nil + queue.removeAll(keepingCapacity: false) primaryProvider.stopPlayback() fallbackProvider?.stopPlayback() + onPlaybackStateChanged?(false) } func resetNarrationHistory() { diff --git a/OrbitTests/OrbitCodexLifecycleTests.swift b/OrbitTests/OrbitCodexLifecycleTests.swift index ecf2176..68f791b 100644 --- a/OrbitTests/OrbitCodexLifecycleTests.swift +++ b/OrbitTests/OrbitCodexLifecycleTests.swift @@ -1,9 +1,82 @@ +import Foundation import Testing @testable import Orbit @MainActor struct OrbitCodexLifecycleTests { + @Test func fiftySequentialRequestsReuseOneThreadGeneration() { + var state = OrbitCodexConversationState() + state.threadStarted("thread-one") + + for _ in 0..<50 { + let requestID = UUID() + state.startRequest(requestID) + state.turnStarted(UUID().uuidString) + #expect(state.activeRequestID == requestID) + _ = state.completeActiveRequest() + #expect(state.threadID == "thread-one") + } + + #expect(state.contextGeneration == 0) + #expect(state.phase == .ready) + } + + @Test func steerPromotionAndRejectionPreserveThreadOwnership() { + var state = OrbitCodexConversationState() + let first = UUID() + let second = UUID() + let third = UUID() + state.threadStarted("thread-one") + state.startRequest(first) + state.turnStarted("turn-one") + + state.queueSteer(second) + state.steerRejected() + #expect(state.activeRequestID == first) + #expect(state.threadID == "thread-one") + + state.queueSteer(third) + state.steerAccepted() + #expect(state.activeRequestID == third) + #expect(state.turnID == "turn-one") + #expect(state.threadID == "thread-one") + } + + @Test func cancellationBeforeTurnIDKeepsWarmThread() { + var state = OrbitCodexConversationState() + let requestID = UUID() + state.threadStarted("thread-one") + state.startRequest(requestID) + state.requestCancellation() + + #expect(state.phase == .cancelling(requestID: requestID, waitingForTurnID: true)) + #expect(state.threadID == "thread-one") + + state.turnStarted("turn-one") + state.requestCancellation() + #expect(state.phase == .cancelling(requestID: requestID, waitingForTurnID: false)) + #expect(state.threadID == "thread-one") + } + + @Test func oldTurnCompletingBeforeSteerStartsPendingRequestOnSameThread() { + var state = OrbitCodexConversationState() + let first = UUID() + let followUp = UUID() + state.threadStarted("thread-one") + state.startRequest(first) + state.turnStarted("turn-one") + state.queueSteer(followUp) + + let completed = state.completeActiveRequest() + + #expect(completed == first) + #expect(state.threadID == "thread-one") + #expect(state.activeRequestID == followUp) + #expect(state.pendingRequestID == nil) + #expect(state.phase == .startingTurn(requestID: followUp)) + } + @Test func authenticatedStartupRequiresAReadyThread() { let unresolved = OrbitCodexStartupReadiness.isResolved( authState: .authenticated(email: nil, plan: nil), diff --git a/OrbitTests/OrbitPermissionTests.swift b/OrbitTests/OrbitPermissionTests.swift new file mode 100644 index 0000000..889f0f3 --- /dev/null +++ b/OrbitTests/OrbitPermissionTests.swift @@ -0,0 +1,172 @@ +import CoreGraphics +import Testing + +@testable import Orbit + +@MainActor +private final class OrbitFakePermissionDriver: OrbitPermissionDriving { + var microphoneStatus: OrbitPermissionStatus = .granted + var hasAccessibilityPermission = false + var hasScreenRecordingPermission = false + var hasScreenContentPermission = false + var microphoneRequestResult = true + var accessibilityDestination: PermissionRequestPresentationDestination = .alreadyGranted + var screenDestination: PermissionRequestPresentationDestination = .alreadyGranted + private(set) var screenContentRequestCount = 0 + + func refresh() {} + func requestMicrophoneAccess() async -> Bool { microphoneRequestResult } + func requestAccessibilityPermission() -> PermissionRequestPresentationDestination { + accessibilityDestination + } + func requestScreenRecordingPermission() -> PermissionRequestPresentationDestination { + screenDestination + } + func requestScreenContentPermission() { + screenContentRequestCount += 1 + } +} + +private struct OrbitEmptySettingsLocator: OrbitSettingsWindowLocating { + @MainActor func frontmostSnapshot() -> OrbitSystemSettingsWindowSnapshot? { nil } +} + +@MainActor +struct OrbitPermissionModelTests { + @Test func deniedMicrophonePromptProducesRecoverableTypedState() async { + let driver = OrbitFakePermissionDriver() + driver.microphoneStatus = .notDetermined + driver.microphoneRequestResult = false + let coordinator = OrbitPermissionCoordinator( + driver: driver, + settingsLocator: OrbitEmptySettingsLocator() + ) + + coordinator.begin(.microphone) + for _ in 0..<20 where coordinator.statusByPermission[.microphone] != .denied { + await Task.yield() + } + + #expect(coordinator.statusByPermission[.microphone] == .denied) + #expect(coordinator.viewState.entryContext == .recovery) + #expect(coordinator.viewState.showsManualFallback) + } + + @Test func screenRecordingGrantFoldsIntoScreenContentProbe() { + let driver = OrbitFakePermissionDriver() + let coordinator = OrbitPermissionCoordinator( + driver: driver, + settingsLocator: OrbitEmptySettingsLocator() + ) + + coordinator.begin(.screenRecording) + + #expect(coordinator.activePermission == .screenRecording) + #expect(coordinator.statusByPermission[.screenRecording] == .waitingForPicker) + #expect(coordinator.statusByPermission[.screenContent] == .waitingForPicker) + #expect(driver.screenContentRequestCount == 1) + } + + @Test func screenContentIsFoldedOutOfVisibleOnboarding() { + #expect(OrbitPermissionKind.screenContent.isVisibleOnboardingStep == false) + #expect(OrbitPermissionKind.screenRecording.isVisibleOnboardingStep) + #expect(OrbitPermissionKind.screenContent.title == OrbitPermissionKind.screenRecording.title) + } + + @Test func snapshotRetainsTypedPermissionStates() { + let snapshot = OrbitPermissionSnapshot( + microphone: .granted, + accessibility: .waitingInSettings, + screenRecording: .restartRequired, + screenContent: .waitingForPicker + ) + + #expect(snapshot[.microphone] == .granted) + #expect(snapshot[.accessibility] == .waitingInSettings) + #expect(snapshot[.screenRecording] == .restartRequired) + #expect(snapshot[.screenContent] == .waitingForPicker) + } + + @Test func guideClampsToNegativeOriginDisplay() { + let visible = CGRect(x: -1920, y: -220, width: 1920, height: 1080) + let snapshot = OrbitSystemSettingsWindowSnapshot( + processIdentifier: 42, + windowFrame: CGRect(x: -1880, y: -180, width: 720, height: 700), + blockingModalFrame: nil, + visibleFrame: visible + ) + + let result = OrbitPermissionGuideAnchorResolver.anchoredFrame(snapshot: snapshot) + + #expect(result.minX >= visible.minX + 10) + #expect(result.maxX <= visible.maxX - 10) + #expect(result.minY >= visible.minY + 10) + #expect(result.maxY <= visible.maxY - 10) + } + + @Test func guideClampsWhenSettingsIsOnVerticalDisplay() { + let visible = CGRect(x: 0, y: 900, width: 1200, height: 1800) + let snapshot = OrbitSystemSettingsWindowSnapshot( + processIdentifier: 42, + windowFrame: CGRect(x: 50, y: 950, width: 680, height: 620), + blockingModalFrame: nil, + visibleFrame: visible + ) + + let result = OrbitPermissionGuideAnchorResolver.anchoredFrame(snapshot: snapshot) + + #expect(result.minX >= visible.minX + 10) + #expect(result.maxX <= visible.maxX - 10) + #expect(result.minY >= visible.minY + 10) + #expect(result.maxY <= visible.maxY - 10) + } + + @Test func oversizedGuideUsesStableVisibleOrigin() { + let visible = CGRect(x: -400, y: 200, width: 320, height: 260) + let result = OrbitPermissionGuideAnchorResolver.clamp( + CGRect(x: -900, y: -300, width: 518, height: 112), + to: visible, + margin: 10 + ) + + #expect(result.minX == visible.minX + 10) + #expect(result.minY >= visible.minY + 10) + #expect(result.maxY <= visible.maxY - 10) + } +} + +#if DEBUG + @MainActor + struct OrbitVisualQAStateTests { + @Test func visualFixtureCatalogCoversEveryRequiredProductArea() { + let values = Set(OrbitVisualQAState.allCases.map(\.rawValue)) + + #expect(values.contains("permission-microphone")) + #expect(values.contains("permission-accessibility")) + #expect(values.contains("permission-screen")) + #expect(values.contains("permission-restart")) + #expect(values.contains("coach-anchored")) + #expect(values.contains("coach-fallback")) + #expect(values.contains("coach-dragging")) + #expect(values.contains("coach-success")) + #expect(values.contains("onboarding-automation")) + #expect(values.contains("onboarding-auth")) + #expect(values.contains("onboarding-local-voice")) + #expect(values.contains("onboarding-cloud-voice")) + #expect(values.contains("onboarding-complete")) + #expect(values.contains("task-listening")) + #expect(values.contains("task-working")) + #expect(values.contains("task-tool-prompt")) + #expect(values.contains("task-team-up")) + #expect(values.contains("task-completed")) + #expect(values.contains("task-interrupted")) + #expect(values.contains("settings-voice")) + #expect(values.contains("settings-cloud-voice-error")) + #expect(values.contains("settings-model")) + #expect(values.contains("settings-unknown-model")) + #expect(values.contains("settings-device-error")) + #expect(values.contains("recovery")) + #expect(values.contains("stress-long-copy")) + } + } +#endif diff --git a/OrbitTests/OrbitTests.swift b/OrbitTests/OrbitTests.swift index 5d92a64..19d1f28 100644 --- a/OrbitTests/OrbitTests.swift +++ b/OrbitTests/OrbitTests.swift @@ -607,6 +607,9 @@ struct OrbitTests { from: "Using the pdf skill" ) let stableSnippet = CodexAppServerActionProvider.speakableCommentarySnippet( + from: "Using the pdf skill to make a polished illustrated PDF and save it for you." + ) + let idleFlushSnippet = CodexAppServerActionProvider.flushableCommentarySnippet( from: "Using the pdf skill to make a polished illustrated PDF and save it for you" ) let completedSentence = CodexAppServerActionProvider.speakableCommentarySnippet( @@ -615,6 +618,7 @@ struct OrbitTests { #expect(tooEarly == nil) #expect(stableSnippet == "Using the pdf skill to make a polished illustrated PDF and save it for you.") + #expect(idleFlushSnippet == "Using the pdf skill to make a polished illustrated PDF and save it for you.") #expect(completedSentence == "Opening the browser now.") } } diff --git a/OrbitTests/OrbitVoiceTests.swift b/OrbitTests/OrbitVoiceTests.swift index 2fb9745..96a6390 100644 --- a/OrbitTests/OrbitVoiceTests.swift +++ b/OrbitTests/OrbitVoiceTests.swift @@ -38,6 +38,37 @@ private final class OrbitVoiceTestProvider: TextToSpeechProvider { private struct OrbitVoiceTestError: Error {} +@MainActor +private final class OrbitBlockingVoiceProvider: TextToSpeechProvider { + let displayName = "Blocking test voice" + let isConfigured = true + let unavailableExplanation: String? = nil + private(set) var spokenTexts: [String] = [] + private(set) var stopCount = 0 + private var firstPlaybackContinuation: CheckedContinuation? + + var isPlaying: Bool { firstPlaybackContinuation != nil } + + func speakText(_ text: String) async throws { + spokenTexts.append(text) + if spokenTexts.count == 1 { + await withCheckedContinuation { continuation in + firstPlaybackContinuation = continuation + } + } + } + + func releaseFirstPlayback() { + firstPlaybackContinuation?.resume() + firstPlaybackContinuation = nil + } + + func stopPlayback() { + stopCount += 1 + releaseFirstPlayback() + } +} + nonisolated private final class OrbitVoiceTestLockedLevel: @unchecked Sendable { private let lock = NSLock() private var value: Float? @@ -78,25 +109,86 @@ struct OrbitNarrationPrimitiveTests { var deduplicator = OrbitNarrationDeduplicator(retentionInterval: 60) let now = Date() - let firstResult = deduplicator.shouldSpeak( + let firstResult = deduplicator.contains( "Opening the report.", turnIdentifier: "turn-a", + source: .firstLine, now: now ) - let duplicateResult = deduplicator.shouldSpeak( - "Opening the report. I will check every page.", + deduplicator.record( + "Opening the report.", turnIdentifier: "turn-a", + source: .firstLine, now: now ) - let otherTurnResult = deduplicator.shouldSpeak( + let duplicateResult = deduplicator.contains( + "Opening the report.", + turnIdentifier: "turn-a", + source: .firstLine, + now: now + ) + let otherTurnResult = deduplicator.contains( "Opening the report.", turnIdentifier: "turn-b", + source: .firstLine, now: now ) - #expect(firstResult) - #expect(!duplicateResult) - #expect(otherTurnResult) + #expect(!firstResult) + #expect(duplicateResult) + #expect(!otherTurnResult) + } + + @Test func firstLineExtractorRequiresAStableBoundaryUntilIdleFlush() { + #expect(OrbitFirstLineExtractor.stableLine(from: "Opening the current report now") == nil) + #expect( + OrbitFirstLineExtractor.stableLine( + from: "Opening the current report now", + allowUnterminated: true + ) == "Opening the current report now." + ) + #expect( + OrbitFirstLineExtractor.stableLine( + from: "## Opening the report.\nThen I will inspect it." + ) == "Opening the report." + ) + } + + @Test func completionRemovesTheFirstLineAlreadyNarrated() { + let remainder = OrbitFirstLineExtractor.removingSpokenPrefix( + "Opening the report.", + from: "Opening the report. I found three issues." + ) + #expect(remainder == "I found three issues.") + } + + @Test func completionQueuesBehindProtectedFirstLineWithoutStoppingIt() async { + let provider = OrbitBlockingVoiceProvider() + let coordinator = OrbitVoiceCoordinator(primary: provider) + coordinator.beginRequest("request-one") + let baselineStops = provider.stopCount + coordinator.enqueueFirstLine("Opening the current report now.", requestIdentifier: "request-one") + + for _ in 0..<20 where provider.spokenTexts.isEmpty { + await Task.yield() + } + #expect(provider.spokenTexts == ["Opening the current report now."]) + + coordinator.enqueueTerminal( + "Opening the current report now. I found three issues.", + source: .completion, + requestIdentifier: "request-one" + ) + #expect(provider.stopCount == baselineStops) + #expect(provider.spokenTexts.count == 1) + + provider.releaseFirstPlayback() + for _ in 0..<40 where provider.spokenTexts.count < 2 { + await Task.yield() + } + + #expect(provider.spokenTexts == ["Opening the current report now.", "I found three issues."]) + #expect(provider.stopCount == baselineStops) } @Test func naturalNoraUsesTheExactShortcutsVoiceIdentifier() { @@ -129,12 +221,13 @@ struct OrbitNarrationPrimitiveTests { guard ProcessInfo.processInfo.environment["ORBIT_RUN_LOCAL_VOICE_TEST"] == "1" else { return } - #expect(await OrbitNoraVoiceAvailability.probe(forceRefresh: true)) - let provider = await NaturalNoraTTSProvider() + let isAvailable = await OrbitNoraVoiceAvailability.probe(forceRefresh: true) + #expect(isAvailable) + let provider = NaturalNoraTTSProvider() try await provider.speakText( "Orbit is using the same local Siri Natural Nora voice as Shortcuts Voice Four." ) - #expect(await !provider.isPlaying) + #expect(!provider.isPlaying) } @Test func microphoneTestTapRunsOutsideMainActor() async throws { @@ -189,11 +282,12 @@ struct OrbitVoiceCoordinatorTests { let coordinator = OrbitVoiceCoordinator(primary: provider) try await coordinator.speak( - OrbitNarrationRequest(text: "Opening the report.", turnIdentifier: "turn-a") + OrbitNarrationRequest(text: "Opening the report.", source: .firstLine, turnIdentifier: "turn-a") ) try await coordinator.speak( OrbitNarrationRequest( - text: "Opening the report. I will inspect it now.", + text: "Opening the report.", + source: .firstLine, turnIdentifier: "turn-a" ) )