diff --git a/.gitignore b/.gitignore index 8a6feed9..7d964c61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,102 +1,136 @@ -# Operating system files +# OS and editor noise .DS_Store Thumbs.db Desktop.ini - -# Editors and IDEs +*.swp +*.swo +*~ .idea/ .vscode/ +!.vscode/extensions.json +!.vscode/settings.json *.code-workspace *.suo *.user -*.swp -*.swo -*~ - -# Local scratch work -.scratch/ -# Environment files and private configuration +# Secrets and local environment .env .env.* !.env.example !.env.sample !.env.template -*.pem +*.local *.key +*.pem +*.crt +*.csr *.p12 *.pfx *.jks credentials*.json secrets*.json -# Logs, temporary files, and local caches +# Logs *.log -*.pid -*.seed -*.tmp -*.temp -*.bak +logs/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* + +# Temporary and cache directories .cache/ +.uv-cache/ .tmp/ tmp/ +temp/ +*.tmp +*.temp +*.bak +.scratch/ -# Rust -target/ -**/*.rs.bk -*.profraw -*.profdata - -# Python bytecode and caches +# Python __pycache__/ *.py[cod] *$py.class +*.so +.Python +.venv/ +.venv-*/ +venv/ +ENV/ +env/ +pip-wheel-metadata/ +*.egg-info/ +.eggs/ +build/ +dist/ +htmlcov/ +.coverage +.coverage.* +coverage.xml .pytest_cache/ .mypy_cache/ -.pyright/ .ruff_cache/ .ty/ +.pyright/ +.pyre/ +.pytype/ .hypothesis/ .tox/ .nox/ -.coverage -.coverage.* -coverage.xml -htmlcov/ +.dmypy.json +dmypy.json +.ipynb_checkpoints/ -# Python environments and packaging output -.venv/ -.venv-*/ -venv/ -env/ -ENV/ -build/ -dist/ -*.egg-info/ -.eggs/ -pip-wheel-metadata/ +# Rust +target/ +**/*.rs.bk +*.profraw +*.profdata -# TypeScript and Node.js dependencies +# JavaScript and TypeScript node_modules/ +.npm/ .pnpm-store/ .yarn/cache/ +.yarn/unplugged/ +.yarn/build-state.yml .yarn/install-state.gz -.npm/ -.eslintcache +dist/ +build/ +coverage/ +.nyc_output/ *.tsbuildinfo - -# TypeScript and frontend build output -lib/ -out/ +*.eslintcache +.vite/ +.turbo/ +.parcel-cache/ +.angular/cache/ .next/ .nuxt/ .svelte-kit/ +.output/ +.vercel/ +.netlify/ +.serverless/ +out/ +lib/ .astro/ -.parcel-cache/ -.turbo/ -.vite/ -coverage/ -# Generated documentation sites +# Static site and docs output public/ site/ +.docusaurus/ +.vitepress/cache/ +.vitepress/dist/ + +# Runtime artifacts +*.pid +*.seed +*.pid.lock + +# Package artifacts +*.tgz +*.tar.gz +*.zip diff --git a/README.md b/README.md index 975d3800..c446a57d 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ OpenShell as the runtime. - `docs/dev-notes/authors.json` contains reusable Dev Notes author metadata. - `docs/documentation/` contains durable guides and references for packages, libraries, benchmarks, and examples introduced in Dev Notes. -- `projects/` contains self-contained research projects and a reusable Python - project template. +- `projects/` contains self-contained research projects, including the reusable + Python project template and the Reachy Mini conversation demo for OpenShell. - `scripts/render-dev-notes.py` renders Dev Notes cards, post bylines, and navigation entries from post front matter. - `zensical.toml` configures the documentation site. diff --git a/docs/assets/reachy-mini-openshell/screenshots/gradio-home.png b/docs/assets/reachy-mini-openshell/screenshots/gradio-home.png new file mode 100644 index 00000000..bbd0cde4 Binary files /dev/null and b/docs/assets/reachy-mini-openshell/screenshots/gradio-home.png differ diff --git a/docs/index.md b/docs/index.md index f7a8dfa9..dd8fd07e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,5 +28,11 @@ hide: Guides and reference material for packages, libraries, benchmarks, and examples introduced in Dev Notes. + + + Projects + Runnable research demos with setup guides, validation commands, and troubleshooting notes, including the Reachy Mini conversation demo. + + diff --git a/docs/projects/index.md b/docs/projects/index.md new file mode 100644 index 00000000..8170fc82 --- /dev/null +++ b/docs/projects/index.md @@ -0,0 +1,21 @@ +# Projects + +Projects are runnable research demos with their own source code, dependencies, +setup guide, validation commands, and troubleshooting notes. + +## Current Projects + +- [Reachy Mini conversation demo](reachy-mini-openshell.md): a simulator-first + Reachy Mini + OpenShell demo with an OpenAI Realtime quick start, Gradio UI, + microphone or text input, optional local speech backends, and Reachy movement + tools. + +## Project Page Standard + +Each project page should help a new contributor answer five questions quickly: + +- What does it run? +- What do I need installed or configured? +- Which command starts the local demo? +- How do I know it is working? +- Where do I look when it fails? diff --git a/docs/projects/reachy-mini-openshell.md b/docs/projects/reachy-mini-openshell.md new file mode 100644 index 00000000..ce2c9642 --- /dev/null +++ b/docs/projects/reachy-mini-openshell.md @@ -0,0 +1,233 @@ +# Reachy Mini Conversation Demo + +This project runs a local Reachy Mini conversation demo for OpenShell. It starts +the Reachy Mini simulator, launches a Gradio browser UI, and lets you talk to +Reachy with either microphone or text input. The preferred first path is +OpenAI Realtime with the simulator. + +Source: `projects/reachy-mini-openshell` + +## Quick Start + +Requirements: + +- macOS +- Python 3.10, 3.11, or 3.12 +- `uv` +- `OPENAI_API_KEY` exported in the shell that starts the app, with access to + the OpenAI Realtime API + +Start here: OpenAI Realtime with the Reachy Mini simulator. + +```sh +cd projects/reachy-mini-openshell +cp .env.example .env +export OPENAI_API_KEY=sk-... +./scripts/start-local.sh +``` + +The launcher syncs dependencies, validates `.env`, starts +`reachy-mini-daemon --sim`, and prints the Gradio URL. It uses + when available and picks the next free port through +`7899` when needed. + +Keep the launcher terminal open. `Ctrl+C` stops the app and the simulator it +started. + +The checked-in `.env.example` already selects `BACKEND_PROVIDER=openai_realtime`. +API keys, base URLs, and model IDs are configured in `.env`, not in the browser +UI. + +## Backend Selection + +Set one backend in `.env`: + +| `BACKEND_PROVIDER` | Use When | +| --- | --- | +| `openai_realtime` | First-time setup and the fastest full voice demo with OpenAI Realtime. | +| `local_stt` | Optional local ASR through Riva ASR NIM or another STT service before Chat Completions and TTS. | +| `hf_realtime` | Optional Pollen/Hugging Face realtime path. | + +Use the single checked-in `.env.example` as the starting point: + +```sh +cp .env.example .env +``` + +Credentials and model routes are read from `.env` and exported environment +variables referenced by `.env`. The browser UI does not accept API keys, base +URLs, or model IDs. + +For OpenAI Realtime, the normal setup uses the global `OPENAI_API_KEY` exported +in the shell that starts the app: + +```dotenv +BACKEND_PROVIDER=openai_realtime +OPENAI_REALTIME_BASE_URL=https://api.openai.com/v1 +OPENAI_REALTIME_MODEL=gpt-realtime-2 +OPENAI_REALTIME_VOICE=cedar +``` + +## Optional Riva ASR + +Use this path after the OpenAI Realtime path is working, or when you explicitly +want microphone input transcribed by Riva before text is sent through Chat +Completions and Reachy tools. + +Requirement: a deployed Riva ASR NIM endpoint reachable from the app host. + +The endpoint must expose: + +- `GET /v1/health/ready` +- `POST /v1/audio/transcriptions` + +Readiness check: + +```sh +curl -X GET http://:9000/v1/health/ready +``` + +Expected response: + +```json +{"status":"ready"} +``` + +Then set the app to use the Riva ASR HTTP transcription route: + +```dotenv +BACKEND_PROVIDER=local_stt + +CHAT_API_KEY=${NVIDIA_INFERENCE_API_KEY} +CHAT_BASE_URL=https://inference-api.nvidia.com/v1 +CHAT_MODEL_NAME=azure/anthropic/claude-opus-4-8 + +STT_API_KEY=not-needed +STT_BASE_URL=http://:9000/v1 +STT_MODEL_NAME=parakeet-1-1b-ctc-en-us + +TTS_API_KEY=${OPENAI_API_KEY} +TTS_BASE_URL=https://api.openai.com/v1 +TTS_MODEL_NAME=gpt-4o-mini-tts +TTS_VOICE=cedar +``` + +This setup uses Riva for ASR. Speech output still uses the configured +OpenAI-compatible `TTS_*` endpoint. Riva TTS NIM exposes a different HTTP +route, `/v1/audio/synthesize`, and is not wired into this app yet. + +`STT_MODEL_NAME` must match an offline model ID served by the Riva ASR endpoint. +If `stt-probe` reports `stt_model_listed=no`, use one of the model IDs reported +by that endpoint. + +The microphone path is: + +```text +microphone -> Riva ASR NIM -> Chat Completions + Reachy tools -> TTS -> Reachy speaks +``` + +For non-Riva STT, keep `BACKEND_PROVIDER=local_stt` and set `STT_BASE_URL` plus +`STT_MODEL_NAME` for that OpenAI-compatible transcription endpoint. + +## What To Expect + +The daemon status endpoint is: + +```text +http://127.0.0.1:8000/api/daemon/status +``` + +A healthy simulator daemon returns JSON with these key fields: + +```json +{ + "type": "daemon_status", + "robot_name": "reachy_mini", + "state": "running", + "simulation_enabled": true, + "no_media": true, + "error": null, + "version": "1.8.0" +} +``` + +The full response contains more fields. `state: "running"` is the important +signal. + +Open the Gradio URL printed by the launcher. The first screen should look like +this: + +![Gradio conversation UI for Talk with Reachy Mini](../assets/reachy-mini-openshell/screenshots/gradio-home.png) + +Use `Microphone` for voice or `Text` for typed prompts. A good first prompt is: + +```text +Hi Reachy, introduce yourself and look around. +``` + +Because the local simulator runs with `--no-media` and the app starts with +`--no-camera`, camera and head-tracking features are disabled. Conversation and +motion tools still work through the simulated daemon. + +## Useful Checks + +Validate configuration without opening the browser: + +```sh +uv run reachy-mini-backend-check +``` + +Call the configured backend: + +```sh +uv run reachy-mini-backend-check --live +``` + +Check Riva/local-STT stages independently: + +```sh +uv run reachy-mini-backend-check --live --stage stt-probe +uv run reachy-mini-backend-check --live --stage chat \ + --seed-text "Reachy, use the sweep_look tool, then tell me what you did." \ + --require-tool +uv run reachy-mini-backend-check --live --stage tts \ + --seed-text "Hello, I am Reachy." +``` + +Run the fake local-STT smoke workflow when Riva or other external STT/TTS +services are not ready: + +```sh +scripts/smoke-local-stt.sh +``` + +## Manual Startup + +Use the launcher for normal development. Manual startup is useful when you need +separate daemon and app terminals. + +Simulator terminal: + +```sh +uv run reachy-mini-daemon --sim --scene minimal --headless --no-media \ + --fastapi-host 127.0.0.1 --fastapi-port 8000 \ + --dataset-update-interval 0 +``` + +App terminal: + +```sh +uv run python -m reachy_mini_conversation_app --gradio --no-camera +``` + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| `BACKEND_PROVIDER` missing | Copy `.env.example` to `.env`, then edit `.env`. | +| App cannot connect to Reachy | Use `./scripts/start-local.sh`, or verify the daemon status endpoint reports `state: "running"`. | +| OpenAI Realtime is not connected | Export `OPENAI_API_KEY` in the same shell that starts the app, then run `uv run reachy-mini-backend-check --live`. | +| Local-STT text returns `404` | Use a plain `CHAT_BASE_URL` ending in `/v1` when required, and set `CHAT_MODEL_NAME` to the provider's exact model ID. | +| Riva/local microphone produces no response | Run the `stt-probe` check and confirm `STT_BASE_URL` exposes `POST /audio/transcriptions`. | +| Riva ASR readiness fails | Check `http://:9000/v1/health/ready`, GPU/container logs, and that the app can reach the host from macOS. | +| Daemon reports `MuJoCo is not installed` | Run `uv sync` from `projects/reachy-mini-openshell`, then start the daemon through `uv run` or the launcher. | diff --git a/projects/README.md b/projects/README.md index 165208cb..c803eff0 100644 --- a/projects/README.md +++ b/projects/README.md @@ -8,3 +8,4 @@ Current projects: - `python-project-template`: Minimal, production-ready Python project scaffold managed with uv. +- `reachy-mini-openshell`: Reachy Mini conversation demo for OpenShell. diff --git a/projects/reachy-mini-openshell/.env.example b/projects/reachy-mini-openshell/.env.example new file mode 100644 index 00000000..e3aa4715 --- /dev/null +++ b/projects/reachy-mini-openshell/.env.example @@ -0,0 +1,56 @@ +# Copy this file to .env, then edit values for the backend you want. +# +# Backend options: +# - openai_realtime: OpenAI Realtime handles speech input, tools, and speech output. +# - local_stt: optional Riva ASR NIM or another STT endpoint -> Chat Completions + Reachy tools -> TTS endpoint. +# - hf_realtime: Pollen/Hugging Face realtime backend. +BACKEND_PROVIDER=openai_realtime + +# OpenAI Realtime backend. +# Uses the globally exported OPENAI_API_KEY by default; no key is required here. +# Uncomment OPENAI_REALTIME_API_KEY only if this app should use a different key +# from the global OpenAI key. +# OPENAI_REALTIME_API_KEY= +OPENAI_REALTIME_BASE_URL=https://api.openai.com/v1 +OPENAI_REALTIME_MODEL=gpt-realtime-2 +OPENAI_REALTIME_VOICE=cedar + +# Optional Riva/local STT cascade backend. Used only when BACKEND_PROVIDER=local_stt. +CHAT_API_KEY=${NVIDIA_INFERENCE_API_KEY} +CHAT_BASE_URL=https://inference-api.nvidia.com/v1 +CHAT_MODEL_NAME=azure/anthropic/claude-opus-4-8 + +STT_API_KEY=not-needed +# Riva ASR NIM defaults to HTTP port 9000 and exposes /v1/audio/transcriptions. +STT_BASE_URL=http://127.0.0.1:9000/v1 +# If you use a Whisper-style endpoint instead, set STT_MODEL_NAME=whisper-1. +STT_MODEL_NAME=parakeet-1-1b-ctc-en-us + +TTS_API_KEY=${OPENAI_API_KEY} +TTS_BASE_URL=https://api.openai.com/v1 +TTS_MODEL_NAME=gpt-4o-mini-tts +TTS_VOICE=cedar + +# Hugging Face realtime backend. Used only when BACKEND_PROVIDER=hf_realtime. +HF_REALTIME_CONNECTION_MODE=deployed +# Optional deployed session broker override. Leave unset for the default Pollen broker. +# HF_REALTIME_SESSION_URL=https://pollen-robotics-reachy-mini-realtime-url.hf.space/session +# For local HF realtime, set: +# HF_REALTIME_CONNECTION_MODE=local +# HF_REALTIME_WS_URL=ws://127.0.0.1:8765/v1/realtime +HF_REALTIME_MODEL= +HF_REALTIME_VOICE=Aiden +HF_TOKEN= + +REALTIME_TRANSCRIPTION_LANGUAGE=en + +# Local STT phrase detection. +MIC_TRANSCRIPTION_RMS_THRESHOLD=500 +MIC_TRANSCRIPTION_MIN_AUDIO_MS=250 +MIC_TRANSCRIPTION_SILENCE_MS=800 +MIC_TRANSCRIPTION_MAX_AUDIO_MS=12000 + +# Used only with --local-vision. +LOCAL_VISION_MODEL=HuggingFaceTB/SmolVLM2-2.2B-Instruct + +HF_HOME=./cache diff --git a/projects/reachy-mini-openshell/.gitignore b/projects/reachy-mini-openshell/.gitignore new file mode 100644 index 00000000..2451acc1 --- /dev/null +++ b/projects/reachy-mini-openshell/.gitignore @@ -0,0 +1,58 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Virtual environments +.venv/ +venv/ +ENV/ +env/ + +# Environment variables +.env + +# Build and distribution +build/ +dist/ +*.egg-info/ +.eggs/ + +# Testing +.pytest_cache/ +.coverage +.hypothesis/ +htmlcov/ +coverage.xml +*.cover + +# Linting and formatting +.ruff_cache/ +.mypy_cache/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo + +# Security +*.key +*.pem +*.crt +*.csr + +# Temporary files +tmp/ +*.log +cache/ + +# macOS +.DS_Store + +# Linux +*~ +.directory +.Trash-* +.nfs* diff --git a/projects/reachy-mini-openshell/LICENSE b/projects/reachy-mini-openshell/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/projects/reachy-mini-openshell/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/projects/reachy-mini-openshell/README.md b/projects/reachy-mini-openshell/README.md new file mode 100644 index 00000000..f9384131 --- /dev/null +++ b/projects/reachy-mini-openshell/README.md @@ -0,0 +1,354 @@ +# Reachy OpenShell + +Reachy Mini conversation demo for OpenShell: Gradio UI, simulator support, +microphone or text input, Reachy movement tools, and selectable model backends. +The default and preferred starting point is OpenAI Realtime. + +Commands: + +- package: `reachy_mini_conversation_app` +- app: `reachy-mini-conversation-app` +- module: `python -m reachy_mini_conversation_app` +- check: `reachy-mini-backend-check` + +## Quick Start + +Start here: OpenAI Realtime + the Reachy Mini simulator. + +Requirements: + +- macOS +- Python 3.10, 3.11, or 3.12. Python 3.12 is recommended. +- `uv` +- `OPENAI_API_KEY` exported in the shell that starts the app, with access to + the OpenAI Realtime API + +From the repository root: + +```bash +cd projects/reachy-mini-openshell +cp .env.example .env +export OPENAI_API_KEY=sk-... +./scripts/start-local.sh +``` + +The launcher creates `.venv`, runs `uv sync`, validates `.env`, starts +`reachy-mini-daemon --sim`, then prints the Gradio URL: +. + +If `7860` is busy, the launcher picks the next free port through `7899`. + +In the UI: + +1. Use `Microphone` for voice. +2. Use `Text` for typed prompts. +3. Try: `Hi Reachy, introduce yourself and look around.` + +Keep the launcher terminal open. `Ctrl+C` stops the app and the simulator it +started. + +The checked-in `.env.example` already selects `BACKEND_PROVIDER=openai_realtime`. +Provider keys, base URLs, and model IDs are configured in `.env`, not in the +browser UI. + +## Backend Selection + +Set exactly one backend in `.env`: + +| `BACKEND_PROVIDER` | Use when | Requires | +| --- | --- | --- | +| `openai_realtime` | First-time setup and the fastest full voice demo. | OpenAI Realtime | +| `local_stt` | Optional local ASR before Chat Completions and TTS. | Riva ASR NIM or compatible STT, Chat Completions, TTS | +| `hf_realtime` | Optional Pollen/Hugging Face realtime path. | Deployed broker or websocket | + +Credentials and model routes come from `.env` or exported variables referenced +by `.env`. They are not entered in the browser UI. + +### OpenAI Realtime + +This is the recommended path for a first run. It handles microphone input, +assistant reasoning, Reachy tool calls, and speech output through one Realtime +session. + +```bash +cp .env.example .env +export OPENAI_API_KEY=sk-... +./scripts/start-local.sh +``` + +```dotenv +BACKEND_PROVIDER=openai_realtime +OPENAI_REALTIME_BASE_URL=https://api.openai.com/v1 +OPENAI_REALTIME_MODEL=gpt-realtime-2 +OPENAI_REALTIME_VOICE=cedar +``` + +Leave `OPENAI_REALTIME_API_KEY` unset unless this app should use a different +key from the exported `OPENAI_API_KEY`. + +### Optional: Riva ASR NIM + Chat + TTS + +Use this path after the OpenAI Realtime path is working, or when you explicitly +want microphone input transcribed by a local ASR service before text is sent +through Chat Completions and Reachy tools. + +Requirement: a deployed Riva ASR NIM endpoint reachable from the app host. + +The endpoint must expose: + +- `GET /v1/health/ready` +- `POST /v1/audio/transcriptions` + +Readiness check: + +```bash +curl -X GET http://:9000/v1/health/ready +``` + +Expected response: + +```json +{"status":"ready"} +``` + +The app uses the ASR NIM HTTP transcription route: + +```text +POST http://:9000/v1/audio/transcriptions +``` + +This setup uses Riva for ASR. Speech output still uses the configured +OpenAI-compatible `TTS_*` endpoint. Riva TTS NIM exposes a different HTTP +route, `/v1/audio/synthesize`, and is not wired into this app yet. + +Flow: + +```text +microphone -> Riva ASR NIM -> Chat Completions + Reachy tools -> TTS -> Reachy speaks +``` + +```bash +cp .env.example .env +``` + +```dotenv +BACKEND_PROVIDER=local_stt + +CHAT_API_KEY=${NVIDIA_INFERENCE_API_KEY} +CHAT_BASE_URL=https://inference-api.nvidia.com/v1 +CHAT_MODEL_NAME=azure/anthropic/claude-opus-4-8 + +STT_API_KEY=not-needed +STT_BASE_URL=http://:9000/v1 +STT_MODEL_NAME=parakeet-1-1b-ctc-en-us + +TTS_API_KEY=${OPENAI_API_KEY} +TTS_BASE_URL=https://api.openai.com/v1 +TTS_MODEL_NAME=gpt-4o-mini-tts +TTS_VOICE=cedar +``` + +Rules: + +- `*_BASE_URL` must be plain URLs, not Markdown links. +- `*_MODEL_NAME` must match the provider's exact model ID. +- `${VAR_NAME}` values expand from the shell environment. +- Use `not-needed` for local no-auth endpoints. +- `STT_MODEL_NAME` must match an offline model ID served by the Riva ASR + endpoint. If `stt-probe` reports `stt_model_listed=no`, use one of the model + IDs reported by that endpoint. +- For a Whisper-compatible endpoint instead of Riva, keep the same backend and + set `STT_BASE_URL` plus `STT_MODEL_NAME=whisper-1`. + +Stage checks: + +```bash +uv run reachy-mini-backend-check --live --stage stt-probe +uv run reachy-mini-backend-check --live --stage chat \ + --seed-text "Reachy, use the sweep_look tool, then tell me what you did." \ + --require-tool +uv run reachy-mini-backend-check --live --stage tts \ + --seed-text "Hello, I am Reachy." +uv run reachy-mini-backend-check --live --stage app-flow --require-tool \ + --seed-text "Reachy, use the sweep_look tool, then tell me what you did." +``` + +Use the fake local-STT smoke test when Riva or other real endpoints are not +ready: + +```bash +scripts/smoke-local-stt.sh +``` + +### Optional: Hugging Face Realtime + +```bash +cp .env.example .env +./scripts/start-local.sh +``` + +```dotenv +BACKEND_PROVIDER=hf_realtime +HF_REALTIME_CONNECTION_MODE=deployed +HF_REALTIME_MODEL= +HF_REALTIME_VOICE=Aiden +``` + +Leave `HF_REALTIME_MODEL` empty to use the backend default. Set `HF_TOKEN` only +when required. + +For a local websocket: + +```dotenv +BACKEND_PROVIDER=hf_realtime +HF_REALTIME_CONNECTION_MODE=local +HF_REALTIME_WS_URL=ws://127.0.0.1:8765/v1/realtime +``` + +## Verify + +Config only: + +```bash +uv run reachy-mini-backend-check +``` + +Live backend connection: + +```bash +uv run reachy-mini-backend-check --live +``` + +Daemon status: + +```text +http://127.0.0.1:8000/api/daemon/status +``` + +Expected fields: + +```json +{ + "type": "daemon_status", + "robot_name": "reachy_mini", + "state": "running", + "simulation_enabled": true, + "no_media": true, + "version": "1.8.0" +} +``` + +The full response contains more fields. `state: running` is the key signal. + +## Run Commands + +Launcher: + +```bash +./scripts/start-local.sh +./scripts/start-local.sh --debug +APP_PORT=7861 ./scripts/start-local.sh +REACHY_SKIP_SYNC=1 ./scripts/start-local.sh +``` + +Manual simulator, in one terminal: + +```bash +uv run reachy-mini-daemon --sim --scene minimal --headless --no-media \ + --fastapi-host 127.0.0.1 --fastapi-port 8000 \ + --dataset-update-interval 0 +``` + +Manual app, in another terminal: + +```bash +uv run python -m reachy_mini_conversation_app --gradio --no-camera +``` + +Use a config file without replacing `.env`: + +```bash +REACHY_MINI_DOTENV_PATH=path/to/alternate.env \ + uv run python -m reachy_mini_conversation_app --gradio --no-camera +``` + +Common app flags: + +- `--gradio`: browser UI +- `--no-camera`: simulator baseline +- `--robot-name `: connect to a matching daemon robot name +- `--debug`: debug logging +- `--local-vision`: local vision model; requires `local_vision` +- `--head-tracker yolo`: YOLO head tracking; requires `yolo_vision` +- `--head-tracker mediapipe`: MediaPipe head tracking; requires + `mediapipe_vision` + +## Customize Reachy + +Profile files: + +```text +src/reachy_mini_conversation_app/profiles/_reachy_mini_conversation_app_locked_profile +``` + +- `instructions.txt`: assistant behavior and personality +- `tools.txt`: allowed profile tools +- `*.py`: profile-local tool implementations + +Current profile tools: + +```text +dance +stop_dance +play_emotion +stop_emotion +sweep_look +``` + +## Optional Vision Extras + +The default install includes the MuJoCo simulator backend. There are no +project-level `backend` or `sim` extras. + +```bash +uv sync --extra local_vision +uv sync --extra yolo_vision +uv sync --extra mediapipe_vision +uv sync --extra all_vision +``` + +## Development + +```bash +uv sync --group dev +uv run ruff check . +uv run ty check +uv run pytest -q +``` + +Useful extras: + +```bash +uv run python -m compileall src tests +uv run python -m reachy_mini_conversation_app --help +uv run reachy-mini-app-assistant check . +``` + +`ty` is the Python type checker for this project. + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| `BACKEND_PROVIDER` missing | Copy `.env.example` to `.env`, then edit it. | +| App cannot connect to Reachy | Start the daemon or use `./scripts/start-local.sh`. Match `--robot-name` when using a custom daemon name. | +| OpenAI Realtime is not connected | Export `OPENAI_API_KEY` in the same shell, then run `uv run reachy-mini-backend-check --live`. | +| Local-STT text returns `404` | Use a plain `CHAT_BASE_URL` and the exact `CHAT_MODEL_NAME` accepted by the provider. | +| Riva/local microphone produces no response | Run `uv run reachy-mini-backend-check --live --stage stt-probe`; check `STT_BASE_URL` includes `/v1` and exposes `POST /audio/transcriptions`. | +| Riva ASR readiness fails | Check `http://:9000/v1/health/ready`, GPU/container logs, and that the app can reach the host from macOS. | +| vLLM STT says audio support is missing | Redeploy the service with vLLM audio support, then rerun `stt-probe`. | +| `uv sync` builds `pygobject` or `pycairo` on macOS | Run `uv cache clean reachy-mini pygobject pycairo`, then `uv sync`. | +| Daemon uses `--no-media` | Start the app with `--no-camera`; the launcher already does this. | + +The checked-in uv resolution targets macOS/Darwin. For Linux deployment, +update `[tool.uv].environments` and regenerate `uv.lock`. diff --git a/projects/reachy-mini-openshell/index.html b/projects/reachy-mini-openshell/index.html new file mode 100644 index 00000000..79fb9161 --- /dev/null +++ b/projects/reachy-mini-openshell/index.html @@ -0,0 +1,42 @@ + + + + + + Reachy OpenShell + + + +
+
+
+
Reachy Mini App
+

Reachy OpenShell

+

A Reachy Mini conversation demo for OpenShell, starting with OpenAI Realtime.

+
+ +
+
+
+

Getting Started

+

Installation

+
+
+

Start the Reachy Mini simulator and browser app locally:

+ 
cp .env.example .env
+export OPENAI_API_KEY=sk-...
+./scripts/start-local.sh
+
+ +
+
+
+

Configuration

+

Provider Configuration

+
+
+

The default .env.example uses OpenAI Realtime. Configure provider credentials and model routing in .env, then choose microphone or text input in the app UI.

+
+
+ + diff --git a/projects/reachy-mini-openshell/pyproject.toml b/projects/reachy-mini-openshell/pyproject.toml new file mode 100644 index 00000000..ce47a1b0 --- /dev/null +++ b/projects/reachy-mini-openshell/pyproject.toml @@ -0,0 +1,77 @@ +[build-system] +requires = [ "setuptools",] +build-backend = "setuptools.build_meta" + +[project] +name = "reachy_mini_conversation_app" +version = "0.3.0" +description = "Reachy Mini conversation demo for OpenShell." +readme = "README.md" +requires-python = ">=3.10,<3.13" +dependencies = [ "aiortc>=1.13.0", "fastrtc>=0.0.34", "gradio==5.50.1.dev1", "huggingface-hub==1.3.0", "opencv-python>=4.12.0.88", "python-dotenv", "openai>=2.1", "reachy_mini_dances_library", "reachy_mini_toolbox", "reachy-mini[mujoco]==1.8.0", "eclipse-zenoh~=1.7.0", "gradio_client>=1.13.3",] +[[project.authors]] +name = "Pollen Robotics" +email = "contact@pollen-robotics.com" + +[dependency-groups] +dev = [ + "pytest", + "pytest-asyncio", + "ruff==0.12.0", + "pre-commit", + "types-requests", + "python-semantic-release>=10.5.3", + "ty>=0.0.53", +] + +[tool.uv] +environments = [ + "sys_platform == 'darwin'", +] + +[project.optional-dependencies] +local_vision = [ "torch>=2.1", "transformers==5.0.0rc2", "num2words",] +yolo_vision = [ "ultralytics", "supervision", "onnxruntime",] +mediapipe_vision = [ "mediapipe==0.10.14",] +all_vision = [ "torch>=2.1", "transformers==5.0.0rc2", "num2words", "ultralytics", "supervision", "onnxruntime", "mediapipe==0.10.14",] + +[project.scripts] +reachy-mini-conversation-app = "reachy_mini_conversation_app.main:main" +reachy-mini-backend-check = "reachy_mini_conversation_app.backend_check:main" + +[tool.setuptools] +include-package-data = true + +[tool.ruff] +line-length = 119 +exclude = [ ".venv", "dist", "build", "**/__pycache__", "*.egg-info", ".pytest_cache",] + +[project.entry-points.reachy_mini_apps] +reachy_mini_conversation_app = "reachy_mini_conversation_app.main:ReachyMiniConversationApp" + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.package-data] +reachy_mini_conversation_app = [ "images/*", "profiles/**/*.py", "profiles/**/*.txt",] + +[tool.ruff.lint] +select = [ "E", "F", "W", "I", "C4", "D",] +ignore = [ "E501", "D100", "D203", "D213",] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +[tool.setuptools.packages.find] +where = [ "src",] + +[tool.ruff.lint.isort] +length-sort = true +lines-after-imports = 2 +no-lines-before = [ "standard-library", "local-folder",] +known-local-folder = [ "reachy_mini_conversation_app",] +known-first-party = [ "reachy_mini", "reachy_mini_dances_library", "reachy_mini_toolbox",] +split-on-trailing-comma = true diff --git a/projects/reachy-mini-openshell/scripts/fake_openai_backend.py b/projects/reachy-mini-openshell/scripts/fake_openai_backend.py new file mode 100755 index 00000000..54f5f459 --- /dev/null +++ b/projects/reachy-mini-openshell/scripts/fake_openai_backend.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Tiny OpenAI-compatible backend for local Reachy conversation smoke tests.""" + +import io +import math +import time +import wave +import argparse +from typing import Any + +import uvicorn +from fastapi import FastAPI, Request, Response + + +def _wav_bytes(text: str, *, sample_rate: int = 24_000, duration_seconds: float = 0.7) -> bytes: + """Return a small mono WAV payload.""" + frequency = 523.25 if "swept" in text.lower() else 440.0 + frame_count = max(1, int(sample_rate * duration_seconds)) + frames = bytearray() + for index in range(frame_count): + sample = int(math.sin(2 * math.pi * frequency * index / sample_rate) * 6000) + frames.extend(sample.to_bytes(2, byteorder="little", signed=True)) + + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(sample_rate) + wav_file.writeframes(bytes(frames)) + return buffer.getvalue() + + +def _message_has_tool_result(messages: list[Any]) -> bool: + """Return whether a Chat Completions request includes a tool result.""" + return any(isinstance(message, dict) and message.get("role") == "tool" for message in messages) + + +def build_app( + *, + transcript: str, + assistant_text: str, + tool_name: str, + call_tool: bool, + audio_duration_seconds: float, +) -> FastAPI: + """Build the fake OpenAI-compatible FastAPI app.""" + app = FastAPI(title="Reachy fake OpenAI-compatible backend") + + @app.get("/v1/models") + async def models() -> dict[str, Any]: + return { + "object": "list", + "data": [ + {"id": "fake-whisper", "object": "model"}, + {"id": "fake-chat", "object": "model"}, + {"id": "fake-tts", "object": "model"}, + ], + } + + @app.post("/v1/audio/transcriptions") + async def transcriptions(request: Request) -> dict[str, str]: + await request.body() + return {"text": transcript} + + @app.post("/v1/audio/speech") + async def speech(request: Request) -> Response: + try: + payload = await request.json() + except Exception: + payload = {} + input_text = str(payload.get("input", "hello")) + return Response( + content=_wav_bytes(input_text, duration_seconds=audio_duration_seconds), + media_type="audio/wav", + ) + + @app.post("/v1/chat/completions") + async def chat_completions(request: Request) -> dict[str, Any]: + payload = await request.json() + messages = payload.get("messages", []) + if not isinstance(messages, list): + messages = [] + + if call_tool and not _message_has_tool_result(messages): + message = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_fake_reachy_tool", + "type": "function", + "function": {"name": tool_name, "arguments": "{}"}, + } + ], + } + finish_reason = "tool_calls" + else: + message = {"role": "assistant", "content": assistant_text} + finish_reason = "stop" + + return { + "id": "chatcmpl-fake-reachy", + "object": "chat.completion", + "created": int(time.time()), + "model": payload.get("model", "fake-chat"), + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + } + + return app + + +def build_parser() -> argparse.ArgumentParser: + """Build CLI parser.""" + parser = argparse.ArgumentParser(description="Run a fake OpenAI-compatible backend for Reachy smoke tests.") + parser.add_argument("--host", default="127.0.0.1", help="host to bind.") + parser.add_argument("--port", type=int, default=8766, help="port to bind.") + parser.add_argument( + "--transcript", + default="Reachy, use the sweep_look tool, then tell me what you did.", + help="transcription text returned by POST /v1/audio/transcriptions.", + ) + parser.add_argument( + "--assistant-text", + default="I swept my gaze and returned to center.", + help="final assistant text returned after a tool result.", + ) + parser.add_argument("--tool-name", default="sweep_look", help="tool name to request from Chat Completions.") + parser.add_argument( + "--no-tool", action="store_true", help="return assistant text immediately without a tool call." + ) + parser.add_argument("--audio-duration", type=float, default=0.7, help="duration in seconds for fake TTS WAVs.") + return parser + + +def main() -> None: + """Run the fake backend.""" + args = build_parser().parse_args() + app = build_app( + transcript=args.transcript, + assistant_text=args.assistant_text, + tool_name=args.tool_name, + call_tool=not args.no_tool, + audio_duration_seconds=args.audio_duration, + ) + uvicorn.run(app, host=args.host, port=args.port, log_level="info") + + +if __name__ == "__main__": + main() diff --git a/projects/reachy-mini-openshell/scripts/smoke-local-stt.sh b/projects/reachy-mini-openshell/scripts/smoke-local-stt.sh new file mode 100755 index 00000000..cf278910 --- /dev/null +++ b/projects/reachy-mini-openshell/scripts/smoke-local-stt.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd -- "${SCRIPT_DIR}/.." && pwd)" + +HOST="${SMOKE_BACKEND_HOST:-127.0.0.1}" +PORT="${SMOKE_BACKEND_PORT:-8766}" +REAL_REACHY="${SMOKE_REAL_REACHY:-0}" +GRADIO_SMOKE="${SMOKE_GRADIO:-0}" +APP_HOST="${SMOKE_APP_HOST:-127.0.0.1}" +APP_PORT="${SMOKE_APP_PORT:-7897}" +ROBOT_NAME="${SMOKE_ROBOT_NAME:-}" +SEED_TEXT="${SMOKE_SEED_TEXT:-Reachy, use the sweep_look tool, then tell me what you did.}" +STARTUP_TIMEOUT="${SMOKE_BACKEND_TIMEOUT:-10}" +APP_STARTUP_TIMEOUT="${SMOKE_APP_TIMEOUT:-20}" +SHUTDOWN_TIMEOUT="${SMOKE_SHUTDOWN_TIMEOUT:-25}" + +SERVER_PID="" +APP_PID="" +SMOKE_ENV="" + +usage() { + cat <<'EOF' +Usage: scripts/smoke-local-stt.sh [--real-reachy] [--gradio] [--robot-name NAME] + +Runs a local OpenAI-compatible fake STT/chat/TTS backend, writes a temporary +local_stt config, and checks the app's microphone app-flow path. + +Options: + --real-reachy Also run the app-flow check through the real Reachy SDK + and movement manager. Requires a running daemon/simulator. + --gradio Also launch the Gradio app, verify the UI labels, and send + a text tool-call prompt through the running app. Requires + a running daemon/simulator. + --robot-name NAME Robot name to pass to reachy-mini-backend-check with + --real-reachy and to the Gradio app smoke when provided. + -h, --help Show this help. + +Environment: + SMOKE_BACKEND_HOST Fake backend host. Default: 127.0.0.1 + SMOKE_BACKEND_PORT Fake backend port. Default: 8766 + SMOKE_APP_HOST Gradio app host for --gradio. Default: 127.0.0.1 + SMOKE_APP_PORT Gradio app port for --gradio. Default: 7897 + SMOKE_SEED_TEXT Seed text for the generated input speech. + SMOKE_BACKEND_TIMEOUT Seconds to wait for the fake backend. Default: 10 + SMOKE_APP_TIMEOUT Seconds to wait for the Gradio app. Default: 20 + SMOKE_SHUTDOWN_TIMEOUT Seconds to wait before forcing shutdown. Default: 25 +EOF +} + +log() { + printf '[reachy-smoke] %s\n' "$*" +} + +terminate_process_tree() { + local pid="$1" + local signal="$2" + local children child + + children="$(pgrep -P "${pid}" 2>/dev/null || true)" + for child in ${children}; do + terminate_process_tree "${child}" "${signal}" + done + kill "-${signal}" "${pid}" >/dev/null 2>&1 || true +} + +wait_for_process_exit() { + local pid="$1" + local timeout_seconds="$2" + local deadline=$((SECONDS + timeout_seconds)) + + while kill -0 "${pid}" >/dev/null 2>&1; do + if (( SECONDS >= deadline )); then + return 1 + fi + sleep 0.2 + done + return 0 +} + +stop_process_tree() { + local pid="$1" + local label="$2" + if [[ -z "${pid}" ]] || ! kill -0 "${pid}" >/dev/null 2>&1; then + return + fi + + terminate_process_tree "${pid}" INT + if ! wait_for_process_exit "${pid}" "${SHUTDOWN_TIMEOUT}"; then + log "Forcing ${label} shutdown" + terminate_process_tree "${pid}" TERM + if ! wait_for_process_exit "${pid}" 3; then + terminate_process_tree "${pid}" KILL + fi + fi + wait "${pid}" >/dev/null 2>&1 || true +} + +cleanup() { + stop_process_tree "${APP_PID}" "Gradio app" + stop_process_tree "${SERVER_PID}" "fake backend" + if [[ -n "${SMOKE_ENV}" ]]; then + rm -f "${SMOKE_ENV}" + fi +} +trap cleanup EXIT INT TERM + +while [[ $# -gt 0 ]]; do + case "$1" in + --real-reachy) + REAL_REACHY=1 + shift + ;; + --gradio) + GRADIO_SMOKE=1 + shift + ;; + --robot-name) + if [[ $# -lt 2 ]]; then + printf 'Missing value for --robot-name\n' >&2 + exit 2 + fi + ROBOT_NAME="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + printf 'Unknown argument: %s\n' "$1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +cd "${PROJECT_DIR}" + +if ! command -v uv >/dev/null 2>&1; then + printf 'Missing required command: uv\n' >&2 + exit 1 +fi + +ensure_url_ready() { + local url="$1" + local timeout_seconds="$2" + local label="$3" + + local deadline=$((SECONDS + timeout_seconds)) + until uv run python - "${url}" <<'PY' >/dev/null 2>&1 +import sys +import urllib.request + +url = sys.argv[1] +try: + with urllib.request.urlopen(url, timeout=0.5) as response: + raise SystemExit(0 if 200 <= response.status < 300 else 1) +except Exception: + raise SystemExit(1) +PY + do + if (( SECONDS >= deadline )); then + printf '%s did not become ready within %s seconds.\n' "${label}" "${timeout_seconds}" >&2 + exit 1 + fi + sleep 0.25 + done +} + +SMOKE_ENV="$(mktemp "${TMPDIR:-/tmp}/reachy-local-stt-smoke.XXXXXX")" +cat >"${SMOKE_ENV}" </dev/null 2>&1 || true + wait "${DAEMON_PID}" >/dev/null 2>&1 || true + fi +} +trap finish EXIT INT TERM + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + printf 'Missing required command: %s\n' "$1" >&2 + exit 1 + fi +} + +daemon_status_ok() { + "${PROJECT_DIR}/.venv/bin/python" - "$DAEMON_HOST" "$DAEMON_PORT" <<'PY' >/dev/null 2>&1 +import json +import sys +import urllib.request + +host, port = sys.argv[1], sys.argv[2] +try: + with urllib.request.urlopen(f"http://{host}:{port}/api/daemon/status", timeout=1.5) as response: + if response.status != 200: + raise SystemExit(1) + payload = json.loads(response.read().decode("utf-8")) + if payload.get("type") == "daemon_status" and payload.get("state") == "running": + raise SystemExit(0) + raise SystemExit(1) +except Exception: + raise SystemExit(1) +PY +} + +pick_app_port() { + if [[ -n "${APP_PORT}" ]]; then + "${PROJECT_DIR}/.venv/bin/python" - "$APP_HOST" "$APP_PORT" <<'PY' +import socket +import sys + +host = sys.argv[1] +port = int(sys.argv[2]) + +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.2) + if sock.connect_ex((host, port)) == 0: + raise SystemExit(f"Requested APP_PORT={port} is already in use") +PY + printf '%s\n' "${APP_PORT}" + return + fi + + "${PROJECT_DIR}/.venv/bin/python" - "$APP_HOST" "$APP_PORT_START" "$APP_PORT_END" <<'PY' +import socket +import sys + +host = sys.argv[1] +start = int(sys.argv[2]) +end = int(sys.argv[3]) + +for port in range(start, end + 1): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.2) + if sock.connect_ex((host, port)) != 0: + print(port) + raise SystemExit(0) + +raise SystemExit(f"No free app port found in {start}-{end}") +PY +} + +require_command uv + +if [[ ! -x "${PROJECT_DIR}/.venv/bin/python" ]]; then + log "Creating virtual environment with Python ${PYTHON_VERSION}" + uv venv --python "${PYTHON_VERSION}" +fi + +if [[ "${REACHY_SKIP_SYNC}" != "1" ]]; then + log "Installing project dependencies with uv sync" + if ! uv sync; then + cat >&2 <<'EOF' + +uv sync failed. This project currently has a platform-specific uv.lock. +After installing dependencies another way, rerun with: + + REACHY_SKIP_SYNC=1 ${LAUNCHER_PATH} + +EOF + exit 1 + fi +fi + +if [[ ! -f ".env" ]]; then + cp .env.example .env + log "Created .env from .env.example. Edit it if you need different provider credentials." +fi + +export PATH="${PROJECT_DIR}/.venv/bin:${PATH}" + +log "Validating conversation backend configuration" +reachy-mini-backend-check + +if daemon_status_ok; then + log "Using existing Reachy daemon at http://${DAEMON_HOST}:${DAEMON_PORT}" +else + log "Starting Reachy simulator daemon at http://${DAEMON_HOST}:${DAEMON_PORT}" + reachy-mini-daemon \ + --sim \ + --scene minimal \ + --headless \ + --no-media \ + --fastapi-host "${DAEMON_HOST}" \ + --fastapi-port "${DAEMON_PORT}" \ + --dataset-update-interval 0 & + DAEMON_PID="$!" + + log "Waiting for simulator daemon" + deadline=$((SECONDS + REACHY_DAEMON_TIMEOUT)) + until daemon_status_ok; do + if (( SECONDS >= deadline )); then + printf 'Reachy simulator daemon did not become ready within %s seconds.\n' "${REACHY_DAEMON_TIMEOUT}" >&2 + exit 1 + fi + sleep 1 + done +fi + +SELECTED_APP_PORT="$(pick_app_port)" +export GRADIO_SERVER_NAME="${APP_HOST}" +export GRADIO_SERVER_PORT="${SELECTED_APP_PORT}" + +log "Starting Reachy conversation app" +log "Open: http://${APP_HOST}:${SELECTED_APP_PORT}/" +"${PROJECT_DIR}/.venv/bin/python" -m reachy_mini_conversation_app --gradio --no-camera "$@" diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/__init__.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/__init__.py new file mode 100644 index 00000000..efa9d4d8 --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/__init__.py @@ -0,0 +1,11 @@ +"""Reachy Mini conversation app fork for OpenShell demos.""" + +from importlib.metadata import PackageNotFoundError, version + + +try: + __version__ = version("reachy_mini_conversation_app") +except PackageNotFoundError: + __version__ = "0.0.0" + +__all__ = ["__version__"] diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/__main__.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/__main__.py new file mode 100644 index 00000000..e5091e7b --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/__main__.py @@ -0,0 +1,5 @@ +from reachy_mini_conversation_app.main import main + + +if __name__ == "__main__": + main() diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/__init__.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/__init__.py new file mode 100644 index 00000000..4eb1e731 --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/__init__.py @@ -0,0 +1 @@ +"""Audio helpers for Reachy Mini speech playback and movement.""" diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/head_wobbler.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/head_wobbler.py new file mode 100644 index 00000000..805d1cdb --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/head_wobbler.py @@ -0,0 +1,181 @@ +"""Moves head given audio samples.""" + +import time +import queue +import base64 +import logging +import threading +from typing import Tuple +from collections.abc import Callable + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini_conversation_app.audio.speech_tapper import HOP_MS, SwayRollRT + + +SAMPLE_RATE = 24000 +MOVEMENT_LATENCY_S = 0.2 # seconds between audio and robot movement +logger = logging.getLogger(__name__) + + +class HeadWobbler: + """Converts audio deltas (base64) into head movement offsets.""" + + def __init__(self, set_speech_offsets: Callable[[Tuple[float, float, float, float, float, float]], None]) -> None: + """Initialize the head wobbler.""" + self._apply_offsets = set_speech_offsets + self._base_ts: float | None = None + self._hops_done: int = 0 + + self.audio_queue: "queue.Queue[Tuple[int, int, NDArray[np.int16]]]" = queue.Queue() + self.sway = SwayRollRT() + + # Synchronization primitives + self._state_lock = threading.Lock() + self._sway_lock = threading.Lock() + self._generation = 0 + + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + def feed(self, delta_b64: str) -> None: + """Thread-safe: push audio into the consumer queue.""" + buf = np.frombuffer(base64.b64decode(delta_b64), dtype=np.int16).reshape(1, -1) + with self._state_lock: + generation = self._generation + self.audio_queue.put((generation, SAMPLE_RATE, buf)) + + def start(self) -> None: + """Start the head wobbler loop in a thread.""" + self._stop_event.clear() + self._thread = threading.Thread(target=self.working_loop, daemon=True) + self._thread.start() + logger.debug("Head wobbler started") + + def stop(self) -> None: + """Stop the head wobbler loop.""" + self._stop_event.set() + if self._thread is not None: + self._thread.join() + logger.debug("Head wobbler stopped") + + def working_loop(self) -> None: + """Convert audio deltas into head movement offsets.""" + hop_dt = HOP_MS / 1000.0 + + logger.debug("Head wobbler thread started") + while not self._stop_event.is_set(): + queue_ref = self.audio_queue + try: + chunk_generation, sr, chunk = queue_ref.get_nowait() # (gen, sr, data) + except queue.Empty: + # avoid while to never exit + time.sleep(MOVEMENT_LATENCY_S) + continue + + try: + with self._state_lock: + current_generation = self._generation + if chunk_generation != current_generation: + continue + + if self._base_ts is None: + with self._state_lock: + if self._base_ts is None: + self._base_ts = time.monotonic() + + pcm = np.asarray(chunk).squeeze(0) + with self._sway_lock: + results = self.sway.feed(pcm, sr) + + i = 0 + while i < len(results): + with self._state_lock: + if self._generation != current_generation: + break + base_ts = self._base_ts + hops_done = self._hops_done + + if base_ts is None: + base_ts = time.monotonic() + with self._state_lock: + if self._base_ts is None: + self._base_ts = base_ts + hops_done = self._hops_done + + target = base_ts + MOVEMENT_LATENCY_S + hops_done * hop_dt + now = time.monotonic() + + if now - target >= hop_dt: + lag_hops = int((now - target) / hop_dt) + drop = min(lag_hops, len(results) - i - 1) + if drop > 0: + with self._state_lock: + self._hops_done += drop + hops_done = self._hops_done + i += drop + continue + + if target > now: + time.sleep(target - now) + with self._state_lock: + if self._generation != current_generation: + break + + r = results[i] + offsets = ( + r["x_mm"] / 1000.0, + r["y_mm"] / 1000.0, + r["z_mm"] / 1000.0, + r["roll_rad"], + r["pitch_rad"], + r["yaw_rad"], + ) + + with self._state_lock: + if self._generation != current_generation: + break + + self._apply_offsets(offsets) + + with self._state_lock: + self._hops_done += 1 + i += 1 + finally: + queue_ref.task_done() + logger.debug("Head wobbler thread exited") + + ''' + def drain_audio_queue(self) -> None: + """Empty the audio queue.""" + try: + while True: + self.audio_queue.get_nowait() + except QueueEmpty: + pass + ''' + + def reset(self) -> None: + """Reset the internal state.""" + with self._state_lock: + self._generation += 1 + self._base_ts = None + self._hops_done = 0 + + # Drain any queued audio chunks from previous generations + drained_any = False + while True: + try: + _, _, _ = self.audio_queue.get_nowait() + except queue.Empty: + break + else: + drained_any = True + self.audio_queue.task_done() + + with self._sway_lock: + self.sway.reset() + + if drained_any: + logger.debug("Head wobbler queue drained during reset") diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/mic_phrase.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/mic_phrase.py new file mode 100644 index 00000000..6c15aeaf --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/mic_phrase.py @@ -0,0 +1,87 @@ +"""Microphone phrase buffering for local-STT transcription.""" + +from typing import Any +from dataclasses import dataclass + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini_conversation_app.audio.pcm import audio_rms, prepare_mono_int16_audio + + +MIC_TRANSCRIPTION_SAMPLE_RATE = 16000 + + +@dataclass(frozen=True) +class MicPhraseConfig: + """Voice activity settings for microphone-to-text phrase detection.""" + + sample_rate: int = MIC_TRANSCRIPTION_SAMPLE_RATE + rms_threshold: float = 500.0 + min_audio_ms: float = 250.0 + silence_ms: float = 800.0 + max_audio_ms: float = 12_000.0 + + +@dataclass(frozen=True) +class MicPhraseResult: + """Result from adding one microphone frame to a phrase buffer.""" + + phrase_audio: NDArray[np.int16] | None + saw_speech: bool + + +class MicPhraseBuffer: + """Accumulate microphone frames into speech phrases.""" + + def __init__(self, config: MicPhraseConfig): + """Initialize an empty phrase buffer.""" + self.config = config + self._buffer: list[NDArray[np.int16]] = [] + self._buffer_ms: float = 0.0 + self._silence_ms: float = 0.0 + self._has_speech: bool = False + + def push_frame(self, frame: tuple[int, NDArray[Any]]) -> MicPhraseResult: + """Add one audio frame and return a completed phrase when ready.""" + audio_frame = prepare_mono_int16_audio(frame, self.config.sample_rate) + if audio_frame.size == 0: + return MicPhraseResult(phrase_audio=None, saw_speech=False) + + frame_ms = len(audio_frame) * 1000.0 / self.config.sample_rate + is_speech = audio_rms(audio_frame) >= self.config.rms_threshold + + if is_speech: + self._has_speech = True + self._silence_ms = 0.0 + elif self._has_speech: + self._silence_ms += frame_ms + else: + return MicPhraseResult(phrase_audio=None, saw_speech=False) + + self._buffer.append(audio_frame) + self._buffer_ms += frame_ms + + should_flush = self._buffer_ms >= self.config.max_audio_ms or ( + self._buffer_ms >= self.config.min_audio_ms and self._silence_ms >= self.config.silence_ms + ) + if not should_flush: + return MicPhraseResult(phrase_audio=None, saw_speech=is_speech) + + return MicPhraseResult(phrase_audio=self.flush(), saw_speech=is_speech) + + def flush(self) -> NDArray[np.int16] | None: + """Return the buffered phrase and reset phrase-detection state.""" + if not self._buffer: + return None + + audio_frame = np.concatenate(self._buffer).astype(np.int16, copy=False) + self.reset() + return audio_frame + + def reset(self) -> None: + """Clear buffered audio and voice activity state.""" + self._buffer = [] + self._buffer_ms = 0.0 + self._silence_ms = 0.0 + self._has_speech = False diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/pcm.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/pcm.py new file mode 100644 index 00000000..84e35147 --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/pcm.py @@ -0,0 +1,141 @@ +"""PCM and WAV helpers shared by realtime and local-STT backends.""" + +import io +import wave +import asyncio +from typing import Any + +import numpy as np +from fastrtc import audio_to_int16 +from numpy.typing import NDArray +from scipy.signal import resample + + +def first_mono_channel(audio_frame: NDArray[Any]) -> NDArray[Any]: + """Return one mono channel from common FastRTC and scipy audio layouts.""" + if audio_frame.ndim == 1: + return audio_frame + + if audio_frame.ndim != 2: + raise ValueError(f"Expected 1D or 2D audio, got shape {audio_frame.shape}") + + rows, columns = audio_frame.shape + if rows == 1: + return audio_frame[0, :] + if columns == 1: + return audio_frame[:, 0] + if rows in {2, 6, 8} and columns > rows: + return audio_frame[0, :] + if columns in {2, 6, 8}: + return audio_frame[:, 0] + if rows < columns: + return audio_frame[0, :] + return audio_frame[:, 0] + + +def coerce_audio_to_int16(audio_frame: NDArray[Any]) -> NDArray[np.int16]: + """Convert supported PCM audio arrays to int16. + + FastRTC accepts normalized float32 audio, while scipy resampling returns + float64 values in the original PCM amplitude range. Preserve that scale + instead of multiplying resampled PCM by 32767 a second time. + """ + if audio_frame.dtype == np.int16: + return audio_frame + + if np.issubdtype(audio_frame.dtype, np.floating): + finite_audio = np.nan_to_num(audio_frame, nan=0.0, posinf=32767.0, neginf=-32768.0) + max_abs = float(np.max(np.abs(finite_audio))) if finite_audio.size else 0.0 + if max_abs <= 1.0: + return audio_to_int16(finite_audio.astype(np.float32, copy=False)) + return np.clip(finite_audio, -32768, 32767).astype(np.int16) + + if np.issubdtype(audio_frame.dtype, np.integer): + return np.clip(audio_frame, -32768, 32767).astype(np.int16) + + raise TypeError(f"Unsupported audio data type: {audio_frame.dtype}") + + +def prepare_mono_int16_audio( + frame: tuple[int, NDArray[Any]], + target_sample_rate: int, +) -> NDArray[np.int16]: + """Convert an incoming audio frame to mono int16 at the requested sample rate.""" + input_sample_rate, audio_frame = frame + mono_audio = first_mono_channel(audio_frame) + + if target_sample_rate != input_sample_rate and len(mono_audio) > 0: + mono_audio = resample(mono_audio, int(len(mono_audio) * target_sample_rate / input_sample_rate)) + + return coerce_audio_to_int16(mono_audio) + + +def audio_rms(audio_frame: NDArray[np.int16]) -> float: + """Return the RMS amplitude for a mono int16 audio frame.""" + if audio_frame.size == 0: + return 0.0 + samples = audio_frame.astype(np.float64) + return float(np.sqrt(np.mean(samples * samples))) + + +def wav_bytes(audio_frame: NDArray[np.int16], sample_rate: int) -> bytes: + """Encode mono int16 PCM audio as a WAV file.""" + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(sample_rate) + wav_file.writeframes(audio_frame.tobytes()) + return buffer.getvalue() + + +def read_wav_audio(wav_payload: bytes) -> tuple[int, NDArray[np.int16]]: + """Decode a mono/stereo 16-bit WAV payload to mono int16 audio.""" + with wave.open(io.BytesIO(wav_payload), "rb") as wav_file: + channels = wav_file.getnchannels() + sample_width = wav_file.getsampwidth() + sample_rate = wav_file.getframerate() + frames = wav_file.readframes(wav_file.getnframes()) + + if sample_width != 2: + raise ValueError(f"Expected 16-bit WAV audio, got sample width {sample_width}") + + audio_frame = np.frombuffer(frames, dtype=np.int16) + if channels > 1: + audio_frame = audio_frame.reshape(-1, channels)[:, 0] + return sample_rate, audio_frame + + +def normalize_wav_bytes(wav_payload: bytes) -> bytes: + """Return a 16-bit mono WAV with concrete RIFF/data sizes.""" + sample_rate, audio_frame = read_wav_audio(wav_payload) + return wav_bytes(audio_frame, sample_rate) + + +def wav_duration_seconds(wav_payload: bytes) -> float: + """Return the duration of a WAV payload.""" + sample_rate, audio_frame = read_wav_audio(wav_payload) + return len(audio_frame) / float(sample_rate) if sample_rate else 0.0 + + +async def binary_response_bytes(response: Any) -> bytes: + """Read bytes from an OpenAI binary response object.""" + content = getattr(response, "content", None) + if isinstance(content, bytes): + return content + + read = getattr(response, "read", None) + if callable(read): + result = read() + if asyncio.iscoroutine(result): + result = await result + if isinstance(result, bytes): + return result + + aread = getattr(response, "aread", None) + if callable(aread): + result = await aread() + if isinstance(result, bytes): + return result + + raise TypeError(f"Unsupported binary response type: {type(response).__name__}") diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/speech_tapper.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/speech_tapper.py new file mode 100644 index 00000000..3f4a2f16 --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/audio/speech_tapper.py @@ -0,0 +1,268 @@ +from __future__ import annotations +import math +from typing import Any, Dict, List +from itertools import islice +from collections import deque + +import numpy as np +from numpy.typing import NDArray + + +# Tunables +SR = 16_000 +FRAME_MS = 20 +HOP_MS = 50 + +SWAY_MASTER = 1.5 +SENS_DB_OFFSET = +4.0 +VAD_DB_ON = -35.0 +VAD_DB_OFF = -45.0 +VAD_ATTACK_MS = 40 +VAD_RELEASE_MS = 250 +ENV_FOLLOW_GAIN = 0.65 + +SWAY_F_PITCH = 2.2 +SWAY_A_PITCH_DEG = 4.5 +SWAY_F_YAW = 0.6 +SWAY_A_YAW_DEG = 7.5 +SWAY_F_ROLL = 1.3 +SWAY_A_ROLL_DEG = 2.25 +SWAY_F_X = 0.35 +SWAY_A_X_MM = 4.5 +SWAY_F_Y = 0.45 +SWAY_A_Y_MM = 3.75 +SWAY_F_Z = 0.25 +SWAY_A_Z_MM = 2.25 + +SWAY_DB_LOW = -46.0 +SWAY_DB_HIGH = -18.0 +LOUDNESS_GAMMA = 0.9 +SWAY_ATTACK_MS = 50 +SWAY_RELEASE_MS = 250 + +# Derived +FRAME = int(SR * FRAME_MS / 1000) +HOP = int(SR * HOP_MS / 1000) +ATTACK_FR = max(1, int(VAD_ATTACK_MS / HOP_MS)) +RELEASE_FR = max(1, int(VAD_RELEASE_MS / HOP_MS)) +SWAY_ATTACK_FR = max(1, int(SWAY_ATTACK_MS / HOP_MS)) +SWAY_RELEASE_FR = max(1, int(SWAY_RELEASE_MS / HOP_MS)) + + +def _rms_dbfs(x: NDArray[np.float32]) -> float: + """Root-mean-square in dBFS for float32 mono array in [-1,1].""" + # numerically stable rms (avoid overflow) + x = x.astype(np.float32, copy=False) + rms = np.sqrt(np.mean(x * x, dtype=np.float32) + 1e-12, dtype=np.float32) + return float(20.0 * math.log10(float(rms) + 1e-12)) + + +def _loudness_gain(db: float, offset: float = SENS_DB_OFFSET) -> float: + """Normalize dB into [0,1] with gamma; clipped to [0,1].""" + t = (db + offset - SWAY_DB_LOW) / (SWAY_DB_HIGH - SWAY_DB_LOW) + if t < 0.0: + t = 0.0 + elif t > 1.0: + t = 1.0 + return t**LOUDNESS_GAMMA if LOUDNESS_GAMMA != 1.0 else t + + +def _to_float32_mono(x: NDArray[Any]) -> NDArray[np.float32]: + """Convert arbitrary PCM array to float32 mono in [-1,1]. + + Accepts shapes: (N,), (1,N), (N,1), (C,N), (N,C). + """ + a = np.asarray(x) + if a.ndim == 0: + return np.zeros(0, dtype=np.float32) + + # If 2D, decide which axis is channels (prefer small first dim) + if a.ndim == 2: + # e.g., (channels, samples) if channels is small (<=8) + if a.shape[0] <= 8 and a.shape[0] <= a.shape[1]: + a = np.mean(a, axis=0) + else: + a = np.mean(a, axis=1) + elif a.ndim > 2: + a = np.mean(a.reshape(a.shape[0], -1), axis=0) + + # Now 1D, cast/scale + if np.issubdtype(a.dtype, np.floating): + return a.astype(np.float32, copy=False) + # integer PCM + info = np.iinfo(a.dtype) + scale = float(max(-info.min, info.max)) + return a.astype(np.float32) / (scale if scale != 0.0 else 1.0) + + +def _resample_linear(x: NDArray[np.float32], sr_in: int, sr_out: int) -> NDArray[np.float32]: + """Lightweight linear resampler for short buffers.""" + if sr_in == sr_out or x.size == 0: + return x + # guard tiny sizes + n_out = int(round(x.size * sr_out / sr_in)) + if n_out <= 1: + return np.zeros(0, dtype=np.float32) + t_in = np.linspace(0.0, 1.0, num=x.size, dtype=np.float32, endpoint=True) + t_out = np.linspace(0.0, 1.0, num=n_out, dtype=np.float32, endpoint=True) + return np.interp(t_out, t_in, x).astype(np.float32, copy=False) + + +class SwayRollRT: + """Feed audio chunks → per-hop sway outputs. + + Usage: + rt = SwayRollRT() + rt.feed(pcm_int16_or_float, sr) -> List[dict] + """ + + def __init__(self, rng_seed: int = 7): + """Initialize state.""" + self._seed = int(rng_seed) + self.samples: deque[float] = deque(maxlen=10 * SR) # sliding window for VAD/env + self.carry: NDArray[np.float32] = np.zeros(0, dtype=np.float32) + + self.vad_on = False + self.vad_above = 0 + self.vad_below = 0 + + self.sway_env = 0.0 + self.sway_up = 0 + self.sway_down = 0 + + rng = np.random.default_rng(self._seed) + self.phase_pitch = float(rng.random() * 2 * math.pi) + self.phase_yaw = float(rng.random() * 2 * math.pi) + self.phase_roll = float(rng.random() * 2 * math.pi) + self.phase_x = float(rng.random() * 2 * math.pi) + self.phase_y = float(rng.random() * 2 * math.pi) + self.phase_z = float(rng.random() * 2 * math.pi) + self.t = 0.0 + + def reset(self) -> None: + """Reset state (VAD/env/buffers/time) but keep initial phases/seed.""" + self.samples.clear() + self.carry = np.zeros(0, dtype=np.float32) + self.vad_on = False + self.vad_above = 0 + self.vad_below = 0 + self.sway_env = 0.0 + self.sway_up = 0 + self.sway_down = 0 + self.t = 0.0 + + def feed(self, pcm: NDArray[Any], sr: int | None) -> List[Dict[str, float]]: + """Stream in PCM chunk. Returns a list of sway dicts, one per hop (HOP_MS). + + Args: + pcm: np.ndarray, shape (N,) or (C,N)/(N,C); int or float. + sr: sample rate of `pcm` (None -> assume SR). + + """ + sr_in = SR if sr is None else int(sr) + x = _to_float32_mono(pcm) + if x.size == 0: + return [] + if sr_in != SR: + x = _resample_linear(x, sr_in, SR) + if x.size == 0: + return [] + + # append to carry and consume fixed HOP chunks + if self.carry.size: + self.carry = np.concatenate([self.carry, x]) + else: + self.carry = x + + out: List[Dict[str, float]] = [] + + while self.carry.size >= HOP: + hop = self.carry[:HOP] + remaining: NDArray[np.float32] = self.carry[HOP:] + self.carry = remaining + + # keep sliding window for VAD/env computation + # (deque accepts any iterable; list() for small HOP is fine) + self.samples.extend(hop.tolist()) + if len(self.samples) < FRAME: + self.t += HOP_MS / 1000.0 + continue + + frame = np.fromiter( + islice(self.samples, len(self.samples) - FRAME, len(self.samples)), + dtype=np.float32, + count=FRAME, + ) + db = _rms_dbfs(frame) + + # VAD with hysteresis + attack/release + if db >= VAD_DB_ON: + self.vad_above += 1 + self.vad_below = 0 + if not self.vad_on and self.vad_above >= ATTACK_FR: + self.vad_on = True + elif db <= VAD_DB_OFF: + self.vad_below += 1 + self.vad_above = 0 + if self.vad_on and self.vad_below >= RELEASE_FR: + self.vad_on = False + + if self.vad_on: + self.sway_up = min(SWAY_ATTACK_FR, self.sway_up + 1) + self.sway_down = 0 + else: + self.sway_down = min(SWAY_RELEASE_FR, self.sway_down + 1) + self.sway_up = 0 + + up = self.sway_up / SWAY_ATTACK_FR + down = 1.0 - (self.sway_down / SWAY_RELEASE_FR) + target = up if self.vad_on else down + self.sway_env += ENV_FOLLOW_GAIN * (target - self.sway_env) + # clamp + if self.sway_env < 0.0: + self.sway_env = 0.0 + elif self.sway_env > 1.0: + self.sway_env = 1.0 + + loud = _loudness_gain(db) * SWAY_MASTER + env = self.sway_env + self.t += HOP_MS / 1000.0 + + # oscillators + pitch = ( + math.radians(SWAY_A_PITCH_DEG) + * loud + * env + * math.sin(2 * math.pi * SWAY_F_PITCH * self.t + self.phase_pitch) + ) + yaw = ( + math.radians(SWAY_A_YAW_DEG) + * loud + * env + * math.sin(2 * math.pi * SWAY_F_YAW * self.t + self.phase_yaw) + ) + roll = ( + math.radians(SWAY_A_ROLL_DEG) + * loud + * env + * math.sin(2 * math.pi * SWAY_F_ROLL * self.t + self.phase_roll) + ) + x_mm = SWAY_A_X_MM * loud * env * math.sin(2 * math.pi * SWAY_F_X * self.t + self.phase_x) + y_mm = SWAY_A_Y_MM * loud * env * math.sin(2 * math.pi * SWAY_F_Y * self.t + self.phase_y) + z_mm = SWAY_A_Z_MM * loud * env * math.sin(2 * math.pi * SWAY_F_Z * self.t + self.phase_z) + + out.append( + { + "pitch_rad": pitch, + "yaw_rad": yaw, + "roll_rad": roll, + "pitch_deg": math.degrees(pitch), + "yaw_deg": math.degrees(yaw), + "roll_deg": math.degrees(roll), + "x_mm": x_mm, + "y_mm": y_mm, + "z_mm": z_mm, + }, + ) + + return out diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/backend_check.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/backend_check.py new file mode 100644 index 00000000..f476aba7 --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/backend_check.py @@ -0,0 +1,774 @@ +"""Backend configuration and endpoint smoke checks for the conversation app.""" + +import asyncio +import argparse +from typing import Any, cast +from pathlib import Path + +import numpy as np +from openai import AsyncOpenAI +from numpy.typing import NDArray + +from reachy_mini_conversation_app.config import ( + config, + load_dotenv_file, + loaded_dotenv_path, +) +from reachy_mini_conversation_app.audio.pcm import ( + wav_bytes, + read_wav_audio, + wav_duration_seconds, + prepare_mono_int16_audio, +) +from reachy_mini_conversation_app.backend_runtime import ( + selected_backend, + backend_config_error, + describe_selected_backend, + local_stt_tts_config_error, + local_stt_stage_config_error, +) +from reachy_mini_conversation_app.tools.core_tools import ToolDependencies +from reachy_mini_conversation_app.local_stt_backend import LocalSTTBackend +from reachy_mini_conversation_app.tools.background_tool_manager import BackgroundToolManager + + +def _required(value: str | None, name: str) -> str: + """Return a required string config value or raise a clear error.""" + if value and value.strip(): + return value.strip() + raise RuntimeError(f"{name} is required") + + +def _backend_config_hint(error: str) -> str | None: + """Return an actionable hint for common backend config errors.""" + if not error.startswith("BACKEND_PROVIDER is missing"): + return None + + loaded_path = loaded_dotenv_path() + if loaded_path is None: + return ( + "No .env file was loaded. Copy .env.example to .env, set BACKEND_PROVIDER to " + "openai_realtime, hf_realtime, or local_stt, or pass --env-file." + ) + + return ( + f"Loaded config from {loaded_path}. Set BACKEND_PROVIDER there to " + "openai_realtime, hf_realtime, or local_stt, or pass --env-file." + ) + + +def _wav_duration_seconds(wav_payload: bytes) -> float: + """Return the duration of a WAV payload.""" + return wav_duration_seconds(wav_payload) + + +def _audio_duration_seconds(sample_rate: int, audio_frame: NDArray[np.int16]) -> float: + """Return the duration of a mono/channel-first int16 frame.""" + if sample_rate <= 0: + return 0.0 + sample_count = audio_frame.shape[-1] if audio_frame.ndim > 1 else len(audio_frame) + return sample_count / float(sample_rate) + + +def _read_wav_audio(wav_payload: bytes) -> tuple[int, NDArray[np.int16]]: + """Decode a WAV payload into mono int16 audio.""" + return read_wav_audio(wav_payload) + + +def _audio_chunks( + audio_frame: NDArray[np.int16], sample_rate: int, *, chunk_ms: float = 100.0 +) -> list[NDArray[np.int16]]: + """Split audio into chunks that resemble browser microphone frames.""" + chunk_size = max(1, int(sample_rate * chunk_ms / 1000.0)) + return [audio_frame[start : start + chunk_size] for start in range(0, len(audio_frame), chunk_size)] + + +def _silence_frame(sample_rate: int, silence_ms: float) -> NDArray[np.int16]: + """Return a mono silence frame long enough to end the mic phrase.""" + sample_count = max(1, int(sample_rate * silence_ms / 1000.0)) + return np.zeros(sample_count, dtype=np.int16) + + +def _tone_wav_bytes() -> bytes: + """Return a tiny placeholder WAV when an endpoint needs valid audio bytes.""" + sample_rate = 16_000 + t = np.linspace(0, 0.25, int(sample_rate * 0.25), endpoint=False) + samples = (np.sin(2 * np.pi * 440 * t) * 2000).astype(np.int16) + return wav_bytes(samples, sample_rate) + + +def _speech_endpoint_error(component: str, context: str, endpoint_path: str, exc: Exception) -> str: + """Return a concise endpoint failure message without exposing secrets.""" + return ( + f"{component} endpoint request failed ({context}). Check the base URL and that the service exposes " + f"{endpoint_path}. {type(exc).__name__}: {exc}" + ) + + +def _model_ids(models_response: Any) -> list[str]: + """Extract model IDs from common OpenAI-compatible /models response shapes.""" + data = ( + models_response.get("data", []) if isinstance(models_response, dict) else getattr(models_response, "data", []) + ) + ids: list[str] = [] + for model in data or []: + model_id = model.get("id") if isinstance(model, dict) else getattr(model, "id", None) + if isinstance(model_id, str) and model_id.strip(): + ids.append(model_id.strip()) + return sorted(set(ids)) + + +def _format_model_ids(model_ids: list[str], *, limit: int = 5) -> str: + """Return a compact non-secret model list for diagnostics.""" + if not model_ids: + return "" + visible = model_ids[:limit] + suffix = f", ... (+{len(model_ids) - limit} more)" if len(model_ids) > limit else "" + return ", ".join(visible) + suffix + + +async def _stt_models_diagnostic_lines() -> list[str]: + """Return diagnostic lines for the STT /models endpoint when available.""" + _required(config.STT_MODEL_NAME, "STT_MODEL_NAME") + client = AsyncOpenAI( + api_key=(config.STT_API_KEY or "not-needed").strip() or "not-needed", + base_url=config.STT_BASE_URL, + ) + try: + models_response = await client.models.list() + except Exception as exc: + return [f"stt_models_endpoint=unavailable ({type(exc).__name__}: {exc})"] + + ids = _model_ids(models_response) + model_listed = config.STT_MODEL_NAME in ids + return [ + "stt_models_endpoint=reachable", + f"stt_model_listed={'yes' if model_listed else 'no'}", + f"stt_available_models={_format_model_ids(ids)}", + ] + + +def _build_fake_local_stt_backend() -> tuple[LocalSTTBackend, BackgroundToolManager, Any]: + """Build local-STT backend dependencies for CLI smoke checks.""" + movement_manager = _FakeMovementManager() + deps = ToolDependencies( + reachy_mini=cast(Any, _FakeReachyMini()), + movement_manager=movement_manager, + ) + tool_manager = BackgroundToolManager() + return ( + LocalSTTBackend( + deps=deps, + tool_manager=tool_manager, + client_factory=AsyncOpenAI, + ), + tool_manager, + movement_manager, + ) + + +def _chat_summary_from_messages(messages: list[dict[str, Any]], *, require_tool: bool) -> tuple[str, list[str]]: + """Return final assistant text and tool result titles from chatbot messages.""" + assistant_text = "" + tool_titles: list[str] = [] + + for message in messages: + content = message.get("content") + if message.get("role") != "assistant" or not isinstance(content, str): + continue + metadata = message.get("metadata") + if isinstance(metadata, dict) and isinstance(metadata.get("title"), str): + tool_titles.append(metadata["title"]) + elif content.startswith("[error]"): + raise RuntimeError(content) + else: + assistant_text = content + + if not assistant_text: + raise RuntimeError("Chat endpoint returned no final assistant response") + if require_tool and not tool_titles: + raise RuntimeError("Chat check did not execute a Reachy tool") + + return assistant_text, tool_titles + + +async def live_realtime_session_check() -> list[str]: + """Exercise the selected realtime backend's session startup path.""" + from reachy_mini_conversation_app.tools.core_tools import ToolDependencies + from reachy_mini_conversation_app.conversation_stream import ConversationStreamHandler + + movement_manager = _FakeMovementManager() + deps = ToolDependencies( + reachy_mini=cast(Any, _FakeReachyMini()), + movement_manager=movement_manager, + ) + handler = ConversationStreamHandler(deps) + + try: + backend = selected_backend() + connected = await handler._ensure_realtime_session("backend-check-realtime-session") + if not connected: + startup_error = getattr(handler, "_startup_error", None) + raise RuntimeError(startup_error or "Realtime session did not connect before the startup timeout") + + return [ + "realtime_session=connected", + f"realtime_model={backend.realtime_model or ''}", + f"realtime_voice={backend.realtime_voice or ''}", + ] + finally: + await handler.shutdown() + startup_task = getattr(handler, "_realtime_startup_task", None) + if startup_task is not None: + try: + await asyncio.wait_for(startup_task, timeout=2.0) + except asyncio.TimeoutError: + startup_task.cancel() + try: + await startup_task + except asyncio.CancelledError: + pass + + +def _stage_name(args: argparse.Namespace) -> str: + """Return the selected live-check stage, preserving --app-flow as a shortcut.""" + stage = getattr(args, "stage", None) + if stage: + return str(stage) + if getattr(args, "app_flow", False): + return "app-flow" + if getattr(args, "live", False) and selected_backend().uses_local_stt: + return "app-flow" + return "chain" + + +def _require_tool_usage_error(args: argparse.Namespace, stage: str) -> str | None: + """Return an error when --require-tool would otherwise be ignored.""" + if not getattr(args, "require_tool", False): + return None + if not getattr(args, "live", False): + return "--require-tool only applies when --live is set." + if not selected_backend().uses_local_stt: + return "--require-tool is only supported for BACKEND_PROVIDER=local_stt --stage chat or --stage app-flow." + if stage not in {"chat", "app-flow"}: + return "--require-tool only applies to --stage chat or --stage app-flow." + return None + + +def _real_reachy_usage_error(args: argparse.Namespace, stage: str) -> str | None: + """Return an error when --real-reachy would otherwise be ignored.""" + if not getattr(args, "real_reachy", False): + return None + if not getattr(args, "live", False) or not selected_backend().uses_local_stt or stage != "app-flow": + return "--real-reachy only applies to BACKEND_PROVIDER=local_stt with --live --stage app-flow." + return None + + +async def _synthesize_input_audio(seed_text: str) -> bytes: + """Use the configured TTS endpoint to generate a speech sample for STT.""" + _required(config.TTS_MODEL_NAME, "TTS_MODEL_NAME") + _required(config.TTS_VOICE, "TTS_VOICE") + backend, tool_manager, _movement_manager = _build_fake_local_stt_backend() + try: + try: + return await backend.synthesize_speech_wav_bytes(seed_text) + except Exception as exc: + raise RuntimeError(_speech_endpoint_error("TTS", backend.tts_context, "POST /audio/speech", exc)) from exc + finally: + await tool_manager.shutdown() + + +async def live_local_stt_probe_check() -> list[str]: + """Probe the local_stt STT endpoint with synthetic audio without requiring recognized text.""" + _required(config.STT_MODEL_NAME, "STT_MODEL_NAME") + model_lines = await _stt_models_diagnostic_lines() + if "stt_model_listed=no" in model_lines: + raise RuntimeError("; ".join(model_lines)) + + input_audio = _tone_wav_bytes() + backend, tool_manager, _movement_manager = _build_fake_local_stt_backend() + try: + try: + transcript = await backend.transcribe_wav_bytes( + input_audio, + filename="backend-check-probe.wav", + ) + except Exception as exc: + raise RuntimeError( + "; ".join( + [ + *model_lines, + _speech_endpoint_error("STT", backend.stt_context, "POST /audio/transcriptions", exc), + ] + ) + ) from exc + finally: + await tool_manager.shutdown() + + return [ + *model_lines, + "input_audio=synthetic probe tone", + f"input_audio_duration={_wav_duration_seconds(input_audio):.2f}s", + "stt_endpoint=reachable", + f"transcript={transcript or ''}", + ] + + +async def live_local_stt_transcription_check(seed_text: str, audio_file: Path | None) -> list[str]: + """Exercise only the local_stt STT endpoint.""" + if audio_file is not None: + input_audio = audio_file.read_bytes() + input_label = str(audio_file) + else: + tts_error = local_stt_tts_config_error() + if tts_error: + raise RuntimeError("--stage stt requires --audio-file unless TTS is configured to synthesize seed audio") + input_audio = await _synthesize_input_audio(seed_text) + input_label = "configured TTS seed" + + _required(config.STT_MODEL_NAME, "STT_MODEL_NAME") + backend, tool_manager, _movement_manager = _build_fake_local_stt_backend() + try: + try: + transcript = await backend.transcribe_wav_bytes( + input_audio or _tone_wav_bytes(), + filename="backend-check.wav", + ) + except Exception as exc: + raise RuntimeError( + _speech_endpoint_error("STT", backend.stt_context, "POST /audio/transcriptions", exc) + ) from exc + finally: + await tool_manager.shutdown() + if not transcript: + raise RuntimeError("STT endpoint returned an empty transcript") + + return [ + f"input_audio={input_label}", + f"input_audio_duration={_wav_duration_seconds(input_audio):.2f}s", + f"transcript={transcript}", + ] + + +async def live_local_stt_chat_check(seed_text: str, *, require_tool: bool) -> list[str]: + """Exercise only the local_stt Chat Completions and Reachy tool path.""" + backend, tool_manager, movement_manager = _build_fake_local_stt_backend() + try: + _required(config.CHAT_MODEL_NAME, "CHAT_MODEL_NAME") + messages = await backend.send_text_message(seed_text) + assistant_text, tool_titles = _chat_summary_from_messages(messages, require_tool=require_tool) + finally: + await tool_manager.shutdown() + + return [ + f"seed_text={seed_text}", + f"assistant_text={assistant_text}", + f"tool_results={', '.join(tool_titles) if tool_titles else ''}", + f"queued_moves={len(movement_manager.queued_moves)}", + ] + + +async def live_local_stt_tts_check(seed_text: str) -> list[str]: + """Exercise only the local_stt TTS endpoint.""" + output_audio = await _synthesize_input_audio(seed_text) + return [ + f"seed_text={seed_text}", + f"output_audio_duration={_wav_duration_seconds(output_audio):.2f}s", + ] + + +async def live_local_stt_check(seed_text: str, audio_file: Path | None) -> list[str]: + """Exercise local_stt endpoints without connecting to a Reachy daemon.""" + if audio_file is not None: + input_audio = audio_file.read_bytes() + input_label = str(audio_file) + else: + input_audio = await _synthesize_input_audio(seed_text) + input_label = "configured TTS seed" + + _required(config.STT_MODEL_NAME, "STT_MODEL_NAME") + backend, tool_manager, _movement_manager = _build_fake_local_stt_backend() + try: + try: + transcript = await backend.transcribe_wav_bytes( + input_audio or _tone_wav_bytes(), + filename="backend-check.wav", + ) + except Exception as exc: + raise RuntimeError( + _speech_endpoint_error("STT", backend.stt_context, "POST /audio/transcriptions", exc) + ) from exc + if not transcript: + raise RuntimeError("STT endpoint returned an empty transcript") + + _required(config.CHAT_MODEL_NAME, "CHAT_MODEL_NAME") + messages = await backend.send_text_message(transcript) + assistant_text, _tool_titles = _chat_summary_from_messages(messages, require_tool=False) + output_audio = await backend.synthesize_speech_wav_bytes(assistant_text) + finally: + await tool_manager.shutdown() + + return [ + f"input_audio={input_label}", + f"input_audio_duration={_wav_duration_seconds(input_audio):.2f}s", + f"transcript={transcript}", + f"assistant_text={assistant_text}", + f"output_audio_duration={_wav_duration_seconds(output_audio):.2f}s", + ] + + +class _FakeReachyMini: + """Small Reachy stand-in for app-flow backend checks.""" + + def get_current_head_pose(self) -> NDArray[np.float64]: + return np.eye(4, dtype=np.float64) + + def get_current_joint_positions(self) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + return np.array([0.0], dtype=np.float64), np.array([0.0, 0.0], dtype=np.float64) + + +class _FakeMovementManager: + """Movement manager stand-in that records queued Reachy moves.""" + + def __init__(self) -> None: + self.queued_moves: list[Any] = [] + self.clear_count = 0 + self.moving_state_durations: list[float] = [] + + def clear_move_queue(self) -> None: + self.clear_count += 1 + self.queued_moves.clear() + + def queue_move(self, move: Any) -> None: + self.queued_moves.append(move) + + def set_moving_state(self, duration: float) -> None: + self.moving_state_durations.append(duration) + + def is_idle(self) -> bool: + return True + + +class _AppFlowDeps: + """Dependencies and cleanup hooks for app-flow checks.""" + + def __init__(self, reachy_mini: Any, movement_manager: Any, *, real_reachy: bool = False): + self.reachy_mini = reachy_mini + self.movement_manager = movement_manager + self.real_reachy = real_reachy + + def start(self) -> None: + if self.real_reachy: + self.movement_manager.start() + + def movement_summary_lines(self) -> list[str]: + if not self.real_reachy: + return [f"queued_moves={len(self.movement_manager.queued_moves)}"] + + status = self.movement_manager.get_status() + loop_frequency = status.get("loop_frequency", {}) if isinstance(status, dict) else {} + return [ + "movement_manager=real", + f"movement_queue_size={status.get('queue_size', '') if isinstance(status, dict) else ''}", + f"movement_loop_samples={loop_frequency.get('samples', '')}", + ] + + def stop(self) -> None: + if self.real_reachy: + self.movement_manager.stop() + try: + self.reachy_mini.client.disconnect() + except Exception: + pass + + +def _build_app_flow_deps(*, real_reachy: bool, robot_name: str | None) -> _AppFlowDeps: + """Build fake or real Reachy dependencies for app-flow checks.""" + if not real_reachy: + movement_manager = _FakeMovementManager() + return _AppFlowDeps( + reachy_mini=cast(Any, _FakeReachyMini()), + movement_manager=movement_manager, + real_reachy=False, + ) + + from reachy_mini import ReachyMini + from reachy_mini_conversation_app.moves import MovementManager + + robot_kwargs = {"robot_name": robot_name} if robot_name else {} + reachy_mini = ReachyMini(**robot_kwargs) + movement_manager = MovementManager(current_robot=reachy_mini) + return _AppFlowDeps(reachy_mini=reachy_mini, movement_manager=movement_manager, real_reachy=True) + + +async def live_local_stt_app_flow_check( + seed_text: str, + audio_file: Path | None, + *, + require_tool: bool, + real_reachy: bool = False, + robot_name: str | None = None, +) -> list[str]: + """Exercise the app handler's mic -> local_stt STT -> chat/tools -> TTS flow.""" + from reachy_mini_conversation_app.tools.core_tools import ToolDependencies + from reachy_mini_conversation_app.conversation_stream import ( + MIC_TRANSCRIPTION_SAMPLE_RATE, + ConversationStreamHandler, + ) + + if audio_file is not None: + input_audio = audio_file.read_bytes() + input_label = str(audio_file) + else: + input_audio = await _synthesize_input_audio(seed_text) + input_label = "configured TTS seed" + + sample_rate, input_frame = _read_wav_audio(input_audio or _tone_wav_bytes()) + prepared_frame = prepare_mono_int16_audio( + (sample_rate, input_frame), + MIC_TRANSCRIPTION_SAMPLE_RATE, + ) + + app_flow_deps = _build_app_flow_deps(real_reachy=real_reachy, robot_name=robot_name) + app_flow_deps.start() + deps = ToolDependencies(reachy_mini=app_flow_deps.reachy_mini, movement_manager=app_flow_deps.movement_manager) + handler = ConversationStreamHandler(deps) + + transcript = "" + assistant_text = "" + tool_titles: list[str] = [] + error_messages: list[str] = [] + output_audio_duration = 0.0 + try: + for chunk in _audio_chunks(prepared_frame, MIC_TRANSCRIPTION_SAMPLE_RATE): + await handler.receive((MIC_TRANSCRIPTION_SAMPLE_RATE, chunk)) + + silence_ms = max(config.MIC_TRANSCRIPTION_SILENCE_MS + 50.0, 1.0) + await handler.receive( + (MIC_TRANSCRIPTION_SAMPLE_RATE, _silence_frame(MIC_TRANSCRIPTION_SAMPLE_RATE, silence_ms)) + ) + + await _wait_for_app_flow_mic_tasks(handler) + + while not handler.output_queue.empty(): + output = handler.output_queue.get_nowait() + if isinstance(output, tuple): + output_sample_rate, audio_frame = output + if isinstance(output_sample_rate, int) and isinstance(audio_frame, np.ndarray): + output_audio_duration += _audio_duration_seconds( + output_sample_rate, + cast(NDArray[np.int16], np.asarray(audio_frame, dtype=np.int16)), + ) + continue + + for message in getattr(output, "args", ()): + if not isinstance(message, dict): + continue + content = message.get("content") + if message.get("role") == "user" and isinstance(content, str): + transcript = content + elif message.get("role") == "assistant" and isinstance(content, str): + metadata = message.get("metadata") + if isinstance(metadata, dict) and isinstance(metadata.get("title"), str): + tool_titles.append(metadata["title"]) + if real_reachy: + await asyncio.sleep(0.2) + elif content.startswith("[error]"): + error_messages.append(content) + elif not content.startswith("[error]"): + assistant_text = content + finally: + await handler.shutdown() + app_flow_deps.stop() + + if error_messages: + raise RuntimeError("; ".join(error_messages)) + if not transcript: + raise RuntimeError("App-flow check did not produce a transcript") + if not assistant_text: + raise RuntimeError("App-flow check did not produce a final assistant response") + if output_audio_duration <= 0: + raise RuntimeError("App-flow check did not produce synthesized audio") + if require_tool and not tool_titles: + raise RuntimeError("App-flow check did not execute a Reachy tool") + + return [ + f"input_audio={input_label}", + f"input_audio_duration={_wav_duration_seconds(input_audio):.2f}s", + f"transcript={transcript}", + f"assistant_text={assistant_text}", + f"tool_results={', '.join(tool_titles) if tool_titles else ''}", + *app_flow_deps.movement_summary_lines(), + f"output_audio_duration={output_audio_duration:.2f}s", + ] + + +async def _wait_for_app_flow_mic_tasks(handler: Any) -> None: + """Wait for the handler's microphone transcription task to finish.""" + for _ in range(50): + tasks = list(getattr(handler, "_mic_transcription_tasks", ())) + if tasks: + await asyncio.gather(*tasks) + return + if not handler.output_queue.empty(): + return + await asyncio.sleep(0.05) + + raise RuntimeError("App-flow check did not flush microphone audio through the receive path") + + +async def run_check(args: argparse.Namespace) -> int: + """Run the selected backend check.""" + env_file = getattr(args, "env_file", None) + if env_file is not None and not load_dotenv_file(env_file): + print(f"[error] Could not load env file: {env_file}") + return 2 + + stage = _stage_name(args) + + for line in describe_selected_backend(): + print(line) + + error = ( + local_stt_stage_config_error(stage) + if getattr(args, "live", False) and selected_backend().uses_local_stt + else backend_config_error() + ) + if error: + print(f"[error] {error}") + hint = _backend_config_hint(error) + if hint: + print(f"[hint] {hint}") + return 2 + + require_tool_error = _require_tool_usage_error(args, stage) + if require_tool_error: + print(f"[error] {require_tool_error}") + return 2 + + real_reachy_error = _real_reachy_usage_error(args, stage) + if real_reachy_error: + print(f"[error] {real_reachy_error}") + return 2 + + print( + f"[ok] local_stt {stage} config is valid" + if getattr(args, "live", False) and selected_backend().uses_local_stt + else "[ok] backend config is valid" + ) + + if not args.live: + return 0 + + backend = selected_backend() + if not backend.uses_local_stt: + if not backend.uses_realtime: + print("[error] --live supports realtime backends and BACKEND_PROVIDER=local_stt") + return 2 + + try: + results = await live_realtime_session_check() + except Exception as e: + print(f"[error] realtime live session check failed: {type(e).__name__}: {e}") + return 1 + + for line in results: + print(line) + print("[ok] realtime live session check completed") + return 0 + + try: + if stage == "stt-probe": + results = await live_local_stt_probe_check() + elif stage == "stt": + results = await live_local_stt_transcription_check(args.seed_text, args.audio_file) + elif stage == "chat": + results = await live_local_stt_chat_check( + args.seed_text, + require_tool=getattr(args, "require_tool", False), + ) + elif stage == "tts": + results = await live_local_stt_tts_check(args.seed_text) + elif stage == "app-flow": + results = await live_local_stt_app_flow_check( + args.seed_text, + args.audio_file, + require_tool=getattr(args, "require_tool", False), + real_reachy=getattr(args, "real_reachy", False), + robot_name=getattr(args, "robot_name", None), + ) + else: + results = await live_local_stt_check(args.seed_text, args.audio_file) + except Exception as e: + print(f"[error] local_stt live {stage} check failed: {type(e).__name__}: {e}") + return 1 + + for line in results: + print(line) + print(f"[ok] local_stt live {stage} check completed") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + """Build the backend check argument parser.""" + parser = argparse.ArgumentParser(description="Validate Reachy conversation backend configuration.") + parser.add_argument( + "--live", + action="store_true", + help="call configured endpoints. Realtime backends validate session startup; local_stt can validate each stage.", + ) + parser.add_argument( + "--seed-text", + default="Hi Reachy, please introduce yourself in one short sentence.", + help="text to synthesize as the input speech sample when --live is used without --audio-file.", + ) + parser.add_argument( + "--audio-file", + type=Path, + help="WAV or provider-supported audio file to send to STT instead of synthesizing seed text.", + ) + parser.add_argument( + "--stage", + choices=["stt-probe", "stt", "chat", "tts", "chain", "app-flow"], + default=None, + help=( + "with --live and BACKEND_PROVIDER=local_stt, check one stage. " + "Defaults to app-flow for local_stt live checks and is ignored by realtime backends." + ), + ) + parser.add_argument( + "--env-file", + type=Path, + help="load this dotenv file for backend validation instead of the default .env search.", + ) + parser.add_argument( + "--app-flow", + action="store_true", + help="shortcut for --stage app-flow.", + ) + parser.add_argument( + "--require-tool", + action="store_true", + help="with --live --stage chat or --live --stage app-flow, fail unless the model executes a Reachy tool.", + ) + parser.add_argument( + "--real-reachy", + action="store_true", + help="with --live --stage app-flow, use the real Reachy SDK/daemon instead of fake tool dependencies.", + ) + parser.add_argument( + "--robot-name", + help="optional Reachy daemon robot name/prefix, used with --real-reachy.", + ) + return parser + + +def main() -> None: + """CLI entry point.""" + parser = build_parser() + args = parser.parse_args() + raise SystemExit(asyncio.run(run_check(args))) + + +if __name__ == "__main__": + main() diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/backend_runtime.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/backend_runtime.py new file mode 100644 index 00000000..f94f3c9f --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/backend_runtime.py @@ -0,0 +1,209 @@ +from typing import Final, Literal +from dataclasses import dataclass + +from reachy_mini_conversation_app.config import ( + BACKEND_LOCAL_STT, + BACKEND_PROVIDERS, + BACKEND_HF_REALTIME, + BACKEND_OPENAI_REALTIME, + HF_REALTIME_CONNECTION_LOCAL, + config, + is_configured_value, + openai_realtime_api_key, +) + + +OPENAI_COMPATIBLE_STREAM_SAMPLE_RATE: Final[int] = 24_000 +HF_REALTIME_STREAM_SAMPLE_RATE: Final[int] = 16_000 + +BackendTransport = Literal["realtime", "local_stt", "unknown"] + + +@dataclass(frozen=True) +class BackendRuntimeSpec: + """Runtime traits derived from the single BACKEND_PROVIDER selector.""" + + provider: str + transport: BackendTransport + stream_sample_rate: int + realtime_model: str = "" + realtime_voice: str = "" + refresh_realtime_client_on_retry: bool = False + + @property + def uses_realtime(self) -> bool: + """Return whether microphone/text input should use a realtime websocket.""" + return self.transport == "realtime" + + @property + def uses_local_stt(self) -> bool: + """Return whether microphone input should use STT plus Chat Completions.""" + return self.transport == "local_stt" + + +def selected_backend() -> BackendRuntimeSpec: + """Return the runtime backend selected by BACKEND_PROVIDER.""" + provider = config.BACKEND_PROVIDER + if provider == BACKEND_OPENAI_REALTIME: + return BackendRuntimeSpec( + provider=provider, + transport="realtime", + stream_sample_rate=OPENAI_COMPATIBLE_STREAM_SAMPLE_RATE, + realtime_model=config.OPENAI_REALTIME_MODEL or "", + realtime_voice=config.OPENAI_REALTIME_VOICE or "", + refresh_realtime_client_on_retry=False, + ) + + if provider == BACKEND_HF_REALTIME: + return BackendRuntimeSpec( + provider=provider, + transport="realtime", + stream_sample_rate=HF_REALTIME_STREAM_SAMPLE_RATE, + realtime_model=(config.HF_REALTIME_MODEL or "").strip(), + realtime_voice=config.HF_REALTIME_VOICE or "", + refresh_realtime_client_on_retry=True, + ) + + if provider == BACKEND_LOCAL_STT: + return BackendRuntimeSpec( + provider=provider, + transport="local_stt", + stream_sample_rate=OPENAI_COMPATIBLE_STREAM_SAMPLE_RATE, + ) + + return BackendRuntimeSpec( + provider=provider, + transport="unknown", + stream_sample_rate=OPENAI_COMPATIBLE_STREAM_SAMPLE_RATE, + ) + + +def configured_marker(value: str | None) -> str: + """Return a display-safe configured/missing marker.""" + return "configured" if is_configured_value(value) else "missing" + + +def local_stt_chat_config_error() -> str | None: + """Return a config error for the local-STT Chat Completions component.""" + if not is_configured_value(config.CHAT_API_KEY): + return ( + "CHAT_API_KEY is missing for BACKEND_PROVIDER=local_stt. If .env uses " + "CHAT_API_KEY=${NVIDIA_INFERENCE_API_KEY}, make sure NVIDIA_INFERENCE_API_KEY is exported " + "in the shell that starts the app." + ) + if not is_configured_value(config.CHAT_BASE_URL): + return "CHAT_BASE_URL is missing for BACKEND_PROVIDER=local_stt." + if not is_configured_value(config.CHAT_MODEL_NAME): + return "CHAT_MODEL_NAME is missing for BACKEND_PROVIDER=local_stt." + return None + + +def local_stt_transcription_config_error() -> str | None: + """Return a config error for the local-STT speech-to-text component.""" + if not is_configured_value(config.STT_API_KEY): + return "STT_API_KEY is missing for BACKEND_PROVIDER=local_stt." + if not is_configured_value(config.STT_BASE_URL): + return "STT_BASE_URL is missing for BACKEND_PROVIDER=local_stt." + if not is_configured_value(config.STT_MODEL_NAME): + return "STT_MODEL_NAME is missing for BACKEND_PROVIDER=local_stt." + return None + + +def local_stt_tts_config_error() -> str | None: + """Return a config error for the local-STT text-to-speech component.""" + if not is_configured_value(config.TTS_API_KEY): + return "TTS_API_KEY is missing for BACKEND_PROVIDER=local_stt." + if not is_configured_value(config.TTS_BASE_URL): + return "TTS_BASE_URL is missing for BACKEND_PROVIDER=local_stt." + if not is_configured_value(config.TTS_MODEL_NAME): + return "TTS_MODEL_NAME is missing for BACKEND_PROVIDER=local_stt." + return None + + +def backend_config_error() -> str | None: + """Return a startup-blocking config error for the selected backend, if any.""" + backend = selected_backend() + expected = ", ".join(sorted(BACKEND_PROVIDERS)) + if not is_configured_value(backend.provider): + return f"BACKEND_PROVIDER is missing; set it to one of {expected}." + + if backend.provider not in BACKEND_PROVIDERS: + return f"Unknown BACKEND_PROVIDER={backend.provider!r}; expected one of {expected}." + + if backend.provider == BACKEND_OPENAI_REALTIME: + if openai_realtime_api_key() is None: + return ( + "OPENAI_API_KEY is missing for BACKEND_PROVIDER=openai_realtime. " + "Export OPENAI_API_KEY in the shell that starts the app; set OPENAI_REALTIME_API_KEY " + "only if this app needs a different OpenAI key." + ) + if not is_configured_value(config.OPENAI_REALTIME_BASE_URL): + return "OPENAI_REALTIME_BASE_URL is missing for BACKEND_PROVIDER=openai_realtime." + if not is_configured_value(backend.realtime_model): + return "OPENAI_REALTIME_MODEL is missing for BACKEND_PROVIDER=openai_realtime." + return None + + if backend.provider == BACKEND_HF_REALTIME: + if config.HF_REALTIME_CONNECTION_MODE == HF_REALTIME_CONNECTION_LOCAL and not is_configured_value( + config.HF_REALTIME_WS_URL + ): + return "HF_REALTIME_WS_URL is missing for HF_REALTIME_CONNECTION_MODE=local." + return None + + if backend.provider == BACKEND_LOCAL_STT: + return local_stt_chat_config_error() or local_stt_transcription_config_error() or local_stt_tts_config_error() + + return None + + +def local_stt_stage_config_error(stage: str) -> str | None: + """Return the config error relevant to a local_stt live-check stage.""" + if stage in {"stt", "stt-probe"}: + return local_stt_transcription_config_error() + if stage == "chat": + return local_stt_chat_config_error() + if stage == "tts": + return local_stt_tts_config_error() + return backend_config_error() + + +def describe_selected_backend() -> list[str]: + """Return a non-secret summary of the selected backend configuration.""" + backend = selected_backend() + lines = [f"backend={backend.provider}"] + if backend.provider == BACKEND_OPENAI_REALTIME: + lines.extend( + [ + f"openai_realtime.base_url={config.OPENAI_REALTIME_BASE_URL}", + f"openai_realtime.model={backend.realtime_model}", + f"openai_realtime.voice={backend.realtime_voice}", + f"openai_realtime.api_key={configured_marker(openai_realtime_api_key())}", + ] + ) + elif backend.provider == BACKEND_HF_REALTIME: + lines.extend( + [ + f"hf_realtime.connection_mode={config.HF_REALTIME_CONNECTION_MODE}", + f"hf_realtime.session_url={config.HF_REALTIME_SESSION_URL}", + f"hf_realtime.ws_url={config.HF_REALTIME_WS_URL or ''}", + f"hf_realtime.model={backend.realtime_model or ''}", + f"hf_realtime.voice={backend.realtime_voice}", + f"hf_realtime.token={configured_marker(config.HF_TOKEN)}", + ] + ) + elif backend.provider == BACKEND_LOCAL_STT: + lines.extend( + [ + f"chat.base_url={config.CHAT_BASE_URL}", + f"chat.model={config.CHAT_MODEL_NAME}", + f"chat.api_key={configured_marker(config.CHAT_API_KEY)}", + f"stt.base_url={config.STT_BASE_URL}", + f"stt.model={config.STT_MODEL_NAME}", + f"stt.api_key={configured_marker(config.STT_API_KEY)}", + f"tts.base_url={config.TTS_BASE_URL}", + f"tts.model={config.TTS_MODEL_NAME}", + f"tts.voice={config.TTS_VOICE}", + f"tts.api_key={configured_marker(config.TTS_API_KEY)}", + ] + ) + return lines diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/camera_worker.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/camera_worker.py new file mode 100644 index 00000000..046bd8db --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/camera_worker.py @@ -0,0 +1,246 @@ +"""Camera worker thread with frame buffering and face tracking. + +Ported from main_works.py camera_worker() function to provide: +- 30Hz+ camera polling with thread-safe frame buffering +- Face tracking integration with smooth interpolation +- Latest frame always available for tools +""" + +import time +import logging +import threading +from typing import Any, List, Tuple + +import numpy as np +from numpy.typing import NDArray +from scipy.spatial.transform import Rotation as R + +from reachy_mini import ReachyMini +from reachy_mini.utils.interpolation import linear_pose_interpolation + + +logger = logging.getLogger(__name__) + + +class CameraWorker: + """Thread-safe camera worker with frame buffering and face tracking.""" + + def __init__(self, reachy_mini: ReachyMini, head_tracker: Any = None) -> None: + """Initialize.""" + self.reachy_mini = reachy_mini + self.head_tracker = head_tracker + + # Thread-safe frame storage + self.latest_frame: NDArray[np.uint8] | None = None + self.frame_lock = threading.Lock() + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + + # Face tracking state + self.is_head_tracking_enabled = True + self.face_tracking_offsets: List[float] = [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ] # x, y, z, roll, pitch, yaw + self.face_tracking_lock = threading.Lock() + + # Face tracking timing variables (same as main_works.py) + self.last_face_detected_time: float | None = None + self.interpolation_start_time: float | None = None + self.interpolation_start_pose: NDArray[np.float64] | None = None + self.face_lost_delay = 2.0 # seconds to wait before starting interpolation + self.interpolation_duration = 1.0 # seconds to interpolate back to neutral + + # Track state changes + self.previous_head_tracking_state = self.is_head_tracking_enabled + + def get_latest_frame(self) -> NDArray[np.uint8] | None: + """Get the latest frame (thread-safe).""" + with self.frame_lock: + if self.latest_frame is None: + return None + # Return a copy in original BGR format (OpenCV native) + return self.latest_frame.copy() + + def get_face_tracking_offsets( + self, + ) -> Tuple[float, float, float, float, float, float]: + """Get current face tracking offsets (thread-safe).""" + with self.face_tracking_lock: + offsets = self.face_tracking_offsets + return (offsets[0], offsets[1], offsets[2], offsets[3], offsets[4], offsets[5]) + + def set_head_tracking_enabled(self, enabled: bool) -> None: + """Enable/disable head tracking.""" + self.is_head_tracking_enabled = enabled + logger.info(f"Head tracking {'enabled' if enabled else 'disabled'}") + + def start(self) -> None: + """Start the camera worker loop in a thread.""" + self._stop_event.clear() + self._thread = threading.Thread(target=self.working_loop, daemon=True) + self._thread.start() + logger.debug("Camera worker started") + + def stop(self) -> None: + """Stop the camera worker loop.""" + self._stop_event.set() + if self._thread is not None: + self._thread.join() + + logger.debug("Camera worker stopped") + + def working_loop(self) -> None: + """Enable the camera worker loop. + + Ported from main_works.py camera_worker() with same logic. + """ + logger.debug("Starting camera working loop") + + # Initialize head tracker if available + neutral_pose = np.eye(4, dtype=np.float64) # Neutral pose (identity matrix) + self.previous_head_tracking_state = self.is_head_tracking_enabled + + while not self._stop_event.is_set(): + try: + current_time = time.time() + + # Get frame from robot + frame = self.reachy_mini.media.get_frame() + + if frame is not None: + # Thread-safe frame storage + with self.frame_lock: + self.latest_frame = frame # .copy() + + # Check if face tracking was just disabled + if self.previous_head_tracking_state and not self.is_head_tracking_enabled: + # Face tracking was just disabled - start interpolation to neutral + self.last_face_detected_time = current_time # Trigger the face-lost logic + self.interpolation_start_time = None # Will be set by the face-lost interpolation + self.interpolation_start_pose = None + + # Update tracking state + self.previous_head_tracking_state = self.is_head_tracking_enabled + + # Handle face tracking if enabled and head tracker available + if self.is_head_tracking_enabled and self.head_tracker is not None: + eye_center, _ = self.head_tracker.get_head_position(frame) + + if eye_center is not None: + # Face detected - immediately switch to tracking + self.last_face_detected_time = current_time + self.interpolation_start_time = None # Stop any interpolation + + # Convert normalized coordinates to pixel coordinates + h, w, _ = frame.shape + eye_center_norm = (eye_center + 1) / 2 + eye_center_pixels = [ + eye_center_norm[0] * w, + eye_center_norm[1] * h, + ] + + # Get the head pose needed to look at the target, but don't perform movement + target_pose = self.reachy_mini.look_at_image( + eye_center_pixels[0], + eye_center_pixels[1], + duration=0.0, + perform_movement=False, + ) + + # Extract translation and rotation from the target pose directly + translation = target_pose[:3, 3] + rotation = R.from_matrix(target_pose[:3, :3]).as_euler("xyz", degrees=False) + + # Scale down translation and rotation because smaller FOV + translation *= 0.6 + rotation *= 0.6 + + # Thread-safe update of face tracking offsets (use pose as-is) + with self.face_tracking_lock: + self.face_tracking_offsets = [ + translation[0], + translation[1], + translation[2], # x, y, z + rotation[0], + rotation[1], + rotation[2], # roll, pitch, yaw + ] + + # No face detected while tracking enabled - set face lost timestamp + elif self.last_face_detected_time is None or self.last_face_detected_time == current_time: + # Only update if we haven't already set a face lost time + # (current_time check prevents overriding the disable-triggered timestamp) + pass + + # Handle smooth interpolation (works for both face-lost and tracking-disabled cases) + if self.last_face_detected_time is not None: + time_since_face_lost = current_time - self.last_face_detected_time + + if time_since_face_lost >= self.face_lost_delay: + # Start interpolation if not already started + if self.interpolation_start_time is None: + self.interpolation_start_time = current_time + # Capture current pose as start of interpolation + with self.face_tracking_lock: + current_translation = self.face_tracking_offsets[:3] + current_rotation_euler = self.face_tracking_offsets[3:] + # Convert to 4x4 pose matrix + pose_matrix = np.eye(4, dtype=np.float64) + pose_matrix[:3, 3] = current_translation + pose_matrix[:3, :3] = R.from_euler( + "xyz", + current_rotation_euler, + ).as_matrix() + self.interpolation_start_pose = pose_matrix + + # Calculate interpolation progress (t from 0 to 1) + interpolation_start_time = self.interpolation_start_time + interpolation_start_pose = self.interpolation_start_pose + if interpolation_start_time is None or interpolation_start_pose is None: + continue + + elapsed_interpolation = current_time - interpolation_start_time + t = min(1.0, elapsed_interpolation / self.interpolation_duration) + + # Interpolate between current pose and neutral pose + interpolated_pose = linear_pose_interpolation( + interpolation_start_pose, + neutral_pose, + t, + ) + + # Extract translation and rotation from interpolated pose + translation = interpolated_pose[:3, 3] + rotation = R.from_matrix(interpolated_pose[:3, :3]).as_euler("xyz", degrees=False) + + # Thread-safe update of face tracking offsets + with self.face_tracking_lock: + self.face_tracking_offsets = [ + translation[0], + translation[1], + translation[2], # x, y, z + rotation[0], + rotation[1], + rotation[2], # roll, pitch, yaw + ] + + # If interpolation is complete, reset timing + if t >= 1.0: + self.last_face_detected_time = None + self.interpolation_start_time = None + self.interpolation_start_pose = None + # else: Keep current offsets (within 2s delay period) + + # Small sleep to prevent excessive CPU usage (same as main_works.py) + time.sleep(0.04) + + except Exception as e: + logger.error(f"Camera worker error: {e}") + time.sleep(0.1) # Longer sleep on error + + logger.debug("Camera worker thread exited") diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/chat_completions.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/chat_completions.py new file mode 100644 index 00000000..882f6d0b --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/chat_completions.py @@ -0,0 +1,270 @@ +"""Chat Completions text/tool loop for the local-STT backend.""" + +import re +import json +import uuid +import asyncio +import logging +from typing import Any, Final, cast + +from reachy_mini_conversation_app.prompts import get_session_instructions +from reachy_mini_conversation_app.tools.core_tools import ( + ToolDependencies, + get_tool_specs, + dispatch_tool_call_with_manager, +) +from reachy_mini_conversation_app.tools.background_tool_manager import BackgroundToolManager + + +logger = logging.getLogger(__name__) + +_RATE_LIMIT_ATTEMPTS: Final[int] = 3 +_RATE_LIMIT_DEFAULT_DELAY: Final[float] = 5.0 +_RATE_LIMIT_MAX_DELAY: Final[float] = 30.0 +_TOOL_ROUND_LIMIT: Final[int] = 5 +_WAIT_RE: Final[re.Pattern[str]] = re.compile( + r"(?:please\s+)?wait\s+(\d+(?:\.\d+)?)\s+seconds?", + re.IGNORECASE, +) + + +def chat_completion_tool_specs() -> list[dict[str, Any]]: + """Convert Realtime-style tool specs to Chat Completions tool specs.""" + chat_tools: list[dict[str, Any]] = [] + for tool in get_tool_specs(): + if tool.get("type") != "function": + continue + chat_tools.append( + { + "type": "function", + "function": { + "name": tool.get("name"), + "description": tool.get("description", ""), + "parameters": tool.get("parameters", {}), + }, + } + ) + return chat_tools + + +def _tool_call_value(tool_call: Any, name: str) -> Any: + """Read a value from either an SDK model object or a dict.""" + if isinstance(tool_call, dict): + return tool_call.get(name) + return getattr(tool_call, name, None) + + +def _message_value(message: Any, name: str) -> Any: + """Read a chat message value from either an SDK model object or a dict.""" + if isinstance(message, dict): + return message.get(name) + return getattr(message, name, None) + + +def _choice_message(choice: Any) -> Any: + """Read a Chat Completions choice message from either an object or a dict.""" + if isinstance(choice, dict): + return choice.get("message") + return getattr(choice, "message", None) + + +def _tool_call_function_value(tool_call: Any, name: str) -> Any: + """Read a function value from either an SDK tool-call object or a dict.""" + function = _tool_call_value(tool_call, "function") + if isinstance(function, dict): + return function.get(name) + return getattr(function, name, None) + + +def _serialize_tool_call(tool_call: Any) -> dict[str, Any]: + """Convert a tool call into the dict shape Chat Completions expects.""" + if hasattr(tool_call, "model_dump"): + return tool_call.model_dump() + if isinstance(tool_call, dict): + return tool_call + return { + "id": getattr(tool_call, "id", None), + "type": getattr(tool_call, "type", "function"), + "function": { + "name": getattr(getattr(tool_call, "function", None), "name", None), + "arguments": getattr(getattr(tool_call, "function", None), "arguments", "{}"), + }, + } + + +def _rate_limit_delay(exc: Exception) -> float | None: + """Return a retry delay when a Chat Completions error is a provider rate limit.""" + status_code = getattr(exc, "status_code", None) + message = str(exc) + is_rate_limit = ( + status_code == 429 + or "ratelimit" in type(exc).__name__.lower() + or "rate limit" in message.lower() + or "throttling" in message.lower() + ) + if not is_rate_limit: + return None + + response = getattr(exc, "response", None) + headers = getattr(response, "headers", None) + if headers is not None: + retry_after = headers.get("retry-after") or headers.get("Retry-After") + if retry_after: + try: + return min(float(retry_after), _RATE_LIMIT_MAX_DELAY) + except (TypeError, ValueError): + pass + + match = _WAIT_RE.search(message) + if match: + try: + return min(float(match.group(1)), _RATE_LIMIT_MAX_DELAY) + except (TypeError, ValueError): + pass + + return _RATE_LIMIT_DEFAULT_DELAY + + +class ChatCompletionRunner: + """Run a Chat Completions turn, including Reachy tool calls.""" + + def __init__( + self, + *, + client: Any, + deps: ToolDependencies, + tool_manager: BackgroundToolManager, + model_name: str, + base_url: str | None, + ) -> None: + """Initialize the runner.""" + self.client = client + self.deps = deps + self.tool_manager = tool_manager + self.model_name = model_name + self.base_url = base_url + + async def create_with_retries(self, operation: str, **kwargs: Any) -> Any: + """Create a Chat Completions response, retrying short provider rate limits.""" + for attempt in range(1, _RATE_LIMIT_ATTEMPTS + 1): + try: + return await self.client.chat.completions.create(**kwargs) + except Exception as e: + delay = _rate_limit_delay(e) + if delay is None or attempt == _RATE_LIMIT_ATTEMPTS: + raise + + logger.warning( + "%s rate-limited by provider; retrying in %.1fs (%d/%d)", + operation, + delay, + attempt, + _RATE_LIMIT_ATTEMPTS, + ) + await asyncio.sleep(delay) + + raise RuntimeError("unreachable Chat Completions retry state") + + async def send_text_message(self, text: str) -> list[dict[str, Any]]: + """Send a user text turn through Chat Completions and Reachy tools.""" + chatbot_messages: list[dict[str, Any]] = [{"role": "user", "content": text}] + chat_messages: list[dict[str, Any]] = [ + {"role": "system", "content": get_session_instructions()}, + {"role": "user", "content": text}, + ] + chat_tools = chat_completion_tool_specs() + + operation = "Chat Completions request" + request_kwargs: dict[str, Any] = { + "model": self.model_name, + "messages": cast(Any, chat_messages), + "tools": cast(Any, chat_tools), + "tool_choice": "auto", + } + + for tool_round in range(_TOOL_ROUND_LIMIT + 1): + try: + completion = await self.create_with_retries(operation, **request_kwargs) + except Exception as e: + logger.exception("%s failed", operation) + chatbot_messages.append( + { + "role": "assistant", + "content": ( + f"[error] {operation} failed " + f"(model={self.model_name!r}, base_url={self.base_url!r}): " + f"{type(e).__name__}: {e}" + ), + } + ) + return chatbot_messages + + choice = completion.choices[0] if completion.choices else None + assistant_message = _choice_message(choice) if choice else None + assistant_content = _message_value(assistant_message, "content") or "" + tool_calls = _message_value(assistant_message, "tool_calls") or [] + + if not tool_calls: + chatbot_messages.append({"role": "assistant", "content": assistant_content or "[no response]"}) + return chatbot_messages + + if tool_round >= _TOOL_ROUND_LIMIT: + chatbot_messages.append( + { + "role": "assistant", + "content": ( + f"[error] Chat Completions exceeded the configured tool round limit ({_TOOL_ROUND_LIMIT})." + ), + } + ) + return chatbot_messages + + chat_messages.append( + { + "role": "assistant", + "content": assistant_content, + "tool_calls": [_serialize_tool_call(tool_call) for tool_call in tool_calls], + } + ) + + for tool_call in tool_calls: + tool_call_id = _tool_call_value(tool_call, "id") or str(uuid.uuid4()) + tool_name = _tool_call_function_value(tool_call, "name") + args_json = _tool_call_function_value(tool_call, "arguments") or "{}" + if not isinstance(tool_name, str): + tool_result = {"error": "tool call did not include a valid function name"} + else: + tool_result = await dispatch_tool_call_with_manager( + tool_name, + args_json, + self.deps, + self.tool_manager, + ) + + tool_result_json = json.dumps(tool_result) + chat_messages.append( + { + "role": "tool", + "tool_call_id": tool_call_id, + "content": tool_result_json, + } + ) + chatbot_messages.append( + { + "role": "assistant", + "content": tool_result_json, + "metadata": { + "title": f"Used tool {tool_name or 'unknown'}", + "status": "done", + }, + } + ) + + operation = "Chat Completions tool follow-up" + request_kwargs = { + "model": self.model_name, + "messages": cast(Any, chat_messages), + "tools": cast(Any, chat_tools), + } + + raise RuntimeError("unreachable Chat Completions tool round state") diff --git a/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/config.py b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/config.py new file mode 100644 index 00000000..d6ab93f2 --- /dev/null +++ b/projects/reachy-mini-openshell/src/reachy_mini_conversation_app/config.py @@ -0,0 +1,499 @@ +import os +import re +import sys +import logging +from typing import Mapping +from pathlib import Path +from dataclasses import dataclass +from urllib.parse import urlsplit, parse_qsl, urlunsplit + +from dotenv import find_dotenv, dotenv_values + + +LOCKED_PROFILE = "_reachy_mini_conversation_app_locked_profile" +DEFAULT_PROFILES_DIRECTORY = Path(__file__).parent / "profiles" + +BACKEND_OPENAI_REALTIME = "openai_realtime" +BACKEND_HF_REALTIME = "hf_realtime" +BACKEND_LOCAL_STT = "local_stt" +BACKEND_PROVIDERS = { + BACKEND_OPENAI_REALTIME, + BACKEND_HF_REALTIME, + BACKEND_LOCAL_STT, +} + +HF_REALTIME_CONNECTION_DEPLOYED = "deployed" +HF_REALTIME_CONNECTION_LOCAL = "local" +HF_REALTIME_SESSION_PROXY_URL = "https://pollen-robotics-reachy-mini-realtime-url.hf.space/session" + +logger = logging.getLogger(__name__) +_MARKDOWN_URL_RE = re.compile(r"^\[(?P