diff --git a/.claude/agents/dhruva-signals.md b/.claude/agents/dhruva-signals.md new file mode 100644 index 0000000..f4a9269 --- /dev/null +++ b/.claude/agents/dhruva-signals.md @@ -0,0 +1,21 @@ +--- +name: dhruva-signals +description: Dhruva's signals agent for Pegasus. Use for work in /signals — behavioral sentiment analysis (Claude), typing dynamics, the :8002 FastAPI service, Twilio SMS alerts, and the collector.js keystroke tracker. MUST stay inside signals/. +tools: Read, Write, Edit, Bash, Grep, Glob +--- + +You are **Dhruva**, the signals engineer on the Pegasus team. + +**Hard boundary:** You may ONLY create/edit files inside `signals/`. Never touch +`frontend/`, `backend/`, `ml/`, or `shared/`. Ask Jason for contract changes. + +**Your service** (FastAPI on :8002): `/health`, `/signals/analyze` (Claude +`claude-sonnet-4-6` sentiment + energy + flags + `combined_signal_score`, with a +lexical fallback when no API key), `/signals/{user_id}`, `/alert` (Twilio SMS on +red; no-ops to console when creds are absent so the demo never crashes). + +You also own `signals/collector.js` (`KeystrokeTracker`), which the frontend +imports — a synced copy lives at `frontend/src/keystroke.js`. Keep them in sync. + +Read `signals/AGENTS.md`, `AGENTS.md`, and `shared/contract.md` first. +Stage only `signals/`. Commit to `feat/dhruva-sig`, PR into `dev`. diff --git a/.claude/agents/imessage-bridge.md b/.claude/agents/imessage-bridge.md new file mode 100644 index 0000000..f5ce526 --- /dev/null +++ b/.claude/agents/imessage-bridge.md @@ -0,0 +1,27 @@ +--- +name: imessage-bridge +description: Agent for the standalone /imessage bridge. Use for work on the macOS iMessage delivery channel — AppleScript send, chat.db polling, the bridge CLI/loop, phone↔user registry. MUST stay inside imessage/ and only call the backend API. +tools: Read, Write, Edit, Bash, Grep, Glob +--- + +You own the **`imessage/` bridge** — a standalone macOS delivery channel for +Pegasus (owner TBD; not one of the four service lanes). + +**Hard boundary:** ONLY create/edit files inside `imessage/`. Never touch +`frontend/`, `backend/`, `ml/`, `signals/`, or `shared/`. Talk to the **backend +only** (:8001): `POST /users`, `GET /stimulus/today`, `POST /checkin`. + +**How it works (macOS, no API):** send iMessages via AppleScript driving +Messages.app (`send.applescript` + `imessage.send`); receive by polling +`~/Library/Messages/chat.db` read-only (`imessage.fetch_new`), decoding +`attributedBody` when the `text` column is NULL. `bridge.py` provides +`register` / `send` / `send-daily` / `listen`. `registry.py` maps phone ⇄ +user_id and auto-registers new senders. + +**Constraints to respect:** needs Full Disk Access + a signed-in iMessage +account on the Mac; AppleScript send syntax is macOS-version-sensitive; no +keystroke dynamics over iMessage (typing metrics sent as 0). Keep it runnable +and never replay old chat history (start from current max ROWID). + +Read `imessage/AGENTS.md`, `AGENTS.md`, and `shared/contract.md` first. +Stage only `imessage/`. diff --git a/.claude/agents/jason-backend.md b/.claude/agents/jason-backend.md new file mode 100644 index 0000000..75075da --- /dev/null +++ b/.claude/agents/jason-backend.md @@ -0,0 +1,22 @@ +--- +name: jason-backend +description: Jason's backend agent for Pegasus. Use for work in /backend and /shared — the :8001 orchestrator, SQLite persistence, daily stimuli, and the API contract. MUST stay inside backend/ and shared/. +tools: Read, Write, Edit, Bash, Grep, Glob +--- + +You are **Jason**, the backend engineer and contract owner on the Pegasus team. + +**Hard boundary:** You may ONLY create/edit files inside `backend/` and +`shared/`. Never touch `frontend/`, `ml/`, or `signals/` — call their HTTP APIs. + +**Your service** (FastAPI on :8001) is the orchestrator and the ONLY service +the frontend talks to: `/health`, `/users`, `/stimulus/today`, `/checkin` +(fans out to Signals :8002 + ML :8003, persists, alerts on red), `/status/{id}`. + +**You own the contract.** `shared/contract.md` is the source of truth. When you +change it, note which service owner is affected so they stay shape-compatible. +Keep `orchestrator.py` degrading gracefully when a downstream service is down — +the demo must never hard-fail. + +Read `backend/AGENTS.md`, `AGENTS.md`, and `shared/contract.md` first. +Stage only `backend/` and `shared/`. Commit to `feat/jason-api`, PR into `dev`. diff --git a/.claude/agents/rishith-ml.md b/.claude/agents/rishith-ml.md new file mode 100644 index 0000000..8794abc --- /dev/null +++ b/.claude/agents/rishith-ml.md @@ -0,0 +1,29 @@ +--- +name: rishith-ml +description: Rishith's ML + Video agent for Pegasus. Use for work in /ml (TRIBE v2 brain prediction, the 4-stream combined burnout scorer, Claude interventions, :8003) and /video (MediaPipe facial + Whisper voice stress, :8004). MUST stay inside ml/ and video/. +tools: Read, Write, Edit, Bash, Grep, Glob +--- + +You are **Rishith**, the ML / brain-pipeline / video engineer on the Pegasus team. + +**Hard boundary:** You may ONLY create/edit files inside `ml/` and `video/`. +Never touch `frontend/`, `backend/`, `signals/`, or `shared/`. Call their HTTP +API or read (never edit) `shared/contract.md`. If you need a contract change, +report it for Jason — don't edit shared/. + +**ML service (:8003):** `/health`, `/predict` (TRIBE v2 healthy-brain prediction ++ per-region activations), `GET /baseline/{stimulus_id}` (cached, for Brain +View), `/score` (merges imessage + typing + facial + voice + tribe streams via +`combined_scorer.py`, renormalizing over whichever are present). Intervention +(`claude_interpreter.py`, `claude-sonnet-4-6`, offline fallback) is part of `/score`. + +**Video service (:8004):** `/health`, `/analyze/video` (facial+voice), +`/analyze/frame`. MediaPipe Face Mesh (`facial_analyzer.py`) + Whisper/librosa +(`voice_analyzer.py`); heavy models load lazily so `/health` is instant. + +**Headline task:** `ml/tribe_inference.py` ships a seeded STUB (`TribeModel._infer`). +Wire real TRIBE v2 inference there, keeping the return shape so the pipeline is +unaffected. Keep both services' `/health` green and runnable at every commit. + +Read `ml/AGENTS.md`, `video/AGENTS.md`, `AGENTS.md`, and `shared/contract.md` +first. Stage only `ml/` and `video/`. Commit to `feat/rishith-ml`, PR into `dev`. diff --git a/.claude/agents/wesley-frontend.md b/.claude/agents/wesley-frontend.md new file mode 100644 index 0000000..90f4f16 --- /dev/null +++ b/.claude/agents/wesley-frontend.md @@ -0,0 +1,23 @@ +--- +name: wesley-frontend +description: Wesley's frontend agent for Pegasus. Use for work in /frontend — the React (Vite) UI on :3000, the check-in flow, keystroke capture, and the Check Engine Light component. MUST stay inside frontend/. +tools: Read, Write, Edit, Bash, Grep, Glob +--- + +You are **Wesley**, the frontend engineer on the Pegasus team. + +**Hard boundary:** You may ONLY create/edit files inside `frontend/`. Never +touch `backend/`, `ml/`, `signals/`, or `shared/`. You talk to the **backend +only** (:8001) through the Vite `/api` proxy (already configured in +`vite.config.js`) — never call ml/signals directly. + +**The UI flow:** create/get user → `GET /api/stimulus/today` → show stimulus → +capture response with `KeystrokeTracker` (`src/keystroke.js`) → `POST +/api/checkin` → render the Check Engine Light (🟢/🟡/🔴) + intervention via +`EngineLight.jsx`. Keep `src/keystroke.js` in sync with Dhruva's +`signals/collector.js`. + +Run with `npm install && npm run dev` (port 3000). Keep it building. + +Read `frontend/AGENTS.md`, `AGENTS.md`, and `shared/contract.md` first. +Stage only `frontend/`. Commit to `feat/wesley-ui`, PR into `dev`. diff --git a/.gitignore b/.gitignore index 4a533e1..91f0ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,14 @@ wandb/ # Logs *.log + +# iMessage bridge runtime (registry.json holds phone numbers — never commit) +imessage/.state.json +imessage/registry.json +imessage/*.scpt + +# Expo / React Native +.expo/ +frontend/.expo/ +frontend/web-build/ +*.tsbuildinfo diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..14bdb75 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,59 @@ +# Pegasus — Agent Rules (root) + +Read this, then read the `AGENTS.md` in **your** folder. Every teammate runs +their own coding agent; these files keep agents in their lane. + +## What Pegasus is +A mental-health "check engine light." Four signal streams (SMS responses, +typing biometrics, video facial/voice, and the in-app check-in) feed +**TRIBE v2** (a brain foundation model deployed on Modal). We compare the user's +real behavioral signals against TRIBE's healthy-brain prediction → deviation = +burnout score → 🟢/🟡/🔴 + a Claude intervention → SMS alert if red. + +## Team & services +| Owner | Folder(s) | Port(s) | +|---------|---------------------------------------------|----------------| +| Rishith | frontend **data layer** + `ml/` + `video/` | 3000, 8003, 8004 | +| Wesley | frontend **screens/components** | 3000 | +| Jason | `backend/` + `shared/` | 8001 | +| Dhruva | `signals/` (Twilio SMS bot + alerts) | 8002 | + +`frontend/` is an **Expo** app shared by Rishith (services/hooks/types) and +Wesley (App.tsx/screens/components/navigation/utils) — see `frontend/AGENTS.md`. +`imessage/` is a legacy macOS bridge, **superseded by Twilio SMS in `signals/`**. + +## The 5 rules (non-negotiable) +1. **Stay in your files.** Only edit what you own (table above + per-folder + AGENTS.md). Need something from another area? **Call its API / import it — never edit it.** +2. **The contract is law.** `shared/contract.md` is the source of truth. Need a + field changed? Jason changes it there first, then tells the owner. +3. **Branch + PR.** Work on `feat/-*`, PR into `dev`. Never push to `main`. + Stage only your files — never `git add .` / `git add -A`. +4. **Stay runnable.** Your service must start on its port and answer `GET /health`. +5. **Sync every ~45 min** (below) so integration never drifts far. + +## The 45-minute sync (everyone, every ~45 min) +```bash +# 1. Push your work + open/merge a PR into dev +git add YOUR_FILES_ONLY/ # e.g. Rishith: ml/ video/ frontend/src/{services,hooks,types}/ +git commit -m "feat: what you did" +git push origin feat/ +# → open PR feat/ → dev on GitHub, merge it + +# 2. Pull everyone else's merged work back into your branch +git checkout dev && git pull origin dev +git checkout feat/ && git merge dev +# keep building +``` + +## Secrets +Each service reads keys from a `.env` in its folder (gitignored). NEVER commit +`.env` or put keys in code. Templates: `*/.env.example`. + +## Run locally +```bash +./dev.sh # ml + signals + backend (+ WITH_VIDEO=1 for video) +cd frontend && npx expo start +``` +The frontend (Expo) talks ONLY to the backend (:8001); backend fans out to +signals + ml; ml calls TRIBE on Modal; video runs facial/voice on :8004. diff --git a/README.md b/README.md index 4b1bae5..ccec9fb 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,141 @@ -# pegasus +# Pegasus -## Branching model +Pegasus is a multimodal burnout detection and mental wellness platform. It works like a check engine light for the mind — passively collecting behavioral signals throughout the day, fusing them into a single wellness score, and surfacing a grounded, personalized intervention when that score drops. + +## Architecture + +Pegasus is composed of four services that communicate over HTTP: + +| Service | Port | Stack | Owner | +|---|---|---|---| +| Backend orchestrator | 8001 | FastAPI, SQLite | Jason | +| Signals processor | 8002 | FastAPI, HuggingFace | Dhruva | +| ML / TRIBE scorer | 8003 | FastAPI, Modal, HuggingFace | Rishith | +| Video analyzer | 8004 | FastAPI | Rishith | + +The Expo React Native mobile app connects to the backend at port 8001. The SMS bot runs alongside the signals service and polls iMessage via a local AppleScript bridge. + +## Features + +### Multimodal Wellness Scoring + +The scoring engine fuses up to five behavioral streams into a single wellness score from 0 to 100 (higher is better). Each stream contributes a burnout sub-score that is weighted and normalized depending on which signals are actually present in a given check-in. + +| Stream | Weight | Signal | +|---|---|---| +| iMessage / text sentiment | 25% | HuggingFace emotion model (j-hartmann/emotion-english-distilroberta-base) | +| Typing biometrics | 20% | WPM, error rate, hesitation count, key hold time, flight time, correction loops | +| Facial stress | 30% | Video frame analysis via the video service | +| Voice stress | 15% | Audio analysis via the video service | +| TRIBE deviation | 10% | Distance from the user's healthy brain-activity baseline | + +The score is inverted from a burnout reading so that rising numbers always mean the user is doing better. Scores above 70 are green, 40-69 are yellow, and below 40 are red. + +### TRIBE Baseline Model + +TRIBE (Temporal Representation of Integrated Brain Events) v2 is a deployed Modal endpoint that models healthy brain activation patterns across six macro-regions: Attention, Auditory, Emotion, Language, Motor, and Visual. For each daily stimulus, Pegasus calls TRIBE once to establish the user's expected healthy-state baseline, then measures deviation from that baseline as a behavioral burnout signal. TRIBE results are cached per stimulus to avoid repeated GPU calls. + +### Crisis and Masking Detection + +The scoring engine applies a keyword override layer on top of the emotion model to catch signals the model tends to under-read: + +- Crisis terms (suicidal ideation, self-harm language) force the iMessage sub-score to 100 regardless of the emotion model output. +- Concern terms (hopeless, burned out, can't cope, etc.) floor the sub-score proportionally to the number of terms present. +- Masking language (clustered over-reassurance such as "I'm fine, totally fine, don't worry") is detected as a tell and pulls the score out of the green even when the emotion model reads positive. + +When the overall score reaches red, the backend routes the user to human support rather than an automated intervention. + +### RAG Intervention Engine + +When the wellness score drops, Pegasus generates a single, warm, specific recommended action using a retrieval-augmented generation pipeline: + +1. A mental health coping corpus is embedded using sentence-transformers/all-MiniLM-L6-v2. +2. The most relevant snippet is retrieved using cosine similarity. +3. A HuggingFace chat model (Qwen/Qwen2.5-7B-Instruct by default) generates a grounded response using only the retrieved content. + +The pipeline degrades gracefully at every step: if HuggingFace is unreachable, the raw retrieved snippet is returned; if the corpus is unavailable, a level-appropriate canned response is used. The intervention call never raises. + +### Signal Collection + +The signals service collects and analyzes behavioral data from multiple sources: + +- **iMessage polling** — an AppleScript bridge reads the local Messages database and extracts response text, response time, and typing metadata. +- **Keystroke biometrics** — a JavaScript collector (collector.js) captures WPM, error rate, hesitation count, key hold duration, inter-key flight time, burst pattern, and correction loops from in-app typing sessions. +- **Sentiment analysis** — HuggingFace Inference API replaces any Claude-based sentiment pipeline; the model degrades to a neutral 0.5 score if the API is unreachable. +- **Linguistic analysis** — structural and lexical features of response text. +- **Temporal analysis** — time-of-day and response latency patterns. + +The signals service maintains a rolling in-memory history of the last 30 readings per user and exposes them to the backend via a typed JSON contract. + +### SMS Bot + +The SMS bot provides a low-friction check-in channel for users who are not on the mobile app: + +- Outbound check-in prompts are sent via Bloo.io SMS. +- Inbound replies are received via a webhook and routed through the full signal analysis and scoring pipeline. +- Automated reminders run on a configurable cadence. +- Stimulus prompts are sent on the same schedule as the in-app experience. + +### Video Check-in + +The mobile app includes a video check-in flow: + +1. The user records a short clip via the in-app camera (60-second maximum, with a countdown ring). +2. The clip is submitted to the video service, which extracts facial stress and voice stress scores. +3. Results are fused into the wellness score and displayed alongside the brain visualization immediately after analysis. + +### Mobile App + +The Expo React Native app is the primary user-facing interface: + +- **Home screen** — an animated breathing orb displays the current wellness score in the user's level color (green, yellow, red), along with the top behavioral indicators and the current intervention suggestion. A red reading triggers a haptic alert and a full-screen support prompt. +- **Check-in screen** — live camera with face-oval overlay, rotating wellness prompts, recording controls, and a post-analysis results view. +- **Brain screen** — a 3D brain visualization that highlights the TRIBE regions currently flagged as elevated. +- **Metrics screen** — historical wellness score charts and breakdown by signal stream. +- **History screen** — a chronological log of past check-ins and score readings. +- **Chat / Talk screen** — a conversational interface for in-app text check-ins. + +The UI uses a liquid glass visual language with level-reactive accent colors throughout. + +### Backend API + +The FastAPI backend at port 8001 is the single point of integration for the mobile app and SMS bot. It persists all readings to SQLite with WAL mode for concurrency, handles first-time-user race conditions, and exposes the following route groups: + +| Route | Purpose | +|---|---| +| `/stimulus` | Deliver the daily check-in prompt | +| `/response` | Accept a text or keystroke-biometric check-in response | +| `/score` | Return the latest wellness score for a user | +| `/brain` | Return TRIBE brain region data for the 3D visualization | +| `/history` | Return the user's check-in history | +| `/metrics` | Return aggregated score and breakdown data for charts | +| `/video` | Accept a video submission and return facial/voice analysis | + +### Infrastructure + +A launchd supervisor script keeps all four services, the iMessage poller, and the ngrok tunnel alive, restarting any process that exits. The sync script handles safe periodic state synchronization without reverting or deleting data. + +## Branch Structure ``` -main (protected — NEVER push directly) - └── dev (integration branch — PRs only) - ├── feat/rishith-ml → /ml/* - ├── feat/wesley-ui → /frontend/* - ├── feat/jason-api → /backend/*, /shared/* - └── feat/dhruva-sig → /signals/* +main + └── dev + ├── feat/jason-api backend/, shared/ + ├── feat/wesley-ui frontend/ + ├── feat/rishith-ml ml/ + └── feat/dhruva-sig signals/ ``` -## Ownership +`feat/integration` and `feat/scaffold` are integration and scaffolding branches used during active development. + +## License + +MIT License + +Copyright (c) 2024 Pegasus Contributors -| Branch | Owner | Scope | -| ----------------- | ------- | ---------------------- | -| `feat/rishith-ml` | Rishith | `ml/` | -| `feat/wesley-ui` | Wesley | `frontend/` | -| `feat/jason-api` | Jason | `backend/`, `shared/` | -| `feat/dhruva-sig` | Dhruva | `signals/` | +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -## Rules +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -- `main` is protected. No direct pushes. Releases only via PR from `dev`. -- `dev` is the integration branch. Feature branches PR into `dev`. -- Stay in your lane: only touch the directories listed for your branch. -- Rebase or merge `dev` into your feature branch regularly to stay current. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/backend/.env.example b/backend/.env.example new file mode 100644 index 0000000..37add83 --- /dev/null +++ b/backend/.env.example @@ -0,0 +1,4 @@ +# Backend service (Jason). Copy to .env and adjust. +SIGNALS_URL=http://localhost:8002 +ML_URL=http://localhost:8003 +PEGASUS_DB=pegasus.db diff --git a/backend/AGENTS.md b/backend/AGENTS.md new file mode 100644 index 0000000..26c998f --- /dev/null +++ b/backend/AGENTS.md @@ -0,0 +1,38 @@ +# `/backend` + `/shared` — Jason's agent + +You are **Jason**. You own **`backend/` and `shared/`**. You build the +orchestrator + persistence on **:8001**, and you are the keeper of the API +contract. Read `../AGENTS.md` first. + +## Never touch +`/frontend`, `/ml`, `/signals`. Call their APIs (you already do, in +`orchestrator.py`). + +## Your endpoints (:8001) — the ONLY service the frontend calls +- `GET /health` +- `POST /users`, `GET /users/{id}` +- `GET /stimulus/today?user_id=` → daily stimulus +- `POST /checkin` → fans out to Signals + ML, stores result, alerts on red +- `GET /status/{user_id}` → latest + history + +## Files +- `main.py` — FastAPI routes. +- `orchestrator.py` — calls Signals (:8002) + ML (:8003), degrades gracefully. +- `db.py` — SQLite (stdlib), tables `users` + `checkins`. +- `stimuli.py` — deterministic daily stimulus pool. +- `../shared/contract.{md,py}` — **source of truth.** When you change it, ping + the affected owner. + +## Run +```bash +cd backend && python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +uvicorn main:app --reload --port 8001 +``` + +## Git +```bash +git checkout feat/jason-api +git add backend/ shared/ # ONLY these +git commit -m "feat(backend): ..." && git push origin feat/jason-api +``` diff --git a/backend/db.py b/backend/db.py new file mode 100644 index 0000000..8842a2b --- /dev/null +++ b/backend/db.py @@ -0,0 +1,89 @@ +"""SQLite persistence for Pegasus backend. Stdlib only, no ORM.""" +from __future__ import annotations + +import json +import os +import sqlite3 +import uuid +from typing import Dict, List, Optional + +DB_PATH = os.getenv("PEGASUS_DB", os.path.join(os.path.dirname(__file__), "pegasus.db")) + + +def _conn() -> sqlite3.Connection: + conn = sqlite3.connect(DB_PATH) + conn.row_factory = sqlite3.Row + return conn + + +def init_db() -> None: + with _conn() as c: + c.executescript( + """ + CREATE TABLE IF NOT EXISTS users ( + user_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + phone TEXT + ); + CREATE TABLE IF NOT EXISTS checkins ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + stimulus_id TEXT, + score INTEGER, + level TEXT, + deviation REAL, + signal_json TEXT, + intervention_json TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """ + ) + + +def create_user(name: str, phone: Optional[str]) -> Dict: + user_id = uuid.uuid4().hex[:12] + with _conn() as c: + c.execute( + "INSERT INTO users (user_id, name, phone) VALUES (?, ?, ?)", + (user_id, name, phone), + ) + return {"user_id": user_id, "name": name, "phone": phone} + + +def get_user(user_id: str) -> Optional[Dict]: + with _conn() as c: + row = c.execute("SELECT * FROM users WHERE user_id = ?", (user_id,)).fetchone() + return dict(row) if row else None + + +def save_checkin(user_id: str, stimulus_id: str, score: Dict, signal: Dict, intervention: Dict) -> None: + with _conn() as c: + c.execute( + """INSERT INTO checkins + (user_id, stimulus_id, score, level, deviation, signal_json, intervention_json) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + ( + user_id, + stimulus_id, + score.get("score"), + score.get("level"), + score.get("deviation"), + json.dumps(signal), + json.dumps(intervention), + ), + ) + + +def get_history(user_id: str, limit: int = 30) -> List[Dict]: + with _conn() as c: + rows = c.execute( + "SELECT * FROM checkins WHERE user_id = ? ORDER BY id DESC LIMIT ?", + (user_id, limit), + ).fetchall() + out = [] + for r in rows: + d = dict(r) + d["signal"] = json.loads(d.pop("signal_json") or "{}") + d["intervention"] = json.loads(d.pop("intervention_json") or "{}") + out.append(d) + return out diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 0000000..60e822f --- /dev/null +++ b/backend/main.py @@ -0,0 +1,88 @@ +"""Pegasus Backend — orchestrator + persistence. Frontend talks ONLY to this. + +Run (from inside /backend): + uvicorn main:app --reload --port 8001 +""" +from __future__ import annotations + +from typing import Optional + +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +import db +import orchestrator +import stimuli + +app = FastAPI(title="Pegasus Backend", version="0.1.0") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], +) + + +@app.on_event("startup") +def _startup(): + db.init_db() + + +class UserIn(BaseModel): + name: str + phone: Optional[str] = None + + +class CheckInIn(BaseModel): + user_id: str + stimulus_id: str + text_response: str + response_time_ms: int = 0 + typing_wpm: int = 0 + error_rate: float = 0.0 + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/users") +def create_user(body: UserIn): + return db.create_user(body.name, body.phone) + + +@app.get("/users/{user_id}") +def get_user(user_id: str): + user = db.get_user(user_id) + if not user: + raise HTTPException(404, "user not found") + return user + + +@app.get("/stimulus/today") +def stimulus_today(user_id: str = "demo"): + return stimuli.stimulus_for(user_id) + + +@app.post("/checkin") +def checkin(body: CheckInIn): + user = db.get_user(body.user_id) + phone = user.get("phone") if user else None + stimulus = stimuli.stimulus_for(body.user_id) + result = orchestrator.run_checkin(body.model_dump(), stimulus, phone) + db.save_checkin(body.user_id, body.stimulus_id, + {"score": result["score"], "level": result["level"], "deviation": result["deviation"]}, + result["signal"], result["intervention"]) + return result + + +@app.get("/status/{user_id}") +def status(user_id: str): + history = db.get_history(user_id) + return {"latest": history[0] if history else None, "history": history} + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8001) diff --git a/backend/orchestrator.py b/backend/orchestrator.py new file mode 100644 index 0000000..d1e63f9 --- /dev/null +++ b/backend/orchestrator.py @@ -0,0 +1,82 @@ +"""Orchestrates a check-in across Signals (:8002) and ML (:8003). + +Each downstream call is defensive: if a service is down, we degrade instead of +500ing the whole pipeline (important for a live demo). +""" +from __future__ import annotations + +import os +from typing import Dict + +import httpx + +SIGNALS_URL = os.getenv("SIGNALS_URL", "http://localhost:8002") +ML_URL = os.getenv("ML_URL", "http://localhost:8003") +TIMEOUT = float(os.getenv("PEGASUS_HTTP_TIMEOUT", "30")) + + +def _post(url: str, payload: Dict) -> Dict: + r = httpx.post(url, json=payload, timeout=TIMEOUT) + r.raise_for_status() + return r.json() + + +def run_checkin(checkin: Dict, stimulus: Dict, phone: str | None) -> Dict: + # 1. Behavioral signal (Signals service) + try: + signal = _post(f"{SIGNALS_URL}/signals/analyze", { + "user_id": checkin["user_id"], + "text": checkin.get("text_response", ""), + "typing_wpm": checkin.get("typing_wpm", 0), + "error_rate": checkin.get("error_rate", 0), + "response_time_ms": checkin.get("response_time_ms", 0), + }) + except Exception: + signal = {"sentiment_score": 0.5, "energy_level": "medium", + "flags": ["signals_unavailable"], "combined_signal_score": 0.5} + + # 2. Healthy-brain prediction (ML / TRIBE v2) + try: + prediction = _post(f"{ML_URL}/predict", {"stimulus": stimulus}) + except Exception: + prediction = {"predicted_engagement": 0.7, "predicted_valence": 0.7, "activation_summary": []} + + # 3. Deviation -> burnout score (ML) + try: + burnout = _post(f"{ML_URL}/score", { + "stimulus": stimulus, "prediction": prediction, + "signal": signal, "checkin": checkin, + }) + except Exception: + burnout = {"score": 0, "level": "green", "deviation": 0.0, "components": {}} + + # 4. Intervention (ML / Claude) + try: + intervention = _post(f"{ML_URL}/intervention", { + "burnout": burnout, "recent_signals": [signal], + }) + except Exception: + intervention = {"message": "Check-in recorded.", "suggested_action": "Take a short break."} + + # 5. Alert on red (Signals / Twilio) + alerted = False + if burnout.get("level") == "red" and phone: + try: + res = _post(f"{SIGNALS_URL}/alert", { + "user_id": checkin["user_id"], "phone": phone, + "score": burnout.get("score"), "level": "red", + "intervention": intervention.get("message", ""), + }) + alerted = bool(res.get("sent")) + except Exception: + alerted = False + + return { + "score": burnout.get("score"), + "level": burnout.get("level"), + "deviation": burnout.get("deviation"), + "components": burnout.get("components", {}), + "signal": signal, + "intervention": intervention, + "alerted": alerted, + } diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..1a4e6e0 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,5 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +httpx==0.28.1 +python-dotenv==1.0.1 diff --git a/backend/stimuli.py b/backend/stimuli.py new file mode 100644 index 0000000..ab2cad0 --- /dev/null +++ b/backend/stimuli.py @@ -0,0 +1,32 @@ +"""Daily stimulus pool. Picks one stimulus per user per day, deterministically +(so a user sees the same prompt all day, and the demo is reproducible).""" +from __future__ import annotations + +import datetime +import hashlib +from typing import Dict, List + +POOL: List[Dict] = [ + {"stimulus_id": "s1", "type": "text", + "content": "A quiet morning, steam rising off a coffee cup by a rain-streaked window.", + "prompt": "Describe what this scene makes you feel, in a few sentences."}, + {"stimulus_id": "s2", "type": "text", + "content": "An empty inbox and a clear calendar for the rest of the afternoon.", + "prompt": "What's the first thing you'd do with this time?"}, + {"stimulus_id": "s3", "type": "text", + "content": "A friend texts: 'Haven't heard from you in a while — you good?'", + "prompt": "Write the reply you'd actually send right now."}, + {"stimulus_id": "s4", "type": "text", + "content": "Sunlight on a trail, the trees just starting to turn.", + "prompt": "What would you want to be doing in this moment?"}, + {"stimulus_id": "s5", "type": "text", + "content": "You just finished something you'd been putting off for weeks.", + "prompt": "How does that land for you? Say more than 'good'."}, +] + + +def stimulus_for(user_id: str, day: datetime.date | None = None) -> Dict: + day = day or datetime.date.today() + seed = f"{user_id}|{day.isoformat()}" + idx = int(hashlib.sha256(seed.encode()).hexdigest(), 16) % len(POOL) + return POOL[idx] diff --git a/dev.sh b/dev.sh new file mode 100755 index 0000000..916d0ec --- /dev/null +++ b/dev.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Start all of Pegasus locally. First run sets up venvs + node_modules. +# ./dev.sh # set up (if needed) + run everything +# ./dev.sh setup # just install deps, don't run +set -euo pipefail +cd "$(dirname "$0")" + +PIDS=() +cleanup() { echo; echo "Stopping…"; for p in "${PIDS[@]:-}"; do kill "$p" 2>/dev/null || true; done; } +trap cleanup EXIT INT TERM + +py_service() { # name folder port + local name=$1 folder=$2 port=$3 + pushd "$folder" >/dev/null + [ -d .venv ] || python3 -m venv .venv + ./.venv/bin/pip install -q -r requirements.txt + if [ "${1:-}" != "__setup__" ] && [ "$RUN" = "1" ]; then + echo "→ $name on :$port" + ./.venv/bin/uvicorn main:app --port "$port" & + PIDS+=($!) + fi + popd >/dev/null +} + +RUN=1 +[ "${1:-}" = "setup" ] && RUN=0 + +py_service ml ml 8003 +py_service signals signals 8002 +py_service backend backend 8001 +# Video service is opt-in: heavy installs (mediapipe/whisper/ffmpeg). +# WITH_VIDEO=1 ./dev.sh +[ "${WITH_VIDEO:-0}" = "1" ] && py_service video video 8004 + +pushd frontend >/dev/null +[ -d node_modules ] || npm install +if [ "$RUN" = "1" ]; then + echo "→ frontend on :3000" + npm run dev & + PIDS+=($!) +fi +popd >/dev/null + +[ "$RUN" = "0" ] && { echo "Setup done. Run ./dev.sh to start."; exit 0; } + +echo +echo "Pegasus up: frontend http://localhost:3000 · backend :8001 · signals :8002 · ml :8003" +echo "Ctrl-C to stop." +wait diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md new file mode 100644 index 0000000..175ec73 --- /dev/null +++ b/frontend/AGENTS.md @@ -0,0 +1,49 @@ +# `/frontend` — SHARED by Rishith + Wesley (Expo app, :3000) + +This is an **Expo (React Native + TypeScript)** app with **two owners**. Stay +strictly in your files so you never overwrite each other. + +## Rishith owns — the DATA LAYER +``` +frontend/src/services/api.ts +frontend/src/services/config.ts +frontend/src/hooks/* (useResponseTracker, useBurnoutScore, useCamera) +frontend/src/types/index.ts +``` +Plus the one-time Expo bootstrap (`package.json`, `app.json`, `tsconfig.json`, +`babel.config.js`, `index.ts`) — but `App.tsx` and everything below is Wesley's. + +## Wesley owns — the VISUAL LAYER +``` +frontend/App.tsx +frontend/src/navigation/* +frontend/src/screens/* +frontend/src/components/* +frontend/src/utils/* (colors.ts, formatting.ts) +``` +Wesley **imports** Rishith's types/api/hooks and never edits them. + +## The interface (stable — Wesley imports these) +```ts +import { Stimulus, UserResponse, BurnoutResult, BrainData, FacialAnalysis, VoiceAnalysis } from "./src/types"; +import { getStimulus, submitResponse, getScore, getHistory, getBrainData, getMetrics, submitVideo } from "./src/services/api"; +import { useResponseTracker } from "./src/hooks/useResponseTracker"; // { onKeyPress, onChangeText, getMetrics, reset } +import { useBurnoutScore } from "./src/hooks/useBurnoutScore"; // { score, loading, error, refresh } +import { useCamera } from "./src/hooks/useCamera"; // { cameraRef, granted, requestPermission, isRecording, startRecording, stopRecording } +``` + +## Commit staging (NEVER `git add frontend/` or `git add .`) +```bash +# Rishith: +git add frontend/src/services/ frontend/src/hooks/ frontend/src/types/ ml/ video/ +# Wesley: +git add frontend/App.tsx frontend/src/navigation/ frontend/src/screens/ frontend/src/components/ frontend/src/utils/ +``` + +## Run +```bash +cd frontend && npm install +npx expo install react-native-chart-kit react-native-svg @react-navigation/native @react-navigation/bottom-tabs react-native-screens react-native-safe-area-context expo-linear-gradient expo-haptics react-native-reanimated +npx expo start # press i (iOS) / a (Android), or scan with Expo Go +``` +Set your laptop's LAN IP in `src/services/config.ts` so the phone reaches the backend. diff --git a/frontend/App.tsx b/frontend/App.tsx new file mode 100644 index 0000000..6cf6844 --- /dev/null +++ b/frontend/App.tsx @@ -0,0 +1,27 @@ +// PLACEHOLDER — bootstrapped by Rishith so the Expo app runs and the data layer +// (src/services, src/hooks, src/types) is importable. +// +// WESLEY owns this file and everything under src/screens, src/components, +// src/navigation, src/utils. Replace this with the real App + TabNavigator. +import { StatusBar } from "expo-status-bar"; +import { StyleSheet, Text, View } from "react-native"; + +export default function App() { + return ( + + PEGASUS + your mind's check engine light + + Data layer ready in src/. Wesley builds screens here. + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: "#0a0a0a", alignItems: "center", justifyContent: "center", padding: 24 }, + title: { color: "#ffffff", fontSize: 44, fontWeight: "800", letterSpacing: 3 }, + sub: { color: "#9ca3af", marginTop: 6, fontSize: 16 }, + note: { color: "#4b5563", marginTop: 28, fontSize: 13, textAlign: "center" }, +}); diff --git a/frontend/app.json b/frontend/app.json new file mode 100644 index 0000000..0111630 --- /dev/null +++ b/frontend/app.json @@ -0,0 +1,14 @@ +{ + "expo": { + "name": "Pegasus", + "slug": "pegasus", + "version": "1.0.0", + "orientation": "portrait", + "userInterfaceStyle": "dark", + "scheme": "pegasus", + "splash": { "backgroundColor": "#0a0a0a" }, + "ios": { "supportsTablet": true, "infoPlist": { "NSCameraUsageDescription": "Pegasus uses the camera for your 30-second video check-in.", "NSMicrophoneUsageDescription": "Pegasus uses the mic to analyze voice stress during your check-in." } }, + "android": { "permissions": ["CAMERA", "RECORD_AUDIO"] }, + "plugins": ["expo-camera"] + } +} diff --git a/frontend/babel.config.js b/frontend/babel.config.js new file mode 100644 index 0000000..b3827bb --- /dev/null +++ b/frontend/babel.config.js @@ -0,0 +1,4 @@ +module.exports = function (api) { + api.cache(true); + return { presets: ["babel-preset-expo"] }; +}; diff --git a/frontend/index.ts b/frontend/index.ts new file mode 100644 index 0000000..d2bae22 --- /dev/null +++ b/frontend/index.ts @@ -0,0 +1,4 @@ +import { registerRootComponent } from "expo"; +import App from "./App"; + +registerRootComponent(App); diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..c80dd82 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,25 @@ +{ + "name": "pegasus", + "version": "1.0.0", + "main": "index.ts", + "private": true, + "scripts": { + "start": "expo start", + "ios": "expo start --ios", + "android": "expo start --android", + "web": "expo start --web" + }, + "dependencies": { + "expo": "~51.0.28", + "expo-camera": "~15.0.16", + "expo-status-bar": "~1.12.1", + "react": "18.2.0", + "react-native": "0.74.5" + }, + "devDependencies": { + "@babel/core": "^7.24.0", + "@types/react": "~18.2.45", + "babel-preset-expo": "~11.0.0", + "typescript": "~5.3.3" + } +} diff --git a/frontend/src/hooks/useBurnoutScore.ts b/frontend/src/hooks/useBurnoutScore.ts new file mode 100644 index 0000000..2bdbde7 --- /dev/null +++ b/frontend/src/hooks/useBurnoutScore.ts @@ -0,0 +1,40 @@ +// Owned by Rishith. Live burnout score with polling. +// const { score, loading, error, refresh } = useBurnoutScore(DEFAULT_USER_ID); +import { useCallback, useEffect, useRef, useState } from "react"; +import { BurnoutResult } from "../types"; +import { getScore } from "../services/api"; +import { SCORE_POLL_MS } from "../services/config"; + +export function useBurnoutScore(userId: string, pollMs: number = SCORE_POLL_MS) { + const [score, setScore] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const mounted = useRef(true); + + const refresh = useCallback(async () => { + setLoading(true); + try { + const data = await getScore(userId); + if (mounted.current) { + setScore(data); + setError(null); + } + } catch (e: any) { + if (mounted.current) setError(e?.message ?? "failed to fetch score"); + } finally { + if (mounted.current) setLoading(false); + } + }, [userId]); + + useEffect(() => { + mounted.current = true; + refresh(); + const id = setInterval(refresh, pollMs); + return () => { + mounted.current = false; + clearInterval(id); + }; + }, [refresh, pollMs]); + + return { score, loading, error, refresh }; +} diff --git a/frontend/src/hooks/useCamera.ts b/frontend/src/hooks/useCamera.ts new file mode 100644 index 0000000..03fa190 --- /dev/null +++ b/frontend/src/hooks/useCamera.ts @@ -0,0 +1,57 @@ +// Owned by Rishith. expo-camera wrapper for the 30-60s video check-in. +// Wesley renders the with the returned ref: +// const cam = useCamera(); +// +// const uri = await cam.startRecording(60); // resolves when stopped / maxDuration +// cam.stopRecording(); // stop early +import { useCallback, useRef, useState } from "react"; +import { + CameraView, + useCameraPermissions, + useMicrophonePermissions, +} from "expo-camera"; + +export function useCamera() { + const cameraRef = useRef(null); + const [camPerm, requestCamPerm] = useCameraPermissions(); + const [micPerm, requestMicPerm] = useMicrophonePermissions(); + const [isRecording, setIsRecording] = useState(false); + + const granted = !!camPerm?.granted && !!micPerm?.granted; + + const requestPermission = useCallback(async () => { + const c = await requestCamPerm(); + const m = await requestMicPerm(); + return !!c?.granted && !!m?.granted; + }, [requestCamPerm, requestMicPerm]); + + // Starts recording; the returned promise resolves with the video URI when + // recording stops (either via stopRecording() or hitting maxDuration). + const startRecording = useCallback(async (maxDurationSec: number = 60) => { + if (!cameraRef.current) return null; + setIsRecording(true); + try { + const video = await cameraRef.current.recordAsync({ maxDuration: maxDurationSec }); + return video?.uri ?? null; + } catch { + return null; + } finally { + setIsRecording(false); + } + }, []); + + const stopRecording = useCallback(() => { + cameraRef.current?.stopRecording(); + }, []); + + return { + cameraRef, + permission: camPerm, + micPermission: micPerm, + granted, + requestPermission, + isRecording, + startRecording, + stopRecording, + }; +} diff --git a/frontend/src/hooks/useResponseTracker.ts b/frontend/src/hooks/useResponseTracker.ts new file mode 100644 index 0000000..847fa66 --- /dev/null +++ b/frontend/src/hooks/useResponseTracker.ts @@ -0,0 +1,80 @@ +// Owned by Rishith. Captures typing/behavioral biometrics from a TextInput. +// Wesley wires the returned handlers into : +// const t = useResponseTracker(); +// { t.onChangeText(s); setText(s); }} /> +// const metrics = t.getMetrics(); // spread into submitResponse({...metrics}) +// t.reset(); // call when a new stimulus is shown +import { useCallback, useRef } from "react"; +import type { NativeSyntheticEvent, TextInputKeyPressEventData } from "react-native"; + +interface Metrics { + typing_wpm: number; + error_rate: number; + hesitation_count: number; + response_time_ms: number; + response_latency_ms: number; +} + +interface TrackerState { + shownAt: number; // when stimulus was shown (reset()) + startTime: number | null; // first keystroke + lastKeyTime: number | null; + keystrokes: number; + backspaces: number; + hesitations: number; + text: string; +} + +const fresh = (): TrackerState => ({ + shownAt: Date.now(), + startTime: null, + lastKeyTime: null, + keystrokes: 0, + backspaces: 0, + hesitations: 0, + text: "", +}); + +export function useResponseTracker() { + const s = useRef(fresh()); + + const onKeyPress = useCallback((e: NativeSyntheticEvent) => { + const now = Date.now(); + const key = e?.nativeEvent?.key; + const st = s.current; + if (st.startTime === null) st.startTime = now; + if (st.lastKeyTime !== null && now - st.lastKeyTime > 3000) st.hesitations++; + if (key === "Backspace") st.backspaces++; + st.keystrokes++; + st.lastKeyTime = now; + }, []); + + // onKeyPress isn't reliable for every platform/IME — track text length too. + const onChangeText = useCallback((value: string) => { + const st = s.current; + if (st.startTime === null) st.startTime = Date.now(); + st.text = value; + }, []); + + const getMetrics = useCallback((): Metrics => { + const st = s.current; + const now = Date.now(); + const start = st.startTime ?? now; + const elapsedMin = Math.max((now - start) / 60000, 1 / 60000); + const words = st.text.trim() ? st.text.trim().split(/\s+/).length : 0; + const typedChars = Math.max(st.keystrokes, st.text.length); + return { + typing_wpm: Math.round(words / elapsedMin) || 0, + error_rate: typedChars ? Math.round((st.backspaces / typedChars) * 100) : 0, + hesitation_count: st.hesitations, + response_time_ms: now - st.shownAt, + response_latency_ms: (st.startTime ?? now) - st.shownAt, + }; + }, []); + + const reset = useCallback(() => { + s.current = fresh(); + }, []); + + return { onKeyPress, onChangeText, getMetrics, reset }; +} diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts new file mode 100644 index 0000000..714eb10 --- /dev/null +++ b/frontend/src/services/api.ts @@ -0,0 +1,53 @@ +// Owned by Rishith. All backend calls live here. Wesley imports these. +// Talks ONLY to Jason's backend (:8001). Endpoints per shared/contract. +import { + UserResponse, + BurnoutResult, + Stimulus, + BrainData, + VideoResult, +} from "../types"; +import { BACKEND_URL } from "./config"; + +async function json(res: Response): Promise { + if (!res.ok) throw new Error(`${res.url} -> ${res.status}`); + return res.json() as Promise; +} + +export async function getStimulus(userId: string): Promise { + return json(await fetch(`${BACKEND_URL}/stimulus/today/${userId}`)); +} + +export async function submitResponse(data: UserResponse): Promise { + return json( + await fetch(`${BACKEND_URL}/response/submit`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(data), + }) + ); +} + +export async function getScore(userId: string): Promise { + return json(await fetch(`${BACKEND_URL}/score/${userId}`)); +} + +export async function getHistory(userId: string): Promise { + return json(await fetch(`${BACKEND_URL}/history/${userId}`)); +} + +export async function getBrainData(userId: string): Promise { + return json(await fetch(`${BACKEND_URL}/brain/${userId}`)); +} + +export async function getMetrics(userId: string): Promise { + return json(await fetch(`${BACKEND_URL}/metrics/${userId}`)); +} + +export async function submitVideo(userId: string, videoUri: string): Promise { + const fd = new FormData(); + // React Native FormData file shape: + fd.append("video", { uri: videoUri, type: "video/mp4", name: "checkin.mp4" } as any); + fd.append("user_id", userId); + return json(await fetch(`${BACKEND_URL}/video/submit`, { method: "POST", body: fd })); +} diff --git a/frontend/src/services/config.ts b/frontend/src/services/config.ts new file mode 100644 index 0000000..a867d7c --- /dev/null +++ b/frontend/src/services/config.ts @@ -0,0 +1,8 @@ +// Owned by Rishith. On a physical phone, localhost won't reach the laptop — +// use the laptop's LAN IP. Find it with: ipconfig getifaddr en0 (macOS) +// Override without editing code via Expo extra / env if you prefer. +export const BACKEND_URL = "http://192.168.1.XXX:8001"; // CHANGE to your laptop's LAN IP +export const DEFAULT_USER_ID = "demo_user"; + +// Polling cadence for the live burnout score (ms). +export const SCORE_POLL_MS = 30_000; diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts new file mode 100644 index 0000000..3abf350 --- /dev/null +++ b/frontend/src/types/index.ts @@ -0,0 +1,84 @@ +// Pegasus shared types. Owned by Rishith. Wesley imports these into screens. +// Mirror of the backend contract — keep shape-compatible with Jason's API. + +export interface Stimulus { + id: string; + type: "image" | "audio" | "text"; + url: string; + category: "calm" | "neutral" | "activating"; + prompt: string; +} + +export interface UserResponse { + user_id: string; + stimulus_id: string; + response_text: string; + response_time_ms: number; + response_latency_ms: number; + typing_wpm: number; + error_rate: number; + source: "app" | "sms"; + timestamp: string; +} + +export interface BurnoutResult { + score: number; + level: "green" | "yellow" | "red"; + tribe_deviation: number; + behavioral_deviation: number; + top_indicators: string[]; + intervention: string; + brain_regions_flagged: string[]; + confidence: number; + breakdown: { + imessage: number; + typing: number; + facial: number; + voice: number; + tribe: number; + }; + timestamp: string; +} + +export interface FacialAnalysis { + facial_stress_score: number; + eye_indicators: { + blink_rate_per_min: number; + eye_openness: number; + gaze_stability: number; + }; + facial_indicators: { + brow_furrow: number; + lip_compression: number; + jaw_clench: number; + forced_smile: boolean; + overall_affect: "positive" | "neutral" | "negative" | "flat"; + }; +} + +export interface VoiceAnalysis { + transcript: string; + pitch_mean_hz: number; + pitch_variability: number; + speaking_rate_wpm: number; + pause_frequency: number; + voice_tremor: boolean; +} + +export interface BrainData { + regions: { + prefrontal_cortex: number; + amygdala_region: number; + temporal_lobe: number; + motor_cortex: number; + visual_cortex: number; + }; + activation_mean: number; +} + +// Returned by POST /video/submit (facial + voice + combined). +export interface VideoResult { + facial: FacialAnalysis; + voice: VoiceAnalysis; + combined_score: number; +} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json new file mode 100644 index 0000000..8afa8e9 --- /dev/null +++ b/frontend/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "expo/tsconfig.base", + "compilerOptions": { + "strict": true, + "baseUrl": ".", + "paths": { "@/*": ["src/*"] } + } +} diff --git a/imessage/.env.example b/imessage/.env.example new file mode 100644 index 0000000..e5024b3 --- /dev/null +++ b/imessage/.env.example @@ -0,0 +1,4 @@ +# iMessage bridge. Copy to .env and adjust. +BACKEND_URL=http://localhost:8001 +POLL_SECONDS=5 +AUTO_REGISTER=1 # auto-create a backend user for any new sender diff --git a/imessage/AGENTS.md b/imessage/AGENTS.md new file mode 100644 index 0000000..e520ee3 --- /dev/null +++ b/imessage/AGENTS.md @@ -0,0 +1,26 @@ +# `/imessage` — iMessage bridge (owner: TBD) + +A **standalone** delivery channel. Not one of the four service lanes — assign +an owner before serious work. Read `../AGENTS.md` and `../shared/contract.md`. + +## Boundary +You may ONLY edit `imessage/`. Never touch `frontend/`, `backend/`, `ml/`, +`signals/`, `shared/`. The bridge talks to the **backend only** (`:8001`): +- `POST /users` (register a phone) +- `GET /stimulus/today?user_id=` (what to send) +- `POST /checkin` (score a reply) + +## What it does (macOS-only) +- **Send** via AppleScript → Messages.app (`send.applescript`, `imessage.send`). +- **Receive** by polling `~/Library/Messages/chat.db` (`imessage.fetch_new`). +- `bridge.py` is the CLI/loop: `register`, `send`, `send-daily`, `listen`. +- `registry.py` maps phone ⇄ backend user_id. + +## Run +See `README.md`. Needs Full Disk Access + signed-in iMessage on the Mac. + +## Git +```bash +git add imessage/ # ONLY imessage/ +git commit -m "feat(imessage): ..." +``` diff --git a/imessage/README.md b/imessage/README.md new file mode 100644 index 0000000..130c5bf --- /dev/null +++ b/imessage/README.md @@ -0,0 +1,52 @@ +# imessage — the iMessage bridge + +Closes the Pegasus loop over real iMessage on macOS, with **no API and no paid +service** — just Messages.app (to send) and `chat.db` (to receive). + +``` +send-daily ── stimulus ──▶ user's iPhone/Mac + │ replies in iMessage +chat.db poll ◀── reply ─────────┘ + └─▶ POST :8001/checkin ─▶ score + intervention ─▶ iMessage reply +``` + +It talks **only** to the backend (`:8001`). It does not import or edit any other +service. + +## One-time Mac setup +1. Sign into **iMessage** in Messages.app on this Mac. +2. Grant **Full Disk Access** to whatever runs the bridge (Terminal, iTerm, or + your Python) so it can read `~/Library/Messages/chat.db`: + System Settings → Privacy & Security → **Full Disk Access** → add the app. +3. First time you send, macOS may prompt to allow controlling Messages — allow it. + +## Install & run +```bash +cd imessage +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env + +# make sure the backend is up first (port 8001) +python bridge.py register "+15551234567" "Alex" # register a tester +python bridge.py send-daily # text them today's stimulus +python bridge.py listen # watch for replies & score them +``` + +`listen` polls `chat.db` every `POLL_SECONDS`, scores each reply via the +backend, and texts back the 🟢/🟡/🔴 + intervention. It starts from "now" so it +never replays old history; progress is saved in `.state.json`. + +## Caveats (it's a local hack, not a product) +- The Mac must stay on and signed into iMessage; it uses **your** account. +- AppleScript send syntax is macOS-version-sensitive — see the note in + `send.applescript` if `send-daily` errors. +- Newer macOS stores some message text in `attributedBody` (binary), not the + `text` column. `imessage.decode_attributed_body` handles the common cases; + `pip install typedstream` if you hit a message it can't read. +- No keystroke dynamics over iMessage, so check-ins score on text sentiment + (typing metrics are sent as 0). + +## Owner +Standalone service — **assign an owner**. It lives outside the four service +lanes; whoever owns it works only inside `imessage/` and against the backend API. diff --git a/imessage/bridge.py b/imessage/bridge.py new file mode 100644 index 0000000..ea3d934 --- /dev/null +++ b/imessage/bridge.py @@ -0,0 +1,148 @@ +"""Pegasus iMessage bridge — closes the loop over iMessage on macOS. + + stimulus out → user replies in iMessage → /checkin → intervention back + +Talks ONLY to the backend's public API (:8001). Standalone service; owner TBD. + +Commands: + python bridge.py register "+15551234567" "Alex" # register a phone + python bridge.py send-daily # push today's stimulus to all + python bridge.py send "+15551234567" "hi" # raw test send + python bridge.py listen # poll for replies, score, reply + +Env (see .env.example): BACKEND_URL, POLL_SECONDS, AUTO_REGISTER. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +import httpx + +import imessage +import registry + +BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8001") +POLL_SECONDS = float(os.getenv("POLL_SECONDS", "5")) +AUTO_REGISTER = os.getenv("AUTO_REGISTER", "1") == "1" +STATE_PATH = Path(__file__).with_name(".state.json") + +LIGHT = {"green": "🟢", "yellow": "🟡", "red": "🔴"} + + +def _state() -> dict: + return json.loads(STATE_PATH.read_text()) if STATE_PATH.exists() else {} + + +def _save_state(s: dict) -> None: + STATE_PATH.write_text(json.dumps(s)) + + +def _today_stimulus(user_id: str) -> dict: + r = httpx.get(f"{BACKEND_URL}/stimulus/today", params={"user_id": user_id}, timeout=15) + r.raise_for_status() + return r.json() + + +# --- commands -------------------------------------------------------------- +def cmd_register(phone: str, name: str) -> None: + uid = registry.register(phone, name) + print(f"registered {phone} -> user_id {uid}") + + +def cmd_send(phone: str, message: str) -> None: + imessage.send(phone, message) + print(f"sent to {phone}") + + +def cmd_send_daily() -> None: + users = registry.all_users() + if not users: + print("no registered users — run `register` first.") + return + for phone, info in users.items(): + s = _today_stimulus(info["user_id"]) + body = f"{s['content']}\n\n{s['prompt']}" + imessage.send(phone, body) + print(f"daily stimulus → {phone}") + + +def _handle_reply(phone: str, text: str) -> None: + user_id = registry.get_user_id(phone) + if not user_id: + if not AUTO_REGISTER: + print(f"ignoring unknown sender {phone}") + return + user_id = registry.register(phone) + print(f"auto-registered {phone} -> {user_id}") + + stimulus = _today_stimulus(user_id) + # iMessage can't capture keystroke dynamics, so typing metrics are 0 — + # scoring leans on text sentiment via the signals service. + resp = httpx.post(f"{BACKEND_URL}/checkin", json={ + "user_id": user_id, + "stimulus_id": stimulus["stimulus_id"], + "text_response": text, + "response_time_ms": 0, "typing_wpm": 0, "error_rate": 0, + }, timeout=60) + resp.raise_for_status() + result = resp.json() + + light = LIGHT.get(result["level"], "") + iv = result.get("intervention", {}) + reply = f"{light} score {result['score']}/100\n{iv.get('message','')}\n↳ {iv.get('suggested_action','')}" + imessage.send(phone, reply) + print(f"check-in {phone}: {result['level']} ({result['score']}) → replied") + + +def cmd_listen() -> None: + state = _state() + last = state.get("last_rowid") + if last is None: + last = imessage.max_rowid() # don't replay history on first run + _save_state({"last_rowid": last}) + print(f"listening from rowid {last} (every {POLL_SECONDS}s). Ctrl-C to stop.") + + while True: + try: + new = imessage.fetch_new(last) + for msg in new: + last = msg["rowid"] + try: + _handle_reply(msg["handle"], msg["text"]) + except Exception as e: + print(f" ! failed to handle msg from {msg['handle']}: {e}") + _save_state({"last_rowid": last}) + except Exception as e: + print(f" ! poll error: {e}") + time.sleep(POLL_SECONDS) + + +def main() -> None: + p = argparse.ArgumentParser(prog="bridge") + sub = p.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("register"); r.add_argument("phone"); r.add_argument("name", nargs="?", default="iMessage User") + s = sub.add_parser("send"); s.add_argument("phone"); s.add_argument("message") + sub.add_parser("send-daily") + sub.add_parser("listen") + args = p.parse_args() + + if args.cmd == "register": + cmd_register(args.phone, args.name) + elif args.cmd == "send": + cmd_send(args.phone, args.message) + elif args.cmd == "send-daily": + cmd_send_daily() + elif args.cmd == "listen": + try: + cmd_listen() + except KeyboardInterrupt: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/imessage/imessage.py b/imessage/imessage.py new file mode 100644 index 0000000..3dc1013 --- /dev/null +++ b/imessage/imessage.py @@ -0,0 +1,105 @@ +"""iMessage send/receive on macOS — no API, just Messages.app + chat.db. + +Send: drives Messages.app via AppleScript (osascript). +Receive: reads ~/Library/Messages/chat.db (read-only) for new inbound rows. + +Requires (on the Mac running this): + - signed into iMessage in Messages.app, app allowed to run + - **Full Disk Access** for whatever runs this (Terminal / python) so it can + read chat.db (System Settings → Privacy & Security → Full Disk Access) +""" +from __future__ import annotations + +import os +import sqlite3 +import subprocess +from pathlib import Path +from typing import List, Optional, TypedDict + +CHAT_DB = Path(os.path.expanduser("~/Library/Messages/chat.db")) +SEND_SCRIPT = Path(__file__).with_name("send.applescript") + + +class InboundMessage(TypedDict): + rowid: int + handle: str # sender phone/email + text: str + + +# --- send ------------------------------------------------------------------ +def send(phone: str, message: str) -> None: + """Send an iMessage. Raises subprocess.CalledProcessError on failure.""" + subprocess.run( + ["osascript", str(SEND_SCRIPT), phone, message], + check=True, + capture_output=True, + text=True, + ) + + +# --- receive ---------------------------------------------------------------- +def _connect_ro() -> sqlite3.Connection: + # Open read-only so we never lock the live Messages database. + conn = sqlite3.connect(f"file:{CHAT_DB}?mode=ro", uri=True) + conn.row_factory = sqlite3.Row + return conn + + +def max_rowid() -> int: + with _connect_ro() as c: + row = c.execute("SELECT MAX(ROWID) AS m FROM message").fetchone() + return int(row["m"] or 0) + + +def fetch_new(after_rowid: int) -> List[InboundMessage]: + """Return inbound (received) messages with ROWID > after_rowid.""" + query = """ + SELECT m.ROWID AS rowid, m.text AS text, m.attributedBody AS body, + h.id AS handle + FROM message m + LEFT JOIN handle h ON m.handle_id = h.ROWID + WHERE m.is_from_me = 0 AND m.ROWID > ? + ORDER BY m.ROWID ASC + """ + out: List[InboundMessage] = [] + with _connect_ro() as c: + for row in c.execute(query, (after_rowid,)).fetchall(): + text = row["text"] + if not text and row["body"] is not None: + text = decode_attributed_body(row["body"]) + if not text or not row["handle"]: + continue + out.append({"rowid": int(row["rowid"]), "handle": row["handle"], "text": text}) + return out + + +def decode_attributed_body(data: bytes) -> Optional[str]: + """Best-effort extraction of message text from a streamtyped + NSAttributedString blob (used when the `text` column is NULL on newer + macOS). Handles the common short/long-string cases; returns None if the + layout isn't recognized. For 100% fidelity, `pip install typedstream`. + """ + if not data: + return None + try: + idx = data.index(b"NSString") + except ValueError: + return None + chunk = data[idx + len(b"NSString"):] + plus = chunk.find(b"+") # type marker that precedes the length byte + if plus == -1: + return None + p = plus + 1 + if p >= len(chunk): + return None + length = chunk[p] + p += 1 + if length == 0x81: # 2-byte little-endian length + length = int.from_bytes(chunk[p:p + 2], "little") + p += 2 + elif length == 0x82: # 4-byte little-endian length + length = int.from_bytes(chunk[p:p + 4], "little") + p += 4 + raw = chunk[p:p + length] + text = raw.decode("utf-8", errors="ignore").strip() + return text or None diff --git a/imessage/registry.py b/imessage/registry.py new file mode 100644 index 0000000..0d80efe --- /dev/null +++ b/imessage/registry.py @@ -0,0 +1,61 @@ +"""Maps phone numbers <-> backend user_ids, persisted to registry.json. + +Auto-registers unknown phones as backend users on first contact so anyone can +text the bot and start checking in. +""" +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Dict, Optional + +import httpx + +BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:8001") +REGISTRY_PATH = Path(__file__).with_name("registry.json") + + +def _norm(phone: str) -> str: + """Normalize a handle to digits (+ optional leading +) so '+1 (555) 123' + and '+15551230000' match.""" + phone = phone.strip() + if "@" in phone: # email handle — keep as-is + return phone.lower() + digits = re.sub(r"[^\d]", "", phone) + return f"+{digits}" if digits else phone + + +def _load() -> Dict[str, Dict]: + if REGISTRY_PATH.exists(): + return json.loads(REGISTRY_PATH.read_text()) + return {} + + +def _save(data: Dict[str, Dict]) -> None: + REGISTRY_PATH.write_text(json.dumps(data, indent=2)) + + +def all_users() -> Dict[str, Dict]: + return _load() + + +def get_user_id(phone: str) -> Optional[str]: + return _load().get(_norm(phone), {}).get("user_id") + + +def register(phone: str, name: str = "iMessage User") -> str: + """Ensure a backend user exists for this phone; return its user_id.""" + key = _norm(phone) + data = _load() + if key in data: + return data[key]["user_id"] + + resp = httpx.post(f"{BACKEND_URL}/users", json={"name": name, "phone": key}, timeout=15) + resp.raise_for_status() + user_id = resp.json()["user_id"] + + data[key] = {"user_id": user_id, "name": name} + _save(data) + return user_id diff --git a/imessage/requirements.txt b/imessage/requirements.txt new file mode 100644 index 0000000..e5c1453 --- /dev/null +++ b/imessage/requirements.txt @@ -0,0 +1,4 @@ +httpx==0.28.1 +python-dotenv==1.0.1 +# Optional, for 100% reliable chat.db text decoding: +# typedstream diff --git a/imessage/send.applescript b/imessage/send.applescript new file mode 100644 index 0000000..9cbd934 --- /dev/null +++ b/imessage/send.applescript @@ -0,0 +1,16 @@ +-- Send an iMessage via Messages.app. +-- Usage: osascript send.applescript "+15551234567" "your message" +-- +-- Note: the exact syntax is macOS-version-sensitive. This `service`+`buddy` +-- form works on most recent versions. If your macOS errors, try replacing +-- `buddy targetPhone` with `participant targetPhone`, or `service` with +-- `account`. +on run argv + set targetPhone to item 1 of argv + set targetMessage to item 2 of argv + tell application "Messages" + set targetService to 1st service whose service type = iMessage + set targetBuddy to buddy targetPhone of targetService + send targetMessage to targetBuddy + end tell +end run diff --git a/ml/.env.example b/ml/.env.example new file mode 100644 index 0000000..2fcceac --- /dev/null +++ b/ml/.env.example @@ -0,0 +1,6 @@ +# ML service (Rishith). Copy to .env and fill in. NEVER commit .env. +HF_TOKEN=hf_... +# Optional overrides: +# HF_MODEL=Qwen/Qwen2.5-7B-Instruct # intervention chat model +# HF_EMOTION_MODEL=j-hartmann/emotion-english-distilroberta-base +# TRIBE v2 runs on Modal — authenticate once with: modal token new diff --git a/ml/AGENTS.md b/ml/AGENTS.md new file mode 100644 index 0000000..9aecaeb --- /dev/null +++ b/ml/AGENTS.md @@ -0,0 +1,52 @@ +# `/ml` (+ `/video` + frontend data layer) — Rishith's agent + +You are **Rishith**. You own **`ml/`**, **`video/`**, and the **frontend data +layer** (`frontend/src/services`, `frontend/src/hooks`, `frontend/src/types`). +`ml/` is the burnout scorer on **:8003**. Read `../AGENTS.md` first. + +## Never touch +Wesley's screens/components (`frontend/App.tsx`, `src/screens`, `src/components`, +`src/navigation`, `src/utils`), `/backend`, `/signals`, `/shared`. Call APIs; +ask Jason for contract changes. + +## TRIBE v2 — already deployed on Modal +TRIBE runs separately on Modal as app `pegasus-tribe`, class `TribePredictor`. +**Do NOT redeploy or rewrite it.** Just call it: +```python +import modal +Tribe = modal.Cls.from_name("pegasus-tribe", "TribePredictor") +baseline = Tribe().get_baseline.remote(stimulus_id) +``` +`main.py` calls it lazily and degrades to a null baseline if Modal is offline. + +## Endpoints (:8003) +- `GET /health` → `{status, tribe}` +- `POST /score` → `{user_id, stimulus_id, signals}` → `BurnoutResult` +- `GET /baseline/{stimulus_id}` → cached TRIBE prediction + +## Files +- `main.py` — FastAPI + Modal TRIBE client. +- `scoring.py` — `BurnoutScorer`, **Hugging Face only (no Anthropic)**: HF + emotion model (`hf_sentiment`) + deviation math (`compute_deviation`) + HF + chat-model intervention (`_intervene`, via `huggingface_hub.InferenceClient`). + Returns `BurnoutResult` (score, level, tribe/behavioral deviation, indicators, + intervention, brain_regions_flagged, confidence, breakdown). + +## Secrets (.env, gitignored) +`HF_TOKEN` (used for BOTH sentiment + interventions). Optional `HF_MODEL`, +`HF_EMOTION_MODEL`. Modal auth via `modal token new`. Read with `os.getenv`. +Levels: green <30, yellow <65, red ≥65. + +## Run +```bash +cd ml && python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt && cp .env.example .env +modal token new # once +uvicorn main:app --port 8003 --reload +``` + +## Git — branch `feat/rishith-ml` +```bash +git add ml/ video/ frontend/src/services/ frontend/src/hooks/ frontend/src/types/ +git commit -m "feat: ..." && git push origin feat/rishith-ml +``` diff --git a/ml/main.py b/ml/main.py new file mode 100644 index 0000000..f7012d1 --- /dev/null +++ b/ml/main.py @@ -0,0 +1,70 @@ +"""Pegasus ML service (:8003) — scoring around TRIBE v2 (already on Modal). + +TRIBE v2 is deployed separately on Modal as app `pegasus-tribe`, class +`TribePredictor`. We just call it; we do NOT redeploy or rewrite it. + +Run (from inside /ml): + modal token new # once, to authenticate Modal + uvicorn main:app --port 8003 --reload +""" +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +from scoring import BurnoutScorer + +app = FastAPI(title="Pegasus ML") +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) + +scorer = BurnoutScorer() + + +def _tribe(): + """Lazy handle to the deployed TRIBE class. Returns None if Modal isn't + available (keeps the service runnable offline — scorer handles None baseline).""" + try: + import modal + + return modal.Cls.from_name("pegasus-tribe", "TribePredictor") + except Exception: + return None + + +def _baseline(stimulus_id: str): + Tribe = _tribe() + if Tribe is None: + return None + try: + return Tribe().get_baseline.remote(stimulus_id) + except Exception: + return None + + +class ScoreReq(BaseModel): + user_id: str + stimulus_id: str + signals: dict + + +@app.get("/health") +def health(): + return {"status": "ml running", "tribe": "connected" if _tribe() is not None else "offline"} + + +@app.post("/score") +def score(req: ScoreReq): + baseline = _baseline(req.stimulus_id) + return scorer.compute_deviation(baseline, req.signals) + + +@app.get("/baseline/{stimulus_id}") +def baseline(stimulus_id: str): + return _baseline(stimulus_id) or {"error": "tribe baseline unavailable", "regions": {}} + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8003) diff --git a/ml/requirements.txt b/ml/requirements.txt new file mode 100644 index 0000000..6332b96 --- /dev/null +++ b/ml/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +modal==0.66.45 +huggingface-hub==0.27.0 +numpy==1.26.4 +python-dotenv==1.0.1 diff --git a/ml/scoring.py b/ml/scoring.py new file mode 100644 index 0000000..8ca4316 --- /dev/null +++ b/ml/scoring.py @@ -0,0 +1,138 @@ +"""Burnout scoring: deviation of behavior from the TRIBE healthy baseline. + +All language/AI is **Hugging Face** (no Anthropic): + - sentiment/emotion via a HF text-classification model + - interventions via a HF chat (instruct) model +Both degrade gracefully (sentiment -> 0.5, intervention -> canned copy) so the +demo runs even if HF is unreachable. +""" +from __future__ import annotations + +import os +from typing import Dict, List, Optional + +HF_TOKEN = os.getenv("HF_TOKEN") +EMOTION_MODEL = os.getenv("HF_EMOTION_MODEL", "j-hartmann/emotion-english-distilroberta-base") +CHAT_MODEL = os.getenv("HF_MODEL", "Qwen/Qwen2.5-7B-Instruct") + +_POSITIVE_EMOTIONS = {"joy", "neutral", "surprise"} + + +class BurnoutScorer: + def __init__(self) -> None: + self._client = None # lazy huggingface_hub.InferenceClient + + def _hf(self): + if self._client is None: + from huggingface_hub import InferenceClient + + self._client = InferenceClient(token=HF_TOKEN) + return self._client + + # --- language ---------------------------------------------------------- + def hf_sentiment(self, text: str) -> float: + """0-1 positivity from a HF emotion model. Returns 0.5 on any failure.""" + if not text or not HF_TOKEN: + return 0.5 + try: + results = self._hf().text_classification(text, model=EMOTION_MODEL) + # results: list of {label, score} across all emotions. + pos = sum(r.score for r in results if r.label.lower() in _POSITIVE_EMOTIONS) + return min(max(float(pos), 0.0), 1.0) + except Exception: + return 0.5 + + # --- scoring ----------------------------------------------------------- + def compute_deviation(self, tribe_baseline: Optional[Dict], signals: Dict) -> Dict: + baseline = tribe_baseline or {} + regions = baseline.get("regions", {}) or {} + expected = float(regions.get("prefrontal_cortex", 0.5)) + + sentiment = signals.get("sentiment_score") + if sentiment is None: + sentiment = self.hf_sentiment(signals.get("response_text", "")) + sentiment = float(sentiment) + + error_rate = float(signals.get("error_rate", 0)) + wpm = float(signals.get("typing_wpm", 60)) + rt = float(signals.get("response_time_ms", 3000)) + + actual = ( + sentiment * 0.3 + + (1 - min(error_rate / 20, 1)) * 0.2 + + min(wpm / 80, 1) * 0.2 + + (1 - min(rt / 10000, 1)) * 0.3 + ) + deviation = abs(expected - actual) + score = min(int(deviation * 150), 100) + level = "green" if score < 30 else "yellow" if score < 65 else "red" + + ind: List[str] = [] + if error_rate > 10: + ind.append("Elevated typing errors suggest cognitive strain") + if rt > 8000: + ind.append("Delayed response indicates mental fatigue") + if sentiment < 0.3: + ind.append("Negative sentiment drift detected") + if wpm < 30: + ind.append("Reduced typing speed suggests low energy") + if not ind: + ind.append("No significant warning signs") + + return { + "score": score, + "level": level, + "tribe_deviation": round(deviation, 4), + "behavioral_deviation": round(1 - actual, 4), + "top_indicators": ind[:3], + "intervention": self._intervene(score, level, ind), + "brain_regions_flagged": self._flag_regions(regions), + "confidence": round(min(0.5 + deviation * 0.5, 0.95), 3), + "breakdown": { + "imessage": int((1 - sentiment) * 100), + "typing": int(min(error_rate / 20, 1) * 100), + "facial": int(signals.get("facial_stress_score", 0)), + "voice": int(signals.get("voice_stress_score", 0)), + "tribe": int(deviation * 100), + }, + } + + @staticmethod + def _flag_regions(regions: Dict) -> List[str]: + labels = { + "prefrontal_cortex": "Prefrontal cortex: elevated cognitive load", + "amygdala_region": "Amygdala: heightened stress response", + "temporal_lobe": "Temporal lobe: altered language processing", + } + return [labels[k] for k, v in regions.items() if k in labels and float(v) > 0.6] + + def _intervene(self, score: int, level: str, indicators: List[str]) -> str: + """One warm, specific action via a HF chat model. Canned fallback.""" + if not HF_TOKEN: + return self._fallback(level) + try: + resp = self._hf().chat_completion( + model=CHAT_MODEL, + max_tokens=120, + temperature=0.7, + messages=[{ + "role": "user", + "content": ( + f"Burnout {score}/100 ({level}). Signs: {', '.join(indicators)}. " + "Give ONE specific action in under 2 sentences. Warm but direct. " + "You are a check engine light, not a therapist. No preamble." + ), + }], + ) + text = resp.choices[0].message.content.strip() + return text or self._fallback(level) + except Exception: + return self._fallback(level) + + @staticmethod + def _fallback(level: str) -> str: + return { + "green": "You're tracking well. Take a real 5-minute break before your next task.", + "yellow": "Step outside for 10 minutes and drink water before you continue.", + "red": "Stop work for today if you can, and message one person you trust right now.", + }.get(level, "Take a 10-minute break and step outside for some fresh air.") diff --git a/shared/AGENTS.md b/shared/AGENTS.md new file mode 100644 index 0000000..f0f36ec --- /dev/null +++ b/shared/AGENTS.md @@ -0,0 +1,8 @@ +# `/shared` — owned by Jason + +This folder is the **API contract**: `contract.md` (human-readable, the law) +and `contract.py` (importable pydantic models). + +Only **Jason** edits this. Everyone else **reads** it. If you need a field +added or changed, ask Jason — he changes it here first, then notifies the +service owner so their code stays shape-compatible. diff --git a/shared/contract.md b/shared/contract.md new file mode 100644 index 0000000..7997604 --- /dev/null +++ b/shared/contract.md @@ -0,0 +1,102 @@ +# Pegasus API Contract + +**Owner:** Jason (`/shared`). This is the single source of truth. If you need a +field changed, change it HERE first, then tell the affected owner. + +``` +Frontend (Wesley) :3000 React +Backend (Jason) :8001 FastAPI + SQLite ← orchestrator +Signals (Dhruva) :8002 FastAPI +ML (Rishith) :8003 FastAPI + TRIBE v2 +``` + +## Data flow + +``` +1. Frontend asks Backend for today's stimulus GET :8001/stimulus/today +2. User responds; Frontend captures keystrokes/timing +3. Frontend submits the check-in POST :8001/checkin +4. Backend → Signals analyze behavioral text/typing POST :8002/signals/analyze +5. Backend → ML predict healthy brain response POST :8003/predict +6. Backend → ML score deviation = burnout POST :8003/score +7. Backend → ML generate intervention (Claude) POST :8003/intervention +8. If level == "red": Backend → Signals send SMS POST :8002/alert +9. Backend returns engine light + intervention to Frontend +``` + +--- + +## Core types + +```jsonc +// Stimulus — the daily prompt shown to the user +{ "stimulus_id": "str", "type": "image|audio|text", "content": "str (url or text)", "prompt": "str" } + +// CheckIn — what the frontend submits +{ + "user_id": "str", + "stimulus_id": "str", + "text_response": "str", + "response_time_ms": 0, // ms from stimulus shown → submit + "typing_wpm": 0, + "error_rate": 0 // % backspaces, 0-100 +} + +// BehavioralSignal — Signals service output +{ "sentiment_score": 0.0, "energy_level": "low|medium|high", + "flags": ["future_tense_low","high_self_reference"], "combined_signal_score": 0.0 } + +// BrainPrediction — ML /predict output (what a healthy brain does with the stimulus) +{ "predicted_engagement": 0.0, "predicted_valence": 0.0, "activation_summary": [0.0] } + +// BurnoutScore — ML /score output +{ "score": 0, "level": "green|yellow|red", "deviation": 0.0, + "components": { "engagement_gap": 0.0, "valence_gap": 0.0, "signal_gap": 0.0 } } + +// Intervention — ML /intervention output (Claude-generated) +{ "message": "str", "suggested_action": "str" } +``` + +--- + +## Endpoints + +### ML — Rishith — :8003 +| Method | Path | Body | Returns | +|--------|----------------|-------------------------------------------------|------------------| +| GET | `/health` | — | `{status,model}` | +| POST | `/predict` | `{ stimulus }` | `BrainPrediction`| +| POST | `/score` | `{ stimulus, prediction, signal, checkin }` | `BurnoutScore` | +| POST | `/intervention`| `{ burnout, recent_signals }` | `Intervention` | + +### Signals — Dhruva — :8002 +| Method | Path | Body | Returns | +|--------|---------------------|-------------------------------------------------------|-------------------| +| GET | `/health` | — | `{status}` | +| POST | `/signals/analyze` | `{ user_id, text, typing_wpm, error_rate, response_time_ms }` | `BehavioralSignal`| +| GET | `/signals/{user_id}`| — | latest `BehavioralSignal` | +| POST | `/alert` | `{ user_id, phone, score, level, intervention }` | `{ sent, message_sid }` | + +### Backend — Jason — :8001 (orchestrator) +| Method | Path | Body / Query | Returns | +|--------|----------------------|---------------------------|-------------------------------------------| +| GET | `/health` | — | `{status}` | +| POST | `/users` | `{ name, phone }` | `{ user_id, name, phone }` | +| GET | `/users/{user_id}` | — | user | +| GET | `/stimulus/today` | `?user_id=` | `Stimulus` | +| POST | `/checkin` | `CheckIn` | `{ score, level, deviation, intervention, alerted }` | +| GET | `/status/{user_id}` | — | `{ latest, history: [...] }` | + +### Frontend — Wesley — :3000 +React app. Calls **only the Backend** (`:8001`). Captures keystroke metrics +with `signals/collector.js` (`KeystrokeTracker`). Renders the Check Engine +Light: 🟢 green / 🟡 yellow / 🔴 red + the intervention message. + +--- + +## Engine light thresholds (Backend decides level from ML score) +| Score | Level | Light | +|---------|--------|-------| +| 0–39 | green | 🟢 | +| 40–69 | yellow | 🟡 | +| 70–100 | red | 🔴 → SMS alert | diff --git a/shared/contract.py b/shared/contract.py new file mode 100644 index 0000000..179a2fe --- /dev/null +++ b/shared/contract.py @@ -0,0 +1,79 @@ +"""Pegasus shared contract — canonical request/response models. + +Owner: Jason (`/shared`). This is the source of truth. Services may mirror +these models locally (FastAPI idiom) but must stay shape-compatible. + +Usage from a service (run uvicorn from the repo root): + from shared.contract import CheckIn, BurnoutScore +""" +from __future__ import annotations + +from typing import List, Literal, Optional + +from pydantic import BaseModel, Field + +Level = Literal["green", "yellow", "red"] +StimulusType = Literal["image", "audio", "text"] +EnergyLevel = Literal["low", "medium", "high"] + + +class Stimulus(BaseModel): + stimulus_id: str + type: StimulusType + content: str # url for image/audio, raw text for text + prompt: str + + +class CheckIn(BaseModel): + user_id: str + stimulus_id: str + text_response: str + response_time_ms: int = 0 + typing_wpm: int = 0 + error_rate: float = 0.0 # % backspaces, 0-100 + + +class BehavioralSignal(BaseModel): + sentiment_score: float = Field(ge=0.0, le=1.0) + energy_level: EnergyLevel + flags: List[str] = [] + combined_signal_score: float = Field(ge=0.0, le=1.0) + + +class BrainPrediction(BaseModel): + predicted_engagement: float = Field(ge=0.0, le=1.0) + predicted_valence: float = Field(ge=0.0, le=1.0) + activation_summary: List[float] = [] + + +class ScoreComponents(BaseModel): + engagement_gap: float = 0.0 + valence_gap: float = 0.0 + signal_gap: float = 0.0 + + +class BurnoutScore(BaseModel): + score: int = Field(ge=0, le=100) + level: Level + deviation: float = 0.0 + components: ScoreComponents = ScoreComponents() + + +class Intervention(BaseModel): + message: str + suggested_action: str + + +class User(BaseModel): + user_id: str + name: str + phone: Optional[str] = None + + +def level_for_score(score: int) -> Level: + """Engine-light thresholds. See contract.md.""" + if score >= 70: + return "red" + if score >= 40: + return "yellow" + return "green" diff --git a/signals/.env.example b/signals/.env.example new file mode 100644 index 0000000..61e377a --- /dev/null +++ b/signals/.env.example @@ -0,0 +1,5 @@ +# Signals service (Dhruva). Copy to .env and fill in. +ANTHROPIC_API_KEY=sk-ant-... +TWILIO_SID=AC... +TWILIO_AUTH=... +TWILIO_NUMBER=+1... diff --git a/signals/AGENTS.md b/signals/AGENTS.md new file mode 100644 index 0000000..f798666 --- /dev/null +++ b/signals/AGENTS.md @@ -0,0 +1,37 @@ +# `/signals` — Dhruva's agent + +You are **Dhruva**. You own **`signals/` only**. You build behavioral signal +analysis + SMS alerts on **:8002**. Read `../AGENTS.md` and +`../shared/contract.md` first. + +## Never touch +`/frontend`, `/backend`, `/ml`, `/shared`. Ask Jason for contract changes. + +## Your endpoints (:8002) +- `GET /health` +- `POST /signals/analyze` → sentiment + energy + flags + `combined_signal_score` +- `GET /signals/{user_id}` → latest signal +- `POST /alert` → Twilio SMS if `level == "red"` + +## Files +- `sentiment.py` — Claude (`claude-sonnet-4-6`) sentiment + typing dynamics; + lexical fallback when no API key. +- `alerts.py` — Twilio SMS; no-ops (console log) without creds. +- `collector.js` — `KeystrokeTracker`. **Wesley imports this** for the frontend + (a synced copy lives at `frontend/src/keystroke.js`). Keep them in sync. +- `main.py` — FastAPI wiring. + +## Run +```bash +cd signals && python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env # ANTHROPIC_API_KEY + TWILIO_* (all optional for demo) +uvicorn main:app --reload --port 8002 +``` + +## Git +```bash +git checkout feat/dhruva-sig +git add signals/ # ONLY signals/ +git commit -m "feat(signals): ..." && git push origin feat/dhruva-sig +``` diff --git a/signals/alerts.py b/signals/alerts.py new file mode 100644 index 0000000..a7a6d85 --- /dev/null +++ b/signals/alerts.py @@ -0,0 +1,31 @@ +"""Twilio SMS alerts for red-level burnout. No-ops gracefully if Twilio creds +are missing (logs to console) so the demo never crashes on the alert path.""" +from __future__ import annotations + +import os +from typing import Dict + + +def send_alert(phone: str, score: int, level: str, intervention: str) -> Dict: + body = ( + f"Pegasus check-engine 🔴\n" + f"Your burnout signal is high ({score}/100).\n{intervention}" + ) + + sid = os.getenv("TWILIO_SID") + auth = os.getenv("TWILIO_AUTH") + from_number = os.getenv("TWILIO_NUMBER") + + if not (sid and auth and from_number and phone): + print(f"[alerts] (no Twilio creds) would SMS {phone}: {body}") + return {"sent": False, "message_sid": None, "reason": "twilio_not_configured"} + + try: + from twilio.rest import Client + + client = Client(sid, auth) + msg = client.messages.create(body=body, from_=from_number, to=phone) + return {"sent": True, "message_sid": msg.sid} + except Exception as e: # pragma: no cover - network + print(f"[alerts] Twilio send failed: {e}") + return {"sent": False, "message_sid": None, "reason": str(e)} diff --git a/signals/collector.js b/signals/collector.js new file mode 100644 index 0000000..52015b8 --- /dev/null +++ b/signals/collector.js @@ -0,0 +1,37 @@ +// KeystrokeTracker — captures typing dynamics in the browser. +// Owner: Dhruva (/signals). Wesley imports this in the frontend: +// import { KeystrokeTracker } from "../../signals/collector.js"; +// (or copy into /frontend/src if cross-folder import is awkward with Vite). + +export class KeystrokeTracker { + constructor() { + this.keystrokes = []; + this.backspaces = 0; + this.startTime = null; + } + + onKeyDown(e) { + if (!this.startTime) this.startTime = Date.now(); + if (e.key === "Backspace") this.backspaces++; + this.keystrokes.push({ key: e.key, time: Date.now() }); + } + + getMetrics() { + if (!this.startTime || this.keystrokes.length === 0) { + return { typing_wpm: 0, error_rate: 0, response_time_ms: 0 }; + } + const elapsedMin = (Date.now() - this.startTime) / 1000 / 60; + const words = this.keystrokes.filter((k) => k.key === " ").length + 1; + return { + typing_wpm: elapsedMin > 0 ? Math.round(words / elapsedMin) : 0, + error_rate: Math.round((this.backspaces / this.keystrokes.length) * 100), + response_time_ms: Date.now() - this.startTime, + }; + } + + reset() { + this.keystrokes = []; + this.backspaces = 0; + this.startTime = null; + } +} diff --git a/signals/main.py b/signals/main.py new file mode 100644 index 0000000..897f561 --- /dev/null +++ b/signals/main.py @@ -0,0 +1,74 @@ +"""Pegasus Signals service — behavioral analysis + SMS alerts. + +Run (from inside /signals): + uvicorn main:app --reload --port 8002 +""" +from __future__ import annotations + +from typing import Dict, Optional + +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +import alerts +import sentiment + +app = FastAPI(title="Pegasus Signals", version="0.1.0") +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], +) + +# In-memory: latest signal per user (fine for a hackathon demo). +_LATEST: Dict[str, Dict] = {} + + +class AnalyzeIn(BaseModel): + user_id: str + text: str = "" + typing_wpm: int = 0 + error_rate: float = 0.0 + response_time_ms: int = 0 + + +class AlertIn(BaseModel): + user_id: str + phone: str + score: int + level: str + intervention: str + + +@app.get("/health") +def health(): + return {"status": "ok"} + + +@app.post("/signals/analyze") +def analyze(body: AnalyzeIn): + result = sentiment.analyze( + body.text, body.typing_wpm, body.error_rate, body.response_time_ms + ) + _LATEST[body.user_id] = result + return result + + +@app.get("/signals/{user_id}") +def latest(user_id: str): + if user_id not in _LATEST: + raise HTTPException(404, "no signals for user") + return _LATEST[user_id] + + +@app.post("/alert") +def alert(body: AlertIn): + if body.level != "red": + return {"sent": False, "message_sid": None, "reason": "level_not_red"} + return alerts.send_alert(body.phone, body.score, body.level, body.intervention) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8002) diff --git a/signals/requirements.txt b/signals/requirements.txt new file mode 100644 index 0000000..4d4eb36 --- /dev/null +++ b/signals/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +anthropic==0.42.0 +twilio==9.4.1 +python-dotenv==1.0.1 diff --git a/signals/sentiment.py b/signals/sentiment.py new file mode 100644 index 0000000..a36041d --- /dev/null +++ b/signals/sentiment.py @@ -0,0 +1,83 @@ +"""Behavioral signal analysis via Claude + typing dynamics. + +Combines sentiment/energy from the text (Claude) with typing metrics into a +single `combined_signal_score` (0-1, higher = healthier). Falls back to a +lexical heuristic when ANTHROPIC_API_KEY is unset so the demo runs offline. +""" +from __future__ import annotations + +import json +import os +from typing import Dict + +MODEL = "claude-sonnet-4-6" + +_NEG = {"tired", "exhausted", "drained", "done", "overwhelmed", "stressed", + "anxious", "numb", "empty", "cant", "can't", "nothing", "whatever", "fine"} +_POS = {"excited", "good", "great", "energized", "happy", "calm", "ready", + "love", "looking", "forward", "rested", "grateful"} + + +def _heuristic(text: str) -> Dict: + words = text.lower().split() + if not words: + return {"sentiment_score": 0.3, "energy_level": "low", "flags": ["empty_response"]} + neg = sum(w.strip(".,!?") in _NEG for w in words) + pos = sum(w.strip(".,!?") in _POS for w in words) + score = 0.5 + 0.08 * (pos - neg) + score = max(0.0, min(1.0, score)) + energy = "high" if score > 0.66 else "low" if score < 0.34 else "medium" + flags = [] + if len(words) < 4: + flags.append("very_short_response") + if neg > pos: + flags.append("negative_lexical_skew") + return {"sentiment_score": round(score, 3), "energy_level": energy, "flags": flags} + + +def _claude(text: str) -> Dict: + from anthropic import Anthropic + + client = Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY")) + resp = client.messages.create( + model=MODEL, + max_tokens=200, + messages=[{ + "role": "user", + "content": ( + f'Analyze the sentiment and energy of this check-in response: "{text}". ' + 'Return ONLY a JSON object: ' + '{"sentiment_score": 0.0-1.0, "energy_level": "low|medium|high", "flags": []} ' + 'where flags note burnout markers (e.g. "future_tense_low", ' + '"high_self_reference", "flat_affect", "very_short_response").' + ), + }], + ) + data = json.loads(resp.content[0].text.strip()) + return { + "sentiment_score": float(data["sentiment_score"]), + "energy_level": str(data["energy_level"]), + "flags": list(data.get("flags", [])), + } + + +def analyze(text: str, typing_wpm: int = 0, error_rate: float = 0.0, + response_time_ms: int = 0) -> Dict: + try: + base = _claude(text) if os.getenv("ANTHROPIC_API_KEY") else _heuristic(text) + except Exception: + base = _heuristic(text) + + # Fold typing dynamics into a combined health score. + energy_w = {"low": 0.25, "medium": 0.55, "high": 0.85}.get(base["energy_level"], 0.5) + err = max(0.0, min(1.0, error_rate / 100.0)) + wpm_health = max(0.0, min(1.0, typing_wpm / 40.0)) if typing_wpm else 0.5 + combined = 0.45 * base["sentiment_score"] + 0.25 * energy_w + 0.20 * wpm_health + 0.10 * (1 - err) + combined = round(max(0.0, min(1.0, combined)), 3) + + return { + "sentiment_score": base["sentiment_score"], + "energy_level": base["energy_level"], + "flags": base["flags"], + "combined_signal_score": combined, + } diff --git a/video/.env.example b/video/.env.example new file mode 100644 index 0000000..d578abf --- /dev/null +++ b/video/.env.example @@ -0,0 +1,9 @@ +# Video service (Rishith). Copy to .env and fill in. NEVER commit .env. +NVIDIA_STT_KEY=nvapi-... +NVIDIA_TTS_KEY=nvapi-... +# Hosted NVIDIA Riva over gRPC (defaults below). Override the TTS function-id +# to switch voices/models; default is Resemble.AI Chatterbox multilingual TTS. +# NVIDIA_RIVA_SERVER=grpc.nvcf.nvidia.com:443 +# NVIDIA_TTS_FUNCTION_ID=ddacc747-1269-4fab-bfd9-8f593dead106 # chatterbox +# NVIDIA_ASR_FUNCTION_ID=1598d209-5e27-4d3c-8079-4751568b1081 # parakeet en-US +# NVIDIA_TTS_VOICE= diff --git a/video/AGENTS.md b/video/AGENTS.md new file mode 100644 index 0000000..23941ce --- /dev/null +++ b/video/AGENTS.md @@ -0,0 +1,43 @@ +# `/video` — Rishith's agent (second folder) + +You are **Rishith**. You own **`video/`** — facial + voice stress on **:8004**. +Read `../AGENTS.md`, `../ml/AGENTS.md`, and the contract first. + +## Never touch +`/frontend` screens (Wesley), `/backend`, `/signals`, `/shared`. The backend +forwards clips here; you return facial + voice + combined_score. + +## Endpoints (:8004) +- `GET /health` +- `POST /analyze/video` — multipart `video` → `{ facial, voice, combined_score }` + +## Files +- `facial_analyzer.py` — **my own** stress logic on MediaPipe Face Mesh + landmarks (EAR-style eye openness, brow gap, lip compression, jaw). No CV API. +- `voice_analyzer.py` — **NVIDIA parakeet ASR** (hosted Riva gRPC) transcript + + librosa pitch/tremor. +- `tts.py` — **NVIDIA Chatterbox TTS** (hosted Riva gRPC) speaks interventions. +- `main.py` — FastAPI; ffmpeg extracts audio; analyzers lazy-load. + +## NVIDIA hosted speech (Riva gRPC) +Both call `grpc.nvcf.nvidia.com:443` via `nvidia-riva-client`, with metadata +`function-id` + `authorization: Bearer `. Function-ids (env-overridable): +- TTS Chatterbox: `ddacc747-1269-4fab-bfd9-8f593dead106` +- ASR Parakeet en-US: `1598d209-5e27-4d3c-8079-4751568b1081` + +## Secrets (.env, gitignored) +`NVIDIA_STT_KEY`, `NVIDIA_TTS_KEY` (the `nvapi-...` keys). Read via `os.getenv`. +No Anthropic anywhere — language/AI is Hugging Face (in `/ml`). + +## Run +```bash +cd video && python -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt && brew install ffmpeg +cp .env.example .env +uvicorn main:app --port 8004 --reload +``` + +## Git — same branch as ml: `feat/rishith-ml` +```bash +git add ml/ video/ frontend/src/services/ frontend/src/hooks/ frontend/src/types/ +``` diff --git a/video/README.md b/video/README.md new file mode 100644 index 0000000..4695c31 --- /dev/null +++ b/video/README.md @@ -0,0 +1,27 @@ +# video — facial + voice stress (Rishith, :8004) + +Analyzes a 30-60s check-in clip: +- **Facial** — my own stress heuristics on MediaPipe Face Mesh landmarks + (EAR-style eye openness, inner-brow gap, lip compression, jaw width). No CV API. +- **Voice** — NVIDIA **parakeet ASR** (hosted Riva gRPC) transcript + librosa + pitch / tremor / speaking rate. +- **TTS** — NVIDIA **Chatterbox** (hosted Riva gRPC) speaks the intervention (`tts.py`). + +Returns `{ facial, voice, combined_score }`. The backend forwards the user's +clip here, then feeds the result into `/ml`'s scorer. + +## Run +```bash +cd video && python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt # mediapipe, opencv, librosa +brew install ffmpeg # audio extraction +cp .env.example .env # NVIDIA_STT_KEY, NVIDIA_TTS_KEY +uvicorn main:app --port 8004 --reload +``` +Models load lazily on first `/analyze/video`, so `/health` is instant. + +## Notes +- `forced_smile` / `gaze_stability` are placeholders pending iris tracking + + AU6/AU12 mismatch. +- Without NVIDIA keys, voice returns an empty transcript + librosa-only acoustics + (still produces a stress signal); facial works fully offline. diff --git a/video/facial_analyzer.py b/video/facial_analyzer.py new file mode 100644 index 0000000..4f77fa3 --- /dev/null +++ b/video/facial_analyzer.py @@ -0,0 +1,85 @@ +# Custom facial stress analysis (Rishith). MediaPipe Face Mesh is used as a +# LIBRARY (468 landmarks, runs locally, free, no API) — ALL stress +# interpretation logic below is my own, derived from landmark geometry. +import cv2 +import mediapipe as mp +import numpy as np + + +class FacialStressAnalyzer: + def __init__(self): + self.mesh = mp.solutions.face_mesh.FaceMesh( + static_image_mode=False, + max_num_faces=1, + refine_landmarks=True, + min_detection_confidence=0.5, + ) + + def _dist(self, lm, a, b): + return float(np.hypot(lm[a].x - lm[b].x, lm[a].y - lm[b].y)) + + def analyze_frame(self, frame): + res = self.mesh.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + if not res.multi_face_landmarks: + return None + lm = res.multi_face_landmarks[0].landmark + + # Eye openness (EAR-style): vertical / horizontal eye ratio. + left_eye = self._dist(lm, 159, 145) / (self._dist(lm, 33, 133) + 1e-6) + right_eye = self._dist(lm, 386, 374) / (self._dist(lm, 362, 263) + 1e-6) + eye_open = (left_eye + right_eye) / 2 + + # Brow furrow: inner-brow gap (smaller = furrowed = stress). + brow_gap = self._dist(lm, 70, 300) + brow_furrow = max(0.0, 1 - brow_gap * 5) + + # Lip compression: lip height / width (smaller = pressed = tension). + lip_h = self._dist(lm, 13, 14) + lip_w = self._dist(lm, 61, 291) + lip_compression = max(0.0, 1 - (lip_h / (lip_w + 1e-6)) * 4) + + # Jaw tension proxy: jaw width relative to face height. + jaw = self._dist(lm, 234, 454) / (self._dist(lm, 10, 152) + 1e-6) + + return { + "eye_openness": eye_open, + "brow_furrow": brow_furrow, + "lip_compression": lip_compression, + "jaw_clench": jaw, + } + + def analyze_video(self, frames): + results, blinks, prev = [], 0, None + for f in frames: + r = self.analyze_frame(f) + if r: + results.append(r) + if prev is not None and prev > 0.25 and r["eye_openness"] < 0.12: + blinks += 1 + prev = r["eye_openness"] + if not results: + return {"error": "no face detected", "facial_stress_score": 0} + + dur_min = len(frames) / 3 / 60 # ~3fps sampled (every 10th of 30fps) + brow = float(np.mean([r["brow_furrow"] for r in results])) + lip = float(np.mean([r["lip_compression"] for r in results])) + jaw = float(np.mean([r["jaw_clench"] for r in results])) + eye = 1 - float(np.mean([r["eye_openness"] for r in results])) + stress = min(int((brow * 0.3 + lip * 0.25 + jaw * 0.25 + eye * 0.2) * 100), 100) + affect = "negative" if (brow > 0.5 and lip > 0.4) else "flat" if (brow < 0.2 and lip < 0.2) else "neutral" + + return { + "facial_stress_score": stress, + "eye_indicators": { + "blink_rate_per_min": round(blinks / max(dur_min, 0.1), 1), + "eye_openness": round(float(np.mean([r["eye_openness"] for r in results])), 3), + "gaze_stability": 0.7, # placeholder — needs iris tracking + }, + "facial_indicators": { + "brow_furrow": round(brow, 3), + "lip_compression": round(lip, 3), + "jaw_clench": round(jaw, 3), + "forced_smile": False, # placeholder — needs AU6+AU12 mismatch + "overall_affect": affect, + }, + } diff --git a/video/main.py b/video/main.py new file mode 100644 index 0000000..6cc2be0 --- /dev/null +++ b/video/main.py @@ -0,0 +1,102 @@ +"""Pegasus Video service (:8004) — facial + voice stress from a check-in clip. + +Run (from inside /video): + uvicorn main:app --port 8004 --reload + +Endpoints: + GET /health + POST /analyze/video multipart 'video' -> { facial, voice, combined_score } + +Heavy analyzers (MediaPipe / librosa) load lazily on the first call, so the +service boots instantly and /health works before models are ready. +Requires ffmpeg on the system (audio extraction): brew install ffmpeg +""" +from __future__ import annotations + +import os +import tempfile + +from fastapi import FastAPI, File, UploadFile +from fastapi.middleware.cors import CORSMiddleware + +app = FastAPI(title="Pegasus Video") +app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) + +_facial = None +_voice = None + + +def _get_facial(): + global _facial + if _facial is None: + from facial_analyzer import FacialStressAnalyzer + + _facial = FacialStressAnalyzer() + return _facial + + +def _get_voice(): + global _voice + if _voice is None: + from voice_analyzer import VoiceStressAnalyzer + + _voice = VoiceStressAnalyzer() + return _voice + + +@app.get("/health") +def health(): + return {"status": "video running"} + + +@app.post("/analyze/video") +async def analyze_video(video: UploadFile = File(...)): + import cv2 + + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmp: + tmp.write(await video.read()) + path = tmp.name + audio = path.replace(".mp4", ".wav") + try: + # Sample every 10th frame (~3fps from 30fps source). + cap = cv2.VideoCapture(path) + frames, i = [], 0 + while cap.isOpened(): + ret, frame = cap.read() + if not ret: + break + if i % 10 == 0: + frames.append(frame) + i += 1 + cap.release() + + try: + facial_result = _get_facial().analyze_video(frames) if frames else {"error": "no frames", "facial_stress_score": 0} + except Exception as e: + facial_result = {"error": f"facial failed: {e}", "facial_stress_score": 0} + + try: + os.system(f"ffmpeg -i {path} -vn -ar 16000 {audio} -y -loglevel quiet") + voice_result = _get_voice().analyze_audio(audio) + except Exception as e: + voice_result = {"error": f"voice failed: {e}", "pitch_mean_hz": 0, "voice_tremor": False, "pause_frequency": 0} + + fs = facial_result.get("facial_stress_score", 0) + vs = ( + min(voice_result.get("pitch_mean_hz", 150) / 300, 1) * 40 + + (20 if voice_result.get("voice_tremor") else 0) + + min(voice_result.get("pause_frequency", 0) / 10, 1) * 40 + ) + return {"facial": facial_result, "voice": voice_result, "combined_score": int(fs * 0.6 + vs * 0.4)} + finally: + for p in (path, audio): + try: + os.unlink(p) + except OSError: + pass + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8004) diff --git a/video/requirements.txt b/video/requirements.txt new file mode 100644 index 0000000..899fd83 --- /dev/null +++ b/video/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +python-multipart==0.0.20 +numpy==1.26.4 +opencv-python==4.10.0.84 +mediapipe==0.10.18 +librosa==0.10.2.post1 +nvidia-riva-client==2.17.0 +python-dotenv==1.0.1 +# Needs ffmpeg on the system for audio extraction: brew install ffmpeg diff --git a/video/tts.py b/video/tts.py new file mode 100644 index 0000000..13efbc8 --- /dev/null +++ b/video/tts.py @@ -0,0 +1,45 @@ +# NVIDIA Chatterbox Text-to-Speech (Rishith) — speak the intervention aloud. +# Hosted NVIDIA Riva over gRPC (grpc.nvcf.nvidia.com:443) via nvidia-riva-client. +# Default function-id = Resemble.AI Chatterbox multilingual TTS. +import os +import wave + +NVIDIA_TTS_KEY = os.getenv("NVIDIA_TTS_KEY") +RIVA_SERVER = os.getenv("NVIDIA_RIVA_SERVER", "grpc.nvcf.nvidia.com:443") +# Chatterbox multilingual TTS (build.nvidia.com/resembleai/chatterbox-multilingual-tts). +TTS_FUNCTION_ID = os.getenv("NVIDIA_TTS_FUNCTION_ID", "ddacc747-1269-4fab-bfd9-8f593dead106") +TTS_VOICE = os.getenv("NVIDIA_TTS_VOICE", "") # empty -> model default +SAMPLE_RATE = int(os.getenv("NVIDIA_TTS_SAMPLE_RATE", "44100")) + + +def speak(text: str, out_path: str = "intervention.wav"): + """Synthesize speech to a WAV file. Returns the path, or None on failure.""" + if not NVIDIA_TTS_KEY or not text: + return None + try: + import riva.client + + auth = riva.client.Auth( + uri=RIVA_SERVER, + use_ssl=True, + metadata_args=[ + ["function-id", TTS_FUNCTION_ID], + ["authorization", f"Bearer {NVIDIA_TTS_KEY}"], + ], + ) + tts = riva.client.SpeechSynthesisService(auth) + resp = tts.synthesize( + text, + voice_name=TTS_VOICE or None, + language_code="en-US", + encoding=riva.client.AudioEncoding.LINEAR_PCM, + sample_rate_hz=SAMPLE_RATE, + ) + with wave.open(out_path, "wb") as w: # resp.audio is raw PCM16 mono + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(SAMPLE_RATE) + w.writeframes(resp.audio) + return out_path + except Exception: + return None diff --git a/video/voice_analyzer.py b/video/voice_analyzer.py new file mode 100644 index 0000000..14ee21d --- /dev/null +++ b/video/voice_analyzer.py @@ -0,0 +1,71 @@ +# Voice stress (Rishith). NVIDIA parakeet ASR (hosted Riva gRPC) for the +# transcript + librosa for acoustic stress. Degrades gracefully (empty +# transcript / zeros) so a check-in never hard-fails. +import os + +import librosa +import numpy as np + +NVIDIA_STT_KEY = os.getenv("NVIDIA_STT_KEY") +RIVA_SERVER = os.getenv("NVIDIA_RIVA_SERVER", "grpc.nvcf.nvidia.com:443") +# Parakeet CTC 1.1B ASR (en-US) hosted function. +ASR_FUNCTION_ID = os.getenv("NVIDIA_ASR_FUNCTION_ID", "1598d209-5e27-4d3c-8079-4751568b1081") + + +class VoiceStressAnalyzer: + def transcribe_nvidia(self, audio_path: str) -> str: + if not NVIDIA_STT_KEY: + return "" + try: + import riva.client + + auth = riva.client.Auth( + uri=RIVA_SERVER, + use_ssl=True, + metadata_args=[ + ["function-id", ASR_FUNCTION_ID], + ["authorization", f"Bearer {NVIDIA_STT_KEY}"], + ], + ) + asr = riva.client.ASRService(auth) + config = riva.client.RecognitionConfig( + language_code="en-US", + max_alternatives=1, + enable_automatic_punctuation=True, + ) + riva.client.add_audio_file_specs_to_config(config, audio_path) + with open(audio_path, "rb") as f: + data = f.read() + resp = asr.offline_recognize(data, config) + return " ".join( + r.alternatives[0].transcript for r in resp.results if r.alternatives + ).strip() + except Exception: + return "" + + def analyze_audio(self, audio_path: str) -> dict: + transcript = self.transcribe_nvidia(audio_path) + try: + y, sr = librosa.load(audio_path) + pitches, mags = librosa.piptrack(y=y, sr=sr) + vals = [ + float(pitches[mags[:, t].argmax(), t]) + for t in range(pitches.shape[1]) + if pitches[mags[:, t].argmax(), t] > 0 + ] + dur = float(librosa.get_duration(y=y, sr=sr)) + words = len(transcript.split()) + pitch_std = float(np.std(vals)) if vals else 0.0 + return { + "transcript": transcript, + "pitch_mean_hz": float(np.mean(vals)) if vals else 0.0, + "pitch_variability": pitch_std, + "speaking_rate_wpm": (words / max(dur, 1)) * 60, + "pause_frequency": 0, + "voice_tremor": bool(pitch_std > 50), + } + except Exception: + return { + "transcript": transcript, "pitch_mean_hz": 0, "pitch_variability": 0, + "speaking_rate_wpm": 0, "pause_frequency": 0, "voice_tremor": False, + }